From 536a20695f8d01be1df7adc3ecc9d9c22b0ed0de Mon Sep 17 00:00:00 2001 From: Kevin O'Brien Date: Fri, 29 May 2020 17:22:13 -0400 Subject: [PATCH] Multi build - add build for IBM Systems Z and P (#157) * Update .travis.yml * add Z and Power builds * Build (#1) * prereq build for Z and Power * prereq build for Z and Power * prereq build for Z and Power * Update MLIR version * Update MLIR version * Update MLIR version * Update MLIR version * Update MLIR version * Update MLIR version * Update MLIR version * Update MLIR version * Update MLIR version * Update MLIR version * Update MLIR version * Update MLIR version * Update MLIR version * Update MLIR version * Update MLIR version * Update MLIR version * Update MLIR version * test build * test build * Update MLIR version * test build * test build * test build * test build * test build * test build * test build * test build * test build * test build * test build * test build * test build * test build --- .travis.yml | 28 +++++++++++++--- docker/{Dockerfile => amd64.Dockerfile} | 2 +- docker/ppc64le.Dockerfile | 12 +++++++ docker/prereq.amd64.Dockerfile | 43 +++++++++++++++++++++++++ docker/prereq.ppc64le.Dockerfile | 43 +++++++++++++++++++++++++ docker/prereq.s390x.Dockerfile | 43 +++++++++++++++++++++++++ docker/s390x.Dockerfile | 12 +++++++ 7 files changed, 178 insertions(+), 5 deletions(-) rename docker/{Dockerfile => amd64.Dockerfile} (84%) create mode 100644 docker/ppc64le.Dockerfile create mode 100644 docker/prereq.amd64.Dockerfile create mode 100644 docker/prereq.ppc64le.Dockerfile create mode 100644 docker/prereq.s390x.Dockerfile create mode 100644 docker/s390x.Dockerfile diff --git a/.travis.yml b/.travis.yml index afd69f9..3ee14b2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,20 +3,40 @@ services: - docker language: cpp dist: bionic -script: - - docker build --tag onnx-mlir-build:initial ./docker +arch: + - s390x + - ppc64le + - amd64 +script: + - echo "CPU Architecture is " $TRAVIS_CPU_ARCH + - echo "commit is " $TRAVIS_COMMIT + - df -h + - if [ "$TRAVIS_COMMIT_MESSAGE" == "Update MLIR version" ]; then + echo "Building Prereq"; + docker build --tag onnxmlirczar/onnx-mlir-llvmimage:$TRAVIS_CPU_ARCH -f ./docker/prereq.$TRAVIS_CPU_ARCH.Dockerfile ./utils; + docker login -u onnxmlirczar -p 143f1da2-332f-45a1-8587-d6cb07c13230 + docker push onnxmlirczar/onnx-mlir-llvmimage:$TRAVIS_CPU_ARCH + exit; + else + echo "Regular Build"; + fi + - docker build --tag onnx-mlir-build:initial --build-arg BASE="onnxmlirczar/onnx-mlir-image:$TRAVIS_CPU_ARCH" -f ./docker/$TRAVIS_CPU_ARCH.Dockerfile ./docker - docker run -itd --name build onnx-mlir-build:initial + - echo "docker is running now" + - df -h - docker cp $(pwd) build:/build/onnx-mlir - cd docker - chmod a+x compile-onnx-mlir.sh - docker cp compile-onnx-mlir.sh build:/usr/bin/compile-onnx-mlir.sh - - docker exec -it build compile-onnx-mlir.sh + - echo "about to execute build inside docker" + - docker exec build df -h + - docker exec build compile-onnx-mlir.sh - docker commit build onnxmlirczar/onnx-mlir-build:latest after_success: - if [ $TRAVIS_PULL_REQUEST == false ] && [ $TRAVIS_BRANCH == "master" ]; then echo "Pushing new build to docker hub"; docker login -u onnxmlirczar -p 143f1da2-332f-45a1-8587-d6cb07c13230; - docker push onnxmlirczar/onnx-mlir-build:latest; + docker push onnxmlirczar/onnx-mlir-build:$TRAVIS_CPU_ARCH; else echo "Skip pushing build"; fi diff --git a/docker/Dockerfile b/docker/amd64.Dockerfile similarity index 84% rename from docker/Dockerfile rename to docker/amd64.Dockerfile index bf85f22..b3ff612 100644 --- a/docker/Dockerfile +++ b/docker/amd64.Dockerfile @@ -1,4 +1,4 @@ -# Grab pre-built LLVM from github packages +# Grab pre-built LLVM from docker hub FROM onnxmlirczar/onnx-mlir-llvmimage:initial WORKDIR /build/bin diff --git a/docker/ppc64le.Dockerfile b/docker/ppc64le.Dockerfile new file mode 100644 index 0000000..9f1e8cb --- /dev/null +++ b/docker/ppc64le.Dockerfile @@ -0,0 +1,12 @@ +# Grab pre-built LLVM from docker hub +FROM onnxmlirczar/onnx-mlir-llvmimage:ppc64le + +WORKDIR /build/bin +WORKDIR /build +ENV HOME=/build +ENV PYENV_ROOT=$HOME/.pyenv +ENV PATH=$PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH +RUN pyenv global 3.7.0 +RUN pyenv rehash + +ENV PATH=$PATH:/build/bin diff --git a/docker/prereq.amd64.Dockerfile b/docker/prereq.amd64.Dockerfile new file mode 100644 index 0000000..5b184ad --- /dev/null +++ b/docker/prereq.amd64.Dockerfile @@ -0,0 +1,43 @@ +FROM ubuntu:focal + +WORKDIR /build + +# install stuff that is needed for compiling LLVM, MLIR and ONNX +RUN apt-get update +RUN DEBIAN_FRONTEND=noninteractive apt-get install -y git cmake ninja-build libprotobuf-dev protobuf-compiler +RUN DEBIAN_FRONTEND=noninteractive \ + apt-get update && \ + apt-get -y install \ + make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev \ + libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev xz-utils \ + libffi-dev liblzma-dev +RUN git clone git://github.com/yyuu/pyenv.git .pyenv +RUN git clone https://github.com/yyuu/pyenv-virtualenv.git ~/.pyenv/plugins/pyenv-virtualenv + +ENV HOME=/build +ENV PYENV_ROOT=$HOME/.pyenv +ENV PATH=$PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH + +RUN pyenv install 3.7.0 + +RUN pyenv global 3.7.0 +RUN pyenv rehash + +# first install MLIR in llvm-project +RUN mkdir bin +ENV PATH=$PATH:/build/bin +COPY clone-mlir.sh bin/clone-mlir.sh +RUN chmod a+x bin/clone-mlir.sh +RUN clone-mlir.sh + +WORKDIR /build/llvm-project/build +RUN cmake -G Ninja ../llvm \ + -DLLVM_ENABLE_PROJECTS=mlir \ + -DLLVM_BUILD_EXAMPLES=ON \ + -DLLVM_TARGETS_TO_BUILD="host" \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_ENABLE_ASSERTIONS=ON \ + -DLLVM_ENABLE_RTTI=ON + +RUN cmake --build . --target -- ${MAKEFLAGS} +RUN cmake --build . --target check-mlir diff --git a/docker/prereq.ppc64le.Dockerfile b/docker/prereq.ppc64le.Dockerfile new file mode 100644 index 0000000..5b184ad --- /dev/null +++ b/docker/prereq.ppc64le.Dockerfile @@ -0,0 +1,43 @@ +FROM ubuntu:focal + +WORKDIR /build + +# install stuff that is needed for compiling LLVM, MLIR and ONNX +RUN apt-get update +RUN DEBIAN_FRONTEND=noninteractive apt-get install -y git cmake ninja-build libprotobuf-dev protobuf-compiler +RUN DEBIAN_FRONTEND=noninteractive \ + apt-get update && \ + apt-get -y install \ + make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev \ + libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev xz-utils \ + libffi-dev liblzma-dev +RUN git clone git://github.com/yyuu/pyenv.git .pyenv +RUN git clone https://github.com/yyuu/pyenv-virtualenv.git ~/.pyenv/plugins/pyenv-virtualenv + +ENV HOME=/build +ENV PYENV_ROOT=$HOME/.pyenv +ENV PATH=$PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH + +RUN pyenv install 3.7.0 + +RUN pyenv global 3.7.0 +RUN pyenv rehash + +# first install MLIR in llvm-project +RUN mkdir bin +ENV PATH=$PATH:/build/bin +COPY clone-mlir.sh bin/clone-mlir.sh +RUN chmod a+x bin/clone-mlir.sh +RUN clone-mlir.sh + +WORKDIR /build/llvm-project/build +RUN cmake -G Ninja ../llvm \ + -DLLVM_ENABLE_PROJECTS=mlir \ + -DLLVM_BUILD_EXAMPLES=ON \ + -DLLVM_TARGETS_TO_BUILD="host" \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_ENABLE_ASSERTIONS=ON \ + -DLLVM_ENABLE_RTTI=ON + +RUN cmake --build . --target -- ${MAKEFLAGS} +RUN cmake --build . --target check-mlir diff --git a/docker/prereq.s390x.Dockerfile b/docker/prereq.s390x.Dockerfile new file mode 100644 index 0000000..75eede8 --- /dev/null +++ b/docker/prereq.s390x.Dockerfile @@ -0,0 +1,43 @@ +FROM ubuntu:focal + +WORKDIR /build + +# install stuff that is needed for compiling LLVM, MLIR and ONNX +RUN apt-get update +RUN DEBIAN_FRONTEND=noninteractive apt-get install -y git cmake ninja-build libprotobuf-dev protobuf-compiler +RUN DEBIAN_FRONTEND=noninteractive \ + apt-get update && \ + apt-get -y install \ + make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev \ + libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev xz-utils \ + libffi-dev liblzma-dev +RUN git clone git://github.com/yyuu/pyenv.git .pyenv +RUN git clone https://github.com/yyuu/pyenv-virtualenv.git ~/.pyenv/plugins/pyenv-virtualenv + +ENV HOME=/build +ENV PYENV_ROOT=$HOME/.pyenv +ENV PATH=$PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH + +RUN pyenv install 3.7.0 + +RUN pyenv global 3.7.0 +RUN pyenv rehash + +# first install MLIR in llvm-project +RUN mkdir bin +ENV PATH=$PATH:/build/bin +COPY clone-mlir.sh bin/clone-mlir.sh +RUN chmod a+x bin/clone-mlir.sh +RUN clone-mlir.sh + +WORKDIR /build/llvm-project/build +RUN cmake -G Ninja ../llvm \ + -DLLVM_ENABLE_PROJECTS=mlir \ + -DLLVM_BUILD_EXAMPLES=ON \ + -DLLVM_TARGETS_TO_BUILD="host" \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_ENABLE_ASSERTIONS=ON \ + -DLLVM_ENABLE_RTTI=ON + +#RUN cmake --build . --target -- ${MAKEFLAGS} +#RUN cmake --build . --target check-mlir diff --git a/docker/s390x.Dockerfile b/docker/s390x.Dockerfile new file mode 100644 index 0000000..09f7ee9 --- /dev/null +++ b/docker/s390x.Dockerfile @@ -0,0 +1,12 @@ +# Grab pre-built LLVM from docker hub +FROM onnxmlirczar/onnx-mlir-llvmimage:s390x + +WORKDIR /build/bin +WORKDIR /build +ENV HOME=/build +ENV PYENV_ROOT=$HOME/.pyenv +ENV PATH=$PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH +RUN pyenv global 3.7.0 +RUN pyenv rehash + +ENV PATH=$PATH:/build/bin