From 38bd77e51ac89629e793293794c2631e1c7ebedd Mon Sep 17 00:00:00 2001 From: Tian Jin Date: Tue, 18 Aug 2020 08:59:50 +0800 Subject: [PATCH] Build llvm nightly prereq images & test onnx-mlir against it regularly. (#268) * Create build-prereq-docker-x86-nightly.yml * Update build-prereq-docker-x86-nightly.yml * Update build-prereq-docker-x86-nightly.yml * Update build-prereq-docker-x86-nightly.yml * Update build-prereq-docker-x86-nightly.yml * Update build-prereq-docker-x86-nightly.yml * Create clone-mlir-master.sh * Update build-prereq-docker-x86-nightly.yml * Update prereq.s390x.Dockerfile * trigger jenkins docker build on s390x * Update prereq.s390x.Dockerfile * Update build-prereq-docker-x86-nightly.yml Run on a schedule (every sunday). * Trigger jenkins * Trigger jenkins build. * check-mlir does not succeed * Trigger jenkins * Trigger Jenkins * Trigger jenkins * Enable nightly build. * Update .travis.yml * Using curly braces to prevent overparse. * Specify base image as a build argument. * Correct repo name. * Build a regular x86 image. * Resotre github action to build night images only. * normalize amd64 -> x64. * In ppc64le, select base image from arg * Under amd64, choose base image from arg. * Using export to make CPU_ARCH persist. * Fix variable definition * Fix bash script. * Rename amd64.Dockerfile to x86.Dockerfile * Allow failures when testing with nightly llvm. * Nit. * Fix allow_failures clause in Travis. * Update .travis.yml * typo fix: fase -> false --- .../build-prereq-docker-x86-nightly.yml | 26 +++++++++++++++++++ .travis.yml | 26 +++++++++++++++---- docker/ppc64le.Dockerfile | 3 ++- docker/prereq.s390x.Dockerfile | 4 +-- docker/s390x.Dockerfile | 3 ++- docker/{amd64.Dockerfile => x86.Dockerfile} | 3 ++- utils/clone-mlir-master.sh | 1 + 7 files changed, 56 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/build-prereq-docker-x86-nightly.yml rename docker/{amd64.Dockerfile => x86.Dockerfile} (83%) create mode 100644 utils/clone-mlir-master.sh diff --git a/.github/workflows/build-prereq-docker-x86-nightly.yml b/.github/workflows/build-prereq-docker-x86-nightly.yml new file mode 100644 index 0000000..874acdc --- /dev/null +++ b/.github/workflows/build-prereq-docker-x86-nightly.yml @@ -0,0 +1,26 @@ +name: Build x86 LLVM-Nightly Prereq Image + +on: + # Allow manually triggering rebuild. + workflow_dispatch: + # Run every week (on Sunday). + schedule: + - cron: '0 0 * * SUN' + +jobs: + build-x86-llvm-nightly-prereq-img: + runs-on: ubuntu-latest + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + - name: build a docker container for the latest llvm/mlir. + env: + DOCKER_HUB_TOKEN: ${{ secrets.DOCKER_HUB_TOKEN }} + run: | + # Swaps utils/clone-mlir for a different shell script that clones the latest version of llvm/mlir. + mv $GITHUB_WORKSPACE/utils/clone-mlir.sh $GITHUB_WORKSPACE/utils/clone-mlir-backup.sh + cp $GITHUB_WORKSPACE/utils/clone-mlir-master.sh $GITHUB_WORKSPACE/utils/clone-mlir.sh + # Build docker image for x86 arch, with the latest llvm/mlir and subsequently publish it. + docker build --tag onnxmlirczar/onnx-mlir-llvmimage:x86-nightly -f $GITHUB_WORKSPACE/.github/workflows/prereq.Dockerfile $GITHUB_WORKSPACE/utils + docker login -u onnxmlirczar -p "$DOCKER_HUB_TOKEN" + docker push onnxmlirczar/onnx-mlir-llvmimage:x86-nightly diff --git a/.travis.yml b/.travis.yml index b11eff6..0508c38 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,11 +7,27 @@ arch: - s390x - ppc64le - amd64 +env: + - NIGHTLY=true + - NIGHTLY=false +jobs: + allow_failures: + - env: NIGHTLY=true script: - - echo "CPU Architecture is " $TRAVIS_CPU_ARCH + - export CPU_ARCH=$TRAVIS_CPU_ARCH + - if [ "$CPU_ARCH" == "amd64" ] ; then + export CPU_ARCH="x86"; + fi + - echo "CPU Architecture is " $CPU_ARCH - echo "commit is " $TRAVIS_COMMIT - df -h - - docker build --tag onnx-mlir-build:initial --build-arg BASE="onnxmlirczar/onnx-mlir-image:$TRAVIS_CPU_ARCH" -f ./docker/$TRAVIS_CPU_ARCH.Dockerfile ./docker + - if [ "$NIGHTLY" = true ] ; then + echo "Using nightly llvm-mlir image, build may fail."; + docker build --tag onnx-mlir-build:initial --build-arg BASE="onnxmlirczar/onnx-mlir-llvmimage:${CPU_ARCH}-nightly" -f ./docker/$CPU_ARCH.Dockerfile ./docker; + else + echo "Using latest compatible llvm-mlir image, build should not fail."; + docker build --tag onnx-mlir-build:initial --build-arg BASE="onnxmlirczar/onnx-mlir-llvmimage:$CPU_ARCH" -f ./docker/$CPU_ARCH.Dockerfile ./docker; + fi - docker run -itd --name build onnx-mlir-build:initial - echo "docker is running now" - df -h @@ -22,12 +38,12 @@ script: - echo "about to execute build inside docker" - docker exec build df -h - docker exec build compile-onnx-mlir.sh - - docker commit build onnxmlirczar/onnx-mlir-build:$TRAVIS_CPU_ARCH + - docker commit build onnxmlirczar/onnx-mlir-build:$CPU_ARCH after_success: - - if [ $TRAVIS_PULL_REQUEST == false ] && [ $TRAVIS_BRANCH == "master" ]; then + - if [ $TRAVIS_PULL_REQUEST == false ] && [ $TRAVIS_BRANCH == "master" ] && [ $NIGHTLY == false ]; then echo "Pushing new build to docker hub"; docker login -u onnxmlirczar -p "$DOCKER_HUB_TOKEN"; - docker push onnxmlirczar/onnx-mlir-build:$TRAVIS_CPU_ARCH; + docker push onnxmlirczar/onnx-mlir-build:$CPU_ARCH; else echo "Skip pushing build"; fi diff --git a/docker/ppc64le.Dockerfile b/docker/ppc64le.Dockerfile index 9f1e8cb..ee56e2c 100644 --- a/docker/ppc64le.Dockerfile +++ b/docker/ppc64le.Dockerfile @@ -1,5 +1,6 @@ +ARG BASE # Grab pre-built LLVM from docker hub -FROM onnxmlirczar/onnx-mlir-llvmimage:ppc64le +FROM $BASE WORKDIR /build/bin WORKDIR /build diff --git a/docker/prereq.s390x.Dockerfile b/docker/prereq.s390x.Dockerfile index 75eede8..2047b24 100644 --- a/docker/prereq.s390x.Dockerfile +++ b/docker/prereq.s390x.Dockerfile @@ -39,5 +39,5 @@ RUN cmake -G Ninja ../llvm \ -DLLVM_ENABLE_ASSERTIONS=ON \ -DLLVM_ENABLE_RTTI=ON -#RUN cmake --build . --target -- ${MAKEFLAGS} -#RUN cmake --build . --target check-mlir +RUN cmake --build . --target -- ${MAKEFLAGS} +# RUN cmake --build . --target check-mlir diff --git a/docker/s390x.Dockerfile b/docker/s390x.Dockerfile index 09f7ee9..ee56e2c 100644 --- a/docker/s390x.Dockerfile +++ b/docker/s390x.Dockerfile @@ -1,5 +1,6 @@ +ARG BASE # Grab pre-built LLVM from docker hub -FROM onnxmlirczar/onnx-mlir-llvmimage:s390x +FROM $BASE WORKDIR /build/bin WORKDIR /build diff --git a/docker/amd64.Dockerfile b/docker/x86.Dockerfile similarity index 83% rename from docker/amd64.Dockerfile rename to docker/x86.Dockerfile index b3ff612..ee56e2c 100644 --- a/docker/amd64.Dockerfile +++ b/docker/x86.Dockerfile @@ -1,5 +1,6 @@ +ARG BASE # Grab pre-built LLVM from docker hub -FROM onnxmlirczar/onnx-mlir-llvmimage:initial +FROM $BASE WORKDIR /build/bin WORKDIR /build diff --git a/utils/clone-mlir-master.sh b/utils/clone-mlir-master.sh new file mode 100644 index 0000000..a2bd0ec --- /dev/null +++ b/utils/clone-mlir-master.sh @@ -0,0 +1 @@ +git clone https://github.com/llvm/llvm-project.git