Build llvm nightly prereq images & test onnx-mlir against it regularly. (#268)

* Create build-prereq-docker-x86-nightly.yml

* Update build-prereq-docker-x86-nightly.yml

* Update build-prereq-docker-x86-nightly.yml

* Update build-prereq-docker-x86-nightly.yml

* Update build-prereq-docker-x86-nightly.yml

* Update build-prereq-docker-x86-nightly.yml

* Create clone-mlir-master.sh

* Update build-prereq-docker-x86-nightly.yml

* Update prereq.s390x.Dockerfile

* trigger jenkins docker build on s390x

* Update prereq.s390x.Dockerfile

* Update build-prereq-docker-x86-nightly.yml

Run on a schedule (every sunday).

* Trigger jenkins

* Trigger jenkins build.

* check-mlir does not succeed

* Trigger jenkins

* Trigger Jenkins

* Trigger jenkins

* Enable nightly build.

* Update .travis.yml

* Using curly braces to prevent overparse.

* Specify base image as a build argument.

* Correct repo name.

* Build a regular x86 image.

* Resotre github action to build night images only.

* normalize amd64 -> x64.

* In ppc64le, select base image from arg

* Under amd64, choose base image from arg.

* Using export to make CPU_ARCH persist.

* Fix variable definition

* Fix bash script.

* Rename amd64.Dockerfile to x86.Dockerfile

* Allow failures when testing with nightly llvm.

* Nit.

* Fix allow_failures clause in Travis.

* Update .travis.yml

* typo fix: fase -> false

.github/workflows/build-prereq-docker-x86-nightly.yml

@@ -0,0 +1,26 @@
+name: Build x86 LLVM-Nightly Prereq Image
+
+on: 
+  # Allow manually triggering rebuild.
+  workflow_dispatch:
+  # Run every week (on Sunday). 
+  schedule:
+    - cron:  '0 0 * * SUN'
+
+jobs:
+  build-x86-llvm-nightly-prereq-img:
+    runs-on: ubuntu-latest
+    steps:
+    # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+    - uses: actions/checkout@v2
+    - name: build a docker container for the latest llvm/mlir.
+      env:
+        DOCKER_HUB_TOKEN: ${{ secrets.DOCKER_HUB_TOKEN }}
+      run: |
+        # Swaps utils/clone-mlir for a different shell script that clones the latest version of llvm/mlir.
+        mv $GITHUB_WORKSPACE/utils/clone-mlir.sh $GITHUB_WORKSPACE/utils/clone-mlir-backup.sh
+        cp $GITHUB_WORKSPACE/utils/clone-mlir-master.sh $GITHUB_WORKSPACE/utils/clone-mlir.sh
+        # Build docker image for x86 arch, with the latest llvm/mlir and subsequently publish it.
+        docker build --tag onnxmlirczar/onnx-mlir-llvmimage:x86-nightly -f $GITHUB_WORKSPACE/.github/workflows/prereq.Dockerfile $GITHUB_WORKSPACE/utils
+        docker login -u onnxmlirczar -p "$DOCKER_HUB_TOKEN" 
+        docker push onnxmlirczar/onnx-mlir-llvmimage:x86-nightly

.travis.yml

@@ -7,11 +7,27 @@ arch:
   - s390x
   - ppc64le
   - amd64
+env:
+  - NIGHTLY=true
+  - NIGHTLY=false
+jobs:
+  allow_failures:
+    - env: NIGHTLY=true
 script:
-  - echo "CPU Architecture is " $TRAVIS_CPU_ARCH
+  - export CPU_ARCH=$TRAVIS_CPU_ARCH
+  - if [ "$CPU_ARCH" == "amd64" ] ; then
+      export CPU_ARCH="x86";
+    fi
+  - echo "CPU Architecture is " $CPU_ARCH
   - echo "commit is " $TRAVIS_COMMIT
   - df -h
-  - docker build --tag onnx-mlir-build:initial --build-arg BASE="onnxmlirczar/onnx-mlir-image:$TRAVIS_CPU_ARCH"  -f ./docker/$TRAVIS_CPU_ARCH.Dockerfile ./docker
+  - if [ "$NIGHTLY" = true ] ; then
+      echo "Using nightly llvm-mlir image, build may fail.";
+      docker build --tag onnx-mlir-build:initial --build-arg BASE="onnxmlirczar/onnx-mlir-llvmimage:${CPU_ARCH}-nightly"  -f ./docker/$CPU_ARCH.Dockerfile ./docker;
+    else
+      echo "Using latest compatible llvm-mlir image, build should not fail.";
+      docker build --tag onnx-mlir-build:initial --build-arg BASE="onnxmlirczar/onnx-mlir-llvmimage:$CPU_ARCH"  -f ./docker/$CPU_ARCH.Dockerfile ./docker;
+    fi
   - docker run -itd --name build onnx-mlir-build:initial
   - echo "docker is running now"
   - df -h
@@ -22,12 +38,12 @@ script:
   - echo "about to execute build inside docker"
   - docker exec build df -h
   - docker exec build compile-onnx-mlir.sh
-  - docker commit build onnxmlirczar/onnx-mlir-build:$TRAVIS_CPU_ARCH
+  - docker commit build onnxmlirczar/onnx-mlir-build:$CPU_ARCH
 after_success: 
-  - if [ $TRAVIS_PULL_REQUEST == false ] && [ $TRAVIS_BRANCH == "master" ]; then
+  - if [ $TRAVIS_PULL_REQUEST == false ] && [ $TRAVIS_BRANCH == "master" ] && [ $NIGHTLY == false ]; then
             echo "Pushing new build to docker hub";
             docker login -u onnxmlirczar -p "$DOCKER_HUB_TOKEN";
-            docker push onnxmlirczar/onnx-mlir-build:$TRAVIS_CPU_ARCH;
+            docker push onnxmlirczar/onnx-mlir-build:$CPU_ARCH;
         else 
             echo "Skip pushing build"; 
         fi

docker/ppc64le.Dockerfile

@@ -1,5 +1,6 @@
+ARG BASE
 # Grab pre-built LLVM from docker hub
-FROM onnxmlirczar/onnx-mlir-llvmimage:ppc64le
+FROM $BASE
 
 WORKDIR /build/bin
 WORKDIR /build

docker/prereq.s390x.Dockerfile

@@ -39,5 +39,5 @@ RUN cmake -G Ninja ../llvm \
    -DLLVM_ENABLE_ASSERTIONS=ON \
    -DLLVM_ENABLE_RTTI=ON
 
-#RUN cmake --build . --target -- ${MAKEFLAGS}
+RUN cmake --build . --target -- ${MAKEFLAGS}
 # RUN cmake --build . --target check-mlir

docker/s390x.Dockerfile

@@ -1,5 +1,6 @@
+ARG BASE
 # Grab pre-built LLVM from docker hub
-FROM onnxmlirczar/onnx-mlir-llvmimage:s390x
+FROM $BASE
 
 WORKDIR /build/bin
 WORKDIR /build

docker/x86.Dockerfile

@@ -1,5 +1,6 @@
+ARG BASE
 # Grab pre-built LLVM from docker hub
-FROM onnxmlirczar/onnx-mlir-llvmimage:initial
+FROM $BASE
 
 WORKDIR /build/bin
 WORKDIR /build

utils/clone-mlir-master.sh

@@ -0,0 +1 @@
+git clone https://github.com/llvm/llvm-project.git