[CI] Fix CI bugs (dmlc#592)

* new jenkins script * fix ci * poke ci * new config * new config * new config * poke ci * poke ci * poke ci * poke ci * poke ci * poke ci * poke ci * update docker image; poke ci * poke ci * poke ci * poke ci * poke ci * poke ci * poke ci * poke ci * poke ci * poke ci * poke ci * poke ci * update image * update image * fix * Windows CI support * typo * typo*2 * missed sh * typo*3 * missed dir
cnavasca95 · Jun 1, 2019 · 5492994 · 5492994
1 parent f728839
commit 5492994
Show file tree

Hide file tree

Showing 14 changed files with 208 additions and 151 deletions.
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -1,97 +1,117 @@
 #!/usr/bin/env groovy
 
-def init_git_submodule() {
+dgl_linux_libs = "build/libdgl.so, build/runUnitTests, python/dgl/_ffi/_cy3/core.cpython-35m-x86_64-linux-gnu.so"
+// Currently DGL on Windows is not working with Cython yet
+dgl_windows_libs = "build\\dgl.dll, build\\runUnitTests.exe"
+
+def init_git() {
+  sh "rm -rf *"
+  checkout scm
   sh "git submodule init"
   sh "git submodule update"
+  sh "ls -lh"
 }
 
-def init_git_submodule_win64() {
+def init_git_win64() {
+  checkout scm
   bat "git submodule init"
   bat "git submodule update"
 }
 
-def build_dgl() {
+// pack libraries for later use
+def pack_lib(name, libs) {
+  echo "Packing ${libs} into ${name}"
+  stash includes: libs, name: name
+}
+
+// unpack libraries saved before
+def unpack_lib(name, libs) {
+  unstash name
+  echo "Unpacked ${libs} from ${name}"
+}
+
+def build_dgl_linux(dev) {
+  init_git()
   sh "bash tests/scripts/build_dgl.sh"
+  pack_lib("dgl-${dev}", dgl_linux_libs)
 }
 
-def build_dgl_win64() {
+def build_dgl_win64(dev) {
   /* Assuming that Windows slaves are already configured with MSBuild VS2017,
    * CMake and Python/pip/setuptools etc. */
+  init_git_win64()
   bat "CALL tests\\scripts\\build_dgl.bat"
+  pack_lib("dgl-${dev}", dgl_windows_libs)
 }
 
-def cpp_unit_test_linux(){
+def cpp_unit_test_linux() {
+  init_git()
+  unpack_lib("dgl-cpu", dgl_linux_libs)
   sh "bash tests/scripts/task_cpp_unit_test.sh"
 }
 
-def cpp_unit_test_windows(){
+def cpp_unit_test_win64() {
+  init_git_win64()
+  unpack_lib("dgl-cpu", dgl_windows_libs)
   bat "CALL tests\\scripts\\task_cpp_unit_test.bat"
 }
 
 def unit_test(backend, dev) {
-  withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build", "PYTHONPATH=${env.WORKSPACE}/python", "DGLBACKEND=${backend}"]) {
+  init_git()
+  unpack_lib("dgl-${dev}", dgl_linux_libs)
+  timeout(time: 2, unit: 'MINUTES') {
     sh "bash tests/scripts/task_unit_test.sh ${backend}"
   }
 }
 
 def unit_test_win64(backend, dev) {
-  withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}\\build", "PYTHONPATH=${env.WORKSPACE}\\python", "DGLBACKEND=${backend}"]) {
+  init_git_win64()
+  unpack_lib("dgl-${dev}", dgl_windows_libs)
+  timeout(time: 2, unit: 'MINUTES') {
     bat "CALL tests\\scripts\\task_unit_test.bat ${backend}"
   }
 }
 
 def example_test(backend, dev) {
-  withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build", "PYTHONPATH=${env.WORKSPACE}/python", "DGLBACKEND=${backend}"]) {
-    dir ("tests/scripts") {
-      sh "bash task_example_test.sh ${dev}"
-    }
+  init_git()
+  unpack_lib("dgl-${dev}", dgl_linux_libs)
+  timeout(time: 20, unit: 'MINUTES') {
+    sh "bash tests/scripts/task_example_test.sh ${dev}"
   }
 }
 
 def example_test_win64(backend, dev) {
-  withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}\\build", "PYTHONPATH=${env.WORKSPACE}\\python", "DGLBACKEND=${backend}"]) {
-    dir ("tests\\scripts") {
-      bat "CALL task_example_test ${dev}"
-    }
+  init_git_win64()
+  unpack_lib("dgl-${dev}", dgl_windows_libs)
+  timeout(time: 20, unit: 'MINUTES') {
+    bat "CALL tests\\scripts\\task_example_test.bat ${dev}"
   }
 }
 
-def pytorch_tutorials() {
-  withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build", "PYTHONPATH=${env.WORKSPACE}/python"]) {
-    dir ("tests/scripts") {
-      sh "bash task_pytorch_tutorial_test.sh"
-    }
+def tutorial_test(backend) {
+  init_git()
+  unpack_lib("dgl-cpu", dgl_linux_libs)
+  timeout(time: 20, unit: 'MINUTES') {
+    sh "bash tests/scripts/task_${backend}_tutorial_test.sh"
   }
 }
 
-def mxnet_tutorials() {
-  withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build", "PYTHONPATH=${env.WORKSPACE}/python", "DGLBACKEND=mxnet"]) {
-    dir("tests/scripts") {
-      sh "bash task_mxnet_tutorial_test.sh"
-    }
-  }
-}
 pipeline {
   agent none
   stages {
     stage("Lint Check") {
-      agent {
-        docker { image "dgllib/dgl-ci-lint" }
-      }
+      agent { docker { image "dgllib/dgl-ci-lint" } }
       steps {
-        init_git_submodule()
+        init_git()
         sh "bash tests/scripts/task_lint.sh"
       }
     }
     stage("Build") {
       parallel {
         stage("CPU Build") {
-          agent {
-            docker { image "dgllib/dgl-ci-cpu" }
-          }
+          agent { docker { image "dgllib/dgl-ci-cpu" } }
           steps {
-            init_git_submodule()
-            build_dgl()
+            build_dgl_linux("cpu")
           }
         }
         stage("GPU Build") {
@@ -102,141 +122,111 @@ pipeline {
             }
           }
           steps {
-            init_git_submodule()
-            build_dgl()
-          }
-        }
-        stage("MXNet CPU Build (temp)") {
-          agent {
-            docker { image "dgllib/dgl-ci-mxnet-cpu" }
-          }
-          steps {
-            init_git_submodule()
-            build_dgl()
+            sh "nvidia-smi"
+            build_dgl_linux("gpu")
           }
         }
-        stage("CPU Build (Win64/PyTorch)") {
-          agent {
-            label "windows"
-          }
+        stage("CPU Build (Win64)") {
+          // Windows build machines are manually added to Jenkins master with
+          // "windows" label as permanent agents.
+          agent { label "windows" }
           steps {
-            init_git_submodule_win64()
-            build_dgl_win64()
+            build_dgl_win64("cpu")
           }
         }
+        // Currently we don't have Windows GPU build machines
       }
     }
     stage("Test") {
       parallel {
-        stage("CPP Test"){
-          stages{
-            stage("CPP Unit Test Linux"){
-              agent {
-                docker {image "dgllib/dgl-ci-cpu"}
-              }
-              steps { 
-                init_git_submodule()
-                cpp_unit_test_linux() 
-              }
-            }
-            stage("CPP Unit Test Windows"){
-              agent {
-                label "windows"
-              }
-              steps {
-                init_git_submodule_win64()
-                cpp_unit_test_windows()
-              }
-            }
+        stage("C++ CPU") {
+          agent { docker { image "dgllib/dgl-ci-cpu" } }
+          steps {
+            cpp_unit_test_linux()
           }
         }
-        stage("Pytorch CPU") {
-          agent {
-            docker { image "dgllib/dgl-ci-cpu" }
+        stage("C++ CPU (Win64)") {
+          agent { label "windows" }
+          steps {
+            cpp_unit_test_win64()
           }
+        }
+        stage("Torch CPU") {
+          agent { docker { image "dgllib/dgl-ci-cpu" } }
           stages {
-            stage("TH CPU unittest") {
-              steps { unit_test("pytorch", "CPU") }
+            stage("Unit test") {
+              steps {
+                unit_test("pytorch", "cpu")
+              }
             }
-            stage("TH CPU example test") {
-              steps { example_test("pytorch", "CPU") }
+            stage("Example test") {
+              steps {
+                example_test("pytorch", "cpu")
+              }
+            }
+            stage("Tutorial test") {
+              steps {
+                tutorial_test("pytorch")
+              }
             }
-          }
-          post {
-            always { junit "*.xml" }
           }
         }
-        stage("Pytorch CPU (Windows)") {
+        stage("Torch CPU (Win64)") {
           agent { label "windows" }
           stages {
-            stage("TH CPU Win64 unittest") {
-              steps { unit_test_win64("pytorch", "CPU") }
+            stage("Unit test") {
+              steps {
+                unit_test_win64("pytorch", "cpu")
+              }
             }
-            stage("TH CPU Win64 example test") {
-              steps { example_test_win64("pytorch", "CPU") }
+            stage("Example test") {
+              steps {
+                example_test_win64("pytorch", "cpu")
+              }
             }
           }
-          post {
-            always { junit "*.xml" }
-          }
         }
-        stage("Pytorch GPU") {
+        stage("Torch GPU") {
           agent {
             docker {
               image "dgllib/dgl-ci-gpu"
               args "--runtime nvidia"
             }
           }
           stages {
-            // TODO: have GPU unittest
-            //stage("TH GPU unittest") {
-            //  steps { pytorch_unit_test("GPU") }
-            //}
-            stage("TH GPU example test") {
-              steps { example_test("pytorch", "GPU") }
+            stage("Unit test") {
+              steps {
+                //unit_test("pytorch", "gpu")
+                sh "nvidia-smi"
+              }
+            }
+            stage("Example test") {
+              steps {
+                example_test("pytorch", "gpu")
+              }
             }
           }
-          // TODO: have GPU unittest
-          //post {
-          //  always { junit "*.xml" }
-          //}
         }
         stage("MXNet CPU") {
-          agent {
-            docker { image "dgllib/dgl-ci-mxnet-cpu" }
-          }
+          agent { docker { image "dgllib/dgl-ci-cpu" } }
           stages {
-            stage("MX Unittest") {
-              options {
-                  timeout(time: 5, unit: 'MINUTES') 
+            stage("Unit test") {
+              steps {
+                unit_test("mxnet", "cpu")
               }
-              steps { unit_test("mxnet", "CPU") }
             }
-          }
-          post {
-            always { junit "*.xml" }
-          }
-        }
-      }
-    }
-    stage("Doc") {
-      parallel {
-        stage("TH Tutorial") {
-          agent {
-            docker { image "dgllib/dgl-ci-cpu" }
-          }
-          steps {
-            pytorch_tutorials()
+            //stage("Example test") {
+            //  steps {
+            //    unit_test("pytorch", "cpu")
+            //  }
+            //}
+            //stage("Tutorial test") {
+            //  steps {
+            //    tutorial_test("mxnet")
+            //  }
+            //}
           }
         }
-        //stage("MX Tutorial") {
-        //  agent {
-        //    docker { image "dgllib/dgl-ci-mxnet-cpu" }
-        //  }
-        //  steps {
-        //    mxnet_tutorials()
-        //  }
-        //}
       }
     }
   }

diff --git a/docker/Dockerfile.ci_cpu b/docker/Dockerfile.ci_cpu
@@ -10,6 +10,14 @@ RUN bash /install/ubuntu_install_core.sh
 COPY install/ubuntu_install_build.sh /install/ubuntu_install_build.sh
 RUN bash /install/ubuntu_install_build.sh
 
+# ANTLR deps
+COPY install/ubuntu_install_java.sh /install/ubuntu_install_java.sh
+RUN bash /install/ubuntu_install_java.sh
+
+COPY install/ubuntu_install_antlr.sh /install/ubuntu_install_antlr.sh
+RUN bash /install/ubuntu_install_antlr.sh
+
+# python
 COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
 RUN bash /install/ubuntu_install_python.sh
 
@@ -18,3 +26,6 @@ RUN bash /install/ubuntu_install_python_package.sh
 
 COPY install/ubuntu_install_torch.sh /install/ubuntu_install_torch.sh
 RUN bash /install/ubuntu_install_torch.sh
+
+COPY install/ubuntu_install_mxnet_cpu.sh /install/ubuntu_install_mxnet_cpu.sh
+RUN bash /install/ubuntu_install_mxnet_cpu.sh
diff --git a/docker/Dockerfile.ci_gpu b/docker/Dockerfile.ci_gpu
@@ -10,6 +10,14 @@ RUN bash /install/ubuntu_install_core.sh
 COPY install/ubuntu_install_build.sh /install/ubuntu_install_build.sh
 RUN bash /install/ubuntu_install_build.sh
 
+# ANTLR deps
+COPY install/ubuntu_install_java.sh /install/ubuntu_install_java.sh
+RUN bash /install/ubuntu_install_java.sh
+
+COPY install/ubuntu_install_antlr.sh /install/ubuntu_install_antlr.sh
+RUN bash /install/ubuntu_install_antlr.sh
+
+# python
 COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
 RUN bash /install/ubuntu_install_python.sh