[CI] Improved CI (dmlc#141)

* change ci * update ci * update ci * update ci * update ci * update ci * update ci * update ci * update ci * update ci * update ci * update ci * update ci * update ci * update ci * update ci * nx package * update ci * update ci * update ci * fix * mx dockerfile by zhengda * python3.6->3.5 * update ci image * add tutorial test * fix ci * fix ssl problem * minor change * small fix on traversal utest * fix syntax * add matplotlib in image * fix * update ci * update ci
aliysefian · Nov 12, 2018 · a95459e · a95459e
1 parent 7cbb83d
commit a95459e
Show file tree

Hide file tree

Showing 12 changed files with 247 additions and 186 deletions.
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -1,176 +1,150 @@
 #!/usr/bin/env groovy
 
 def init_git_submodule() {
-    sh 'git submodule init'
-    sh 'git submodule update'
+  sh "git submodule init"
+  sh "git submodule update"
 }
 
 def setup() {
-    sh 'easy_install nose'
-    init_git_submodule()
+  init_git_submodule()
 }
 
 def build_dgl() {
-    sh 'if [ -d build ]; then rm -rf build; fi; mkdir build'
-    dir('python') {
-        sh 'python3 setup.py install'
-    }
-    dir ('build') {
-        sh 'cmake ..'
-        sh 'make -j4'
-    }
+  sh "if [ -d build ]; then rm -rf build; fi; mkdir build"
+  dir("python") {
+    sh "python3 setup.py install"
+  }
+  dir ("build") {
+    sh "cmake .."
+    sh "make -j4"
+  }
 }
 
-def pytorch_unit_test() {
-    withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build"]) {
-        sh 'nosetests tests -v --with-xunit'
-        sh 'nosetests tests/pytorch -v --with-xunit'
-        sh 'nosetests tests/graph_index -v --with-xunit'
-    }
+def pytorch_unit_test(dev) {
+  withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build", "PYTHONPATH=${env.WORKSPACE}/python"]) {
+    sh "python3 -m nose -v --with-xunit tests"
+    sh "python3 -m nose -v --with-xunit tests/pytorch"
+    sh "python3 -m nose -v --with-xunit tests/graph_index"
+  }
 }
 
-def mxnet_unit_test() {
-    withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build"]) {
-        sh 'nosetests tests/mxnet -v --with-xunit'
-    }
+def mxnet_unit_test(dev) {
+  withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build", "PYTHONPATH=${env.WORKSPACE}/python"]) {
+    sh "python3 -m nose -v --with-xunit tests/mxnet"
+  }
 }
 
 def example_test(dev) {
-    dir ('tests/scripts') {
-        withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build"]) {
-            sh "./test_examples.sh ${dev}"
-        }
+  dir ("tests/scripts") {
+    withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build", "PYTHONPATH=${env.WORKSPACE}/python"]) {
+      sh "./task_example_test.sh ${dev}"
+    }
+  }
+}
+
+def pytorch_tutorials() {
+  dir ("tests/scripts") {
+    withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build", "PYTHONPATH=${env.WORKSPACE}/python"]) {
+      sh "./task_tutorial_test.sh"
     }
+  }
 }
 
 pipeline {
-    agent none
-    stages {
-        stage('Lint Check') {
-            agent {
-                docker {
-                    image 'lingfanyu/dgl-lint'
-                }
+  agent none
+  stages {
+    stage("Lint Check") {
+      agent { docker { image "dgllib/dgl-ci-lint" } }
+      steps {
+        init_git_submodule()
+        sh "tests/scripts/task_lint.sh"
+      }
+    }
+    stage("Build") {
+      parallel {
+        stage("CPU Build") {
+          agent { docker { image "dgllib/dgl-ci-cpu" } }
+          steps {
+            setup()
+            build_dgl()
+          }
+        }
+        stage("GPU Build") {
+          agent {
+            docker {
+              image "dgllib/dgl-ci-gpu"
+              args "--runtime nvidia"
+            }
+          }
+          steps {
+            setup()
+            build_dgl()
+          }
+        }
+        stage("MXNet CPU Build (temp)") {
+          agent { docker { image "dgllib/dgl-ci-mxnet-cpu" } }
+          steps {
+            setup()
+            build_dgl()
+          }
+        }
+      }
+    }
+    stage("Test") {
+      parallel {
+        stage("Pytorch CPU") {
+          agent { docker { image "dgllib/dgl-ci-cpu" } }
+          stages {
+            stage("TH CPU unittest") {
+              steps { pytorch_unit_test("CPU") }
             }
-            stages {
-                stage('CHECK') {
-                    steps {
-                        init_git_submodule()
-                        sh 'tests/scripts/task_lint.sh'
-                    }
-                }
+            stage("TH CPU example test") {
+              steps { example_test("CPU") }
             }
+          }
+          post {
+            always { junit "*.xml" }
+          }
         }
-        stage('Build and Test on Pytorch') {
-            parallel {
-                stage('CPU') {
-                    agent {
-                        docker {
-                            image 'lingfanyu/dgl-cpu'
-                        }
-                    }
-                    stages {
-                        stage('SETUP') {
-                            steps {
-                                setup()
-                            }
-                        }
-                        stage('BUILD') {
-                            steps {
-                                build_dgl()
-                            }
-                        }
-                        stage('UNIT TEST') {
-                            steps {
-                                pytorch_unit_test()
-                            }
-                        }
-                        stage('EXAMPLE TEST') {
-                            steps {
-                                example_test('CPU')
-                            }
-                        }
-                    }
-                    post {
-                        always {
-                            junit '*.xml'
-                        }
-                    }
-                }
-                stage('GPU') {
-                    agent {
-                        docker {
-                            image 'lingfanyu/dgl-gpu'
-                            args '--runtime nvidia'
-                        }
-                    }
-                    stages {
-                        stage('SETUP') {
-                            steps {
-                                setup()
-                            }
-                        }
-                        stage('BUILD') {
-                            steps {
-                                build_dgl()
-                            }
-                        }
-                        stage('UNIT TEST') {
-                            steps {
-                                pytorch_unit_test()
-                            }
-                        }
-                        stage('EXAMPLE TEST') {
-                            steps {
-                                example_test('GPU')
-                            }
-                        }
-                    }
-                    post {
-                        always {
-                            junit '*.xml'
-                        }
-                    }
-                }
+        stage("Pytorch GPU") {
+          agent {
+            docker {
+              image "dgllib/dgl-ci-gpu"
+              args "--runtime nvidia"
             }
+          }
+          stages {
+            // TODO: have GPU unittest
+            //stage("TH GPU unittest") {
+            //  steps { pytorch_unit_test("GPU") }
+            //}
+            stage("TH GPU example test") {
+              steps { example_test("GPU") }
+            }
+          }
+          // TODO: have GPU unittest
+          //post {
+          //  always { junit "*.xml" }
+          //}
         }
-        stage('Build and Test on MXNet') {
-            parallel {
-                stage('CPU') {
-                    agent {
-                        docker {
-                            image 'zhengda1936/dgl-mxnet-cpu:v3'
-                        }
-                    }
-                    stages {
-                        stage('SETUP') {
-                            steps {
-                                setup()
-                            }
-                        }
-                        stage('BUILD') {
-                            steps {
-                                build_dgl()
-                            }
-                        }
-                        stage('UNIT TEST') {
-                            steps {
-                                mxnet_unit_test()
-                            }
-                        }
-                        stage('EXAMPLE TEST') {
-                            steps {
-                                example_test('CPU')
-                            }
-                        }
-                    }
-                    post {
-                        always {
-                            junit '*.xml'
-                        }
-                    }
-                }
+        stage("MXNet CPU") {
+          agent { docker { image "dgllib/dgl-ci-mxnet-cpu" } }
+          stages {
+            stage("MX Unittest") {
+              steps { mxnet_unit_test("CPU") }
             }
+          }
+          post {
+            always { junit "*.xml" }
+          }
         }
+      }
+    }
+    stage("Doc") {
+      agent { docker { image "dgllib/dgl-ci-cpu" } }
+      steps {
+        pytorch_tutorials()
+      }
     }
+  }
 }
diff --git a/docker/Dockerfile.ci_cpu_mxnet b/docker/Dockerfile.ci_cpu_mxnet
@@ -0,0 +1,25 @@
+# CI docker GPU env
+FROM nvidia/cuda:9.0-cudnn7-devel
+
+# Base scripts
+RUN apt-get update --fix-missing
+
+COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh
+RUN bash /install/ubuntu_install_core.sh
+
+COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
+RUN bash /install/ubuntu_install_python.sh
+
+COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh
+RUN bash /install/ubuntu_install_python_package.sh
+
+COPY install/ubuntu_install_mxnet_cpu.sh /install/ubuntu_install_mxnet_cpu.sh
+RUN bash /install/ubuntu_install_mxnet_cpu.sh
+
+# Environment variables
+ENV PATH=/usr/local/nvidia/bin:${PATH}
+ENV PATH=/usr/local/cuda/bin:${PATH}
+ENV CPLUS_INCLUDE_PATH=/usr/local/cuda/include:${CPLUS_INCLUDE_PATH}
+ENV C_INCLUDE_PATH=/usr/local/cuda/include:${C_INCLUDE_PATH}
+ENV LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/nvidia/lib64:${LIBRARY_PATH}
+ENV LD_LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/nvidia/lib64:${LD_LIBRARY_PATH}
diff --git a/docker/Dockerfile.ci_gpu_mxnet b/docker/Dockerfile.ci_gpu_mxnet
@@ -0,0 +1,25 @@
+# CI docker GPU env
+FROM nvidia/cuda:9.0-cudnn7-devel
+
+# Base scripts
+RUN apt-get update --fix-missing
+
+COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh
+RUN bash /install/ubuntu_install_core.sh
+
+COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
+RUN bash /install/ubuntu_install_python.sh
+
+COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh
+RUN bash /install/ubuntu_install_python_package.sh
+
+COPY install/ubuntu_install_mxnet_gpu.sh /install/ubuntu_install_mxnet_gpu.sh
+RUN bash /install/ubuntu_install_mxnet_gpu.sh
+
+ # Environment variables
+ENV PATH=/usr/local/nvidia/bin:${PATH}
+ENV PATH=/usr/local/cuda/bin:${PATH}
+ENV CPLUS_INCLUDE_PATH=/usr/local/cuda/include:${CPLUS_INCLUDE_PATH}
+ENV C_INCLUDE_PATH=/usr/local/cuda/include:${C_INCLUDE_PATH}
+ENV LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/nvidia/lib64:${LIBRARY_PATH}
+ENV LD_LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/nvidia/lib64:${LD_LIBRARY_PATH}
diff --git a/docker/README.md b/docker/README.md
@@ -8,3 +8,9 @@ docker build -t dgl-gpu -f Dockerfile.ci_gpu .
 
 ### Lint image
 docker build -t dgl-lint -f Dockerfile.ci_lint .
+
+### CPU MXNet image
+docker build -t dgl-mxnet-cpu -f Dockerfile.ci_cpu_mxnet .
+
+### GPU MXNet image
+docker build -t dgl-mxnet-gpu -f Dockerfile.ci_gpu_mxnet .
diff --git a/docker/install/ubuntu_install_core.sh b/docker/install/ubuntu_install_core.sh
@@ -1,3 +1,4 @@
 # install libraries for building c++ core on ubuntu
 apt update && apt install -y --no-install-recommends --force-yes \
-        apt-utils git build-essential make cmake wget unzip sudo libz-dev libxml2-dev
+        apt-utils git build-essential make cmake wget unzip sudo \
+        libz-dev libxml2-dev libopenblas-dev libopencv-dev ca-certificates
diff --git a/docker/install/ubuntu_install_mxnet_cpu.sh b/docker/install/ubuntu_install_mxnet_cpu.sh
@@ -0,0 +1,4 @@
+git clone --recursive -b dgl_graph https://github.com/zheng-da/incubator-mxnet.git
+cd incubator-mxnet
+make -j4 USE_OPENCV=1 USE_BLAS=openblas USE_MKLDNN=1 USE_DIST_KVSTORE=1
+pip3 install -e python
diff --git a/docker/install/ubuntu_install_mxnet_gpu.sh b/docker/install/ubuntu_install_mxnet_gpu.sh
@@ -0,0 +1,5 @@
+git clone --recursive -b dgl_graph https://github.com/zheng-da/incubator-mxnet.git
+cd incubator-mxnet
+make -j4 USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda \
+  USE_CUDNN=1 USE_MKLDNN=1 USE_DIST_KVSTORE=1
+pip3 install -e python