[Backend] TF backend (dmlc#978)

* tf * add builtin support * fiix * pytest * fix * fix * fix some bugs * fix selecting * fix todo * fix test * fix test fail in tf * fix * fix * fix gather row * fix gather row * log backend * fix gather row * fix gather row * fix for pytorch * fix * fix * fix * fix * fix * fix tests * fix * fix * fix * fix * fix * fix * fix convert * fix * fix * fix * fix inplace * add alignment setting * add debug option * Revert "add alignment setting" This reverts commit ec63fb3. * tf ci * fix lint * fix lint * add tfdlpack * fix type * add env * fix backend * fix * fix tests * remove one_hot * remove comment * remove comment * fix * use pip to install all * fix test * fix base * fix * fix * add skip * upgrade cmake * change version * change ci * fix * fix * fix * fix * fix seg fault * fix * fix python version * fix * try fix * fix * fix * tf takes longer time in ci * change py version * fix * fix * fix oom * change kg env * change kg env * 啊啊啊啊啊啊啊啊啊啊啊啊啊啊啊啊啊啊啊啊啊啊啊啊啊啊啊啊啊啊啊啊啊啊啊啊啊啊 * 我再也不搞各种乱七八糟环境了…… * use pytest * Chang image
ghostintheshellarise · Dec 20, 2019 · d30a69b · d30a69b
1 parent cf9ba90
commit d30a69b
Show file tree

Hide file tree

Showing 46 changed files with 1,479 additions and 244 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -25,6 +25,10 @@ endif()
 dgl_option(USE_CUDA "Build with CUDA" OFF)
 dgl_option(USE_OPENMP "Build with OpenMP" ON)
 dgl_option(BUILD_CPP_TEST "Build cpp unittest executables" OFF)
+# Set debug compile option for gdb, only happens when -DCMAKE_BUILD_TYPE=DEBUG
+if (NOT MSVC)
+ set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 -g3 -ggdb")
+endif(NOT MSVC)
 
 if(USE_CUDA)
   message(STATUS "Build with CUDA support")
@@ -83,7 +87,6 @@ endif(USE_OPENMP)
 
 # configure minigun
 add_definitions(-DENABLE_PARTIAL_FRONTIER=0)  # disable minigun partial frontier compile
-
 # Source file lists
 file(GLOB DGL_SRC
   src/*.cc

diff --git a/Jenkinsfile b/Jenkinsfile
@@ -1,6 +1,6 @@
 #!/usr/bin/env groovy
 
-dgl_linux_libs = "build/libdgl.so, build/runUnitTests, python/dgl/_ffi/_cy3/core.cpython-35m-x86_64-linux-gnu.so"
+dgl_linux_libs = "build/libdgl.so, build/runUnitTests, python/dgl/_ffi/_cy3/core.cpython-36m-x86_64-linux-gnu.so"
 // Currently DGL on Windows is not working with Cython yet
 dgl_win64_libs = "build\\dgl.dll, build\\runUnitTests.exe"
 
@@ -56,7 +56,7 @@ def cpp_unit_test_win64() {
 def unit_test_linux(backend, dev) {
   init_git()
   unpack_lib("dgl-${dev}-linux", dgl_linux_libs)
-  timeout(time: 5, unit: 'MINUTES') {
+  timeout(time: 10, unit: 'MINUTES') {
     sh "bash tests/scripts/task_unit_test.sh ${backend} ${dev}"
   }
 }
@@ -119,7 +119,7 @@ pipeline {
     stage("Build") {
       parallel {
         stage("CPU Build") {
-          agent { docker { image "dgllib/dgl-ci-cpu" } }
+          agent { docker { image "dgllib/dgl-ci-cpu:conda" } }
           steps {
             build_dgl_linux("cpu")
           }
@@ -132,7 +132,7 @@ pipeline {
         stage("GPU Build") {
           agent {
             docker {
-              image "dgllib/dgl-ci-gpu"
+              image "dgllib/dgl-ci-gpu:conda"
               args "--runtime nvidia"
             }
           }
@@ -165,7 +165,7 @@ pipeline {
     stage("Test") {
       parallel {
         stage("C++ CPU") {
-          agent { docker { image "dgllib/dgl-ci-cpu" } }
+          agent { docker { image "dgllib/dgl-ci-cpu:conda" } }
           steps {
             cpp_unit_test_linux()
           }
@@ -186,8 +186,43 @@ pipeline {
             }
           }
         }
+        stage("Tensorflow CPU") {
+          agent { docker { image "dgllib/dgl-ci-cpu:conda" } }
+          stages {
+            stage("Unit test") {
+              steps {
+                unit_test_linux("tensorflow", "cpu")
+              }
+            }
+          }
+          post {
+            always {
+              cleanWs disableDeferredWipeout: true, deleteDirs: true
+            }
+          }
+        }
+        stage("Tensorflow GPU") {
+          agent { 
+            docker { 
+              image "dgllib/dgl-ci-gpu:conda" 
+              args "--runtime nvidia"
+            }
+          }
+          stages {
+            stage("Unit test") {
+              steps {
+                unit_test_linux("tensorflow", "gpu")
+              }
+            }
+          }
+          post {
+            always {
+              cleanWs disableDeferredWipeout: true, deleteDirs: true
+            }
+          }
+        }
         stage("Torch CPU") {
-          agent { docker { image "dgllib/dgl-ci-cpu" } }
+          agent { docker { image "dgllib/dgl-ci-cpu:conda" } }
           stages {
             stage("Unit test") {
               steps {
@@ -234,7 +269,7 @@ pipeline {
         stage("Torch GPU") {
           agent {
             docker {
-              image "dgllib/dgl-ci-gpu"
+              image "dgllib/dgl-ci-gpu:conda"
               args "--runtime nvidia"
             }
           }
@@ -258,7 +293,7 @@ pipeline {
           }
         }
         stage("MXNet CPU") {
-          agent { docker { image "dgllib/dgl-ci-cpu" } }
+          agent { docker { image "dgllib/dgl-ci-cpu:conda" } }
           stages {
             stage("Unit test") {
               steps {
@@ -280,7 +315,7 @@ pipeline {
         stage("MXNet GPU") {
           agent {
             docker {
-              image "dgllib/dgl-ci-gpu"
+              image "dgllib/dgl-ci-gpu:conda"
               args "--runtime nvidia"
             }
           }
@@ -303,7 +338,7 @@ pipeline {
     stage("App") {
       parallel {
         stage("Knowledge Graph CPU") {
-          agent { docker { image "dgllib/dgl-ci-cpu:torch-1.2.0" } }
+          agent { docker { image "dgllib/dgl-ci-cpu:conda" } }
           stages {
             stage("Torch test") {
               steps {
@@ -325,7 +360,7 @@ pipeline {
         stage("Knowledge Graph GPU") {
           agent {
             docker {
-              image "dgllib/dgl-ci-gpu:torch-1.2.0"
+              image "dgllib/dgl-ci-gpu:conda"
               args "--runtime nvidia"
             }
           }

diff --git a/apps/kg/eval.py b/apps/kg/eval.py
@@ -2,17 +2,18 @@
 from dataloader import get_dataset
 
 import argparse
-import torch.multiprocessing as mp
 import os
 import logging
 import time
 import pickle
 
 backend = os.environ.get('DGLBACKEND', 'pytorch')
 if backend.lower() == 'mxnet':
+    import multiprocessing as mp
     from train_mxnet import load_model_from_checkpoint
     from train_mxnet import test
 else:
+    import torch.multiprocessing as mp
     from train_pytorch import load_model_from_checkpoint
     from train_pytorch import test
 

diff --git a/docker/Dockerfile.ci_cpu b/docker/Dockerfile.ci_cpu
@@ -10,22 +10,19 @@ RUN bash /install/ubuntu_install_core.sh
 COPY install/ubuntu_install_build.sh /install/ubuntu_install_build.sh
 RUN bash /install/ubuntu_install_build.sh
 
-# ANTLR deps
-COPY install/ubuntu_install_java.sh /install/ubuntu_install_java.sh
-RUN bash /install/ubuntu_install_java.sh
+# python
+COPY install/ubuntu_install_conda.sh /install/ubuntu_install_conda.sh
+RUN bash /install/ubuntu_install_conda.sh
 
-COPY install/ubuntu_install_antlr.sh /install/ubuntu_install_antlr.sh
-RUN bash /install/ubuntu_install_antlr.sh
+ENV CONDA_ALWAYS_YES="true"
 
-# python
-COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
-RUN bash /install/ubuntu_install_python.sh
+COPY install/conda_env/torch_cpu.yml /install/conda_env/torch_cpu.yml
+RUN ["/bin/bash", "-i", "-c", "conda env create -f /install/conda_env/torch_cpu.yml"]
 
-COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh
-RUN bash /install/ubuntu_install_python_package.sh
+COPY install/conda_env/tensorflow_cpu.yml /install/conda_env/tensorflow_cpu.yml
+RUN ["/bin/bash", "-i", "-c", "conda env create -f /install/conda_env/tensorflow_cpu.yml"]
 
-COPY install/ubuntu_install_torch.sh /install/ubuntu_install_torch.sh
-RUN bash /install/ubuntu_install_torch.sh
+COPY install/conda_env/mxnet_cpu.yml /install/conda_env/mxnet_cpu.yml
+RUN ["/bin/bash", "-i", "-c", "conda env create -f /install/conda_env/mxnet_cpu.yml"]
 
-COPY install/ubuntu_install_mxnet_cpu.sh /install/ubuntu_install_mxnet_cpu.sh
-RUN bash /install/ubuntu_install_mxnet_cpu.sh
+ENV CONDA_ALWAYS_YES=
diff --git a/docker/Dockerfile.ci_cpu_torch_1.2.0 b/docker/Dockerfile.ci_cpu_torch_1.2.0
@@ -10,25 +10,13 @@ RUN bash /install/ubuntu_install_core.sh
 COPY install/ubuntu_install_build.sh /install/ubuntu_install_build.sh
 RUN bash /install/ubuntu_install_build.sh
 
-# ANTLR deps
-COPY install/ubuntu_install_java.sh /install/ubuntu_install_java.sh
-RUN bash /install/ubuntu_install_java.sh
-
-COPY install/ubuntu_install_antlr.sh /install/ubuntu_install_antlr.sh
-RUN bash /install/ubuntu_install_antlr.sh
-
 # python
-COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
-RUN bash /install/ubuntu_install_python.sh
-
-COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh
-RUN bash /install/ubuntu_install_python_package.sh
+COPY install/ubuntu_install_conda.sh /install/ubuntu_install_conda.sh
+RUN bash /install/ubuntu_install_conda.sh
 
-COPY install/ubuntu_install_torch_1.2.0.sh /install/ubuntu_install_torch.sh
-RUN bash /install/ubuntu_install_torch.sh
+ENV CONDA_ALWAYS_YES="true"
 
-COPY install/ubuntu_install_mxnet_cpu.sh /install/ubuntu_install_mxnet_cpu.sh
-RUN bash /install/ubuntu_install_mxnet_cpu.sh
+COPY install/conda_env/kg_cpu.yml /install/conda_env/kg_cpu.yml
+RUN ["/bin/bash", "-i", "-c", "conda env create -f /install/conda_env/kg_cpu.yml"]
 
-COPY install/FB15k.zip /data/kg/FB15k.zip
-RUN cd /data/kg && unzip FB15k.zip
+ENV CONDA_ALWAYS_YES=
diff --git a/docker/Dockerfile.ci_gpu b/docker/Dockerfile.ci_gpu
@@ -1,7 +1,6 @@
 # CI docker GPU env
-FROM nvidia/cuda:9.0-cudnn7-devel
+FROM nvidia/cuda:10.1-cudnn7-devel-ubuntu16.04
 
-# Base scripts
 RUN apt-get update --fix-missing
 
 COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh
@@ -10,25 +9,22 @@ RUN bash /install/ubuntu_install_core.sh
 COPY install/ubuntu_install_build.sh /install/ubuntu_install_build.sh
 RUN bash /install/ubuntu_install_build.sh
 
-# ANTLR deps
-COPY install/ubuntu_install_java.sh /install/ubuntu_install_java.sh
-RUN bash /install/ubuntu_install_java.sh
+# python
+COPY install/ubuntu_install_conda.sh /install/ubuntu_install_conda.sh
+RUN bash /install/ubuntu_install_conda.sh
 
-COPY install/ubuntu_install_antlr.sh /install/ubuntu_install_antlr.sh
-RUN bash /install/ubuntu_install_antlr.sh
+ENV CONDA_ALWAYS_YES="true"
 
-# python
-COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
-RUN bash /install/ubuntu_install_python.sh
+COPY install/conda_env/torch_gpu.yml /install/conda_env/torch_gpu.yml
+RUN ["/bin/bash", "-i", "-c", "conda env create -f /install/conda_env/torch_gpu.yml"]
 
-COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh
-RUN bash /install/ubuntu_install_python_package.sh
+COPY install/conda_env/tensorflow_gpu.yml /install/conda_env/tensorflow_gpu.yml
+RUN ["/bin/bash", "-i", "-c", "conda env create -f /install/conda_env/tensorflow_gpu.yml"]
 
-COPY install/ubuntu_install_torch.sh /install/ubuntu_install_torch.sh
-RUN bash /install/ubuntu_install_torch.sh
+COPY install/conda_env/mxnet_gpu.yml /install/conda_env/mxnet_gpu.yml
+RUN ["/bin/bash", "-i", "-c", "conda env create -f /install/conda_env/mxnet_gpu.yml"]
 
-COPY install/ubuntu_install_mxnet_gpu.sh /install/ubuntu_install_mxnet_gpu.sh
-RUN bash /install/ubuntu_install_mxnet_gpu.sh
+ENV CONDA_ALWAYS_YES=
 
 # Environment variables
 ENV PATH=/usr/local/nvidia/bin:${PATH}
@@ -37,3 +33,5 @@ ENV CPLUS_INCLUDE_PATH=/usr/local/cuda/include:${CPLUS_INCLUDE_PATH}
 ENV C_INCLUDE_PATH=/usr/local/cuda/include:${C_INCLUDE_PATH}
 ENV LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/nvidia/lib64:${LIBRARY_PATH}
 ENV LD_LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/nvidia/lib64:${LD_LIBRARY_PATH}
+ENV CUDA_VISIBLE_DEVICES=0
+ENV TF_FORCE_GPU_ALLOW_GROWTH=true
diff --git a/docker/Dockerfile.ci_gpu_torch_1.2.0 b/docker/Dockerfile.ci_gpu_torch_1.2.0
@@ -1,7 +1,6 @@
 # CI docker GPU env
-FROM nvidia/cuda:9.0-cudnn7-devel
+FROM nvidia/cuda:10.1-cudnn7-devel-ubuntu16.04
 
-# Base scripts
 RUN apt-get update --fix-missing
 
 COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh
@@ -10,25 +9,16 @@ RUN bash /install/ubuntu_install_core.sh
 COPY install/ubuntu_install_build.sh /install/ubuntu_install_build.sh
 RUN bash /install/ubuntu_install_build.sh
 
-# ANTLR deps
-COPY install/ubuntu_install_java.sh /install/ubuntu_install_java.sh
-RUN bash /install/ubuntu_install_java.sh
-
-COPY install/ubuntu_install_antlr.sh /install/ubuntu_install_antlr.sh
-RUN bash /install/ubuntu_install_antlr.sh
-
 # python
-COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
-RUN bash /install/ubuntu_install_python.sh
+COPY install/ubuntu_install_conda.sh /install/ubuntu_install_conda.sh
+RUN bash /install/ubuntu_install_conda.sh
 
-COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh
-RUN bash /install/ubuntu_install_python_package.sh
+ENV CONDA_ALWAYS_YES="true"
 
-COPY install/ubuntu_install_torch_1.2.0.sh /install/ubuntu_install_torch.sh
-RUN bash /install/ubuntu_install_torch.sh
+COPY install/conda_env/kg_gpu.yml /install/conda_env/kg_gpu.yml
+RUN ["/bin/bash", "-i", "-c", "conda env create -f /install/conda_env/kg_gpu.yml"]
 
-COPY install/ubuntu_install_mxnet_gpu.sh /install/ubuntu_install_mxnet_gpu.sh
-RUN bash /install/ubuntu_install_mxnet_gpu.sh
+ENV CONDA_ALWAYS_YES=
 
 COPY install/FB15k.zip /data/kg/FB15k.zip
 RUN cd /data/kg && unzip FB15k.zip

diff --git a/docker/install/conda_env/kg_cpu.yml b/docker/install/conda_env/kg_cpu.yml
@@ -0,0 +1,18 @@
+name: kg-ci
+dependencies:
+  - python=3.6.9
+  - pip
+  - pip:
+    - torch
+    - torchvision
+    - mxnet
+    - pytest
+  - nose
+  - numpy
+  - cython
+  - scipy
+  - networkx
+  - matplotlib
+  - nltk
+  - requests[security]
+  - tqdm
diff --git a/docker/install/conda_env/kg_gpu.yml b/docker/install/conda_env/kg_gpu.yml
@@ -0,0 +1,18 @@
+name: kg-ci
+dependencies:
+  - python=3.6.9
+  - pip
+  - pip:
+    - torch
+    - torchvision
+    - mxnet-cu101
+    - pytest
+  - nose
+  - numpy
+  - cython
+  - scipy
+  - networkx
+  - matplotlib
+  - nltk
+  - requests[security]
+  - tqdm
diff --git a/docker/install/conda_env/mxnet_cpu.yml b/docker/install/conda_env/mxnet_cpu.yml
@@ -1,8 +1,10 @@
 name: mxnet-ci
 dependencies:
+  - python=3.6.9
   - pip
   - pip:
     - mxnet
+    - pytest
   - nose
   - numpy
   - cython

diff --git a/docker/install/conda_env/mxnet_gpu.yml b/docker/install/conda_env/mxnet_gpu.yml
@@ -1,9 +1,10 @@
 name: mxnet-ci
 dependencies:
-  - cudatoolkit = 9.0
+  - python=3.6.9
   - pip
   - pip:
-    - mxnet-cu90
+    - mxnet-cu101
+    - pytest
   - nose
   - numpy
   - cython