diff --git a/Jenkinsfile b/Jenkinsfile index 1780246ea992..ab47664bd787 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,176 +1,150 @@ #!/usr/bin/env groovy def init_git_submodule() { - sh 'git submodule init' - sh 'git submodule update' + sh "git submodule init" + sh "git submodule update" } def setup() { - sh 'easy_install nose' - init_git_submodule() + init_git_submodule() } def build_dgl() { - sh 'if [ -d build ]; then rm -rf build; fi; mkdir build' - dir('python') { - sh 'python3 setup.py install' - } - dir ('build') { - sh 'cmake ..' - sh 'make -j4' - } + sh "if [ -d build ]; then rm -rf build; fi; mkdir build" + dir("python") { + sh "python3 setup.py install" + } + dir ("build") { + sh "cmake .." + sh "make -j4" + } } -def pytorch_unit_test() { - withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build"]) { - sh 'nosetests tests -v --with-xunit' - sh 'nosetests tests/pytorch -v --with-xunit' - sh 'nosetests tests/graph_index -v --with-xunit' - } +def pytorch_unit_test(dev) { + withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build", "PYTHONPATH=${env.WORKSPACE}/python"]) { + sh "python3 -m nose -v --with-xunit tests" + sh "python3 -m nose -v --with-xunit tests/pytorch" + sh "python3 -m nose -v --with-xunit tests/graph_index" + } } -def mxnet_unit_test() { - withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build"]) { - sh 'nosetests tests/mxnet -v --with-xunit' - } +def mxnet_unit_test(dev) { + withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build", "PYTHONPATH=${env.WORKSPACE}/python"]) { + sh "python3 -m nose -v --with-xunit tests/mxnet" + } } def example_test(dev) { - dir ('tests/scripts') { - withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build"]) { - sh "./test_examples.sh ${dev}" - } + dir ("tests/scripts") { + withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build", "PYTHONPATH=${env.WORKSPACE}/python"]) { + sh "./task_example_test.sh ${dev}" + } + } +} + +def pytorch_tutorials() { + dir ("tests/scripts") { + withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build", "PYTHONPATH=${env.WORKSPACE}/python"]) { + sh "./task_tutorial_test.sh" } + } } pipeline { - agent none - stages { - stage('Lint Check') { - agent { - docker { - image 'lingfanyu/dgl-lint' - } + agent none + stages { + stage("Lint Check") { + agent { docker { image "dgllib/dgl-ci-lint" } } + steps { + init_git_submodule() + sh "tests/scripts/task_lint.sh" + } + } + stage("Build") { + parallel { + stage("CPU Build") { + agent { docker { image "dgllib/dgl-ci-cpu" } } + steps { + setup() + build_dgl() + } + } + stage("GPU Build") { + agent { + docker { + image "dgllib/dgl-ci-gpu" + args "--runtime nvidia" + } + } + steps { + setup() + build_dgl() + } + } + stage("MXNet CPU Build (temp)") { + agent { docker { image "dgllib/dgl-ci-mxnet-cpu" } } + steps { + setup() + build_dgl() + } + } + } + } + stage("Test") { + parallel { + stage("Pytorch CPU") { + agent { docker { image "dgllib/dgl-ci-cpu" } } + stages { + stage("TH CPU unittest") { + steps { pytorch_unit_test("CPU") } } - stages { - stage('CHECK') { - steps { - init_git_submodule() - sh 'tests/scripts/task_lint.sh' - } - } + stage("TH CPU example test") { + steps { example_test("CPU") } } + } + post { + always { junit "*.xml" } + } } - stage('Build and Test on Pytorch') { - parallel { - stage('CPU') { - agent { - docker { - image 'lingfanyu/dgl-cpu' - } - } - stages { - stage('SETUP') { - steps { - setup() - } - } - stage('BUILD') { - steps { - build_dgl() - } - } - stage('UNIT TEST') { - steps { - pytorch_unit_test() - } - } - stage('EXAMPLE TEST') { - steps { - example_test('CPU') - } - } - } - post { - always { - junit '*.xml' - } - } - } - stage('GPU') { - agent { - docker { - image 'lingfanyu/dgl-gpu' - args '--runtime nvidia' - } - } - stages { - stage('SETUP') { - steps { - setup() - } - } - stage('BUILD') { - steps { - build_dgl() - } - } - stage('UNIT TEST') { - steps { - pytorch_unit_test() - } - } - stage('EXAMPLE TEST') { - steps { - example_test('GPU') - } - } - } - post { - always { - junit '*.xml' - } - } - } + stage("Pytorch GPU") { + agent { + docker { + image "dgllib/dgl-ci-gpu" + args "--runtime nvidia" } + } + stages { + // TODO: have GPU unittest + //stage("TH GPU unittest") { + // steps { pytorch_unit_test("GPU") } + //} + stage("TH GPU example test") { + steps { example_test("GPU") } + } + } + // TODO: have GPU unittest + //post { + // always { junit "*.xml" } + //} } - stage('Build and Test on MXNet') { - parallel { - stage('CPU') { - agent { - docker { - image 'zhengda1936/dgl-mxnet-cpu:v3' - } - } - stages { - stage('SETUP') { - steps { - setup() - } - } - stage('BUILD') { - steps { - build_dgl() - } - } - stage('UNIT TEST') { - steps { - mxnet_unit_test() - } - } - stage('EXAMPLE TEST') { - steps { - example_test('CPU') - } - } - } - post { - always { - junit '*.xml' - } - } - } + stage("MXNet CPU") { + agent { docker { image "dgllib/dgl-ci-mxnet-cpu" } } + stages { + stage("MX Unittest") { + steps { mxnet_unit_test("CPU") } } + } + post { + always { junit "*.xml" } + } } + } + } + stage("Doc") { + agent { docker { image "dgllib/dgl-ci-cpu" } } + steps { + pytorch_tutorials() + } } + } } diff --git a/docker/Dockerfile.ci_cpu_mxnet b/docker/Dockerfile.ci_cpu_mxnet new file mode 100644 index 000000000000..d35e6c9f693f --- /dev/null +++ b/docker/Dockerfile.ci_cpu_mxnet @@ -0,0 +1,25 @@ +# CI docker GPU env +FROM nvidia/cuda:9.0-cudnn7-devel + +# Base scripts +RUN apt-get update --fix-missing + +COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh +RUN bash /install/ubuntu_install_core.sh + +COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh +RUN bash /install/ubuntu_install_python.sh + +COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh +RUN bash /install/ubuntu_install_python_package.sh + +COPY install/ubuntu_install_mxnet_cpu.sh /install/ubuntu_install_mxnet_cpu.sh +RUN bash /install/ubuntu_install_mxnet_cpu.sh + +# Environment variables +ENV PATH=/usr/local/nvidia/bin:${PATH} +ENV PATH=/usr/local/cuda/bin:${PATH} +ENV CPLUS_INCLUDE_PATH=/usr/local/cuda/include:${CPLUS_INCLUDE_PATH} +ENV C_INCLUDE_PATH=/usr/local/cuda/include:${C_INCLUDE_PATH} +ENV LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/nvidia/lib64:${LIBRARY_PATH} +ENV LD_LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/nvidia/lib64:${LD_LIBRARY_PATH} diff --git a/docker/Dockerfile.ci_gpu_mxnet b/docker/Dockerfile.ci_gpu_mxnet new file mode 100644 index 000000000000..02ed60c39b31 --- /dev/null +++ b/docker/Dockerfile.ci_gpu_mxnet @@ -0,0 +1,25 @@ +# CI docker GPU env +FROM nvidia/cuda:9.0-cudnn7-devel + +# Base scripts +RUN apt-get update --fix-missing + +COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh +RUN bash /install/ubuntu_install_core.sh + +COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh +RUN bash /install/ubuntu_install_python.sh + +COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh +RUN bash /install/ubuntu_install_python_package.sh + +COPY install/ubuntu_install_mxnet_gpu.sh /install/ubuntu_install_mxnet_gpu.sh +RUN bash /install/ubuntu_install_mxnet_gpu.sh + + # Environment variables +ENV PATH=/usr/local/nvidia/bin:${PATH} +ENV PATH=/usr/local/cuda/bin:${PATH} +ENV CPLUS_INCLUDE_PATH=/usr/local/cuda/include:${CPLUS_INCLUDE_PATH} +ENV C_INCLUDE_PATH=/usr/local/cuda/include:${C_INCLUDE_PATH} +ENV LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/nvidia/lib64:${LIBRARY_PATH} +ENV LD_LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/nvidia/lib64:${LD_LIBRARY_PATH} diff --git a/docker/README.md b/docker/README.md index fb79a6a64a0c..7f628a9f1cc0 100644 --- a/docker/README.md +++ b/docker/README.md @@ -8,3 +8,9 @@ docker build -t dgl-gpu -f Dockerfile.ci_gpu . ### Lint image docker build -t dgl-lint -f Dockerfile.ci_lint . + +### CPU MXNet image +docker build -t dgl-mxnet-cpu -f Dockerfile.ci_cpu_mxnet . + +### GPU MXNet image +docker build -t dgl-mxnet-gpu -f Dockerfile.ci_gpu_mxnet . diff --git a/docker/install/ubuntu_install_core.sh b/docker/install/ubuntu_install_core.sh index 97d30f6e7aa9..d46c8bc8bc35 100644 --- a/docker/install/ubuntu_install_core.sh +++ b/docker/install/ubuntu_install_core.sh @@ -1,3 +1,4 @@ # install libraries for building c++ core on ubuntu apt update && apt install -y --no-install-recommends --force-yes \ - apt-utils git build-essential make cmake wget unzip sudo libz-dev libxml2-dev + apt-utils git build-essential make cmake wget unzip sudo \ + libz-dev libxml2-dev libopenblas-dev libopencv-dev ca-certificates diff --git a/docker/install/ubuntu_install_mxnet_cpu.sh b/docker/install/ubuntu_install_mxnet_cpu.sh new file mode 100644 index 000000000000..8f0e6a961db3 --- /dev/null +++ b/docker/install/ubuntu_install_mxnet_cpu.sh @@ -0,0 +1,4 @@ +git clone --recursive -b dgl_graph https://github.com/zheng-da/incubator-mxnet.git +cd incubator-mxnet +make -j4 USE_OPENCV=1 USE_BLAS=openblas USE_MKLDNN=1 USE_DIST_KVSTORE=1 +pip3 install -e python diff --git a/docker/install/ubuntu_install_mxnet_gpu.sh b/docker/install/ubuntu_install_mxnet_gpu.sh new file mode 100644 index 000000000000..413c1fdcefba --- /dev/null +++ b/docker/install/ubuntu_install_mxnet_gpu.sh @@ -0,0 +1,5 @@ +git clone --recursive -b dgl_graph https://github.com/zheng-da/incubator-mxnet.git +cd incubator-mxnet +make -j4 USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda \ + USE_CUDNN=1 USE_MKLDNN=1 USE_DIST_KVSTORE=1 +pip3 install -e python diff --git a/docker/install/ubuntu_install_python.sh b/docker/install/ubuntu_install_python.sh index 55a144ed47fc..67d7bd15332b 100644 --- a/docker/install/ubuntu_install_python.sh +++ b/docker/install/ubuntu_install_python.sh @@ -1,13 +1,14 @@ # install python and pip, don't modify this, modify install_python_package.sh -# apt-get update && apt-get install -y python-dev python-pip +apt-get update +apt-get install -y python-dev python3-dev -# python 3.6 -apt-get update && yes | apt-get install software-properties-common -add-apt-repository ppa:jonathonf/python-3.6 -apt-get update && apt-get install -y python3.6 python3.6-dev -rm -f /usr/bin/python3 && ln -s /usr/bin/python3.6 /usr/bin/python3 - -# Install pip +# install pip cd /tmp && wget https://bootstrap.pypa.io/get-pip.py -# python2 get-pip.py -python3.6 get-pip.py +python2 get-pip.py +python3 get-pip.py + +# santiy check +python2 --version +python3 --version +pip2 --version +pip3 --version diff --git a/docker/install/ubuntu_install_python_package.sh b/docker/install/ubuntu_install_python_package.sh index e248f7580790..c78112c40598 100644 --- a/docker/install/ubuntu_install_python_package.sh +++ b/docker/install/ubuntu_install_python_package.sh @@ -1,7 +1,7 @@ # install libraries for python package on ubuntu -# pip2 install pylint numpy cython scipy nltk requests[security] -pip3 install pylint numpy cython scipy nltk requests[security] +pip2 install nose numpy cython scipy networkx matplotlib nltk requests[security] +pip3 install nose numpy cython scipy networkx matplotlib nltk requests[security] # install DL Framework -# pip2 install torch torchvision +pip2 install torch torchvision pip3 install torch torchvision diff --git a/tests/pytorch/test_traversal.py b/tests/pytorch/test_traversal.py index a399d843f1df..1e52b6e4f698 100644 --- a/tests/pytorch/test_traversal.py +++ b/tests/pytorch/test_traversal.py @@ -16,37 +16,42 @@ def toset(x): return set(x.tolist()) -def test_bfs_nodes(n=1000): - g_nx = nx.random_tree(n) - g = dgl.DGLGraph() - g.from_networkx(g_nx) +def test_bfs(n=1000): + def _bfs_nx(g_nx, src): + edges = nx.bfs_edges(g_nx, src) + layers_nx = [set([src])] + edges_nx = [] + frontier = set() + edge_frontier = set() + for u, v in edges: + if u in layers_nx[-1]: + frontier.add(v) + edge_frontier.add(g.edge_id(u, v)) + else: + layers_nx.append(frontier) + edges_nx.append(edge_frontier) + frontier = set([v]) + edge_frontier = set([g.edge_id(u, v)]) + layers_nx.append(frontier) + edges_nx.append(edge_frontier) + return layers_nx, edges_nx + g = dgl.DGLGraph() + a = sp.random(n, n, 10 / n, data_rvs=lambda n: np.ones(n)) + g.from_scipy_sparse_matrix(a) + g_nx = g.to_networkx() src = random.choice(range(n)) - - edges = nx.bfs_edges(g_nx, src) - layers_nx = [set([src])] - edges_nx = [] - frontier = set() - edge_frontier = set() - for u, v in edges: - if u in layers_nx[-1]: - frontier.add(v) - edge_frontier.add(g.edge_id(u, v)) - else: - layers_nx.append(frontier) - edges_nx.append(edge_frontier) - frontier = set([v]) - edge_frontier = set([g.edge_id(u, v)]) - layers_nx.append(frontier) - edges_nx.append(edge_frontier) - + layers_nx, _ = _bfs_nx(g_nx, src) layers_dgl = dgl.bfs_nodes_generator(g, src) - assert len(layers_dgl) == len(layers_nx) assert all(toset(x) == y for x, y in zip(layers_dgl, layers_nx)) + g_nx = nx.random_tree(n, seed=42) + g = dgl.DGLGraph() + g.from_networkx(g_nx) + src = 0 + _, edges_nx = _bfs_nx(g_nx, src) edges_dgl = dgl.bfs_edges_generator(g, src) - assert len(edges_dgl) == len(edges_nx) assert all(toset(x) == y for x, y in zip(edges_dgl, edges_nx)) @@ -114,6 +119,6 @@ def combine_frontiers(sol): if __name__ == '__main__': - test_bfs_nodes() + test_bfs() test_topological_nodes() test_dfs_labeled_edges() diff --git a/tests/scripts/test_examples.sh b/tests/scripts/task_example_test.sh similarity index 100% rename from tests/scripts/test_examples.sh rename to tests/scripts/task_example_test.sh diff --git a/tests/scripts/task_tutorial_test.sh b/tests/scripts/task_tutorial_test.sh new file mode 100755 index 000000000000..e45d8f967a93 --- /dev/null +++ b/tests/scripts/task_tutorial_test.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# The working directory for this script will be "tests/scripts" + +function fail { + echo FAIL: $@ + exit -1 +} + +export MPLBACKEND=Agg + +for f in $(find "../../tutorials" -name "*.py") +do + echo "Running tutorial ${f} ..." + python3 $f || fail "run ${f}" +done