[CI] Change tests for flexibility

* change ci image

* fix

* force bash

* fix

* fix python version

* fix

* fix

* fix

* update gpu

* cuda

* jenkins

* fix build sh

* fix

* Revert "fix"

This reverts commit 6b09191.

* try fix

* fix

* Revert "fix"

This reverts commit e42c303.

* try fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix tests

* try fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix ctx problem

* fix many tests

* fix typo

* add backend

* move to pytorch folder

* fix?

* fix ci

* try skip

* try false

* try?

* try?

* Revert to 7d9a52f

* fix various

* fix lint

* Revert Jenkinsfile

* revert env

* revert env

* address comment

* remove file

docker/install/conda_env/mxnet_cpu.yml

@@ -0,0 +1,14 @@
+name: mxnet-ci
+dependencies:
+  - pip
+  - pip:
+    - mxnet
+  - nose
+  - numpy
+  - cython
+  - scipy
+  - networkx
+  - matplotlib
+  - nltk
+  - requests[security]
+  - tqdm

docker/install/conda_env/mxnet_gpu.yml

@@ -0,0 +1,15 @@
+name: mxnet-ci
+dependencies:
+  - cudatoolkit = 9.0
+  - pip
+  - pip:
+    - mxnet-cu90
+  - nose
+  - numpy
+  - cython
+  - scipy
+  - networkx
+  - matplotlib
+  - nltk
+  - requests[security]
+  - tqdm

docker/install/conda_env/tensorflow_cpu.yml

@@ -0,0 +1,15 @@
+name: tensorflow-ci
+dependencies:
+  - pip
+  - pip:
+    - tf-nightly == 2.1.0.dev20191125
+    - tfdlpack
+  - nose
+  - numpy
+  - cython
+  - scipy
+  - networkx
+  - matplotlib
+  - nltk
+  - requests[security]
+  - tqdm

docker/install/conda_env/tensorflow_gpu.yml

@@ -0,0 +1,17 @@
+
+name: tensorflow-ci
+dependencies:
+  - cudatoolkit = 10.1
+  - pip
+  - pip:
+    - tf-nightly-gpu == 2.1.0.dev20191125
+    - tfdlpack-gpu
+  - nose
+  - numpy
+  - cython
+  - scipy
+  - networkx
+  - matplotlib
+  - nltk
+  - requests[security]
+  - tqdm

docker/install/conda_env/torch_cpu.yml

@@ -0,0 +1,17 @@
+name: pytorch-ci
+channels:
+  - pytorch
+dependencies:
+  - python = 3.6
+  - pytorch = 1.0.1
+  - pip
+  - torchvision
+  - nose
+  - numpy
+  - cython
+  - scipy
+  - networkx
+  - matplotlib
+  - nltk
+  - requests[security]
+  - tqdm

docker/install/conda_env/torch_gpu.yml

@@ -0,0 +1,18 @@
+name: pytorch-ci
+channels:
+  - pytorch
+dependencies:
+  - python = 3.6
+  - pytorch = 1.0.1
+  - cudatoolkit = 9.2
+  - pip
+  - torchvision
+  - nose
+  - numpy
+  - cython
+  - scipy
+  - networkx
+  - matplotlib
+  - nltk
+  - requests[security]
+  - tqdm

docker/install/ubuntu_install_conda.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+export LANG=C.UTF-8 LC_ALL=C.UTF-8
+export PATH=/opt/conda/bin:$PATH
+
+apt-get update --fix-missing && \
+    apt-get install -y wget bzip2 ca-certificates curl git && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-4.5.11-Linux-x86_64.sh -O ~/miniconda.sh && \
+    /bin/bash ~/miniconda.sh -b -p /opt/conda && \
+    rm ~/miniconda.sh && \
+    /opt/conda/bin/conda clean -tipsy && \
+    ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \
+    echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \
+    echo "conda activate base" >> ~/.bashrc
+
+export TINI_VERSION=v0.16.1
+source ~/.bashrc

python/dgl/__init__.py

@@ -3,6 +3,10 @@
 # This initializes Winsock and performs cleanup at termination as required
 import socket
 
+# Need to ensure that the backend framework is imported before load dgl libs,
+# otherwise weird cuda problem happens
+from .backend import load_backend
+
 from . import function
 from . import nn
 from . import contrib
@@ -14,7 +18,6 @@
 from ._ffi.base import DGLError, __version__
 
 from .base import ALL, NTYPE, NID, ETYPE, EID
-from .backend import load_backend
 from .batched_graph import *
 from .batched_heterograph import *
 from .convert import *

python/dgl/backend/__init__.py

@@ -35,6 +35,8 @@ def load_backend(mod_name):
             setattr(thismod,
                     'reverse_data_type_dict',
                     {v: k for k, v in data_type_dict.items()})
+            # log backend name
+            setattr(thismod, 'backend_name', mod_name)
         else:
             # load functions
             if api in mod.__dict__:

tests/compute/test_basics.py

@@ -126,6 +126,7 @@ def _pfc(x):
     v = F.tensor([3, 4, 5])
     assert _pfc(g.edges[u, v].data['l']) == [1., 1., 1.]
 
+
 def test_batch_setter_autograd():
     g = generate_graph(grad=True)
     h1 = g.ndata['h']
@@ -135,7 +136,7 @@ def test_batch_setter_autograd():
     with F.record_grad():
         g.nodes[v].data['h'] = hh
         h2 = g.ndata['h']
-    F.backward(h2, F.ones((10, D)) * 2)
+        F.backward(h2, F.ones((10, D)) * 2)
     assert F.array_equal(F.grad(h1)[:,0], F.tensor([2., 0., 0., 2., 2., 2., 2., 2., 0., 2.]))
     assert F.array_equal(F.grad(hh)[:,0], F.tensor([2., 2., 2.]))
 
@@ -246,7 +247,7 @@ def _check_nx_feature(nxg, nf, ef):
     nxg = nx.cycle_graph(5)
     nxg.remove_nodes_from([0, 4])
     for u in nxg.nodes():
-        nxg.node[u]['h'] = F.tensor([u])
+        nxg.nodes[u]['h'] = F.tensor([u])
     for u, v, d in nxg.edges(data=True):
         d['h'] = F.tensor([u, v])
 

tests/compute/test_filter.py

@@ -1,15 +1,18 @@
 from dgl.graph import DGLGraph
 import backend as F
+import numpy as np
 
 def test_filter():
     g = DGLGraph()
     g.add_nodes(4)
     g.add_edges([0,1,2,3], [1,2,3,0])
 
-    n_repr = F.zeros((4, 5))
-    e_repr = F.zeros((4, 5))
+    n_repr = np.zeros((4, 5))
+    e_repr = np.zeros((4, 5))
     n_repr[[1, 3]] = 1
     e_repr[[1, 3]] = 1
+    n_repr = F.copy_to(F.zerocopy_from_numpy(n_repr), F.ctx())
+    e_repr = F.copy_to(F.zerocopy_from_numpy(e_repr), F.ctx())
 
     g.ndata['a'] = n_repr
     g.edata['a'] = e_repr

tests/compute/test_frame.py

@@ -2,6 +2,8 @@
 from dgl.frame import Frame, FrameRef
 from dgl.utils import Index, toindex
 import backend as F
+import dgl
+import unittest
 
 N = 10
 D = 5
@@ -181,7 +183,7 @@ def test_row2():
         rowid = Index(F.tensor([0, 2]))
         rows = f[rowid]
         y = rows['a1']
-    F.backward(y, F.ones((len(rowid), D)))
+        F.backward(y, F.ones((len(rowid), D)))
     assert F.allclose(F.grad(c1)[:,0], F.tensor([1., 0., 1., 0., 0., 0., 0., 0., 0., 0.]))
 
     f['a1'] = F.attach_grad(f['a1'])
@@ -191,7 +193,7 @@ def test_row2():
         rowid = Index(F.tensor([8, 2, 2, 1]))
         rows = f[rowid]
         y = rows['a1']
-    F.backward(y, F.ones((len(rowid), D)))
+        F.backward(y, F.ones((len(rowid), D)))
     assert F.allclose(F.grad(c1)[:,0], F.tensor([0., 1., 2., 0., 0., 0., 0., 0., 1., 0.]))
 
     f['a1'] = F.attach_grad(f['a1'])
@@ -205,7 +207,7 @@ def test_row2():
                 }
         f[rowid] = vals
         c11 = f['a1']
-    F.backward(c11, F.ones((N, D)))
+        F.backward(c11, F.ones((N, D)))
     assert F.allclose(F.grad(c1)[:,0], F.tensor([0., 1., 0., 1., 0., 1., 1., 1., 1., 1.]))
     assert F.allclose(F.grad(vals['a1']), F.ones((len(rowid), D)))
     assert F.is_no_grad(vals['a2'])
@@ -231,6 +233,8 @@ def test_row3():
     for k, v in f.items():
         assert F.allclose(v, data[k][newidx])
 
+
+@unittest.skipIf(dgl.backend.backend_name == "tensorflow", reason="TF doesn't support inplace update")
 def test_row4():
     # test updating row with empty frame but has preset num_rows
     f = FrameRef(Frame(num_rows=5))
@@ -240,6 +244,7 @@ def test_row4():
     ans[F.tensor([0, 2, 4])] = F.ones((3, 2))
     assert F.allclose(f['h'], ans)
 
+@unittest.skipIf(dgl.backend.backend_name == "tensorflow", reason="TF doesn't support inplace update")
 def test_sharing():
     data = Frame(create_test_data())
     f1 = FrameRef(data, index=toindex([0, 1, 2, 3]))
@@ -267,6 +272,7 @@ def test_sharing():
     F.narrow_row_set(f2_a1, 0, 2, F.ones([2, D]))
     assert F.allclose(f2['a1'], f2_a1)
 
+@unittest.skipIf(dgl.backend.backend_name == "tensorflow", reason="TF doesn't support inplace update")
 def test_slicing():
     data = Frame(create_test_data(grad=True))
     f1 = FrameRef(data, index=toindex(slice(1, 5)))
@@ -314,6 +320,7 @@ def test_add_rows():
     ans = F.cat([F.zeros((4, 5)), F.ones((4, 5))], 0)
     assert F.allclose(f1['y'], ans)
 
+@unittest.skipIf(dgl.backend.backend_name == "tensorflow", reason="TF doesn't support inplace update")
 def test_inplace():
     f = FrameRef(Frame(create_test_data()))
     print(f.schemes)

tests/compute/test_graph_index.py

@@ -43,7 +43,7 @@ def sort_edges(edges):
     edges = [e.tousertensor() for e in edges]
     if np.prod(edges[2].shape) > 0:
         val, idx = F.sort_1d(edges[2])
-        return (edges[0][idx], edges[1][idx], edges[2][idx])
+        return (F.gather_row(edges[0], idx), F.gather_row(edges[1], idx), F.gather_row(edges[2], idx))
     else:
         return (edges[0], edges[1], edges[2])
 
@@ -124,8 +124,8 @@ def test_node_subgraph():
     subig = ig.node_subgraph(utils.toindex(randv))
     check_basics(subg.graph, subig.graph)
     check_graph_equal(subg.graph, subig.graph)
-    assert F.sum(map_to_subgraph_nid(subg, utils.toindex(randv1[0:10])).tousertensor()
-            == map_to_subgraph_nid(subig, utils.toindex(randv1[0:10])).tousertensor(), 0) == 10
+    assert F.asnumpy(map_to_subgraph_nid(subg, utils.toindex(randv1[0:10])).tousertensor()
+            == map_to_subgraph_nid(subig, utils.toindex(randv1[0:10])).tousertensor()).sum(0).item() == 10
 
     # node_subgraphs
     randvs = []

tests/compute/test_hetero_basics.py

@@ -5,6 +5,7 @@
 import dgl
 import networkx as nx
 from collections import defaultdict as ddict
+import unittest
 
 D = 5
 reduce_msg_shapes = set()
@@ -148,7 +149,7 @@ def test_batch_setter_autograd():
     with F.record_grad():
         g.nodes[v].data['h'] = hh
         h2 = g.ndata['h']
-    F.backward(h2, F.ones((10, D)) * 2)
+        F.backward(h2, F.ones((10, D)) * 2)
     assert F.array_equal(F.grad(h1)[:,0], F.tensor([2., 0., 0., 2., 2., 2., 2., 2., 0., 2.]))
     assert F.array_equal(F.grad(hh)[:,0], F.tensor([2., 2., 2.]))
 
@@ -257,7 +258,7 @@ def _check_nx_feature(nxg, nf, ef):
     nxg = nx.cycle_graph(5)
     nxg.remove_nodes_from([0, 4])
     for u in nxg.nodes():
-        nxg.node[u]['h'] = F.tensor([u])
+        nxg.nodes[u]['h'] = F.tensor([u])
     for u, v, d in nxg.edges(data=True):
         d['h'] = F.tensor([u, v])
 
@@ -599,6 +600,7 @@ def test_repr():
     repr_string = G.__repr__()
     print(repr_string)
 
+@unittest.skipIf(dgl.backend.backend_name == "tensorflow", reason="will core dump")
 def test_group_apply_edges():
     def edge_udf(edges):
         h = F.sum(edges.data['feat'] * (edges.src['h'] + edges.dst['h']), dim=2)
@@ -784,7 +786,7 @@ def test_issue_1088():
     test_update_all_0deg()
     test_pull_0deg()
     test_send_multigraph()
-    #test_dynamic_addition()
+    test_dynamic_addition()
     test_repr()
     test_group_apply_edges()
     test_local_var()

tests/compute/test_heterograph.py

@@ -6,9 +6,9 @@
 import itertools
 import backend as F
 import networkx as nx
+import unittest
 from dgl import DGLError
 
-
 def create_test_heterograph():
     # test heterograph from the docstring, plus a user -- wishes -- game relation
     # 3 users, 2 games, 2 developers
@@ -546,11 +546,11 @@ def check_mapping(g, fg):
             src_fg, dst_fg = fg.find_edges([i])
             # TODO(gq): I feel this code is quite redundant; can we just add new members (like
             # "induced_srcid") to returned heterograph object and not store them as features?
-            assert src_g == fg.nodes[SRC].data[dgl.NID][src_fg]
-            tid = F.asnumpy(fg.nodes[SRC].data[dgl.NTYPE][src_fg])[0]
+            assert src_g == F.gather_row(fg.nodes[SRC].data[dgl.NID], src_fg)[0]
+            tid = F.asnumpy(F.gather_row(fg.nodes[SRC].data[dgl.NTYPE], src_fg)).item()
             assert g.canonical_etypes[etype][0] == g.ntypes[tid]
-            assert dst_g == fg.nodes[DST].data[dgl.NID][dst_fg]
-            tid = F.asnumpy(fg.nodes[DST].data[dgl.NTYPE][dst_fg])[0]
+            assert dst_g == F.gather_row(fg.nodes[DST].data[dgl.NID], dst_fg)[0]
+            tid = F.asnumpy(F.gather_row(fg.nodes[DST].data[dgl.NTYPE], dst_fg)).item()
             assert g.canonical_etypes[etype][2] == g.ntypes[tid]
 
     # check for wildcard slices
@@ -782,8 +782,8 @@ def _check_typed_subgraph1(g, sg):
             assert F.array_equal(dst_sg, dst_g)
         assert F.array_equal(sg.nodes['user'].data['h'], g.nodes['user'].data['h'])
         assert F.array_equal(sg.edges['follows'].data['h'], g.edges['follows'].data['h'])
-        g.nodes['user'].data['h'][2] = F.randn((5,))
-        g.edges['follows'].data['h'][1] = F.randn((4,))
+        g.nodes['user'].data['h'] = F.scatter_row(g.nodes['user'].data['h'], F.tensor([2]), F.randn((1, 5)))
+        g.edges['follows'].data['h'] = F.scatter_row(g.edges['follows'].data['h'], F.tensor([1]), F.randn((1, 4)))
         assert F.array_equal(sg.nodes['user'].data['h'], g.nodes['user'].data['h'])
         assert F.array_equal(sg.edges['follows'].data['h'], g.edges['follows'].data['h'])
 
@@ -934,6 +934,8 @@ def afunc(nodes):
         fail = True
     assert fail
 
+
+@unittest.skipIf(dgl.backend.backend_name == "tensorflow", reason="Core dump")
 def test_level2():
     #edges = {
     #    'follows': ([0, 1], [1, 2]),

tests/compute/test_index.py

@@ -3,7 +3,9 @@
 from dgl.utils import toindex
 import numpy as np
 import backend as F
+import unittest
 
+@unittest.skipIf(dgl.backend.backend_name == "tensorflow", reason="TF doesn't support inplace update")
 def test_dlpack():
     # test dlpack conversion.
     def nd2th():