Merge branch 'cpp' of https://github.com/jermainewang/dgl into line-g…

…raph

Jenkinsfile

@@ -1,33 +1,95 @@
 pipeline {
-    agent {
-        docker {
-            image 'pytorch/pytorch'
-        }
-    }
+    agent none
     stages {
-        stage('SETUP') {
-            steps {
-                sh 'easy_install nose'
-                sh 'apt-get update && apt-get install -y libxml2-dev'
-            }
-        }
-        stage('BUILD') {
-            steps {
-                dir('python') {
-                    sh 'python setup.py install'
+        stage('Build and Test') {
+            parallel {
+                stage('CPU') {
+                    agent {
+                        docker {
+                            image 'lingfanyu/dgl-cpu'
+                            args '-u root'
+                        }
+                    }
+                    stages {
+                        stage('SETUP') {
+                            steps {
+                                sh 'easy_install nose'
+                                sh 'git submodule init'
+                                sh 'git submodule update'
+                            }
+                        }
+                        stage('BUILD') {
+                            steps {
+                                sh 'if [ -d build ]; then rm -rf build; fi; mkdir build'
+                                dir('python') {
+                                    sh 'python3 setup.py install'
+                                }
+                                dir ('build') {
+                                    sh 'cmake ..'
+                                    sh 'make -j$(nproc)'
+                                }
+                            }
+                        }
+                        stage('TEST') {
+                            steps {
+                                withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build"]) {
+                                    sh 'echo $DGL_LIBRARY_PATH'
+                                    sh 'nosetests tests -v --with-xunit'
+                                    sh 'nosetests tests/pytorch -v --with-xunit'
+                                }
+                            }
+                        }
+                    }
+                    post {
+                        always {
+                            junit '*.xml'
+                        }
+                    }
+                }
+                stage('GPU') {
+                    agent {
+                        docker {
+                            image 'lingfanyu/dgl-gpu'
+                            args '--runtime nvidia -u root'
+                        }
+                    }
+                    stages {
+                        stage('SETUP') {
+                            steps {
+                                sh 'easy_install nose'
+                                sh 'git submodule init'
+                                sh 'git submodule update'
+                            }
+                        }
+                        stage('BUILD') {
+                            steps {
+                                sh 'if [ -d build ]; then rm -rf build; fi; mkdir build'
+                                dir('python') {
+                                    sh 'python3 setup.py install'
+                                }
+                                dir ('build') {
+                                    sh 'cmake ..'
+                                    sh 'make -j$(nproc)'
+                                }
+                            }
+                        }
+                        stage('TEST') {
+                            steps {
+                                withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build"]) {
+                                    sh 'echo $DGL_LIBRARY_PATH'
+                                    sh 'nosetests tests -v --with-xunit'
+                                    sh 'nosetests tests/pytorch -v --with-xunit'
+                                }
+                            }
+                        }
+                    }
+                    post {
+                        always {
+                            junit '*.xml'
+                        }
+                    }
                 }
             }
         }
-        stage('TEST') {
-            steps {
-                sh 'nosetests tests -v --with-xunit'
-                sh 'nosetests tests/pytorch -v --with-xunit'
-            }
-        }
-    }
-    post {
-        always {
-            junit '*.xml'
-        }
     }
 }

README.md

@@ -12,7 +12,7 @@ Show below, there are three sets of APIs for different models.
 
 ## For Model developers
 - Always choose the API at the *highest* possible level.
-- Refer to [the default modules](examples/pytorch/util.py) to see how to register message and node update functions as well as readout functions; note how you can control sharing of parameters by adding a counter.
+- Refer to the [GCN example](examples/pytorch/gcn/gcn_batch.py) to see how to register message and node update functions;
 
 ## How to build (the `cpp` branch)
 

docker/Dockerfile.ci_cpu

@@ -0,0 +1,14 @@
+# CI docker CPU env
+# Adapted from github.com/dmlc/tvm/docker/Dockerfile.ci_cpu
+FROM ubuntu:16.04
+
+RUN apt-get update --fix-missing
+
+COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh
+RUN bash /install/ubuntu_install_core.sh
+
+COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
+RUN bash /install/ubuntu_install_python.sh
+
+COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh
+RUN bash /install/ubuntu_install_python_package.sh

docker/Dockerfile.ci_gpu

@@ -0,0 +1,22 @@
+# CI docker GPU env
+FROM nvidia/cuda:9.0-cudnn7-devel
+
+# Base scripts
+RUN apt-get update --fix-missing
+
+COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh
+RUN bash /install/ubuntu_install_core.sh
+
+COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
+RUN bash /install/ubuntu_install_python.sh
+
+COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh
+RUN bash /install/ubuntu_install_python_package.sh
+
+# Environment variables
+ENV PATH=/usr/local/nvidia/bin:${PATH}
+ENV PATH=/usr/local/cuda/bin:${PATH}
+ENV CPLUS_INCLUDE_PATH=/usr/local/cuda/include:${CPLUS_INCLUDE_PATH}
+ENV C_INCLUDE_PATH=/usr/local/cuda/include:${C_INCLUDE_PATH}
+ENV LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/nvidia/lib64:${LIBRARY_PATH}
+ENV LD_LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/nvidia/lib64:${LD_LIBRARY_PATH}

docker/README.md

@@ -0,0 +1,7 @@
+## Build docker image for CI
+
+### CPU image
+docker build -t dgl-cpu -f Dockerfile.ci_cpu .
+
+### GPU image
+docker build -t dgl-gpu -f Dockerfile.ci_gpu .

docker/install/ubuntu_install_core.sh

@@ -0,0 +1,3 @@
+# install libraries for building c++ core on ubuntu
+apt update && apt install -y --no-install-recommends --force-yes \
+        apt-utils git build-essential make cmake wget unzip sudo libz-dev libxml2-dev

docker/install/ubuntu_install_python.sh

@@ -0,0 +1,13 @@
+# install python and pip, don't modify this, modify install_python_package.sh
+# apt-get update && apt-get install -y python-dev python-pip
+
+# python 3.6
+apt-get update && yes | apt-get install software-properties-common
+add-apt-repository ppa:jonathonf/python-3.6
+apt-get update && apt-get install -y python3.6 python3.6-dev
+rm -f /usr/bin/python3 && ln -s /usr/bin/python3.6 /usr/bin/python3
+
+# Install pip
+cd /tmp && wget https://bootstrap.pypa.io/get-pip.py
+# python2 get-pip.py
+python3.6 get-pip.py

docker/install/ubuntu_install_python_package.sh

@@ -0,0 +1,7 @@
+# install libraries for python package on ubuntu
+# pip2 install pylint numpy cython scipy nltk requests[security]
+pip3 install pylint numpy cython scipy nltk requests[security]
+
+# install DL Framework
+# pip2 install torch torchvision
+pip3 install torch torchvision

examples/pytorch/tree_lstm/train.py

@@ -91,7 +91,7 @@ def _batcher(trees):
                 mean_dur = np.mean(dur)
                 print("Epoch {:05d} | Step {:05d} | Loss {:.4f} | "
                       "Acc {:.4f} | Time(s) {:.4f} | Trees/s {:.4f}".format(
-                    epoch, step, loss.item(), acc.item() / args.batch_size,
+                    epoch, step, loss.item(), acc.item() / len(label),
                     mean_dur, args.batch_size / mean_dur))
         print("Epoch time(s):", time.time() - t_epoch)
 

examples/pytorch/tree_lstm/tree_lstm.py

@@ -23,7 +23,7 @@ def __init__(self, x_size, h_size):
         self.ut = 0.
 
     def message_func(self, src, edge):
-        return src
+        return {'h' : src['h'], 'c' : src['c']}
 
     def reduce_func(self, node, msgs):
         # equation (2)
@@ -90,9 +90,9 @@ def forward(self, graph, zero_initializer, h=None, c=None, iterator=None, train=
         """
         g = graph
         n = g.number_of_nodes()
-        g.register_message_func(self.cell.message_func, batchable=True)
-        g.register_reduce_func(self.cell.reduce_func, batchable=True)
-        g.register_apply_node_func(self.cell.apply_func, batchable=True)
+        g.register_message_func(self.cell.message_func)
+        g.register_reduce_func(self.cell.reduce_func)
+        g.register_apply_node_func(self.cell.apply_func)
         # feed embedding
         wordid = g.pop_n_repr('x')
         mask = (wordid != dgl.data.SST.PAD_WORD)

python/dgl/batched_graph.py

@@ -174,7 +174,7 @@ def unbatch(graph):
     return [DGLGraph(graph_data=pttns[i],
                      node_frame=node_frames[i],
                      edge_frame=edge_frames[i]) for i in range(bsize)]
-    
+
 def batch(graph_list, node_attrs=ALL, edge_attrs=ALL):
     """Batch a list of DGLGraphs into one single graph.
 

python/dgl/graph.py

@@ -55,7 +55,7 @@ def __init__(self,
 
     def add_nodes(self, num, reprs=None):
         """Add nodes.
-        
+
         Parameters
         ----------
         num : int
@@ -70,7 +70,7 @@ def add_nodes(self, num, reprs=None):
 
     def add_edge(self, u, v, reprs=None):
         """Add one edge.
-        
+
         Parameters
         ----------
         u : int
@@ -86,7 +86,7 @@ def add_edge(self, u, v, reprs=None):
 
     def add_edges(self, u, v, reprs=None):
         """Add many edges.
-        
+
         Parameters
         ----------
         u : list, tensor
@@ -154,7 +154,7 @@ def has_node(self, vid):
             True if the node exists
         """
         return self.has_node(vid)
-    
+
     def __contains__(self, vid):
         """Same as has_node."""
         return self.has_node(vid)
@@ -291,7 +291,7 @@ def in_edges(self, v):
         ----------
         v : int, list, tensor
             The node(s).
-        
+
         Returns
         -------
         tensor
@@ -312,7 +312,7 @@ def out_edges(self, v):
         ----------
         v : int, list, tensor
             The node(s).
-        
+
         Returns
         -------
         tensor
@@ -333,7 +333,7 @@ def edges(self, sorted=False):
         ----------
         sorted : bool
             True if the returned edges are sorted by their src and dst ids.
-        
+
         Returns
         -------
         tensor
@@ -432,7 +432,7 @@ def from_networkx(self, nx_graph, node_attrs=None, edge_attrs=None):
 
         If 'id' edge attribute exists, the edge will be added follows
         the edge id order. Otherwise, order is undefined.
-        
+
         Parameters
         ----------
         nx_graph : networkx.DiGraph
@@ -1165,22 +1165,18 @@ def propagate(self,
         kwargs : keyword arguments, optional
             Arguments for pre-defined iterators.
         """
-        if isinstance(iterator, str):
+        if isinstance(traverser, str):
             # TODO Call pre-defined routine to unroll the computation.
             raise RuntimeError('Not implemented.')
         else:
             # NOTE: the iteration can return multiple edges at each step.
-            for u, v in iterator:
+            for u, v in traverser:
                 self.send_and_recv(u, v,
                         message_func, reduce_func, apply_node_func)
 
     def subgraph(self, nodes):
         """Generate the subgraph among the given nodes.
 
-        The generated graph contains only the graph structure. The node/edge
-        features are not shared implicitly. Use `copy_from` to get node/edge
-        features from parent graph.
-
         Parameters
         ----------
         nodes : list, or iterable