Introduce neocons.rest.nodes/create-batch

ChangeLog.md

@@ -1,11 +1,20 @@
 ## Changes between Neocons 1.0.0-rc2 and 1.0.0-rc3
 
+### Batch creation of nodes
+
+A new function, `clojurewerkz.neocons.rest.nodes/create-batch`, can be used to efficiently insert a large number of nodes
+at the same time (up to hundreds of thousands or millions).
+
+It returns a lazy sequence of results, which both makes it more memory efficient and may require forcing the evaluation
+with `clojure.core/doall` in some cases.
+
+
 ### Unique indexes
 
-`clojurewerkz.neocons.rest.nodes/add-to-index` now accepts a new configuration option: `:unique`, which makes
+`clojurewerkz.neocons.rest.nodes/create-index` now accepts a new configuration option: `:unique`, which makes
 the index unique (that allows/guarantees only one entry per key).
 
-`clojurewerkz.neocons.rest.relationships/add-to-index` works the same way.
+`clojurewerkz.neocons.rest.relationships/create-index` works the same way.
 
 
 ## Changes between Neocons 1.0.0-rc1 and 1.0.0-rc2

project.clj

@@ -18,6 +18,7 @@
                    :edge-features  :edge-features
                    ;; assorted examples (extra integration tests)
                    :examples       :examples
+                   :batching       :batching
                    :all            (constantly true)}
   :source-paths ["src/clojure"]
   :profiles       {:1.4 {:dependencies [[org.clojure/clojure "1.4.0"]]}

src/clojure/clojurewerkz/neocons/rest/nodes.clj

@@ -42,6 +42,20 @@
          (add-to-index node idx k v))
        node)))
 
+(defn create-batch
+  "Does an efficient batch insert of multiple nodes. Use it if you need to insert tens of hundreds of thousands
+   of nodes.
+
+   This function returns a lazy sequence of results, so you may need to force it using clojure.core/doall"
+  [xs]
+  (let [batched (doall (reduce (fn [acc x]
+                          (conj acc {:body   x
+                                     :to     "/node"
+                                     :method "POST"})) [] xs))
+        {:keys [status headers body]} (rest/POST (:batch-uri rest/*endpoint*) :body (json/json-str batched))
+        payload                       (map :body (json/read-json body true))]
+    (map instantiate-node-from payload)))
+
 (defn get
   "Fetches a node by id"
   [^long id]

test/clojurewerkz/neocons/rest/test/batch_operations_test.clj

@@ -0,0 +1,17 @@
+(ns clojurewerkz.neocons.rest.test.batch-operations-test
+  (:require [clojurewerkz.neocons.rest               :as neorest]
+            [clojurewerkz.neocons.rest.nodes         :as nn])
+  (:use clojure.test))
+
+(neorest/connect! "http://localhost:7474/db/data/")
+
+
+(deftest ^{:batching true} test-basic-batching-of-inserts
+  (let [n     100
+        rng   (range 0 n)
+        xs    (doall (map (fn [x] {:n x}) rng))
+        nodes (doall (nn/create-batch xs))]
+    ;; ensure that we aren't tripped by laziness
+    (is (= (count nodes) n))
+    (is (= (count (map :id nodes)) n))
+    (nn/delete-many (vec nodes))))