Vector docs and cache improvements (Pometry#1391)

* add vector docs and new cache improvements * add rust docs * small fix for the docs * update enron notebook * add some interesting queries to the enron notebook * remove list of individuals from enron notebook * minor fixes for enron notebook * add RAG example to enron notebook * remove last cell from enron notebook * add override_cache parameter to vectorise * fix cache references in the docs * small fix for python vectorise doc * small fix in find_top_k function doc
D4rkisek · Dec 7, 2023 · 1caff59 · 1caff59
1 parent f723661
commit 1caff59
Show file tree

Hide file tree

Showing 11 changed files with 1,156 additions and 64 deletions.
diff --git a/examples/py/enron/embedding-cache b/examples/py/enron/embedding-cache
diff --git a/examples/py/enron/enron-vectors.ipynb b/examples/py/enron/enron-vectors.ipynb
diff --git a/examples/py/enron/nx.html b/examples/py/enron/nx.html
@@ -0,0 +1,155 @@
+<html>
+    <head>
+        <meta charset="utf-8">
+
+            <script src="lib/bindings/utils.js"></script>
+            <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/vis-network/9.1.2/dist/dist/vis-network.min.css" integrity="sha512-WgxfT5LWjfszlPHXRmBWHkV2eceiWTOBvrKCNbdgDYTHrT2AeLCGbF4sZlZw3UMN3WtL0tGUoIAKsu8mllg/XA==" crossorigin="anonymous" referrerpolicy="no-referrer" />
+            <script src="https://cdnjs.cloudflare.com/ajax/libs/vis-network/9.1.2/dist/vis-network.min.js" integrity="sha512-LnvoEWDFrqGHlHmDD2101OrLcbsfkrzoSpvtSQtxK3RMnRV0eOkhhBN2dXHKRrUU8p2DGRTk35n4O8nWSVe1mQ==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
+
+
+<center>
+<h1></h1>
+</center>
+
+<!-- <link rel="stylesheet" href="../node_modules/vis/dist/vis.min.css" type="text/css" />
+<script type="text/javascript" src="../node_modules/vis/dist/vis.js"> </script>-->
+        <link
+          href="https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bootstrap.min.css"
+          rel="stylesheet"
+          integrity="sha384-eOJMYsd53ii+scO/bJGFsiCZc+5NDVN2yr8+0RDqr0Ql0h+rP48ckxlpbzKgwra6"
+          crossorigin="anonymous"
+        />
+        <script
+          src="https://cdn.jsdelivr.net/npm/[email protected]/dist/js/bootstrap.bundle.min.js"
+          integrity="sha384-JEW9xMcG8R+pH31jmWH6WWP0WintQrMb4s7ZOdauHnUtxwoG2vI5DkLtS3qm9Ekf"
+          crossorigin="anonymous"
+        ></script>
+
+
+        <center>
+          <h1></h1>
+        </center>
+        <style type="text/css">
+
+             #mynetwork {
+                 width: 100%;
+                 height: 600px;
+                 background-color: #ffffff;
+                 border: 1px solid lightgray;
+                 position: relative;
+                 float: left;
+             }
+
+
+
+
+
+
+        </style>
+    </head>
+
+
+    <body>
+        <div class="card" style="width: 100%">
+
+
+            <div id="mynetwork" class="card-body"></div>
+        </div>
+
+
+
+
+        <script type="text/javascript">
+
+              // initialize global variables.
+              var edges;
+              var nodes;
+              var allNodes;
+              var allEdges;
+              var nodeColors;
+              var originalNodes;
+              var network;
+              var container;
+              var options, data;
+              var filter = {
+                  item : '',
+                  property : '',
+                  value : []
+              };
+
+
+
+
+
+              // This method is responsible for drawing the graph, returns the drawn network
+              function drawGraph() {
+                  var container = document.getElementById('mynetwork');
+
+
+
+                  // parsing and collecting nodes and edges from the python
+                  nodes = new vis.DataSet([{"color": "#97c2fc", "id": 6076003423996713654, "image": "https://cdn-icons-png.flaticon.com/512/7584/7584620.png", "label": "greg.whalley", "shape": "dot"}, {"color": "#97c2fc", "id": 3706582846319807979, "image": "https://cdn-icons-png.flaticon.com/512/7584/7584620.png", "label": "michael.norris", "shape": "dot"}, {"color": "#97c2fc", "id": 5350559918789963524, "image": "https://cdn-icons-png.flaticon.com/512/7584/7584620.png", "label": "w..pereira", "shape": "dot"}, {"color": "#97c2fc", "id": 4166911038501177767, "image": "https://cdn-icons-png.flaticon.com/512/7584/7584620.png", "label": "kenneth.lay", "shape": "dot"}, {"color": "#97c2fc", "id": 15621484258632277716, "image": "https://cdn-icons-png.flaticon.com/512/7584/7584620.png", "label": "a..howard", "shape": "dot"}, {"color": "#97c2fc", "id": 11195289028662148947, "image": "https://cdn-icons-png.flaticon.com/512/7584/7584620.png", "label": "rod.hayslett", "shape": "dot"}, {"color": "#97c2fc", "id": 7142029215401505166, "image": "https://cdn-icons-png.flaticon.com/512/7584/7584620.png", "label": "stanley.horton", "shape": "dot"}, {"color": "#97c2fc", "id": 1598159658826880704, "image": "https://cdn-icons-png.flaticon.com/512/7584/7584620.png", "label": "khortongolf", "shape": "dot"}, {"color": "#97c2fc", "id": 3135071489733153058, "image": "https://cdn-icons-png.flaticon.com/512/7584/7584620.png", "label": "betty.alexander", "shape": "dot"}, {"color": "#97c2fc", "id": 4916201938344921332, "image": "https://cdn-icons-png.flaticon.com/512/7584/7584620.png", "label": "deane.pierce", "shape": "dot"}, {"color": "#97c2fc", "id": 8336200709397585546, "image": "https://cdn-icons-png.flaticon.com/512/7584/7584620.png", "label": "a..hughes", "shape": "dot"}]);
+                  edges = new vis.DataSet([{"arrowStrikethrough": false, "color": "#FF0000", "from": 6076003423996713654, "title": "", "to": 3706582846319807979, "value": 1}, {"arrowStrikethrough": false, "color": "#FF0000", "from": 5350559918789963524, "title": "", "to": 4166911038501177767, "value": 1}, {"arrowStrikethrough": false, "color": "#FF0000", "from": 15621484258632277716, "title": "", "to": 11195289028662148947, "value": 1}, {"arrowStrikethrough": false, "color": "#FF0000", "from": 3706582846319807979, "title": "", "to": 4166911038501177767, "value": 1}, {"arrowStrikethrough": false, "color": "#FF0000", "from": 7142029215401505166, "title": "", "to": 1598159658826880704, "value": 1}, {"arrowStrikethrough": false, "color": "#FF0000", "from": 3135071489733153058, "title": "", "to": 4166911038501177767, "value": 1}, {"arrowStrikethrough": false, "color": "#FF0000", "from": 4916201938344921332, "title": "", "to": 4166911038501177767, "value": 1}, {"arrowStrikethrough": false, "color": "#FF0000", "from": 8336200709397585546, "title": "", "to": 7142029215401505166, "value": 1}]);
+
+                  nodeColors = {};
+                  allNodes = nodes.get({ returnType: "Object" });
+                  for (nodeId in allNodes) {
+                    nodeColors[nodeId] = allNodes[nodeId].color;
+                  }
+                  allEdges = edges.get({ returnType: "Object" });
+                  // adding nodes and edges to the graph
+                  data = {nodes: nodes, edges: edges};
+
+                  var options = {
+    "configure": {
+        "enabled": false
+    },
+    "edges": {
+        "color": {
+            "inherit": true
+        },
+        "smooth": {
+            "enabled": true,
+            "type": "dynamic"
+        }
+    },
+    "interaction": {
+        "dragNodes": true,
+        "hideEdgesOnDrag": false,
+        "hideNodesOnDrag": false
+    },
+    "physics": {
+        "enabled": true,
+        "stabilization": {
+            "enabled": true,
+            "fit": true,
+            "iterations": 1000,
+            "onlyDynamicEdges": false,
+            "updateInterval": 50
+        }
+    }
+};
+
+
+
+
+
+
+                  network = new vis.Network(container, data, options);
+
+
+
+
+
+
+
+
+
+
+                  return network;
+
+              }
+              drawGraph();
+        </script>
+    </body>
+</html>
diff --git a/raphtory-graphql/src/server.rs b/raphtory-graphql/src/server.rs
@@ -94,6 +94,7 @@ impl RaphtoryServer {
                 .vectorise_with_template(
                     Box::new(embedding.clone()),
                     Some(graph_cache),
+                    true,
                     template.clone(),
                     true,
                 )

diff --git a/raphtory/src/python/graph/views/graph_view.rs b/raphtory/src/python/graph/views/graph_view.rs
@@ -308,11 +308,24 @@ impl PyGraphView {
         GraphIndex::new(self.graph.clone())
     }
 
-    #[pyo3(signature = (embedding, cache = None, node_document = None, edge_document = None, verbose = false))]
+    /// Create a VectorisedGraph from the current graph
+    ///
+    /// Args:
+    ///   embedding (Callable[[list], list]): the embedding function to translate documents to embeddings
+    ///   cache (str): the file to be used as a cache to avoid calling the embedding function (optional)
+    ///   overwrite_cache (bool): whether or not to overwrite the cache if there are new embeddings (optional)
+    ///   node_document (str): the property name to be used as document for nodes (optional)
+    ///   edge_document (str): the property name to be used as document for edges (optional)
+    ///   verbose (bool): whether or not to print logs reporting the progress
+    ///   
+    /// Returns:
+    ///   A VectorisedGraph with all the documents/embeddings computed and with an initial empty selection
+    #[pyo3(signature = (embedding, cache = None, overwrite_cache = false, node_document = None, edge_document = None, verbose = false))]
     fn vectorise(
         &self,
         embedding: &PyFunction,
         cache: Option<String>,
+        overwrite_cache: bool,
         node_document: Option<String>,
         edge_document: Option<String>,
         verbose: bool,
@@ -323,7 +336,13 @@ impl PyGraphView {
         let template = PyDocumentTemplate::new(node_document, edge_document);
         execute_async_task(move || async move {
             graph
-                .vectorise_with_template(Box::new(embedding.clone()), cache, template, verbose)
+                .vectorise_with_template(
+                    Box::new(embedding.clone()),
+                    cache,
+                    overwrite_cache,
+                    template,
+                    verbose,
+                )
                 .await
         })
     }