Feature: Initial Hnsw implementation (surrealdb#3353)

cackle.toml

@@ -1309,3 +1309,27 @@ allow_apis = [
 
 [pkg.phf_macros]
 allow_proc_macro = true
+
+[pkg.rawpointer]
+allow_unsafe = true
+
+[pkg.matrixmultiply]
+allow_unsafe = true
+
+[pkg.approx]
+allow_unsafe = true
+
+[pkg.num-complex]
+allow_unsafe = true
+
+[pkg.ndarray]
+allow_unsafe = true
+
+[pkg.ndarray-stats]
+allow_unsafe = true
+
+[pkg.noisy_float]
+allow_unsafe = true
+
+[pkg.linfa-linalg]
+allow_unsafe = true

core/Cargo.toml

@@ -40,7 +40,7 @@ kv-fdb-7_1 = ["foundationdb/fdb-7_1", "kv-fdb", "dep:tempfile", "dep:ext-sort"]
 kv-surrealkv = ["dep:surrealkv", "tokio/time", "dep:tempfile", "dep:ext-sort"]
 scripting = ["dep:js"]
 http = ["dep:reqwest"]
-ml = ["dep:surrealml", "dep:ndarray"]
+ml = ["dep:surrealml"]
 jwks = ["dep:reqwest"]
 arbitrary = [
     "dep:arbitrary",
@@ -86,6 +86,7 @@ futures = "0.3.29"
 fuzzy-matcher = "0.3.7"
 geo = { version = "0.27.0", features = ["use-serde"] }
 geo-types = { version = "0.7.12", features = ["arbitrary"] }
+hashbrown = { version = "0.14.5", features = ["serde"] }
 hex = { version = "0.4.3" }
 indxdb = { version = "0.4.0", optional = true }
 ipnet = "2.9.0"
@@ -102,9 +103,12 @@ js = { version = "0.6.2", package = "rquickjs", features = [
 ], optional = true }
 jsonwebtoken = { version = "8.3.0-surreal.1", package = "surrealdb-jsonwebtoken" }
 lexicmp = "0.1.0"
+linfa-linalg = "=0.1.0"
 md-5 = "0.10.6"
 nanoid = "0.4.0"
-ndarray = { version = "0.15.6", optional = true }
+ndarray = { version = "=0.15.6", features = ["serde"] }
+ndarray-stats = "=0.5.1"
+num-traits = "0.2.18"
 nom = { version = "7.1.3", features = ["alloc"] }
 num_cpus = "1.16.0"
 object_store = { version = "0.8.0", optional = false }
@@ -152,6 +156,7 @@ url = "2.5.0"
 [dev-dependencies]
 criterion = { version = "0.5.1", features = ["async_tokio"] }
 env_logger = "0.10.1"
+flate2 = "1.0.28"
 pprof = { version = "0.13.0", features = ["flamegraph", "criterion"] }
 serial_test = "2.0.0"
 temp-dir = "0.1.11"

core/src/doc/index.rs

@@ -9,7 +9,7 @@ use crate::idx::IndexKeyBase;
 use crate::key;
 use crate::kvs::TransactionType;
 use crate::sql::array::Array;
-use crate::sql::index::{Index, MTreeParams, SearchParams};
+use crate::sql::index::{HnswParams, Index, MTreeParams, SearchParams};
 use crate::sql::statements::DefineIndexStatement;
 use crate::sql::{Part, Thing, Value};
 use reblessive::tree::Stk;
@@ -65,6 +65,7 @@ impl<'a> Document<'a> {
 					Index::Idx => ic.index_non_unique(txn).await?,
 					Index::Search(p) => ic.index_full_text(stk, ctx, txn, p).await?,
 					Index::MTree(p) => ic.index_mtree(stk, ctx, txn, p).await?,
+					Index::Hnsw(p) => ic.index_hnsw(ctx, p).await?,
 				};
 			}
 		}
@@ -407,4 +408,18 @@ impl<'a> IndexOperation<'a> {
 		}
 		mt.finish(&mut tx).await
 	}
+
+	async fn index_hnsw(&mut self, ctx: &Context<'_>, p: &HnswParams) -> Result<(), Error> {
+		let hnsw = ctx.get_index_stores().get_index_hnsw(self.opt, self.ix, p).await;
+		let mut hnsw = hnsw.write().await;
+		// Delete the old index data
+		if let Some(o) = self.o.take() {
+			hnsw.remove_document(self.rid, &o)?;
+		}
+		// Create the new index data
+		if let Some(n) = self.n.take() {
+			hnsw.index_document(self.rid, &n)?;
+		}
+		Ok(())
+	}
 }