feat: Add nitro engine settings

SamPatt · Dec 8, 2023 · 19637c4 · 19637c4
1 parent 48fd8de
commit 19637c4
Showing 1 changed file with 6 additions and 0 deletions.
diff --git a/extensions/inference-nitro-extension/nitro.json b/extensions/inference-nitro-extension/nitro.json
@@ -0,0 +1,6 @@
+{
+    "ctx_len": 2048,
+    "ngl": 100,
+    "cont_batching": false,
+    "embedding": false
+}