Feat: Adjust model hub v0.4.13 (janhq#2879)

* fix: correct phi3 * redundant phi2 dolphin * add: hermes llama3 * add: ngl settings * correct ctx len * correct ngl * correct maxlen + ngl * disable phi3 * add ngl * add ngl * add ngl * add ngl * add ngl * add ngl * add ngl * remove redundant hermes pro * add ngl * add ngl * add ngl * remove miqu * add ngl * add ngl * add ngl * add ngl * remove redundant * add ngl * add ngl * add ngl * add ngl * add ngl * add ngl * add ngl * add ngl * add ngl * version package bump * feat: resolve issue of cannot found model in the extensions due to the removal * feat: completely remove hermes-pro-7b * feat: completely remove openhermes-neural-7b and miqu-70b, and add llama3-hermes-8b via renaming from Rex * fix: correct description --------- Co-authored-by: Van-QA <[email protected]>
rogershi-dev · May 13, 2024 · 1e0d4f3 · 1e0d4f3
1 parent 6af4a2d
commit 1e0d4f3
Show file tree

Hide file tree

Showing 30 changed files with 66 additions and 178 deletions.
diff --git a/extensions/inference-nitro-extension/package.json b/extensions/inference-nitro-extension/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@janhq/inference-nitro-extension",
   "productName": "Nitro Inference Engine",
-  "version": "1.0.5",
+  "version": "1.0.6",
   "description": "This extension embeds Nitro, a lightweight (3mb) inference engine written in C++. See https://nitro.jan.ai.\nAdditional dependencies could be installed to run without Cuda Toolkit installation.",
   "main": "dist/index.js",
   "node": "dist/node/index.cjs.js",

diff --git a/extensions/inference-nitro-extension/resources/models/codeninja-1.0-7b/model.json b/extensions/inference-nitro-extension/resources/models/codeninja-1.0-7b/model.json
@@ -14,7 +14,8 @@
   "settings": {
     "ctx_len": 8192,
     "prompt_template": "GPT4 Correct User: {prompt}<|end_of_turn|>GPT4 Correct Assistant:",
-    "llama_model_path": "codeninja-1.0-openchat-7b.Q4_K_M.gguf"
+    "llama_model_path": "codeninja-1.0-openchat-7b.Q4_K_M.gguf",
+    "ngl": 32
   },
   "parameters": {
     "temperature": 0.7,

diff --git a/extensions/inference-nitro-extension/resources/models/command-r-34b/model.json b/extensions/inference-nitro-extension/resources/models/command-r-34b/model.json
@@ -14,7 +14,8 @@
     "settings": {
       "ctx_len": 131072,
       "prompt_template": "<|START_OF_TURN_TOKEN|><|USER_TOKEN|>{prompt}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>",
-      "llama_model_path": "c4ai-command-r-v01-Q4_K_M.gguf"
+      "llama_model_path": "c4ai-command-r-v01-Q4_K_M.gguf",
+      "ngl": 40
     },
     "parameters": {
       "temperature": 0.7,

diff --git a/extensions/inference-nitro-extension/resources/models/deepseek-coder-1.3b/model.json b/extensions/inference-nitro-extension/resources/models/deepseek-coder-1.3b/model.json
@@ -14,7 +14,8 @@
   "settings": {
     "ctx_len": 16384,
     "prompt_template": "### Instruction:\n{prompt}\n### Response:",
-    "llama_model_path": "deepseek-coder-1.3b-instruct.Q8_0.gguf"
+    "llama_model_path": "deepseek-coder-1.3b-instruct.Q8_0.gguf",
+    "ngl": 24
   },
   "parameters": {
     "temperature": 0.7,

diff --git a/extensions/inference-nitro-extension/resources/models/deepseek-coder-34b/model.json b/extensions/inference-nitro-extension/resources/models/deepseek-coder-34b/model.json
@@ -14,7 +14,8 @@
   "settings": {
     "ctx_len": 16384,
     "prompt_template": "### Instruction:\n{prompt}\n### Response:",
-    "llama_model_path": "deepseek-coder-33b-instruct.Q4_K_M.gguf"
+    "llama_model_path": "deepseek-coder-33b-instruct.Q4_K_M.gguf",
+    "ngl": 62
   },
   "parameters": {
     "temperature": 0.7,

diff --git a/extensions/inference-nitro-extension/resources/models/dolphin-phi-2/model.json b/extensions/inference-nitro-extension/resources/models/dolphin-phi-2/model.json
diff --git a/extensions/inference-nitro-extension/resources/models/gemma-2b/model.json b/extensions/inference-nitro-extension/resources/models/gemma-2b/model.json
@@ -14,7 +14,8 @@
   "settings": {
     "ctx_len": 8192,
     "prompt_template": "<start_of_turn>user\n{prompt}<end_of_turn>\n<start_of_turn>model",
-    "llama_model_path": "gemma-2b-it-q4_k_m.gguf"
+    "llama_model_path": "gemma-2b-it-q4_k_m.gguf",
+    "ngl": 18
   },
   "parameters": {
     "temperature": 0.7,

diff --git a/extensions/inference-nitro-extension/resources/models/gemma-7b/model.json b/extensions/inference-nitro-extension/resources/models/gemma-7b/model.json
@@ -14,7 +14,8 @@
   "settings": {
     "ctx_len": 8192,
     "prompt_template": "<start_of_turn>user\n{prompt}<end_of_turn>\n<start_of_turn>model",
-    "llama_model_path": "gemma-7b-it-q4_K_M.gguf"
+    "llama_model_path": "gemma-7b-it-q4_K_M.gguf",
+    "ngl": 28
   },
   "parameters": {
     "temperature": 0.7,

diff --git a/extensions/inference-nitro-extension/resources/models/llama2-chat-70b/model.json b/extensions/inference-nitro-extension/resources/models/llama2-chat-70b/model.json
@@ -14,7 +14,8 @@
   "settings": {
     "ctx_len": 4096,
     "prompt_template": "[INST] <<SYS>>\n{system_message}<</SYS>>\n{prompt}[/INST]",
-    "llama_model_path": "llama-2-70b-chat.Q4_K_M.gguf"
+    "llama_model_path": "llama-2-70b-chat.Q4_K_M.gguf",
+    "ngl": 80
   },
   "parameters": {
     "temperature": 0.7,

diff --git a/extensions/inference-nitro-extension/resources/models/llama2-chat-7b/model.json b/extensions/inference-nitro-extension/resources/models/llama2-chat-7b/model.json
@@ -14,7 +14,8 @@
   "settings": {
     "ctx_len": 4096,
     "prompt_template": "[INST] <<SYS>>\n{system_message}<</SYS>>\n{prompt}[/INST]",
-    "llama_model_path": "llama-2-7b-chat.Q4_K_M.gguf"
+    "llama_model_path": "llama-2-7b-chat.Q4_K_M.gguf",
+    "ngl": 32
   },
   "parameters": {
     "temperature": 0.7,

diff --git a/extensions/inference-nitro-extension/resources/models/llama3-8b-instruct/model.json b/extensions/inference-nitro-extension/resources/models/llama3-8b-instruct/model.json
@@ -8,19 +8,20 @@
     "id": "llama3-8b-instruct",
     "object": "model",
     "name": "Llama 3 8B Q4",
-    "version": "1.0",
+    "version": "1.1",
     "description": "Meta's Llama 3 excels at general usage situations, including chat, general world knowledge, and coding.",
     "format": "gguf",
     "settings": {
       "ctx_len": 8192,
       "prompt_template": "<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\n{system_message}<|eot_id|><|start_header_id|>user<|end_header_id|>\n\n{prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n",
-      "llama_model_path": "Meta-Llama-3-8B-Instruct-Q4_K_M.gguf"
+      "llama_model_path": "Meta-Llama-3-8B-Instruct-Q4_K_M.gguf",
+      "ngl": 32
     },
     "parameters": {
       "temperature": 0.7,
       "top_p": 0.95,
       "stream": true,
-      "max_tokens": 4096,
+      "max_tokens": 8192,
       "stop": ["<|end_of_text|>","<|eot_id|>"],
       "frequency_penalty": 0,
       "presence_penalty": 0

diff --git a/...resources/models/hermes-pro-7b/model.json → ...ources/models/llama3-hermes-8b/model.json b/...resources/models/hermes-pro-7b/model.json → ...ources/models/llama3-hermes-8b/model.json
@@ -1,35 +1,38 @@
 {
     "sources": [
       {
-        "filename": "Hermes-2-Pro-Mistral-7B.Q4_K_M.gguf",
-        "url": "https://huggingface.co/NousResearch/Hermes-2-Pro-Mistral-7B-GGUF/resolve/main/Hermes-2-Pro-Mistral-7B.Q4_K_M.gguf"
+        "filename": "Hermes-2-Pro-Llama-3-8B-Q4_K_M.gguf",
+        "url": "https://huggingface.co/NousResearch/Hermes-2-Pro-Llama-3-8B-GGUF/resolve/main/Hermes-2-Pro-Llama-3-8B-Q4_K_M.gguf"
       }
     ],
-    "id": "hermes-pro-7b",
+    "id": "llama3-hermes-8b",
     "object": "model",
-    "name": "Hermes Pro 7B Q4",
+    "name": "Hermes Pro Llama 3 8B Q4",
     "version": "1.1",
-    "description": "Hermes Pro is superior in Roleplaying, Reasoning and Explaining problem.",
+    "description": "Hermes Pro is well-designed for General chat and JSON output.",
     "format": "gguf",
     "settings": {
-      "ctx_len": 4096,
+      "ctx_len": 8192,
       "prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant",
-      "llama_model_path": "Hermes-2-Pro-Mistral-7B.Q4_K_M.gguf"
+      "llama_model_path": "Hermes-2-Pro-Llama-3-8B-Q4_K_M.gguf",
+      "ngl": 32
     },
     "parameters": {
       "temperature": 0.7,
       "top_p": 0.95,
       "stream": true,
-      "max_tokens": 4096,
+      "max_tokens": 8192,
       "stop": [],
       "frequency_penalty": 0,
       "presence_penalty": 0
     },
     "metadata": {
       "author": "NousResearch",
-      "tags": ["7B", "Finetuned"],
-      "size": 4370000000
+      "tags": [
+        "7B",
+        "Finetuned"
+      ],
+      "size": 4920000000
     },
     "engine": "nitro"
   }
-
diff --git a/extensions/inference-nitro-extension/resources/models/llamacorn-1.1b/model.json b/extensions/inference-nitro-extension/resources/models/llamacorn-1.1b/model.json
@@ -14,7 +14,8 @@
     "settings": {
       "ctx_len": 2048,
       "prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant",
-      "llama_model_path": "llamacorn-1.1b-chat.Q8_0.gguf"
+      "llama_model_path": "llamacorn-1.1b-chat.Q8_0.gguf",
+      "ngl": 22
     },
     "parameters": {
       "temperature": 0.7,

diff --git a/extensions/inference-nitro-extension/resources/models/miqu-70b/model.json b/extensions/inference-nitro-extension/resources/models/miqu-70b/model.json
diff --git a/extensions/inference-nitro-extension/resources/models/mistral-ins-7b-q4/model.json b/extensions/inference-nitro-extension/resources/models/mistral-ins-7b-q4/model.json
@@ -14,7 +14,8 @@
   "settings": {
     "ctx_len": 32768,
     "prompt_template": "[INST] {prompt} [/INST]",
-    "llama_model_path": "mistral-7b-instruct-v0.2.Q4_K_M.gguf"
+    "llama_model_path": "mistral-7b-instruct-v0.2.Q4_K_M.gguf",
+    "ngl": 32
   },
   "parameters": {
     "temperature": 0.7,

diff --git a/extensions/inference-nitro-extension/resources/models/mixtral-8x7b-instruct/model.json b/extensions/inference-nitro-extension/resources/models/mixtral-8x7b-instruct/model.json
@@ -14,7 +14,8 @@
   "settings": {
     "ctx_len": 32768,
     "prompt_template": "[INST] {prompt} [/INST]",
-    "llama_model_path": "mixtral-8x7b-instruct-v0.1.Q4_K_M.gguf"
+    "llama_model_path": "mixtral-8x7b-instruct-v0.1.Q4_K_M.gguf",
+    "ngl": 100
   },
   "parameters": {
     "temperature": 0.7,

diff --git a/extensions/inference-nitro-extension/resources/models/noromaid-7b/model.json b/extensions/inference-nitro-extension/resources/models/noromaid-7b/model.json
@@ -14,7 +14,8 @@
   "settings": {
     "ctx_len": 32768,
     "prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant",
-    "llama_model_path": "Noromaid-7B-0.4-DPO.q4_k_m.gguf"
+    "llama_model_path": "Noromaid-7B-0.4-DPO.q4_k_m.gguf",
+    "ngl": 32
   },
   "parameters": {
     "temperature": 0.7,

diff --git a/extensions/inference-nitro-extension/resources/models/openchat-3.5-7b/model.json b/extensions/inference-nitro-extension/resources/models/openchat-3.5-7b/model.json
@@ -14,7 +14,8 @@
   "settings": {
     "ctx_len": 8192,
     "prompt_template": "GPT4 Correct User: {prompt}<|end_of_turn|>GPT4 Correct Assistant:",
-    "llama_model_path": "openchat-3.5-0106.Q4_K_M.gguf"
+    "llama_model_path": "openchat-3.5-0106.Q4_K_M.gguf",
+    "ngl": 32
   },
   "parameters": {
     "temperature": 0.7,

diff --git a/extensions/inference-nitro-extension/resources/models/openhermes-neural-7b/model.json b/extensions/inference-nitro-extension/resources/models/openhermes-neural-7b/model.json
diff --git a/extensions/inference-nitro-extension/resources/models/phi3-3.8b/model.json b/extensions/inference-nitro-extension/resources/models/phi3-3.8b/model.json
diff --git a/extensions/inference-nitro-extension/resources/models/phind-34b/model.json b/extensions/inference-nitro-extension/resources/models/phind-34b/model.json
@@ -14,7 +14,8 @@
   "settings": {
     "ctx_len": 16384,
     "prompt_template": "### System Prompt\n{system_message}\n### User Message\n{prompt}\n### Assistant",
-    "llama_model_path": "phind-codellama-34b-v2.Q4_K_M.gguf"
+    "llama_model_path": "phind-codellama-34b-v2.Q4_K_M.gguf",
+    "ngl": 48
   },
   "parameters": {
     "temperature": 0.7,

diff --git a/extensions/inference-nitro-extension/resources/models/qwen-7b/model.json b/extensions/inference-nitro-extension/resources/models/qwen-7b/model.json
@@ -14,7 +14,8 @@
   "settings": {
     "ctx_len": 32768,
     "prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant",
-    "llama_model_path": "qwen1_5-7b-chat-q4_k_m.gguf"
+    "llama_model_path": "qwen1_5-7b-chat-q4_k_m.gguf",
+    "ngl": 32
   },
   "parameters": {
     "temperature": 0.7,

diff --git a/extensions/inference-nitro-extension/resources/models/stable-zephyr-3b/model.json b/extensions/inference-nitro-extension/resources/models/stable-zephyr-3b/model.json
@@ -14,7 +14,8 @@
     "settings": {
       "ctx_len": 4096,
       "prompt_template": "<|user|>\n{prompt}<|endoftext|>\n<|assistant|>",
-      "llama_model_path": "stablelm-zephyr-3b.Q8_0.gguf"
+      "llama_model_path": "stablelm-zephyr-3b.Q8_0.gguf",
+      "ngl": 32
     },
     "parameters": {
       "temperature": 0.7,

diff --git a/extensions/inference-nitro-extension/resources/models/stealth-v1.2-7b/model.json b/extensions/inference-nitro-extension/resources/models/stealth-v1.2-7b/model.json
@@ -14,7 +14,8 @@
   "settings": {
     "ctx_len": 32768,
     "prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant",
-    "llama_model_path": "stealth-v1.3.Q4_K_M.gguf"
+    "llama_model_path": "stealth-v1.3.Q4_K_M.gguf",
+    "ngl": 32
   },
   "parameters": {
     "temperature": 0.7,

diff --git a/extensions/inference-nitro-extension/resources/models/tinyllama-1.1b/model.json b/extensions/inference-nitro-extension/resources/models/tinyllama-1.1b/model.json
@@ -14,7 +14,8 @@
   "settings": {
     "ctx_len": 4096,
     "prompt_template": "<|system|>\n{system_message}<|user|>\n{prompt}<|assistant|>",
-    "llama_model_path": "tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf"
+    "llama_model_path": "tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf",
+    "ngl": 22
   },
   "parameters": {
     "temperature": 0.7,