forked from janhq/jan
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request janhq#3751 from janhq/chore/add-new-models
chore: add llama3.2 and qwen2.5 models
- Loading branch information
Showing
16 changed files
with
278 additions
and
22 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
35 changes: 35 additions & 0 deletions
35
extensions/inference-nitro-extension/resources/models/llama3.2-1b-instruct/model.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| { | ||
| "sources": [ | ||
| { | ||
| "filename": "Llama-3.2-1B-Instruct-Q8_0.gguf", | ||
| "url": "https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q8_0.gguf" | ||
| } | ||
| ], | ||
| "id": "llama3.2-1b-instruct", | ||
| "object": "model", | ||
| "name": "Llama 3.2 1B Instruct Q8", | ||
| "version": "1.0", | ||
| "description": "Meta's Llama 3.2 excels at general usage situations, including chat, general world knowledge, and coding.", | ||
| "format": "gguf", | ||
| "settings": { | ||
| "ctx_len": 131072, | ||
| "prompt_template": "<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\n{system_message}<|eot_id|><|start_header_id|>user<|end_header_id|>\n\n{prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", | ||
| "llama_model_path": "Llama-3.2-1B-Instruct-Q8_0.gguf", | ||
| "ngl": 33 | ||
| }, | ||
| "parameters": { | ||
| "temperature": 0.7, | ||
| "top_p": 0.95, | ||
| "stream": true, | ||
| "max_tokens": 8192, | ||
| "stop": ["<|end_of_text|>", "<|eot_id|>", "<|eom_id|>"], | ||
| "frequency_penalty": 0, | ||
| "presence_penalty": 0 | ||
| }, | ||
| "metadata": { | ||
| "author": "MetaAI", | ||
| "tags": ["1B", "Featured"], | ||
| "size": 1320000000 | ||
| }, | ||
| "engine": "nitro" | ||
| } |
35 changes: 35 additions & 0 deletions
35
extensions/inference-nitro-extension/resources/models/llama3.2-3b-instruct/model.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| { | ||
| "sources": [ | ||
| { | ||
| "filename": "Llama-3.2-3B-Instruct-Q8_0.gguf", | ||
| "url": "https://huggingface.co/bartowski/Llama-3.2-3B-Instruct-GGUF/resolve/main/Llama-3.2-3B-Instruct-Q8_0.gguf" | ||
| } | ||
| ], | ||
| "id": "llama3.2-3b-instruct", | ||
| "object": "model", | ||
| "name": "Llama 3.2 3B Instruct Q8", | ||
| "version": "1.0", | ||
| "description": "Meta's Llama 3.2 excels at general usage situations, including chat, general world knowledge, and coding.", | ||
| "format": "gguf", | ||
| "settings": { | ||
| "ctx_len": 131072, | ||
| "prompt_template": "<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\n{system_message}<|eot_id|><|start_header_id|>user<|end_header_id|>\n\n{prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", | ||
| "llama_model_path": "Llama-3.2-3B-Instruct-Q8_0.gguf", | ||
| "ngl": 33 | ||
| }, | ||
| "parameters": { | ||
| "temperature": 0.7, | ||
| "top_p": 0.95, | ||
| "stream": true, | ||
| "max_tokens": 8192, | ||
| "stop": ["<|end_of_text|>", "<|eot_id|>", "<|eom_id|>"], | ||
| "frequency_penalty": 0, | ||
| "presence_penalty": 0 | ||
| }, | ||
| "metadata": { | ||
| "author": "MetaAI", | ||
| "tags": ["3B", "Featured"], | ||
| "size": 3420000000 | ||
| }, | ||
| "engine": "nitro" | ||
| } |
36 changes: 36 additions & 0 deletions
36
extensions/inference-nitro-extension/resources/models/qwen2.5-14b-instruct/model.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,36 @@ | ||
| { | ||
| "sources": [ | ||
| { | ||
| "filename": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", | ||
| "url": "https://huggingface.co/bartowski/Qwen2.5-14B-Instruct-GGUF/resolve/main/Qwen2.5-14B-Instruct-Q4_K_M.gguf" | ||
| } | ||
| ], | ||
| "id": "qwen2.5-14b-instruct", | ||
| "object": "model", | ||
| "name": "Qwen2.5 14B Instruct Q4", | ||
| "version": "1.0", | ||
| "description": "The Qwen 2.5 family is specifically designed to excel in math, coding, and other computational tasks", | ||
| "format": "gguf", | ||
| "settings": { | ||
| "ctx_len": 32768, | ||
| "prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant", | ||
| "llama_model_path": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", | ||
| "ngl": 49 | ||
| }, | ||
| "parameters": { | ||
| "temperature": 0.7, | ||
| "top_p": 0.95, | ||
| "stream": true, | ||
| "max_tokens": 32768, | ||
| "stop": ["<|endoftext|>", "<|im_end|>"], | ||
| "frequency_penalty": 0, | ||
| "presence_penalty": 0 | ||
| }, | ||
| "metadata": { | ||
| "author": "QwenLM", | ||
| "tags": ["14B", "Featured"], | ||
| "size": 8990000000 | ||
| }, | ||
| "engine": "nitro" | ||
| } | ||
|
|
36 changes: 36 additions & 0 deletions
36
extensions/inference-nitro-extension/resources/models/qwen2.5-32b-instruct/model.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,36 @@ | ||
| { | ||
| "sources": [ | ||
| { | ||
| "filename": "Qwen2.5-32B-Instruct-Q4_K_M.gguf", | ||
| "url": "https://huggingface.co/bartowski/Qwen2.5-32B-Instruct-GGUF/resolve/main/Qwen2.5-32B-Instruct-Q4_K_M.gguf" | ||
| } | ||
| ], | ||
| "id": "qwen2.5-32b-instruct", | ||
| "object": "model", | ||
| "name": "Qwen2.5 32B Instruct Q4", | ||
| "version": "1.0", | ||
| "description": "The Qwen 2.5 family is specifically designed to excel in math, coding, and other computational tasks", | ||
| "format": "gguf", | ||
| "settings": { | ||
| "ctx_len": 32768, | ||
| "prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant", | ||
| "llama_model_path": "Qwen2.5-32B-Instruct-Q4_K_M.gguf", | ||
| "ngl": 65 | ||
| }, | ||
| "parameters": { | ||
| "temperature": 0.7, | ||
| "top_p": 0.95, | ||
| "stream": true, | ||
| "max_tokens": 32768, | ||
| "stop": ["<|endoftext|>", "<|im_end|>"], | ||
| "frequency_penalty": 0, | ||
| "presence_penalty": 0 | ||
| }, | ||
| "metadata": { | ||
| "author": "QwenLM", | ||
| "tags": ["32B"], | ||
| "size": 19900000000 | ||
| }, | ||
| "engine": "nitro" | ||
| } | ||
|
|
36 changes: 36 additions & 0 deletions
36
extensions/inference-nitro-extension/resources/models/qwen2.5-72b-instruct/model.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,36 @@ | ||
| { | ||
| "sources": [ | ||
| { | ||
| "filename": "Qwen2.5-72B-Instruct-Q4_K_M.gguf", | ||
| "url": "https://huggingface.co/bartowski/Qwen2.5-72B-Instruct-GGUF/resolve/main/Qwen2.5-72B-Instruct-Q4_K_M.gguf" | ||
| } | ||
| ], | ||
| "id": "qwen2.5-72b-instruct", | ||
| "object": "model", | ||
| "name": "Qwen2.5 72B Instruct Q4", | ||
| "version": "1.0", | ||
| "description": "The Qwen 2.5 family is specifically designed to excel in math, coding, and other computational tasks", | ||
| "format": "gguf", | ||
| "settings": { | ||
| "ctx_len": 32768, | ||
| "prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant", | ||
| "llama_model_path": "Qwen2.5-72B-Instruct-Q4_K_M.gguf", | ||
| "ngl": 81 | ||
| }, | ||
| "parameters": { | ||
| "temperature": 0.7, | ||
| "top_p": 0.95, | ||
| "stream": true, | ||
| "max_tokens": 32768, | ||
| "stop": ["<|endoftext|>", "<|im_end|>"], | ||
| "frequency_penalty": 0, | ||
| "presence_penalty": 0 | ||
| }, | ||
| "metadata": { | ||
| "author": "QwenLM", | ||
| "tags": ["72B"], | ||
| "size": 47400000000 | ||
| }, | ||
| "engine": "nitro" | ||
| } | ||
|
|
36 changes: 36 additions & 0 deletions
36
extensions/inference-nitro-extension/resources/models/qwen2.5-7b-instruct/model.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,36 @@ | ||
| { | ||
| "sources": [ | ||
| { | ||
| "filename": "Qwen2.5-7B-Instruct-Q4_K_M.gguf", | ||
| "url": "https://huggingface.co/bartowski/Qwen2.5-7B-Instruct-GGUF/resolve/main/Qwen2.5-7B-Instruct-Q4_K_M.gguf" | ||
| } | ||
| ], | ||
| "id": "qwen2.5-7b-instruct", | ||
| "object": "model", | ||
| "name": "Qwen2.5 7B Instruct Q4", | ||
| "version": "1.0", | ||
| "description": "The Qwen 2.5 family is specifically designed to excel in math, coding, and other computational tasks", | ||
| "format": "gguf", | ||
| "settings": { | ||
| "ctx_len": 32768, | ||
| "prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant", | ||
| "llama_model_path": "Qwen2.5-7B-Instruct-Q4_K_M.gguf", | ||
| "ngl": 29 | ||
| }, | ||
| "parameters": { | ||
| "temperature": 0.7, | ||
| "top_p": 0.95, | ||
| "stream": true, | ||
| "max_tokens": 32768, | ||
| "stop": ["<|endoftext|>", "<|im_end|>"], | ||
| "frequency_penalty": 0, | ||
| "presence_penalty": 0 | ||
| }, | ||
| "metadata": { | ||
| "author": "QwenLM", | ||
| "tags": ["7B", "Featured"], | ||
| "size": 4680000000 | ||
| }, | ||
| "engine": "nitro" | ||
| } | ||
|
|
36 changes: 36 additions & 0 deletions
36
extensions/inference-nitro-extension/resources/models/qwen2.5-coder-7b-instruct/model.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,36 @@ | ||
| { | ||
| "sources": [ | ||
| { | ||
| "filename": "Qwen2.5-Coder-7B-Instruct-Q4_K_M.gguf", | ||
| "url": "https://huggingface.co/bartowski/Qwen2.5-Coder-7B-Instruct-GGUF/resolve/main/Qwen2.5-Coder-7B-Instruct-Q4_K_M.gguf" | ||
| } | ||
| ], | ||
| "id": "qwen2.5-coder-7b-instruct", | ||
| "object": "model", | ||
| "name": "Qwen2.5 Coder 7B Instruct Q4", | ||
| "version": "1.0", | ||
| "description": "Qwen2.5-Coder is the latest series of Code-Specific Qwen large language models. Significantly improvements in code generation, code reasoning and code fixing.", | ||
| "format": "gguf", | ||
| "settings": { | ||
| "ctx_len": 32768, | ||
| "prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant", | ||
| "llama_model_path": "Qwen2.5-Coder-7B-Instruct-Q4_K_M.gguf", | ||
| "ngl": 29 | ||
| }, | ||
| "parameters": { | ||
| "temperature": 0.7, | ||
| "top_p": 0.95, | ||
| "stream": true, | ||
| "max_tokens": 32768, | ||
| "stop": ["<|endoftext|>", "<|im_end|>"], | ||
| "frequency_penalty": 0, | ||
| "presence_penalty": 0 | ||
| }, | ||
| "metadata": { | ||
| "author": "QwenLM", | ||
| "tags": ["7B", "Featured"], | ||
| "size": 4680000000 | ||
| }, | ||
| "engine": "nitro" | ||
| } | ||
|
|
Oops, something went wrong.