Skip to content

Commit

Permalink
Chore: aya update (janhq#2941)
Browse files Browse the repository at this point in the history
* init

* init

* fix: correct format

* version bump

* add: aya 8b, aya 35b, phi3

* fix: stop token

* fix: stop token
hahuyhoang411 authored May 24, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
1 parent 20c9c3f commit 25daba9
Showing 4 changed files with 78 additions and 2 deletions.
2 changes: 1 addition & 1 deletion extensions/inference-nitro-extension/package.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "@janhq/inference-cortex-extension",
"productName": "Cortex Inference Engine",
"version": "1.0.9",
"version": "1.0.10",
"description": "This extension embeds cortex.cpp, a lightweight inference engine written in C++. See https://nitro.jan.ai.\nAdditional dependencies could be installed to run without Cuda Toolkit installation.",
"main": "dist/index.js",
"node": "dist/node/index.cjs.js",
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
{
"sources": [
{
"filename": "aya-23-35B-Q4_K_M.gguf",
"url": "https://huggingface.co/bartowski/aya-23-35B-GGUF/resolve/main/aya-23-35B-Q4_K_M.gguf"
}
],
"id": "aya-23-35b",
"object": "model",
"name": "Aya 23 35B Q4",
"version": "1.0",
"description": "Aya 23 can talk upto 23 languages fluently.",
"format": "gguf",
"settings": {
"ctx_len": 8192,
"prompt_template": "<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>{system_prompt}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|USER_TOKEN|>{prompt}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>",
"llama_model_path": "aya-23-35B-Q4_K_M.gguf",
"ngl": 40
},
"parameters": {
"temperature": 0.7,
"top_p": 0.95,
"stream": true,
"max_tokens": 8192,
"frequency_penalty": 0,
"presence_penalty": 0,
"stop": ["<|END_OF_TURN_TOKEN|>"]
},
"metadata": {
"author": "CohereForAI",
"tags": ["34B", "Finetuned"],
"size": 21556982144
},
"engine": "nitro"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
{
"sources": [
{
"filename": "aya-23-8B-Q4_K_M.gguf",
"url": "https://huggingface.co/bartowski/aya-23-8B-GGUF/resolve/main/aya-23-8B-Q4_K_M.gguf"
}
],
"id": "aya-23-8b",
"object": "model",
"name": "Aya 23 8B Q4",
"version": "1.0",
"description": "Aya 23 can talk upto 23 languages fluently.",
"format": "gguf",
"settings": {
"ctx_len": 8192,
"prompt_template": "<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>{system_prompt}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|USER_TOKEN|>{prompt}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>",
"llama_model_path": "aya-23-8B-Q4_K_M.gguf",
"ngl": 32
},
"parameters": {
"temperature": 0.7,
"top_p": 0.95,
"stream": true,
"max_tokens": 8192,
"frequency_penalty": 0,
"presence_penalty": 0,
"stop": ["<|END_OF_TURN_TOKEN|>"]
},
"metadata": {
"author": "CohereForAI",
"tags": ["7B", "Finetuned","Featured"],
"size": 5056982144
},
"engine": "nitro"
}
8 changes: 7 additions & 1 deletion extensions/inference-nitro-extension/rollup.config.ts
Original file line number Diff line number Diff line change
@@ -35,6 +35,9 @@ const wizardcoder13bJson = require('./resources/models/wizardcoder-13b/model.jso
const yi34bJson = require('./resources/models/yi-34b/model.json')
const llama3Json = require('./resources/models/llama3-8b-instruct/model.json')
const llama3Hermes8bJson = require('./resources/models/llama3-hermes-8b/model.json')
const aya8bJson = require('./resources/models/aya-23-8b/model.json')
const aya35bJson = require('./resources/models/aya-23-35b/model.json')
const phimediumJson = require('./resources/models/phi3-medium/model.json')

export default [
{
@@ -76,7 +79,10 @@ export default [
wizardcoder13bJson,
yi34bJson,
llama3Json,
llama3Hermes8bJson
llama3Hermes8bJson,
phimediumJson,
aya8bJson,
aya35bJson
]),
NODE: JSON.stringify(`${packageJson.name}/${packageJson.node}`),
DEFAULT_SETTINGS: JSON.stringify(defaultSettingJson),

0 comments on commit 25daba9

Please sign in to comment.