feat: support multiple model binaries (janhq#1659)

* feat: Support multiple model binaries * fix: Update downloadModel with multiple binaries handler * feat: Add 3 models with multiple binaries * chore: fix model download * fix: model file lookup & model path * chore: add .prettierrc * chore: refactor docs * chore: bump model version * fix(capybara): add filename * fix(codeninja): add file name + llama model path * fix(default): add llama model path * fix(deepseek coder): add filename * fix(deepseek 33B): add filename * fix(dolphin mixtral): add filename * fix(llama2-chat): add filename * fix(llama2-70B): add filename * fix(mistral 7b): add filename + model path * fix(bakllava): correct size model * fix(llava-7b): correct size model * fix(llava-13b): correct size model * fix(mixtral-8x7b): add file name + modelpath * fix(noramaid-7b): add file name + modelpath * fix(openchat-7b): add file name + modelpath * fix(openhermes-7b): add file name + modelpath * fix(phi2-3b): add file name + modelpath * fix(phind): add file name + modelpath * fix(solarslerp): add file name + modelpath * fix(starling): add file name + modelpath * fix(stealth): add file name + modelpath * fix(tinyllama): add file name + modelpath * fix(trinity): add file name + modelpath * fix(tulu): add file name + modelpath * fix(wizardcoder): add file name + modelpath * fix(yi): add file name + modelpath * update from source -> sources Signed-off-by: James <[email protected]> --------- Signed-off-by: James <[email protected]> Co-authored-by: hiro <[email protected]> Co-authored-by: hahuyhoang411 <[email protected]> Co-authored-by: James <[email protected]>
SUPERMASSlVE · Jan 25, 2024 · 0e48be6 · 0e48be6
1 parent 3b8e2c5
commit 0e48be6
Show file tree

Hide file tree

Showing 53 changed files with 1,117 additions and 775 deletions.
diff --git a/core/.prettierignore b/core/.prettierignore
@@ -0,0 +1,5 @@
+.next/
+node_modules/
+dist/
+*.hbs
+*.mdx
diff --git a/core/src/node/api/common/builder.ts b/core/src/node/api/common/builder.ts
@@ -265,19 +265,22 @@ export const downloadModel = async (
   const modelBinaryPath = join(directoryPath, modelId)
 
   const request = require('request')
-  const rq = request({ url: model.source_url, strictSSL, proxy })
   const progress = require('request-progress')
-  progress(rq, {})
-    .on('progress', function (state: any) {
-      console.log('progress', JSON.stringify(state, null, 2))
-    })
-    .on('error', function (err: Error) {
-      console.error('error', err)
-    })
-    .on('end', function () {
-      console.log('end')
-    })
-    .pipe(fs.createWriteStream(modelBinaryPath))
+
+  for (const source of model.sources) {
+    const rq = request({ url: source, strictSSL, proxy })
+    progress(rq, {})
+      .on('progress', function (state: any) {
+        console.debug('progress', JSON.stringify(state, null, 2))
+      })
+      .on('error', function (err: Error) {
+        console.error('error', err)
+      })
+      .on('end', function () {
+        console.debug('end')
+      })
+      .pipe(fs.createWriteStream(modelBinaryPath))
+  }
 
   return {
     message: `Starting download ${modelId}`,

diff --git a/core/src/types/model/modelEntity.ts b/core/src/types/model/modelEntity.ts
@@ -21,6 +21,11 @@ export enum InferenceEngine {
   hf_endpoint = 'hf_endpoint',
 }
 
+export type ModelArtifact = {
+  filename: string
+  url: string
+}
+
 /**
  * Model type defines the shape of a model object.
  * @stored
@@ -45,7 +50,7 @@ export type Model = {
   /**
    * The model download source. It can be an external url or a local filepath.
    */
-  source_url: string
+  sources: ModelArtifact[]
 
   /**
    * The model identifier, which can be referenced in the API endpoints.
@@ -107,6 +112,8 @@ export type ModelSettingParams = {
   system_prompt?: string
   ai_prompt?: string
   user_prompt?: string
+  llama_model_path?: string
+  mmproj?: string
 }
 
 /**

diff --git a/docs/docs/docs/03-engineering/models.md b/docs/docs/docs/03-engineering/models.md
@@ -56,44 +56,50 @@ jan/                               # Jan root folder
 
 - Each `model` folder contains a `model.json` file, which is a representation of a model.
 - `model.json` contains metadata and default parameters used to run a model.
-- The only required field is `source_url`.
 
 ### Example
 
 Here's a standard example `model.json` for a GGUF model.
 
 ```js
 {
-"id": "zephyr-7b",                  // Defaults to foldername
-"object": "model",                  // Defaults to "model"
-"source_url": "https://huggingface.co/TheBloke/zephyr-7B-beta-GGUF/blob/main/zephyr-7b-beta.Q4_K_M.gguf",
-"name": "Zephyr 7B",                // Defaults to foldername
-"owned_by": "you",                  // Defaults to "you"
-"version": "1",                     // Defaults to 1
-"created": 1231231,                 // Defaults to file creation time
-"description": null,                // Defaults to null
-"state": enum[null, "ready"]
-"format": "ggufv3",                 // Defaults to "ggufv3"
-"engine": "nitro",                // engine_id specified in jan/engine folder
-"engine_parameters": {              // Engine parameters inside model.json can override
-    "ctx_len": 2048,                // the value inside the base engine.json
+  "id": "zephyr-7b",        // Defaults to foldername
+  "object": "model",        // Defaults to "model"
+  "sources": [
+    {
+      "filename": "zephyr-7b-beta.Q4_K_M.gguf",
+      "url": "https://huggingface.co/TheBloke/zephyr-7B-beta-GGUF/blob/main/zephyr-7b-beta.Q4_K_M.gguf"
+    }
+  ],
+  "name": "Zephyr 7B",      // Defaults to foldername
+  "owned_by": "you",        // Defaults to "you"
+  "version": "1",           // Defaults to 1
+  "created": 1231231,       // Defaults to file creation time
+  "description": null,      // Defaults to null
+  "format": "ggufv3",       // Defaults to "ggufv3"
+  "engine": "nitro",        // engine_id specified in jan/engine folder
+  "engine_parameters": {
+    // Engine parameters inside model.json can override
+    "ctx_len": 4096,        // the value inside the base engine.json
     "ngl": 100,
     "embedding": true,
-    "n_parallel": 4,
-},
-"model_parameters": {                     // Models are called parameters
+    "n_parallel": 4
+  },
+  "model_parameters": {
+    // Models are called parameters
     "stream": true,
-    "max_tokens": 2048,
-    "stop": ["<endofstring>"],      // This usually can be left blank, only used with specific need from model author
+    "max_tokens": 4096,
+    "stop": ["<endofstring>"], // This usually can be left blank, only used with specific need from model author
     "frequency_penalty": 0,
     "presence_penalty": 0,
     "temperature": 0.7,
     "top_p": 0.95
-},
-"metadata": {},                     // Defaults to {}
-"assets": [                         // Defaults to current dir
-    "file://.../zephyr-7b-q4_k_m.bin",
-]
+  },
+  "metadata": {},           // Defaults to {}
+  "assets": [
+    // Defaults to current dir
+    "file://.../zephyr-7b-q4_k_m.bin"
+  ]
 }
 ```
 

diff --git a/docs/docs/guides/04-using-models/02-import-manually.mdx b/docs/docs/guides/04-using-models/02-import-manually.mdx
@@ -31,7 +31,6 @@ In this section, we will show you how to import a GGUF model from [HuggingFace](
 
 ## Manually Importing a Downloaded Model (nightly versions and v0.4.4+)
 
-
 ### 1. Create a Model Folder
 
 Navigate to the `~/jan/models` folder. You can find this folder by going to `App Settings` > `Advanced` > `Open App Directory`.
@@ -92,7 +91,7 @@ Drag and drop your model binary into this folder, ensuring the `modelname.gguf`
 
 #### 3. Voila
 
-If your model doesn't show up in the Model Selector in conversations, please restart the app. 
+If your model doesn't show up in the Model Selector in conversations, please restart the app.
 
 If that doesn't work, please feel free to join our [Discord community](https://discord.gg/Dt7MxDyNNZ) for support, updates, and discussions.
 
@@ -190,14 +189,18 @@ Edit `model.json` and include the following configurations:
 - Ensure the filename must be `model.json`.
 - Ensure the `id` property matches the folder name you created.
 - Ensure the GGUF filename should match the `id` property exactly.
-- Ensure the `source_url` property is the direct binary download link ending in `.gguf`. In HuggingFace, you can find the direct links in the `Files and versions` tab.
+- Ensure the `source.url` property is the direct binary download link ending in `.gguf`. In HuggingFace, you can find the direct links in the `Files and versions` tab.
 - Ensure you are using the correct `prompt_template`. This is usually provided in the HuggingFace model's description page.
-- Ensure the `state` property is set to `ready`.
 
 ```json title="model.json"
 {
   // highlight-start
-  "source_url": "https://huggingface.co/janhq/trinity-v1-GGUF/resolve/main/trinity-v1.Q4_K_M.gguf",
+  "sources": [
+    {
+      "filename": "trinity-v1.Q4_K_M.gguf",
+      "url": "https://huggingface.co/janhq/trinity-v1-GGUF/resolve/main/trinity-v1.Q4_K_M.gguf"
+    }
+  ],
   "id": "trinity-v1-7b",
   // highlight-end
   "object": "model",
@@ -208,7 +211,8 @@ Edit `model.json` and include the following configurations:
   "settings": {
     "ctx_len": 4096,
     // highlight-next-line
-    "prompt_template": "{system_message}\n### Instruction:\n{prompt}\n### Response:"
+    "prompt_template": "{system_message}\n### Instruction:\n{prompt}\n### Response:",
+    "llama_model_path": "trinity-v1.Q4_K_M.gguf"
   },
   "parameters": {
     "max_tokens": 4096
@@ -218,9 +222,7 @@ Edit `model.json` and include the following configurations:
     "tags": ["7B", "Merged"],
     "size": 4370000000
   },
-  "engine": "nitro",
-  // highlight-next-line
-  "state": "ready"
+  "engine": "nitro"
 }
 ```
 

diff --git a/docs/docs/guides/04-using-models/03-integrate-with-remote-server.mdx b/docs/docs/guides/04-using-models/03-integrate-with-remote-server.mdx
@@ -40,7 +40,12 @@ Navigate to the `~/jan/models` folder. Create a folder named `gpt-3.5-turbo-16k`
 
 ```json title="~/jan/models/gpt-3.5-turbo-16k/model.json"
 {
-  "source_url": "https://openai.com",
+  "sources": [
+    {
+      "filename": "openai",
+      "url": "https://openai.com"
+    }
+  ],
   // highlight-next-line
   "id": "gpt-3.5-turbo-16k",
   "object": "model",
@@ -55,8 +60,7 @@ Navigate to the `~/jan/models` folder. Create a folder named `gpt-3.5-turbo-16k`
     "author": "OpenAI",
     "tags": ["General", "Big Context Length"]
   },
-  "engine": "openai",
-  "state": "ready"
+  "engine": "openai"
   // highlight-end
 }
 ```
@@ -118,7 +122,12 @@ Navigate to the `~/jan/models` folder. Create a folder named `mistral-ins-7b-q4`
 
 ```json title="~/jan/models/mistral-ins-7b-q4/model.json"
 {
-  "source_url": "https://jan.ai",
+  "sources": [
+    {
+      "filename": "janai",
+      "url": "https://jan.ai"
+    }
+  ],
   // highlight-next-line
   "id": "mistral-ins-7b-q4",
   "object": "model",
@@ -134,8 +143,7 @@ Navigate to the `~/jan/models` folder. Create a folder named `mistral-ins-7b-q4`
     "tags": ["remote", "awesome"]
   },
   // highlight-start
-  "engine": "openai",
-  "state": "ready"
+  "engine": "openai"
   // highlight-end
 }
 ```

diff --git a/docs/docs/guides/07-integrations/02-integrate-openrouter.mdx b/docs/docs/guides/07-integrations/02-integrate-openrouter.mdx
@@ -49,7 +49,12 @@ Navigate to the `~/jan/models` folder. Create a folder named `<openrouter-modeln
 
 ```json title="~/jan/models/openrouter-dolphin-mixtral-8x7b/model.json"
 {
-  "source_url": "https://openrouter.ai/",
+  "sources": [
+    {
+      "filename": "openrouter",
+      "url": "https://openrouter.ai/"
+    }
+  ],
   "id": "cognitivecomputations/dolphin-mixtral-8x7b",
   "object": "model",
   "name": "Dolphin 2.6 Mixtral 8x7B",
@@ -63,8 +68,7 @@ Navigate to the `~/jan/models` folder. Create a folder named `<openrouter-modeln
     "tags": ["General", "Big Context Length"]
   },
   // highlight-start
-  "engine": "openai",
-  "state": "ready"
+  "engine": "openai"
   // highlight-end
 }
 ```

diff --git a/docs/docs/guides/07-integrations/03-integrate-azure-openai-service.mdx b/docs/docs/guides/07-integrations/03-integrate-azure-openai-service.mdx
@@ -59,7 +59,12 @@ Navigate to the `~/jan/models` folder. Create a folder named `<your-deployment-n
 
 ```json title="~/jan/models/gpt-35-hieu-jan/model.json"
 {
-  "source_url": "https://hieujan.openai.azure.com",
+  "sources": [
+    {
+      "filename": "azure_openai",
+      "url": "https://hieujan.openai.azure.com"
+    }
+  ],
   // highlight-next-line
   "id": "gpt-35-hieu-jan",
   "object": "model",
@@ -75,8 +80,7 @@ Navigate to the `~/jan/models` folder. Create a folder named `<your-deployment-n
     "tags": ["General", "Big Context Length"]
   },
   // highlight-start
-  "engine": "openai",
-  "state": "ready"
+  "engine": "openai"
   // highlight-end
 }
 ```