feat: Jan can see (janhq#2069)

* feat: jan can see feat: Add GPT-4 Vision model (Preview) fix: Add visionModel as property in ModelInfo fix: Fix condition to load local messages in useSetActiveThread hook feat: Enable Image as input for chat fix: Update model parameters in JSON files for remote GPT models fix: Add thread as optional fix: Add support for message as image fix: Linter fix: Update proxyModel to proxy_model and add textModel chore: Change proxyModel to proxy_model fix: Update settings with visionModel and textModel fix: vision model passed through the retrieval tool fix: linter * fix: could not load image and request is not able to be sent --------- Co-authored-by: Louis <[email protected]>
Bura56 · Mar 5, 2024 · e6c1020 · e6c1020
1 parent 1584f0d
commit e6c1020
Show file tree

Hide file tree

Showing 18 changed files with 238 additions and 57 deletions.
diff --git a/core/src/types/model/modelEntity.ts b/core/src/types/model/modelEntity.ts
@@ -7,7 +7,7 @@ export type ModelInfo = {
   settings: ModelSettingParams
   parameters: ModelRuntimeParams
   engine?: InferenceEngine
-  proxyEngine?: InferenceEngine
+  proxy_model?: InferenceEngine
 }
 
 /**
@@ -93,12 +93,7 @@ export type Model = {
    */
   engine: InferenceEngine
 
-  proxyEngine?: InferenceEngine
-
-  /**
-   * Is multimodal or not.
-   */
-  visionModel?: boolean
+  proxy_model?: InferenceEngine
 }
 
 export type ModelMetadata = {
@@ -124,6 +119,8 @@ export type ModelSettingParams = {
   llama_model_path?: string
   mmproj?: string
   cont_batching?: boolean
+  vision_model?: boolean
+  text_model?: boolean
 }
 
 /**

diff --git a/extensions/assistant-extension/src/index.ts b/extensions/assistant-extension/src/index.ts
@@ -10,6 +10,7 @@ import {
   executeOnMain,
   AssistantExtension,
   AssistantEvent,
+  ChatCompletionMessageContentType,
 } from '@janhq/core'
 
 export default class JanAssistantExtension extends AssistantExtension {
@@ -86,7 +87,7 @@ export default class JanAssistantExtension extends AssistantExtension {
           NODE,
           'toolRetrievalIngestNewDocument',
           docFile,
-          data.model?.proxyEngine
+          data.model?.proxy_model
         )
       }
     } else if (
@@ -105,7 +106,7 @@ export default class JanAssistantExtension extends AssistantExtension {
         ...data,
         model: {
           ...data.model,
-          engine: data.model.proxyEngine,
+          engine: data.model.proxy_model,
         },
       }
       events.emit(MessageEvent.OnMessageSent, output)
@@ -168,7 +169,7 @@ export default class JanAssistantExtension extends AssistantExtension {
       ...data,
       model: {
         ...data.model,
-        engine: data.model.proxyEngine,
+        engine: data.model.proxy_model,
       },
     }
     events.emit(MessageEvent.OnMessageSent, output)

diff --git a/models/bakllava-1/model.json b/models/bakllava-1/model.json
@@ -16,6 +16,8 @@
   "description": "BakLlava 1 can bring vision understanding to Jan",
   "format": "gguf",
   "settings": {
+    "vision_model": true,
+    "text_model": false,
     "ctx_len": 4096,
     "prompt_template": "\n### Instruction:\n{prompt}\n### Response:\n",
     "llama_model_path": "ggml-model-q5_k.gguf",

diff --git a/models/gpt-3.5-turbo-16k-0613/model.json b/models/gpt-3.5-turbo-16k-0613/model.json
@@ -11,7 +11,10 @@
   "description": "OpenAI GPT 3.5 Turbo 16k 0613 model is extremely good",
   "format": "api",
   "settings": {},
-  "parameters": {},
+  "parameters": {
+    "max_tokens": 4096,
+    "temperature": 0.7
+  },
   "metadata": {
     "author": "OpenAI",
     "tags": ["General", "Big Context Length"]

diff --git a/models/gpt-3.5-turbo/model.json b/models/gpt-3.5-turbo/model.json
@@ -11,7 +11,10 @@
   "description": "OpenAI GPT 3.5 Turbo model is extremely good",
   "format": "api",
   "settings": {},
-  "parameters": {},
+  "parameters": {
+    "max_tokens": 4096,
+    "temperature": 0.7
+  },
   "metadata": {
     "author": "OpenAI",
     "tags": ["General", "Big Context Length"]

diff --git a/models/gpt-4-vision-preview/model.json b/models/gpt-4-vision-preview/model.json
@@ -0,0 +1,26 @@
+{
+  "sources": [
+    {
+      "url": "https://openai.com"
+    }
+  ],
+  "id": "gpt-4-vision-preview",
+  "object": "model",
+  "name": "OpenAI GPT 4 with Vision (Preview)",
+  "version": "1.0",
+  "description": "OpenAI GPT 4 with Vision model is extremely good in preview",
+  "format": "api",
+  "settings": {
+    "vision_model": true,
+    "textModel": false
+  },
+  "parameters": {
+    "max_tokens": 4096,
+    "temperature": 0.7
+  },
+  "metadata": {
+    "author": "OpenAI",
+    "tags": ["General", "Big Context Length", "Vision"]
+  },
+  "engine": "openai"
+}
diff --git a/models/gpt-4/model.json b/models/gpt-4/model.json
@@ -11,7 +11,10 @@
   "description": "OpenAI GPT 4 model is extremely good",
   "format": "api",
   "settings": {},
-  "parameters": {},
+  "parameters": {
+    "max_tokens": 4096,
+    "temperature": 0.7
+  },
   "metadata": {
     "author": "OpenAI",
     "tags": ["General", "Big Context Length"]

diff --git a/models/llava-1.5-13b-q5/model.json b/models/llava-1.5-13b-q5/model.json
@@ -12,10 +12,12 @@
   "id": "llava-1.5-13b-q5",
   "object": "model",
   "name": "LlaVa 1.5 13B Q5 K",
-  "version": "1.0",
+  "version": "1.1",
   "description": "LlaVa 1.5 can bring vision understanding to Jan",
   "format": "gguf",
   "settings": {
+    "vision_model": true,
+    "text_model": false,
     "ctx_len": 4096,
     "prompt_template": "\n### Instruction:\n{prompt}\n### Response:\n",
     "llama_model_path": "ggml-model-q5_k.gguf",

diff --git a/models/llava-1.5-7b-q5/model.json b/models/llava-1.5-7b-q5/model.json
@@ -12,10 +12,12 @@
   "id": "llava-1.5-7b-q5",
   "object": "model",
   "name": "LlaVa 1.5 7B Q5 K",
-  "version": "1.0",
+  "version": "1.1",
   "description": "LlaVa 1.5 can bring vision understanding to Jan",
   "format": "gguf",
   "settings": {
+    "vision_model": true,
+    "text_model": false,
     "ctx_len": 4096,
     "prompt_template": "\n### Instruction:\n{prompt}\n### Response:\n",
     "llama_model_path": "ggml-model-q5_k.gguf",

diff --git a/web/containers/DropdownListSidebar/index.tsx b/web/containers/DropdownListSidebar/index.tsx
@@ -139,11 +139,16 @@ const DropdownListSidebar = ({
 
         // Update model parameter to the thread file
         if (model)
-          updateModelParameter(activeThread.id, {
-            params: modelParams,
-            modelId: model.id,
-            engine: model.engine,
-          })
+          updateModelParameter(
+            activeThread.id,
+            {
+              params: modelParams,
+              modelId: model.id,
+              engine: model.engine,
+            },
+            // Overwrite the existing model parameter
+            true
+          )
       }
     },
     [

diff --git a/web/hooks/usePath.ts b/web/hooks/usePath.ts
@@ -1,4 +1,9 @@
-import { openFileExplorer, joinPath, getJanDataFolderPath } from '@janhq/core'
+import {
+  openFileExplorer,
+  joinPath,
+  getJanDataFolderPath,
+  baseName,
+} from '@janhq/core'
 import { useAtomValue } from 'jotai'
 
 import { selectedModelAtom } from '@/containers/DropdownListSidebar'
@@ -78,6 +83,8 @@ export const usePath = () => {
 
     const userSpace = await getJanDataFolderPath()
     let filePath = undefined
+
+    id = await baseName(id)
     filePath = await joinPath(['threads', `${activeThread.id}/files`, `${id}`])
     if (!filePath) return
     const fullPath = await joinPath([userSpace, filePath])

diff --git a/web/hooks/useSendChatMessage.ts b/web/hooks/useSendChatMessage.ts
@@ -30,7 +30,7 @@ import {
   fileUploadAtom,
 } from '@/containers/Providers/Jotai'
 
-import { getBase64 } from '@/utils/base64'
+import { compressImage, getBase64 } from '@/utils/base64'
 import { toRuntimeParams, toSettingParams } from '@/utils/modelParam'
 
 import { loadModelErrorAtom, useActiveModel } from './useActiveModel'
@@ -169,12 +169,22 @@ export default function useSendChatMessage() {
     setCurrentPrompt('')
     setEditPrompt('')
 
-    const base64Blob = fileUpload[0]
-      ? await getBase64(fileUpload[0].file).then()
+    let base64Blob = fileUpload[0]
+      ? await getBase64(fileUpload[0].file)
       : undefined
 
+    const fileContentType = fileUpload[0]?.type
+
     const msgId = ulid()
 
+    const isDocumentInput = base64Blob && fileContentType === 'pdf'
+    const isImageInput = base64Blob && fileContentType === 'image'
+
+    if (isImageInput && base64Blob) {
+      // Compress image
+      base64Blob = await compressImage(base64Blob, 512)
+    }
+
     const messages: ChatCompletionMessage[] = [
       activeThread.assistants[0]?.instructions,
     ]
@@ -202,13 +212,23 @@ export default function useSendChatMessage() {
                         type: ChatCompletionMessageContentType.Text,
                         text: prompt,
                       },
-                      {
-                        type: ChatCompletionMessageContentType.Doc,
-                        doc_url: {
-                          url: `threads/${activeThread.id}/files/${msgId}.pdf`,
-                        },
-                      },
-                    ]
+                      isDocumentInput
+                        ? {
+                            type: ChatCompletionMessageContentType.Doc,
+                            doc_url: {
+                              url: `threads/${activeThread.id}/files/${msgId}.pdf`,
+                            },
+                          }
+                        : null,
+                      isImageInput
+                        ? {
+                            type: ChatCompletionMessageContentType.Image,
+                            image_url: {
+                              url: base64Blob,
+                            },
+                          }
+                        : null,
+                    ].filter((e) => e !== null)
                   : prompt,
             } as ChatCompletionMessage,
           ])
@@ -226,8 +246,13 @@ export default function useSendChatMessage() {
     ) {
       modelRequest = {
         ...modelRequest,
-        engine: InferenceEngine.tool_retrieval_enabled,
-        proxyEngine: modelRequest.engine,
+        // Tool retrieval support document input only for now
+        ...(isDocumentInput
+          ? {
+              engine: InferenceEngine.tool_retrieval_enabled,
+              proxy_model: modelRequest.engine,
+            }
+          : {}),
       }
     }
     const messageRequest: MessageRequest = {

diff --git a/web/hooks/useSetActiveThread.ts b/web/hooks/useSetActiveThread.ts
@@ -21,17 +21,17 @@ export default function useSetActiveThread() {
 
   const setActiveThread = async (thread: Thread) => {
     // Load local messages only if there are no messages in the state
-    if (!readyMessageThreads[thread.id]) {
-      const messages = await getLocalThreadMessage(thread.id)
-      setThreadMessage(thread.id, messages)
+    if (!readyMessageThreads[thread?.id]) {
+      const messages = await getLocalThreadMessage(thread?.id)
+      setThreadMessage(thread?.id, messages)
     }
 
-    setActiveThreadId(thread.id)
+    setActiveThreadId(thread?.id)
     const modelParams: ModelParams = {
-      ...thread.assistants[0]?.model?.parameters,
-      ...thread.assistants[0]?.model?.settings,
+      ...thread?.assistants[0]?.model?.parameters,
+      ...thread?.assistants[0]?.model?.settings,
     }
-    setThreadModelParams(thread.id, modelParams)
+    setThreadModelParams(thread?.id, modelParams)
   }
 
   return { setActiveThread }

diff --git a/web/hooks/useUpdateModelParameters.ts b/web/hooks/useUpdateModelParameters.ts
@@ -37,7 +37,8 @@ export default function useUpdateModelParameters() {
 
   const updateModelParameter = async (
     threadId: string,
-    settings: UpdateModelParameter
+    settings: UpdateModelParameter,
+    overwrite: boolean = false
   ) => {
     const thread = threads.find((thread) => thread.id === threadId)
     if (!thread) {
@@ -66,8 +67,14 @@ export default function useUpdateModelParameters() {
         const runtimeParams = toRuntimeParams(updatedModelParams)
         const settingParams = toSettingParams(updatedModelParams)
 
-        assistant.model.parameters = runtimeParams
-        assistant.model.settings = settingParams
+        assistant.model.parameters = {
+          ...(overwrite ? {} : assistant.model.parameters),
+          ...runtimeParams,
+        }
+        assistant.model.settings = {
+          ...(overwrite ? {} : assistant.model.settings),
+          ...settingParams,
+        }
         if (selectedModel) {
           assistant.model.id = settings.modelId ?? selectedModel?.id
           assistant.model.engine = settings.engine ?? selectedModel?.engine