fix: image and attachments

core/src/types/assistant/assistantEntity.ts

@@ -36,3 +36,10 @@ export type Assistant = {
   /** Represents the metadata of the object. */
   metadata?: Record<string, unknown>
 }
+
+export interface CodeInterpreterTool {
+  /**
+   * The type of tool being defined: `code_interpreter`
+   */
+  type: 'code_interpreter'
+}

core/src/types/message/messageEntity.ts

@@ -1,3 +1,4 @@
+import { CodeInterpreterTool } from '../assistant'
 import { ChatCompletionMessage, ChatCompletionRole } from '../inference'
 import { ModelInfo } from '../model'
 import { Thread } from '../thread'
@@ -15,6 +16,10 @@ export type ThreadMessage = {
   thread_id: string
   /** The assistant id of this thread. **/
   assistant_id?: string
+  /**
+   * A list of files attached to the message, and the tools they were added to.
+   */
+  attachments?: Array<Attachment> | null
   /** The role of the author of this message. **/
   role: ChatCompletionRole
   /** The content of this message. **/
@@ -52,6 +57,11 @@ export type MessageRequest = {
    */
   assistantId?: string
 
+  /**
+   * A list of files attached to the message, and the tools they were added to.
+   */
+  attachments: Array<Attachment> | null
+
   /** Messages for constructing a chat completion request **/
   messages?: ChatCompletionMessage[]
 
@@ -98,7 +108,6 @@ export enum ErrorCode {
 export enum ContentType {
   Text = 'text',
   Image = 'image_url',
-  Pdf = 'pdf',
 }
 
 /**
@@ -108,8 +117,15 @@ export enum ContentType {
 export type ContentValue = {
   value: string
   annotations: string[]
-  name?: string
-  size?: number
+}
+
+/**
+ * The `ImageContentValue` type defines the shape of a content value object of image type
+ * @data_transfer_object
+ */
+export type ImageContentValue = {
+  detail?: string
+  url?: string
 }
 
 /**
@@ -118,5 +134,37 @@ export type ContentValue = {
  */
 export type ThreadContent = {
   type: ContentType
-  text: ContentValue
+  text?: ContentValue
+  image_url?: ImageContentValue
+}
+
+export interface Attachment {
+  /**
+   * The ID of the file to attach to the message.
+   */
+  file_id?: string
+
+  /**
+   * The tools to add this file to.
+   */
+  tools?: Array<CodeInterpreterTool | Attachment.AssistantToolsFileSearchTypeOnly>
+}
+
+export namespace Attachment {
+  export interface AssistantToolsFileSearchTypeOnly {
+    /**
+     * The type of tool being defined: `file_search`
+     */
+    type: 'file_search'
+  }
+}
+
+/**
+ * On an incomplete message, details about why the message is incomplete.
+ */
+export interface IncompleteDetails {
+  /**
+   * The reason the message is incomplete.
+   */
+  reason: 'content_filter' | 'max_tokens' | 'run_cancelled' | 'run_expired' | 'run_failed'
 }

extensions/assistant-extension/src/index.ts

@@ -141,7 +141,7 @@ export default class JanAssistantExtension extends AssistantExtension {
           top_k: 2,
           chunk_size: 1024,
           chunk_overlap: 64,
-          retrieval_template: `Use the following pieces of context to answer the question at the end. If you don't know the answer, just say that you don't know, don't try to make up an answer.
+          retrieval_template: `Use the following pieces of context to answer the question at the end.
 ----------------
 CONTEXT: {CONTEXT}
 ----------------

extensions/assistant-extension/src/node/index.ts

@@ -9,13 +9,14 @@ export function toolRetrievalUpdateTextSplitter(
   retrieval.updateTextSplitter(chunkSize, chunkOverlap)
 }
 export async function toolRetrievalIngestNewDocument(
+  thread: string,
   file: string,
   model: string,
   engine: string,
   useTimeWeighted: boolean
 ) {
-  const filePath = path.join(getJanDataFolderPath(), normalizeFilePath(file))
-  const threadPath = path.dirname(filePath.replace('files', ''))
+  const threadPath = path.join(getJanDataFolderPath(), 'threads', thread)
+  const filePath = path.join(getJanDataFolderPath(), 'files', file)
   retrieval.updateEmbeddingEngine(model, engine)
   return retrieval
     .ingestAgentKnowledge(filePath, `${threadPath}/memory`, useTimeWeighted)

extensions/assistant-extension/src/tools/retrieval.ts

@@ -35,6 +35,7 @@ export class RetrievalTool extends InferenceTool {
         await executeOnMain(
           NODE,
           'toolRetrievalIngestNewDocument',
+          data.thread?.id,
           docFile,
           data.model?.id,
           data.model?.engine,

web/containers/ErrorMessage/index.tsx

@@ -89,7 +89,9 @@ const ErrorMessage = ({ message }: { message: ThreadMessage }) => {
               </span>
             ) : (
               <>
-                <AutoLink text={message.content[0].text.value} />
+                {message?.content[0]?.text?.value && (
+                  <AutoLink text={message?.content[0]?.text?.value} />
+                )}
                 {defaultDesc()}
               </>
             )}

web/containers/Providers/Jotai.tsx

@@ -8,7 +8,7 @@ import { FileInfo } from '@/types/file'
 
 export const editPromptAtom = atom<string>('')
 export const currentPromptAtom = atom<string>('')
-export const fileUploadAtom = atom<FileInfo[]>([])
+export const fileUploadAtom = atom<FileInfo | undefined>()
 
 export const searchAtom = atom<string>('')
 

web/helpers/atoms/Model.atom.test.ts

@@ -58,7 +58,9 @@ describe('Model.atom.ts', () => {
         setAtom.current({ id: '1' } as any)
       })
       expect(getAtom.current).toEqual([{ id: '1' }])
-      reset.current([])
+      act(() => {
+        reset.current([])
+      })
     })
   })
 
@@ -83,7 +85,9 @@ describe('Model.atom.ts', () => {
         removeAtom.current('1')
       })
       expect(getAtom.current).toEqual([])
-      reset.current([])
+      act(() => {
+        reset.current([])
+      })
     })
   })
 
@@ -113,7 +117,9 @@ describe('Model.atom.ts', () => {
         removeAtom.current('1')
       })
       expect(getAtom.current).toEqual([])
-      reset.current([])
+      act(() => {
+        reset.current([])
+      })
     })
   })
 

web/hooks/useCreateNewThread.ts

@@ -20,6 +20,7 @@ import { fileUploadAtom } from '@/containers/Providers/Jotai'
 import { toaster } from '@/containers/Toast'
 
 import { isLocalEngine } from '@/utils/modelEngine'
+
 import { useActiveModel } from './useActiveModel'
 import useRecommendedModel from './useRecommendedModel'
 
@@ -168,7 +169,7 @@ export const useCreateNewThread = () => {
       })
 
       // Delete the file upload state
-      setFileUpload([])
+      setFileUpload(undefined)
       setActiveThread(createdThread)
     } catch (ex) {
       return toaster({

web/hooks/useDropModelBinaries.test.ts

@@ -1,3 +1,6 @@
+/**
+ * @jest-environment jsdom
+ */
 // useDropModelBinaries.test.ts
 
 import { renderHook, act } from '@testing-library/react'
@@ -18,6 +21,7 @@ jest.mock('jotai', () => ({
 jest.mock('uuid')
 jest.mock('@/utils/file')
 jest.mock('@/containers/Toast')
+jest.mock("@uppy/core")
 
 describe('useDropModelBinaries', () => {
   const mockSetImportingModels = jest.fn()

web/hooks/useSendChatMessage.ts

@@ -134,11 +134,9 @@ export default function useSendChatMessage() {
     setCurrentPrompt('')
     setEditPrompt('')
 
-    let base64Blob = fileUpload[0]
-      ? await getBase64(fileUpload[0].file)
-      : undefined
+    let base64Blob = fileUpload ? await getBase64(fileUpload.file) : undefined
 
-    if (base64Blob && fileUpload[0]?.type === 'image') {
+    if (base64Blob && fileUpload?.type === 'image') {
       // Compress image
       base64Blob = await compressImage(base64Blob, 512)
     }
@@ -171,7 +169,7 @@ export default function useSendChatMessage() {
     ).addSystemMessage(activeAssistantRef.current?.instructions)
 
     if (!isResend) {
-      requestBuilder.pushMessage(prompt, base64Blob, fileUpload[0]?.type)
+      requestBuilder.pushMessage(prompt, base64Blob, fileUpload)
 
       // Build Thread Message to persist
       const threadMessageBuilder = new ThreadMessageBuilder(
@@ -207,7 +205,7 @@ export default function useSendChatMessage() {
       selectedModelRef.current?.id ?? activeAssistantRef.current?.model.id
 
     if (base64Blob) {
-      setFileUpload([])
+      setFileUpload(undefined)
     }
 
     if (modelRef.current?.id !== modelId && modelId) {
@@ -222,9 +220,7 @@ export default function useSendChatMessage() {
     // Process message request with Assistants tools
     const request = await ToolManager.instance().process(
       requestBuilder.build(),
-      activeThreadRef.current.assistants?.flatMap(
-        (assistant) => assistant.tools ?? []
-      ) ?? []
+      activeAssistantRef?.current.tools ?? []
     )
 
     // Request for inference

web/jest.config.js

@@ -37,5 +37,5 @@ const config = {
 // module.exports = createJestConfig(config)
 module.exports = async () => ({
   ...(await createJestConfig(config)()),
-  transformIgnorePatterns: ['/node_modules/(?!(layerr)/)'],
+  transformIgnorePatterns: ['/node_modules/(?!(layerr|nanoid|@uppy|preact)/)'],
 })

web/package.json

@@ -17,6 +17,9 @@
     "@janhq/core": "link:./core",
     "@janhq/joi": "link:./joi",
     "@tanstack/react-virtual": "^3.10.9",
+    "@uppy/core": "^4.3.0",
+    "@uppy/react": "^4.0.4",
+    "@uppy/xhr-upload": "^4.2.3",
     "autoprefixer": "10.4.16",
     "class-variance-authority": "^0.7.0",
     "framer-motion": "^10.16.4",

web/screens/Thread/ThreadCenterPanel/ChatInput/index.tsx

@@ -24,6 +24,7 @@ import { useActiveModel } from '@/hooks/useActiveModel'
 
 import useSendChatMessage from '@/hooks/useSendChatMessage'
 
+import { uploader } from '@/utils/file'
 import { isLocalEngine } from '@/utils/modelEngine'
 
 import FileUploadPreview from '../FileUploadPreview'
@@ -71,6 +72,7 @@ const ChatInput = () => {
   const activeAssistant = useAtomValue(activeAssistantAtom)
   const { stopInference } = useActiveModel()
 
+  const upload = uploader()
   const [activeTabThreadRightPanel, setActiveTabThreadRightPanel] = useAtom(
     activeTabThreadRightPanelAtom
   )
@@ -104,18 +106,26 @@ const ChatInput = () => {
   const handleFileChange = (event: React.ChangeEvent<HTMLInputElement>) => {
     const file = event.target.files?.[0]
     if (!file) return
-    setFileUpload([{ file: file, type: 'pdf' }])
+    upload.addFile(file)
+    upload.upload().then((data) => {
+      setFileUpload({
+        file: file,
+        type: 'pdf',
+        id: data?.successful?.[0]?.response?.body?.id,
+        name: data?.successful?.[0]?.response?.body?.filename,
+      })
+    })
   }
 
   const handleImageChange = (event: React.ChangeEvent<HTMLInputElement>) => {
     const file = event.target.files?.[0]
     if (!file) return
-    setFileUpload([{ file: file, type: 'image' }])
+    setFileUpload({ file: file, type: 'image' })
   }
 
   const renderPreview = (fileUpload: any) => {
-    if (fileUpload.length > 0) {
-      if (fileUpload[0].type === 'image') {
+    if (fileUpload) {
+      if (fileUpload.type === 'image') {
         return <ImageUploadPreview file={fileUpload[0].file} />
       } else {
         return <FileUploadPreview />
@@ -132,7 +142,7 @@ const ChatInput = () => {
             'relative mb-1 max-h-[400px] resize-none rounded-lg border border-[hsla(var(--app-border))] p-3 pr-20',
             'focus-within:outline-none focus-visible:outline-0 focus-visible:ring-1 focus-visible:ring-[hsla(var(--primary-bg))] focus-visible:ring-offset-0',
             'overflow-y-auto',
-            fileUpload.length && 'rounded-t-none',
+            fileUpload && 'rounded-t-none',
             experimentalFeature && 'pl-10',
             activeSettingInputBox && 'pb-14 pr-16'
           )}
@@ -154,7 +164,7 @@ const ChatInput = () => {
                 className="absolute left-3 top-2.5"
                 onClick={(e) => {
                   if (
-                    fileUpload.length > 0 ||
+                    !!fileUpload ||
                     (activeAssistant?.tools &&
                       !activeAssistant?.tools[0]?.enabled &&
                       !activeAssistant?.model.settings?.vision_model)
@@ -178,12 +188,12 @@ const ChatInput = () => {
             }
             content={
               <>
-                {fileUpload.length > 0 ||
+                {!!fileUpload ||
                   (activeAssistant?.tools &&
                     !activeAssistant?.tools[0]?.enabled &&
                     !activeAssistant?.model.settings?.vision_model && (
                       <>
-                        {fileUpload.length !== 0 && (
+                        {!!fileUpload && (
                           <span>
                             Currently, we only support 1 attachment at the same
                             time.