fix: override cpu_threads setting from model.json (janhq#2789)

hansongwang · Apr 23, 2024 · da161cd · da161cd
1 parent e54e7c0
commit da161cd
Show file tree

Hide file tree

Showing 6 changed files with 58 additions and 40 deletions.
diff --git a/core/src/node/api/restful/helper/startStopModel.ts b/core/src/node/api/restful/helper/startStopModel.ts
@@ -63,11 +63,11 @@ const runModel = async (modelId: string, settingParams?: ModelSettingParams): Pr
 
   const nitroResourceProbe = await getSystemResourceInfo()
   const nitroModelSettings: NitroModelSettings = {
+    // This is critical and requires real CPU physical core count (or performance core)
+    cpu_threads: Math.max(1, nitroResourceProbe.numCpuPhysicalCore),
     ...modelMetadata.settings,
     ...settingParams,
     llama_model_path: modelBinaryPath,
-    // This is critical and requires real CPU physical core count (or performance core)
-    cpu_threads: Math.max(1, nitroResourceProbe.numCpuPhysicalCore),
     ...(modelMetadata.settings.mmproj && {
       mmproj: join(modelFolderFullPath, modelMetadata.settings.mmproj),
     }),

diff --git a/extensions/assistant-extension/src/node/engine.ts b/extensions/assistant-extension/src/node/engine.ts
@@ -15,6 +15,8 @@ export const readEmbeddingEngine = (engineName: string) => {
     const settingDirectoryPath = path.join(
       getJanDataFolderPath(),
       'settings',
+      '@janhq',
+      // TODO: James - To be removed
       engineName === 'openai'
         ? 'inference-openai-extension'
         : 'inference-groq-extension',

diff --git a/extensions/inference-nitro-extension/src/node/index.ts b/extensions/inference-nitro-extension/src/node/index.ts
@@ -131,10 +131,11 @@ async function loadModel(
     if (!llama_model_path) return Promise.reject('No GGUF model file found')
 
     currentSettings = {
+      cpu_threads: Math.max(1, nitroResourceProbe.numCpuPhysicalCore),
+      // model.settings can override the default settings
       ...params.model.settings,
       llama_model_path,
       // This is critical and requires real CPU physical core count (or performance core)
-      cpu_threads: Math.max(1, nitroResourceProbe.numCpuPhysicalCore),
       ...(params.model.settings.mmproj && {
         mmproj: path.isAbsolute(params.model.settings.mmproj)
           ? params.model.settings.mmproj

diff --git a/extensions/monitoring-extension/src/node/logger.ts b/extensions/monitoring-extension/src/node/logger.ts
@@ -67,54 +67,54 @@ export class FileLogger extends Logger {
     const size = maxFileSizeBytes ?? 1 * 1024 * 1024 // 1 MB
     const days = daysToKeep ?? 7 // 7 days
     const logDirectory = path.join(getJanDataFolderPath(), 'logs')
-
     // Perform log cleaning
     const currentDate = new Date()
-    fs.readdir(logDirectory, (err, files) => {
-      if (err) {
-        console.error('Error reading log directory:', err)
-        return
-      }
-
-      files.forEach((file) => {
-        const filePath = path.join(logDirectory, file)
-        fs.stat(filePath, (err, stats) => {
-          if (err) {
-            console.error('Error getting file stats:', err)
-            return
-          }
+    if (fs.existsSync(logDirectory))
+      fs.readdir(logDirectory, (err, files) => {
+        if (err) {
+          console.error('Error reading log directory:', err)
+          return
+        }
+
+        files.forEach((file) => {
+          const filePath = path.join(logDirectory, file)
+          fs.stat(filePath, (err, stats) => {
+            if (err) {
+              console.error('Error getting file stats:', err)
+              return
+            }
 
-          // Check size
-          if (stats.size > size) {
-            fs.unlink(filePath, (err) => {
-              if (err) {
-                console.error('Error deleting log file:', err)
-                return
-              }
-              console.debug(
-                `Deleted log file due to exceeding size limit: ${filePath}`
-              )
-            })
-          } else {
-            // Check age
-            const creationDate = new Date(stats.ctime)
-            const daysDifference = Math.floor(
-              (currentDate.getTime() - creationDate.getTime()) /
-                (1000 * 3600 * 24)
-            )
-            if (daysDifference > days) {
+            // Check size
+            if (stats.size > size) {
               fs.unlink(filePath, (err) => {
                 if (err) {
                   console.error('Error deleting log file:', err)
                   return
                 }
-                console.debug(`Deleted old log file: ${filePath}`)
+                console.debug(
+                  `Deleted log file due to exceeding size limit: ${filePath}`
+                )
               })
+            } else {
+              // Check age
+              const creationDate = new Date(stats.ctime)
+              const daysDifference = Math.floor(
+                (currentDate.getTime() - creationDate.getTime()) /
+                  (1000 * 3600 * 24)
+              )
+              if (daysDifference > days) {
+                fs.unlink(filePath, (err) => {
+                  if (err) {
+                    console.error('Error deleting log file:', err)
+                    return
+                  }
+                  console.debug(`Deleted old log file: ${filePath}`)
+                })
+              }
             }
-          }
+          })
         })
       })
-    })
 
     // Schedule the next execution with doubled delays
     this.timeout = setTimeout(

diff --git a/web/containers/ServerLogs/index.tsx b/web/containers/ServerLogs/index.tsx
@@ -97,7 +97,7 @@ const ServerLogs = (props: ServerLogsProps) => {
         </div>
       </div>
       <div className="overflow-hidden">
-        {logs.length > 1 ? (
+        {logs.length > 0 ? (
           <div className="h-full overflow-auto">
             <code className="inline-block whitespace-pre-line text-xs">
               {logs.slice(-limit).map((log, i) => {

diff --git a/web/screens/Chat/ModelSetting/predefinedComponent.ts b/web/screens/Chat/ModelSetting/predefinedComponent.ts
@@ -165,6 +165,21 @@ export const presetConfiguration: Record<string, SettingComponentProps> = {
     requireModelReload: true,
     configType: 'setting',
   },
+  cpu_threads: {
+    key: 'cpu_threads',
+    title: 'CPU Threads',
+    description:
+      'Determines CPU inference threads, limited by hardware and OS. (Maximum determined by system)',
+    controllerType: 'slider',
+    controllerProps: {
+      min: 0,
+      max: 128,
+      step: 1,
+      value: 1,
+    },
+    requireModelReload: true,
+    configType: 'setting',
+  },
   // assistant
   chunk_size: {
     key: 'chunk_size',