From 3412a23654acd2cf6bb8d683f7da8442a6cef6d8 Mon Sep 17 00:00:00 2001 From: Louis Date: Thu, 15 Feb 2024 08:38:05 +0700 Subject: [PATCH] chore: prettier fix (#2019) --- .devcontainer/devcontainer.json | 6 +- core/jest.config.js | 2 +- core/rollup.config.ts | 2 +- core/src/api/index.ts | 5 +- core/src/extension.ts | 18 +- core/src/extensions/assistant.ts | 12 +- core/src/extensions/conversational.ts | 2 +- core/src/extensions/index.ts | 10 +- core/src/extensions/inference.ts | 8 +- core/src/extensions/model.ts | 20 +- core/src/extensions/monitoring.ts | 10 +- core/src/node/api/common/adapter.ts | 8 +- core/src/node/api/restful/v1.ts | 2 +- core/src/node/extension/extension.ts | 2 +- core/src/node/extension/store.ts | 61 +++---- core/src/node/helper/path.ts | 2 +- core/src/node/helper/resource.ts | 6 +- core/src/types/assistant/assistantEvent.ts | 7 +- core/src/types/message/messageRequestType.ts | 8 +- core/src/types/model/modelInterface.ts | 2 +- core/tslint.json | 7 +- electron/main.ts | 4 +- electron/merge-latest-ymls.js | 4 +- electron/sign.js | 88 ++++----- electron/utils/disposable.ts | 6 +- electron/utils/setup.ts | 12 +- .../assistant-extension/rollup.config.ts | 36 ++-- .../src/@types/global.d.ts | 6 +- extensions/assistant-extension/src/index.ts | 157 ++++++++-------- .../assistant-extension/src/node/engine.ts | 16 +- .../assistant-extension/src/node/index.ts | 28 +-- .../src/node/tools/retrieval/index.ts | 80 ++++---- extensions/assistant-extension/tsconfig.json | 4 +- .../webpack.config.js | 20 +- .../inference-nitro-extension/README.md | 5 +- .../rollup.config.ts | 38 ++-- .../src/@types/global.d.ts | 12 +- .../src/helpers/sse.ts | 56 +++--- .../inference-nitro-extension/src/index.ts | 171 +++++++++--------- .../src/node/execute.ts | 60 +++--- .../src/node/nvidia.ts | 162 +++++++++-------- .../src/@types/global.d.ts | 38 ++-- .../src/helpers/sse.ts | 70 +++---- .../inference-openai-extension/src/index.ts | 134 +++++++------- .../inference-openai-extension/tsconfig.json | 4 +- .../webpack.config.js | 26 +-- .../src/@types/global.d.ts | 4 +- .../src/helpers/sse.ts | 54 +++--- .../src/index.ts | 116 ++++++------ .../tsconfig.json | 4 +- .../webpack.config.js | 24 +-- extensions/model-extension/tsconfig.json | 4 +- extensions/model-extension/webpack.config.js | 2 +- .../src/@types/global.d.ts | 2 +- extensions/monitoring-extension/src/index.ts | 6 +- extensions/monitoring-extension/src/module.ts | 91 ++++++---- .../monitoring-extension/webpack.config.js | 22 +-- server/helpers/setup.ts | 38 ++-- server/index.ts | 106 +++++------ server/main.ts | 8 +- server/middleware/s3.ts | 52 +++--- uikit/types/declaration.d.ts | 6 +- web/app/error.tsx | 1 - web/containers/Layout/BottomBar/index.tsx | 7 - web/screens/Chat/index.tsx | 3 +- 65 files changed, 995 insertions(+), 992 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index f980b9df73..db1eed38df 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,4 +1,4 @@ { - "name": "jan", - "image": "node:20" -} \ No newline at end of file + "name": "jan", + "image": "node:20" +} diff --git a/core/jest.config.js b/core/jest.config.js index fb03768fec..c18f550916 100644 --- a/core/jest.config.js +++ b/core/jest.config.js @@ -4,4 +4,4 @@ module.exports = { moduleNameMapper: { '@/(.*)': '/src/$1', }, -} \ No newline at end of file +} diff --git a/core/rollup.config.ts b/core/rollup.config.ts index 98cb1f2a44..ebea8e2377 100644 --- a/core/rollup.config.ts +++ b/core/rollup.config.ts @@ -55,7 +55,7 @@ export default [ 'http', 'os', 'util', - 'child_process' + 'child_process', ], watch: { include: 'src/node/**', diff --git a/core/src/api/index.ts b/core/src/api/index.ts index 0a160df393..6760207580 100644 --- a/core/src/api/index.ts +++ b/core/src/api/index.ts @@ -125,8 +125,5 @@ export const CoreRoutes = [ ...Object.values(FileManagerRoute), ] -export const APIRoutes = [ - ...CoreRoutes, - ...Object.values(NativeRoute), -] +export const APIRoutes = [...CoreRoutes, ...Object.values(NativeRoute)] export const APIEvents = [...Object.values(AppEvent), ...Object.values(DownloadEvent)] diff --git a/core/src/extension.ts b/core/src/extension.ts index 0b7f9b7fc1..3528f581cc 100644 --- a/core/src/extension.ts +++ b/core/src/extension.ts @@ -1,13 +1,13 @@ export enum ExtensionTypeEnum { - Assistant = "assistant", - Conversational = "conversational", - Inference = "inference", - Model = "model", - SystemMonitoring = "systemMonitoring", + Assistant = 'assistant', + Conversational = 'conversational', + Inference = 'inference', + Model = 'model', + SystemMonitoring = 'systemMonitoring', } export interface ExtensionType { - type(): ExtensionTypeEnum | undefined; + type(): ExtensionTypeEnum | undefined } /** * Represents a base extension. @@ -20,16 +20,16 @@ export abstract class BaseExtension implements ExtensionType { * Undefined means its not extending any known extension by the application. */ type(): ExtensionTypeEnum | undefined { - return undefined; + return undefined } /** * Called when the extension is loaded. * Any initialization logic for the extension should be put here. */ - abstract onLoad(): void; + abstract onLoad(): void /** * Called when the extension is unloaded. * Any cleanup logic for the extension should be put here. */ - abstract onUnload(): void; + abstract onUnload(): void } diff --git a/core/src/extensions/assistant.ts b/core/src/extensions/assistant.ts index ba345711ae..5c3114f41b 100644 --- a/core/src/extensions/assistant.ts +++ b/core/src/extensions/assistant.ts @@ -1,5 +1,5 @@ -import { Assistant, AssistantInterface } from "../index"; -import { BaseExtension, ExtensionTypeEnum } from "../extension"; +import { Assistant, AssistantInterface } from '../index' +import { BaseExtension, ExtensionTypeEnum } from '../extension' /** * Assistant extension for managing assistants. @@ -10,10 +10,10 @@ export abstract class AssistantExtension extends BaseExtension implements Assist * Assistant extension type. */ type(): ExtensionTypeEnum | undefined { - return ExtensionTypeEnum.Assistant; + return ExtensionTypeEnum.Assistant } - abstract createAssistant(assistant: Assistant): Promise; - abstract deleteAssistant(assistant: Assistant): Promise; - abstract getAssistants(): Promise; + abstract createAssistant(assistant: Assistant): Promise + abstract deleteAssistant(assistant: Assistant): Promise + abstract getAssistants(): Promise } diff --git a/core/src/extensions/conversational.ts b/core/src/extensions/conversational.ts index 4319784c35..a49a4e6895 100644 --- a/core/src/extensions/conversational.ts +++ b/core/src/extensions/conversational.ts @@ -14,7 +14,7 @@ export abstract class ConversationalExtension * Conversation extension type. */ type(): ExtensionTypeEnum | undefined { - return ExtensionTypeEnum.Conversational; + return ExtensionTypeEnum.Conversational } abstract getThreads(): Promise diff --git a/core/src/extensions/index.ts b/core/src/extensions/index.ts index 1796c16187..5223345489 100644 --- a/core/src/extensions/index.ts +++ b/core/src/extensions/index.ts @@ -2,24 +2,24 @@ * Conversational extension. Persists and retrieves conversations. * @module */ -export { ConversationalExtension } from "./conversational"; +export { ConversationalExtension } from './conversational' /** * Inference extension. Start, stop and inference models. */ -export { InferenceExtension } from "./inference"; +export { InferenceExtension } from './inference' /** * Monitoring extension for system monitoring. */ -export { MonitoringExtension } from "./monitoring"; +export { MonitoringExtension } from './monitoring' /** * Assistant extension for managing assistants. */ -export { AssistantExtension } from "./assistant"; +export { AssistantExtension } from './assistant' /** * Model extension for managing models. */ -export { ModelExtension } from "./model"; +export { ModelExtension } from './model' diff --git a/core/src/extensions/inference.ts b/core/src/extensions/inference.ts index c551d108f5..e8e51f9eb9 100644 --- a/core/src/extensions/inference.ts +++ b/core/src/extensions/inference.ts @@ -1,5 +1,5 @@ -import { InferenceInterface, MessageRequest, ThreadMessage } from "../index"; -import { BaseExtension, ExtensionTypeEnum } from "../extension"; +import { InferenceInterface, MessageRequest, ThreadMessage } from '../index' +import { BaseExtension, ExtensionTypeEnum } from '../extension' /** * Inference extension. Start, stop and inference models. @@ -9,8 +9,8 @@ export abstract class InferenceExtension extends BaseExtension implements Infere * Inference extension type. */ type(): ExtensionTypeEnum | undefined { - return ExtensionTypeEnum.Inference; + return ExtensionTypeEnum.Inference } - abstract inference(data: MessageRequest): Promise; + abstract inference(data: MessageRequest): Promise } diff --git a/core/src/extensions/model.ts b/core/src/extensions/model.ts index 30aa5b6ba2..df7d14f421 100644 --- a/core/src/extensions/model.ts +++ b/core/src/extensions/model.ts @@ -1,5 +1,5 @@ -import { BaseExtension, ExtensionTypeEnum } from "../extension"; -import { Model, ModelInterface } from "../index"; +import { BaseExtension, ExtensionTypeEnum } from '../extension' +import { Model, ModelInterface } from '../index' /** * Model extension for managing models. @@ -9,16 +9,16 @@ export abstract class ModelExtension extends BaseExtension implements ModelInter * Model extension type. */ type(): ExtensionTypeEnum | undefined { - return ExtensionTypeEnum.Model; + return ExtensionTypeEnum.Model } abstract downloadModel( model: Model, - network?: { proxy: string; ignoreSSL?: boolean }, - ): Promise; - abstract cancelModelDownload(modelId: string): Promise; - abstract deleteModel(modelId: string): Promise; - abstract saveModel(model: Model): Promise; - abstract getDownloadedModels(): Promise; - abstract getConfiguredModels(): Promise; + network?: { proxy: string; ignoreSSL?: boolean } + ): Promise + abstract cancelModelDownload(modelId: string): Promise + abstract deleteModel(modelId: string): Promise + abstract saveModel(model: Model): Promise + abstract getDownloadedModels(): Promise + abstract getConfiguredModels(): Promise } diff --git a/core/src/extensions/monitoring.ts b/core/src/extensions/monitoring.ts index 2de9b9ae56..ba193f0f4d 100644 --- a/core/src/extensions/monitoring.ts +++ b/core/src/extensions/monitoring.ts @@ -1,5 +1,5 @@ -import { BaseExtension, ExtensionTypeEnum } from "../extension"; -import { MonitoringInterface } from "../index"; +import { BaseExtension, ExtensionTypeEnum } from '../extension' +import { MonitoringInterface } from '../index' /** * Monitoring extension for system monitoring. @@ -10,9 +10,9 @@ export abstract class MonitoringExtension extends BaseExtension implements Monit * Monitoring extension type. */ type(): ExtensionTypeEnum | undefined { - return ExtensionTypeEnum.SystemMonitoring; + return ExtensionTypeEnum.SystemMonitoring } - abstract getResourcesInfo(): Promise; - abstract getCurrentLoad(): Promise; + abstract getResourcesInfo(): Promise + abstract getCurrentLoad(): Promise } diff --git a/core/src/node/api/common/adapter.ts b/core/src/node/api/common/adapter.ts index b1013f6d2b..56f4cedb35 100644 --- a/core/src/node/api/common/adapter.ts +++ b/core/src/node/api/common/adapter.ts @@ -1,4 +1,10 @@ -import { AppRoute, DownloadRoute, ExtensionRoute, FileManagerRoute, FileSystemRoute } from '../../../api' +import { + AppRoute, + DownloadRoute, + ExtensionRoute, + FileManagerRoute, + FileSystemRoute, +} from '../../../api' import { Downloader } from '../processors/download' import { FileSystem } from '../processors/fs' import { Extension } from '../processors/extension' diff --git a/core/src/node/api/restful/v1.ts b/core/src/node/api/restful/v1.ts index 474b1f3550..5eb8f50679 100644 --- a/core/src/node/api/restful/v1.ts +++ b/core/src/node/api/restful/v1.ts @@ -6,7 +6,7 @@ import { handleRequests } from './app/handlers' export const v1Router = async (app: HttpServer) => { // MARK: Public API Routes app.register(commonRouter) - + // MARK: Internal Application Routes handleRequests(app) diff --git a/core/src/node/extension/extension.ts b/core/src/node/extension/extension.ts index aeb0277c0b..1f8dfa3ec2 100644 --- a/core/src/node/extension/extension.ts +++ b/core/src/node/extension/extension.ts @@ -104,7 +104,7 @@ export default class Extension { await pacote.extract( this.specifier, join(ExtensionManager.instance.getExtensionsPath() ?? '', this.name ?? ''), - this.installOptions, + this.installOptions ) // Set the url using the custom extensions protocol diff --git a/core/src/node/extension/store.ts b/core/src/node/extension/store.ts index b92b3fd62f..93b1aeb2b6 100644 --- a/core/src/node/extension/store.ts +++ b/core/src/node/extension/store.ts @@ -1,6 +1,6 @@ -import { writeFileSync } from "fs"; -import Extension from "./extension"; -import { ExtensionManager } from "./manager"; +import { writeFileSync } from 'fs' +import Extension from './extension' +import { ExtensionManager } from './manager' /** * @module store @@ -11,7 +11,7 @@ import { ExtensionManager } from "./manager"; * Register of installed extensions * @type {Object.} extension - List of installed extensions */ -const extensions: Record = {}; +const extensions: Record = {} /** * Get a extension from the stored extensions. @@ -21,10 +21,10 @@ const extensions: Record = {}; */ export function getExtension(name: string) { if (!Object.prototype.hasOwnProperty.call(extensions, name)) { - throw new Error(`Extension ${name} does not exist`); + throw new Error(`Extension ${name} does not exist`) } - return extensions[name]; + return extensions[name] } /** @@ -33,7 +33,7 @@ export function getExtension(name: string) { * @alias extensionManager.getAllExtensions */ export function getAllExtensions() { - return Object.values(extensions); + return Object.values(extensions) } /** @@ -42,7 +42,7 @@ export function getAllExtensions() { * @alias extensionManager.getActiveExtensions */ export function getActiveExtensions() { - return Object.values(extensions).filter((extension) => extension.active); + return Object.values(extensions).filter((extension) => extension.active) } /** @@ -53,9 +53,9 @@ export function getActiveExtensions() { * @alias extensionManager.removeExtension */ export function removeExtension(name: string, persist = true) { - const del = delete extensions[name]; - if (persist) persistExtensions(); - return del; + const del = delete extensions[name] + if (persist) persistExtensions() + return del } /** @@ -65,10 +65,10 @@ export function removeExtension(name: string, persist = true) { * @returns {void} */ export function addExtension(extension: Extension, persist = true) { - if (extension.name) extensions[extension.name] = extension; + if (extension.name) extensions[extension.name] = extension if (persist) { - persistExtensions(); - extension.subscribe("pe-persist", persistExtensions); + persistExtensions() + extension.subscribe('pe-persist', persistExtensions) } } @@ -77,14 +77,11 @@ export function addExtension(extension: Extension, persist = true) { * @returns {void} */ export function persistExtensions() { - const persistData: Record = {}; + const persistData: Record = {} for (const name in extensions) { - persistData[name] = extensions[name]; + persistData[name] = extensions[name] } - writeFileSync( - ExtensionManager.instance.getExtensionsFile(), - JSON.stringify(persistData), - ); + writeFileSync(ExtensionManager.instance.getExtensionsFile(), JSON.stringify(persistData)) } /** @@ -95,26 +92,28 @@ export function persistExtensions() { * @alias extensionManager.installExtensions */ export async function installExtensions(extensions: any) { - const installed: Extension[] = []; + const installed: Extension[] = [] for (const ext of extensions) { // Set install options and activation based on input type - const isObject = typeof ext === "object"; - const spec = isObject ? [ext.specifier, ext] : [ext]; - const activate = isObject ? ext.activate !== false : true; + const isObject = typeof ext === 'object' + const spec = isObject ? [ext.specifier, ext] : [ext] + const activate = isObject ? ext.activate !== false : true // Install and possibly activate extension - const extension = new Extension(...spec); - if(!extension.origin) { continue } - await extension._install(); - if (activate) extension.setActive(true); + const extension = new Extension(...spec) + if (!extension.origin) { + continue + } + await extension._install() + if (activate) extension.setActive(true) // Add extension to store if needed - addExtension(extension); - installed.push(extension); + addExtension(extension) + installed.push(extension) } // Return list of all installed extensions - return installed; + return installed } /** diff --git a/core/src/node/helper/path.ts b/core/src/node/helper/path.ts index b07f297f4b..c20889f4c9 100644 --- a/core/src/node/helper/path.ts +++ b/core/src/node/helper/path.ts @@ -1,4 +1,4 @@ -import { join } from "path" +import { join } from 'path' /** * Normalize file path diff --git a/core/src/node/helper/resource.ts b/core/src/node/helper/resource.ts index 8e3cf96aa8..04a7d512af 100644 --- a/core/src/node/helper/resource.ts +++ b/core/src/node/helper/resource.ts @@ -1,6 +1,6 @@ -import { SystemResourceInfo } from "../../types" -import { physicalCpuCount } from "./config" -import { log, logServer } from "./log" +import { SystemResourceInfo } from '../../types' +import { physicalCpuCount } from './config' +import { log, logServer } from './log' export const getSystemResourceInfo = async (): Promise => { const cpu = await physicalCpuCount() diff --git a/core/src/types/assistant/assistantEvent.ts b/core/src/types/assistant/assistantEvent.ts index f8f3e6ad03..8c32f5d37a 100644 --- a/core/src/types/assistant/assistantEvent.ts +++ b/core/src/types/assistant/assistantEvent.ts @@ -2,7 +2,6 @@ * The `EventName` enumeration contains the names of all the available events in the Jan platform. */ export enum AssistantEvent { - /** The `OnAssistantsUpdate` event is emitted when the assistant list is updated. */ - OnAssistantsUpdate = 'OnAssistantsUpdate', - } - \ No newline at end of file + /** The `OnAssistantsUpdate` event is emitted when the assistant list is updated. */ + OnAssistantsUpdate = 'OnAssistantsUpdate', +} diff --git a/core/src/types/message/messageRequestType.ts b/core/src/types/message/messageRequestType.ts index 51be51996a..cbb4cf4217 100644 --- a/core/src/types/message/messageRequestType.ts +++ b/core/src/types/message/messageRequestType.ts @@ -1,5 +1,5 @@ export enum MessageRequestType { - Thread = 'Thread', - Assistant = 'Assistant', - Summary = 'Summary', -} \ No newline at end of file + Thread = 'Thread', + Assistant = 'Assistant', + Summary = 'Summary', +} diff --git a/core/src/types/model/modelInterface.ts b/core/src/types/model/modelInterface.ts index 74a479f3cb..93d5867eeb 100644 --- a/core/src/types/model/modelInterface.ts +++ b/core/src/types/model/modelInterface.ts @@ -10,7 +10,7 @@ export interface ModelInterface { * @param network - Optional object to specify proxy/whether to ignore SSL certificates. * @returns A Promise that resolves when the model has been downloaded. */ - downloadModel(model: Model, network?: { ignoreSSL?: boolean, proxy?: string }): Promise + downloadModel(model: Model, network?: { ignoreSSL?: boolean; proxy?: string }): Promise /** * Cancels the download of a specific model. diff --git a/core/tslint.json b/core/tslint.json index 398a416704..6543a641a1 100644 --- a/core/tslint.json +++ b/core/tslint.json @@ -1,6 +1,3 @@ { - "extends": [ - "tslint-config-standard", - "tslint-config-prettier" - ] -} \ No newline at end of file + "extends": ["tslint-config-standard", "tslint-config-prettier"] +} diff --git a/electron/main.ts b/electron/main.ts index 884c0e55fe..13e181cdf4 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -1,4 +1,4 @@ -import { app, BrowserWindow } from 'electron' +import { app, BrowserWindow, shell } from 'electron' import { join } from 'path' /** * Managers @@ -77,7 +77,7 @@ function createMainWindow() { /* Open external links in the default browser */ mainWindow.webContents.setWindowOpenHandler(({ url }) => { - require('electron').shell.openExternal(url) + shell.openExternal(url) return { action: 'deny' } }) diff --git a/electron/merge-latest-ymls.js b/electron/merge-latest-ymls.js index 8172a31768..ee8caf825d 100644 --- a/electron/merge-latest-ymls.js +++ b/electron/merge-latest-ymls.js @@ -9,7 +9,9 @@ const file3 = args[2] // check that all arguments are present and throw error instead if (!file1 || !file2 || !file3) { - throw new Error('Please provide 3 file paths as arguments: path to file1, to file2 and destination path') + throw new Error( + 'Please provide 3 file paths as arguments: path to file1, to file2 and destination path' + ) } const doc1 = yaml.load(fs.readFileSync(file1, 'utf8')) diff --git a/electron/sign.js b/electron/sign.js index 6e973eb6e3..73afedc4ef 100644 --- a/electron/sign.js +++ b/electron/sign.js @@ -1,44 +1,48 @@ -const { exec } = require('child_process'); - - -function sign({ path, name, certUrl, clientId, tenantId, clientSecret, certName, timestampServer, version }) { - return new Promise((resolve, reject) => { - - const command = `azuresigntool.exe sign -kvu "${certUrl}" -kvi "${clientId}" -kvt "${tenantId}" -kvs "${clientSecret}" -kvc "${certName}" -tr "${timestampServer}" -v "${path}"`; - - - exec(command, (error, stdout, stderr) => { - if (error) { - console.error(`Error: ${error}`); - return reject(error); - } - console.log(`stdout: ${stdout}`); - console.error(`stderr: ${stderr}`); - resolve(); - }); - }); +const { exec } = require('child_process') + +function sign({ + path, + name, + certUrl, + clientId, + tenantId, + clientSecret, + certName, + timestampServer, + version, +}) { + return new Promise((resolve, reject) => { + const command = `azuresigntool.exe sign -kvu "${certUrl}" -kvi "${clientId}" -kvt "${tenantId}" -kvs "${clientSecret}" -kvc "${certName}" -tr "${timestampServer}" -v "${path}"` + + exec(command, (error, stdout, stderr) => { + if (error) { + console.error(`Error: ${error}`) + return reject(error) + } + console.log(`stdout: ${stdout}`) + console.error(`stderr: ${stderr}`) + resolve() + }) + }) } - -exports.default = async function(options) { - - const certUrl = process.env.AZURE_KEY_VAULT_URI; - const clientId = process.env.AZURE_CLIENT_ID; - const tenantId = process.env.AZURE_TENANT_ID; - const clientSecret = process.env.AZURE_CLIENT_SECRET; - const certName = process.env.AZURE_CERT_NAME; - const timestampServer = 'http://timestamp.globalsign.com/tsa/r6advanced1'; - - - await sign({ - path: options.path, - name: "jan-win-x64", - certUrl, - clientId, - tenantId, - clientSecret, - certName, - timestampServer, - version: options.version - }); -}; +exports.default = async function (options) { + const certUrl = process.env.AZURE_KEY_VAULT_URI + const clientId = process.env.AZURE_CLIENT_ID + const tenantId = process.env.AZURE_TENANT_ID + const clientSecret = process.env.AZURE_CLIENT_SECRET + const certName = process.env.AZURE_CERT_NAME + const timestampServer = 'http://timestamp.globalsign.com/tsa/r6advanced1' + + await sign({ + path: options.path, + name: 'jan-win-x64', + certUrl, + clientId, + tenantId, + clientSecret, + certName, + timestampServer, + version: options.version, + }) +} diff --git a/electron/utils/disposable.ts b/electron/utils/disposable.ts index 462f7e3e51..59018a7751 100644 --- a/electron/utils/disposable.ts +++ b/electron/utils/disposable.ts @@ -1,8 +1,8 @@ export function dispose(requiredModules: Record) { for (const key in requiredModules) { - const module = requiredModules[key]; - if (typeof module["dispose"] === "function") { - module["dispose"](); + const module = requiredModules[key] + if (typeof module['dispose'] === 'function') { + module['dispose']() } } } diff --git a/electron/utils/setup.ts b/electron/utils/setup.ts index 887c3c2b7a..01b0b31da2 100644 --- a/electron/utils/setup.ts +++ b/electron/utils/setup.ts @@ -1,9 +1,9 @@ import { app } from 'electron' export const setupCore = async () => { - // Setup core api for main process - global.core = { - // Define appPath function for app to retrieve app path globaly - appPath: () => app.getPath('userData') - } -} \ No newline at end of file + // Setup core api for main process + global.core = { + // Define appPath function for app to retrieve app path globaly + appPath: () => app.getPath('userData'), + } +} diff --git a/extensions/assistant-extension/rollup.config.ts b/extensions/assistant-extension/rollup.config.ts index 1e3c38fabf..d3c39cab2d 100644 --- a/extensions/assistant-extension/rollup.config.ts +++ b/extensions/assistant-extension/rollup.config.ts @@ -1,22 +1,22 @@ -import resolve from "@rollup/plugin-node-resolve"; -import commonjs from "@rollup/plugin-commonjs"; -import sourceMaps from "rollup-plugin-sourcemaps"; -import typescript from "rollup-plugin-typescript2"; -import json from "@rollup/plugin-json"; -import replace from "@rollup/plugin-replace"; +import resolve from '@rollup/plugin-node-resolve' +import commonjs from '@rollup/plugin-commonjs' +import sourceMaps from 'rollup-plugin-sourcemaps' +import typescript from 'rollup-plugin-typescript2' +import json from '@rollup/plugin-json' +import replace from '@rollup/plugin-replace' -const packageJson = require("./package.json"); +const packageJson = require('./package.json') -const pkg = require("./package.json"); +const pkg = require('./package.json') export default [ { input: `src/index.ts`, - output: [{ file: pkg.main, format: "es", sourcemap: true }], + output: [{ file: pkg.main, format: 'es', sourcemap: true }], // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') external: [], watch: { - include: "src/**", + include: 'src/**', }, plugins: [ replace({ @@ -35,7 +35,7 @@ export default [ // which external modules to include in the bundle // https://github.com/rollup/rollup-plugin-node-resolve#usage resolve({ - extensions: [".js", ".ts", ".svelte"], + extensions: ['.js', '.ts', '.svelte'], }), // Resolve source maps to the original source @@ -44,15 +44,11 @@ export default [ }, { input: `src/node/index.ts`, - output: [{ dir: "dist/node", format: "cjs", sourcemap: false }], + output: [{ dir: 'dist/node', format: 'cjs', sourcemap: false }], // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') - external: [ - "@janhq/core/node", - "path", - "hnswlib-node", - ], + external: ['@janhq/core/node', 'path', 'hnswlib-node'], watch: { - include: "src/node/**", + include: 'src/node/**', }, // inlineDynamicImports: true, plugins: [ @@ -68,11 +64,11 @@ export default [ // which external modules to include in the bundle // https://github.com/rollup/rollup-plugin-node-resolve#usage resolve({ - extensions: [".ts", ".js", ".json"], + extensions: ['.ts', '.js', '.json'], }), // Resolve source maps to the original source // sourceMaps(), ], }, -]; +] diff --git a/extensions/assistant-extension/src/@types/global.d.ts b/extensions/assistant-extension/src/@types/global.d.ts index dc11709a4f..bc97157cdf 100644 --- a/extensions/assistant-extension/src/@types/global.d.ts +++ b/extensions/assistant-extension/src/@types/global.d.ts @@ -1,3 +1,3 @@ -declare const NODE: string; -declare const EXTENSION_NAME: string; -declare const VERSION: string; +declare const NODE: string +declare const EXTENSION_NAME: string +declare const VERSION: string diff --git a/extensions/assistant-extension/src/index.ts b/extensions/assistant-extension/src/index.ts index 785b3768e0..0a5319c8a7 100644 --- a/extensions/assistant-extension/src/index.ts +++ b/extensions/assistant-extension/src/index.ts @@ -10,57 +10,56 @@ import { executeOnMain, AssistantExtension, AssistantEvent, -} from "@janhq/core"; +} from '@janhq/core' export default class JanAssistantExtension extends AssistantExtension { - private static readonly _homeDir = "file://assistants"; - private static readonly _threadDir = "file://threads"; + private static readonly _homeDir = 'file://assistants' + private static readonly _threadDir = 'file://threads' - controller = new AbortController(); - isCancelled = false; - retrievalThreadId: string | undefined = undefined; + controller = new AbortController() + isCancelled = false + retrievalThreadId: string | undefined = undefined async onLoad() { // making the assistant directory const assistantDirExist = await fs.existsSync( JanAssistantExtension._homeDir - ); + ) if ( localStorage.getItem(`${EXTENSION_NAME}-version`) !== VERSION || !assistantDirExist ) { - if (!assistantDirExist) - await fs.mkdirSync(JanAssistantExtension._homeDir); + if (!assistantDirExist) await fs.mkdirSync(JanAssistantExtension._homeDir) // Write assistant metadata - await this.createJanAssistant(); + await this.createJanAssistant() // Finished migration - localStorage.setItem(`${EXTENSION_NAME}-version`, VERSION); + localStorage.setItem(`${EXTENSION_NAME}-version`, VERSION) // Update the assistant list - events.emit(AssistantEvent.OnAssistantsUpdate, {}); + events.emit(AssistantEvent.OnAssistantsUpdate, {}) } // Events subscription events.on(MessageEvent.OnMessageSent, (data: MessageRequest) => JanAssistantExtension.handleMessageRequest(data, this) - ); + ) events.on(InferenceEvent.OnInferenceStopped, () => { - JanAssistantExtension.handleInferenceStopped(this); - }); + JanAssistantExtension.handleInferenceStopped(this) + }) } private static async handleInferenceStopped(instance: JanAssistantExtension) { - instance.isCancelled = true; - instance.controller?.abort(); + instance.isCancelled = true + instance.controller?.abort() } private static async handleMessageRequest( data: MessageRequest, instance: JanAssistantExtension ) { - instance.isCancelled = false; - instance.controller = new AbortController(); + instance.isCancelled = false + instance.controller = new AbortController() if ( data.model?.engine !== InferenceEngine.tool_retrieval_enabled || @@ -69,26 +68,26 @@ export default class JanAssistantExtension extends AssistantExtension { // That could lead to an issue where thread stuck at generating response !data.thread?.assistants[0]?.tools ) { - return; + return } - const latestMessage = data.messages[data.messages.length - 1]; + const latestMessage = data.messages[data.messages.length - 1] // 1. Ingest the document if needed if ( latestMessage && latestMessage.content && - typeof latestMessage.content !== "string" && + typeof latestMessage.content !== 'string' && latestMessage.content.length > 1 ) { - const docFile = latestMessage.content[1]?.doc_url?.url; + const docFile = latestMessage.content[1]?.doc_url?.url if (docFile) { await executeOnMain( NODE, - "toolRetrievalIngestNewDocument", + 'toolRetrievalIngestNewDocument', docFile, data.model?.proxyEngine - ); + ) } } else if ( // Check whether we need to ingest document or not @@ -97,7 +96,7 @@ export default class JanAssistantExtension extends AssistantExtension { await joinPath([ JanAssistantExtension._threadDir, data.threadId, - "memory", + 'memory', ]) )) ) { @@ -108,61 +107,61 @@ export default class JanAssistantExtension extends AssistantExtension { ...data.model, engine: data.model.proxyEngine, }, - }; - events.emit(MessageEvent.OnMessageSent, output); - return; + } + events.emit(MessageEvent.OnMessageSent, output) + return } // 2. Load agent on thread changed if (instance.retrievalThreadId !== data.threadId) { - await executeOnMain(NODE, "toolRetrievalLoadThreadMemory", data.threadId); + await executeOnMain(NODE, 'toolRetrievalLoadThreadMemory', data.threadId) - instance.retrievalThreadId = data.threadId; + instance.retrievalThreadId = data.threadId // Update the text splitter await executeOnMain( NODE, - "toolRetrievalUpdateTextSplitter", + 'toolRetrievalUpdateTextSplitter', data.thread.assistants[0].tools[0]?.settings?.chunk_size ?? 4000, data.thread.assistants[0].tools[0]?.settings?.chunk_overlap ?? 200 - ); + ) } // 3. Using the retrieval template with the result and query if (latestMessage.content) { const prompt = - typeof latestMessage.content === "string" + typeof latestMessage.content === 'string' ? latestMessage.content - : latestMessage.content[0].text; + : latestMessage.content[0].text // Retrieve the result const retrievalResult = await executeOnMain( NODE, - "toolRetrievalQueryResult", + 'toolRetrievalQueryResult', prompt - ); - console.debug("toolRetrievalQueryResult", retrievalResult); + ) + console.debug('toolRetrievalQueryResult', retrievalResult) // Update message content if (data.thread?.assistants[0]?.tools && retrievalResult) data.messages[data.messages.length - 1].content = data.thread.assistants[0].tools[0].settings?.retrieval_template - ?.replace("{CONTEXT}", retrievalResult) - .replace("{QUESTION}", prompt); + ?.replace('{CONTEXT}', retrievalResult) + .replace('{QUESTION}', prompt) } // Filter out all the messages that are not text data.messages = data.messages.map((message) => { if ( message.content && - typeof message.content !== "string" && + typeof message.content !== 'string' && (message.content.length ?? 0) > 0 ) { return { ...message, content: [message.content[0]], - }; + } } - return message; - }); + return message + }) // 4. Reroute the result to inference engine const output = { @@ -171,8 +170,8 @@ export default class JanAssistantExtension extends AssistantExtension { ...data.model, engine: data.model.proxyEngine, }, - }; - events.emit(MessageEvent.OnMessageSent, output); + } + events.emit(MessageEvent.OnMessageSent, output) } /** @@ -184,88 +183,88 @@ export default class JanAssistantExtension extends AssistantExtension { const assistantDir = await joinPath([ JanAssistantExtension._homeDir, assistant.id, - ]); - if (!(await fs.existsSync(assistantDir))) await fs.mkdirSync(assistantDir); + ]) + if (!(await fs.existsSync(assistantDir))) await fs.mkdirSync(assistantDir) // store the assistant metadata json const assistantMetadataPath = await joinPath([ assistantDir, - "assistant.json", - ]); + 'assistant.json', + ]) try { await fs.writeFileSync( assistantMetadataPath, JSON.stringify(assistant, null, 2) - ); + ) } catch (err) { - console.error(err); + console.error(err) } } async getAssistants(): Promise { // get all the assistant directories // get all the assistant metadata json - const results: Assistant[] = []; + const results: Assistant[] = [] const allFileName: string[] = await fs.readdirSync( JanAssistantExtension._homeDir - ); + ) for (const fileName of allFileName) { const filePath = await joinPath([ JanAssistantExtension._homeDir, fileName, - ]); + ]) - if (filePath.includes(".DS_Store")) continue; + if (filePath.includes('.DS_Store')) continue const jsonFiles: string[] = (await fs.readdirSync(filePath)).filter( - (file: string) => file === "assistant.json" - ); + (file: string) => file === 'assistant.json' + ) if (jsonFiles.length !== 1) { // has more than one assistant file -> ignore - continue; + continue } const content = await fs.readFileSync( await joinPath([filePath, jsonFiles[0]]), - "utf-8" - ); + 'utf-8' + ) const assistant: Assistant = - typeof content === "object" ? content : JSON.parse(content); + typeof content === 'object' ? content : JSON.parse(content) - results.push(assistant); + results.push(assistant) } - return results; + return results } async deleteAssistant(assistant: Assistant): Promise { - if (assistant.id === "jan") { - return Promise.reject("Cannot delete Jan Assistant"); + if (assistant.id === 'jan') { + return Promise.reject('Cannot delete Jan Assistant') } // remove the directory const assistantDir = await joinPath([ JanAssistantExtension._homeDir, assistant.id, - ]); - await fs.rmdirSync(assistantDir); - return Promise.resolve(); + ]) + await fs.rmdirSync(assistantDir) + return Promise.resolve() } private async createJanAssistant(): Promise { const janAssistant: Assistant = { - avatar: "", + avatar: '', thread_location: undefined, - id: "jan", - object: "assistant", + id: 'jan', + object: 'assistant', created_at: Date.now(), - name: "Jan", - description: "A default assistant that can use all downloaded models", - model: "*", - instructions: "", + name: 'Jan', + description: 'A default assistant that can use all downloaded models', + model: '*', + instructions: '', tools: [ { - type: "retrieval", + type: 'retrieval', enabled: false, settings: { top_k: 2, @@ -283,8 +282,8 @@ Helpful Answer:`, ], file_ids: [], metadata: undefined, - }; + } - await this.createAssistant(janAssistant); + await this.createAssistant(janAssistant) } } diff --git a/extensions/assistant-extension/src/node/engine.ts b/extensions/assistant-extension/src/node/engine.ts index 54b2a6ba16..70d02af1f5 100644 --- a/extensions/assistant-extension/src/node/engine.ts +++ b/extensions/assistant-extension/src/node/engine.ts @@ -1,13 +1,13 @@ -import fs from "fs"; -import path from "path"; -import { getJanDataFolderPath } from "@janhq/core/node"; +import fs from 'fs' +import path from 'path' +import { getJanDataFolderPath } from '@janhq/core/node' // Sec: Do not send engine settings over requests // Read it manually instead export const readEmbeddingEngine = (engineName: string) => { const engineSettings = fs.readFileSync( - path.join(getJanDataFolderPath(), "engines", `${engineName}.json`), - "utf-8", - ); - return JSON.parse(engineSettings); -}; + path.join(getJanDataFolderPath(), 'engines', `${engineName}.json`), + 'utf-8' + ) + return JSON.parse(engineSettings) +} diff --git a/extensions/assistant-extension/src/node/index.ts b/extensions/assistant-extension/src/node/index.ts index c308a2d578..d52a4b23e3 100644 --- a/extensions/assistant-extension/src/node/index.ts +++ b/extensions/assistant-extension/src/node/index.ts @@ -1,39 +1,39 @@ -import { getJanDataFolderPath, normalizeFilePath } from "@janhq/core/node"; -import { retrieval } from "./tools/retrieval"; -import path from "path"; +import { getJanDataFolderPath, normalizeFilePath } from '@janhq/core/node' +import { retrieval } from './tools/retrieval' +import path from 'path' export function toolRetrievalUpdateTextSplitter( chunkSize: number, chunkOverlap: number ) { - retrieval.updateTextSplitter(chunkSize, chunkOverlap); + retrieval.updateTextSplitter(chunkSize, chunkOverlap) } export async function toolRetrievalIngestNewDocument( file: string, engine: string ) { - const filePath = path.join(getJanDataFolderPath(), normalizeFilePath(file)); - const threadPath = path.dirname(filePath.replace("files", "")); - retrieval.updateEmbeddingEngine(engine); + const filePath = path.join(getJanDataFolderPath(), normalizeFilePath(file)) + const threadPath = path.dirname(filePath.replace('files', '')) + retrieval.updateEmbeddingEngine(engine) return retrieval .ingestAgentKnowledge(filePath, `${threadPath}/memory`) .catch((err) => { - console.error(err); - }); + console.error(err) + }) } export async function toolRetrievalLoadThreadMemory(threadId: string) { return retrieval .loadRetrievalAgent( - path.join(getJanDataFolderPath(), "threads", threadId, "memory") + path.join(getJanDataFolderPath(), 'threads', threadId, 'memory') ) .catch((err) => { - console.error(err); - }); + console.error(err) + }) } export async function toolRetrievalQueryResult(query: string) { return retrieval.generateResult(query).catch((err) => { - console.error(err); - }); + console.error(err) + }) } diff --git a/extensions/assistant-extension/src/node/tools/retrieval/index.ts b/extensions/assistant-extension/src/node/tools/retrieval/index.ts index b302915790..e58ec0c46c 100644 --- a/extensions/assistant-extension/src/node/tools/retrieval/index.ts +++ b/extensions/assistant-extension/src/node/tools/retrieval/index.ts @@ -1,80 +1,80 @@ -import { RecursiveCharacterTextSplitter } from "langchain/text_splitter"; -import { formatDocumentsAsString } from "langchain/util/document"; -import { PDFLoader } from "langchain/document_loaders/fs/pdf"; +import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter' +import { formatDocumentsAsString } from 'langchain/util/document' +import { PDFLoader } from 'langchain/document_loaders/fs/pdf' -import { HNSWLib } from "langchain/vectorstores/hnswlib"; +import { HNSWLib } from 'langchain/vectorstores/hnswlib' -import { OpenAIEmbeddings } from "langchain/embeddings/openai"; -import { readEmbeddingEngine } from "../../engine"; +import { OpenAIEmbeddings } from 'langchain/embeddings/openai' +import { readEmbeddingEngine } from '../../engine' export class Retrieval { - public chunkSize: number = 100; - public chunkOverlap?: number = 0; - private retriever: any; + public chunkSize: number = 100 + public chunkOverlap?: number = 0 + private retriever: any - private embeddingModel?: OpenAIEmbeddings = undefined; - private textSplitter?: RecursiveCharacterTextSplitter; + private embeddingModel?: OpenAIEmbeddings = undefined + private textSplitter?: RecursiveCharacterTextSplitter constructor(chunkSize: number = 4000, chunkOverlap: number = 200) { - this.updateTextSplitter(chunkSize, chunkOverlap); + this.updateTextSplitter(chunkSize, chunkOverlap) } public updateTextSplitter(chunkSize: number, chunkOverlap: number): void { - this.chunkSize = chunkSize; - this.chunkOverlap = chunkOverlap; + this.chunkSize = chunkSize + this.chunkOverlap = chunkOverlap this.textSplitter = new RecursiveCharacterTextSplitter({ chunkSize: chunkSize, chunkOverlap: chunkOverlap, - }); + }) } public updateEmbeddingEngine(engine: string): void { // Engine settings are not compatible with the current embedding model params // Switch case manually for now - const settings = readEmbeddingEngine(engine); - if (engine === "nitro") { + const settings = readEmbeddingEngine(engine) + if (engine === 'nitro') { this.embeddingModel = new OpenAIEmbeddings( - { openAIApiKey: "nitro-embedding" }, + { openAIApiKey: 'nitro-embedding' }, // TODO: Raw settings - { basePath: "http://127.0.0.1:3928/v1" }, - ); + { basePath: 'http://127.0.0.1:3928/v1' } + ) } else { // Fallback to OpenAI Settings this.embeddingModel = new OpenAIEmbeddings({ openAIApiKey: settings.api_key, - }); + }) } } public ingestAgentKnowledge = async ( filePath: string, - memoryPath: string, + memoryPath: string ): Promise => { const loader = new PDFLoader(filePath, { splitPages: true, - }); - if (!this.embeddingModel) return Promise.reject(); - const doc = await loader.load(); - const docs = await this.textSplitter!.splitDocuments(doc); - const vectorStore = await HNSWLib.fromDocuments(docs, this.embeddingModel); - return vectorStore.save(memoryPath); - }; + }) + if (!this.embeddingModel) return Promise.reject() + const doc = await loader.load() + const docs = await this.textSplitter!.splitDocuments(doc) + const vectorStore = await HNSWLib.fromDocuments(docs, this.embeddingModel) + return vectorStore.save(memoryPath) + } public loadRetrievalAgent = async (memoryPath: string): Promise => { - if (!this.embeddingModel) return Promise.reject(); - const vectorStore = await HNSWLib.load(memoryPath, this.embeddingModel); - this.retriever = vectorStore.asRetriever(2); - return Promise.resolve(); - }; + if (!this.embeddingModel) return Promise.reject() + const vectorStore = await HNSWLib.load(memoryPath, this.embeddingModel) + this.retriever = vectorStore.asRetriever(2) + return Promise.resolve() + } public generateResult = async (query: string): Promise => { if (!this.retriever) { - return Promise.resolve(" "); + return Promise.resolve(' ') } - const relevantDocs = await this.retriever.getRelevantDocuments(query); - const serializedDoc = formatDocumentsAsString(relevantDocs); - return Promise.resolve(serializedDoc); - }; + const relevantDocs = await this.retriever.getRelevantDocuments(query) + const serializedDoc = formatDocumentsAsString(relevantDocs) + return Promise.resolve(serializedDoc) + } } -export const retrieval = new Retrieval(); \ No newline at end of file +export const retrieval = new Retrieval() diff --git a/extensions/assistant-extension/tsconfig.json b/extensions/assistant-extension/tsconfig.json index d3794cace9..e425358c35 100644 --- a/extensions/assistant-extension/tsconfig.json +++ b/extensions/assistant-extension/tsconfig.json @@ -14,7 +14,7 @@ "outDir": "dist", "importHelpers": true, "typeRoots": ["node_modules/@types"], - "skipLibCheck": true, + "skipLibCheck": true }, - "include": ["src"], + "include": ["src"] } diff --git a/extensions/conversational-extension/webpack.config.js b/extensions/conversational-extension/webpack.config.js index 36e3382953..a3eb873d71 100644 --- a/extensions/conversational-extension/webpack.config.js +++ b/extensions/conversational-extension/webpack.config.js @@ -1,27 +1,27 @@ -const path = require("path"); -const webpack = require("webpack"); +const path = require('path') +const webpack = require('webpack') module.exports = { experiments: { outputModule: true }, - entry: "./src/index.ts", // Adjust the entry point to match your project's main file - mode: "production", + entry: './src/index.ts', // Adjust the entry point to match your project's main file + mode: 'production', module: { rules: [ { test: /\.tsx?$/, - use: "ts-loader", + use: 'ts-loader', exclude: /node_modules/, }, ], }, output: { - filename: "index.js", // Adjust the output file name as needed - path: path.resolve(__dirname, "dist"), - library: { type: "module" }, // Specify ESM output format + filename: 'index.js', // Adjust the output file name as needed + path: path.resolve(__dirname, 'dist'), + library: { type: 'module' }, // Specify ESM output format }, plugins: [new webpack.DefinePlugin({})], resolve: { - extensions: [".ts", ".js"], + extensions: ['.ts', '.js'], fallback: { path: require.resolve('path-browserify'), }, @@ -31,4 +31,4 @@ module.exports = { minimize: false, }, // Add loaders and other configuration as needed for your project -}; +} diff --git a/extensions/inference-nitro-extension/README.md b/extensions/inference-nitro-extension/README.md index 455783efb1..f499e0b9c5 100644 --- a/extensions/inference-nitro-extension/README.md +++ b/extensions/inference-nitro-extension/README.md @@ -64,10 +64,10 @@ There are a few things to keep in mind when writing your plugin code: In `index.ts`, you will see that the extension function will return a `Promise`. ```typescript - import { core } from "@janhq/core"; + import { core } from '@janhq/core' function onStart(): Promise { - return core.invokePluginFunc(MODULE_PATH, "run", 0); + return core.invokePluginFunc(MODULE_PATH, 'run', 0) } ``` @@ -75,4 +75,3 @@ There are a few things to keep in mind when writing your plugin code: [documentation](https://github.com/janhq/jan/blob/main/core/README.md). So, what are you waiting for? Go ahead and start customizing your plugin! - diff --git a/extensions/inference-nitro-extension/rollup.config.ts b/extensions/inference-nitro-extension/rollup.config.ts index 77a9fb208a..ec8943f9cd 100644 --- a/extensions/inference-nitro-extension/rollup.config.ts +++ b/extensions/inference-nitro-extension/rollup.config.ts @@ -1,34 +1,34 @@ -import resolve from "@rollup/plugin-node-resolve"; -import commonjs from "@rollup/plugin-commonjs"; -import sourceMaps from "rollup-plugin-sourcemaps"; -import typescript from "rollup-plugin-typescript2"; -import json from "@rollup/plugin-json"; -import replace from "@rollup/plugin-replace"; -const packageJson = require("./package.json"); +import resolve from '@rollup/plugin-node-resolve' +import commonjs from '@rollup/plugin-commonjs' +import sourceMaps from 'rollup-plugin-sourcemaps' +import typescript from 'rollup-plugin-typescript2' +import json from '@rollup/plugin-json' +import replace from '@rollup/plugin-replace' +const packageJson = require('./package.json') -const pkg = require("./package.json"); +const pkg = require('./package.json') export default [ { input: `src/index.ts`, - output: [{ file: pkg.main, format: "es", sourcemap: true }], + output: [{ file: pkg.main, format: 'es', sourcemap: true }], // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') external: [], watch: { - include: "src/**", + include: 'src/**', }, plugins: [ replace({ NODE: JSON.stringify(`${packageJson.name}/${packageJson.node}`), INFERENCE_URL: JSON.stringify( process.env.INFERENCE_URL || - "http://127.0.0.1:3928/inferences/llamacpp/chat_completion" + 'http://127.0.0.1:3928/inferences/llamacpp/chat_completion' ), TROUBLESHOOTING_URL: JSON.stringify( - "https://jan.ai/guides/troubleshooting" + 'https://jan.ai/guides/troubleshooting' ), JAN_SERVER_INFERENCE_URL: JSON.stringify( - "http://localhost:1337/v1/chat/completions" + 'http://localhost:1337/v1/chat/completions' ), }), // Allow json resolution @@ -42,7 +42,7 @@ export default [ // which external modules to include in the bundle // https://github.com/rollup/rollup-plugin-node-resolve#usage resolve({ - extensions: [".js", ".ts", ".svelte"], + extensions: ['.js', '.ts', '.svelte'], }), // Resolve source maps to the original source @@ -52,12 +52,12 @@ export default [ { input: `src/node/index.ts`, output: [ - { file: "dist/node/index.cjs.js", format: "cjs", sourcemap: true }, + { file: 'dist/node/index.cjs.js', format: 'cjs', sourcemap: true }, ], // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') - external: ["@janhq/core/node"], + external: ['@janhq/core/node'], watch: { - include: "src/node/**", + include: 'src/node/**', }, plugins: [ // Allow json resolution @@ -70,11 +70,11 @@ export default [ // which external modules to include in the bundle // https://github.com/rollup/rollup-plugin-node-resolve#usage resolve({ - extensions: [".ts", ".js", ".json"], + extensions: ['.ts', '.js', '.json'], }), // Resolve source maps to the original source sourceMaps(), ], }, -]; +] diff --git a/extensions/inference-nitro-extension/src/@types/global.d.ts b/extensions/inference-nitro-extension/src/@types/global.d.ts index 7a4fb4805a..3a3d2aa325 100644 --- a/extensions/inference-nitro-extension/src/@types/global.d.ts +++ b/extensions/inference-nitro-extension/src/@types/global.d.ts @@ -1,13 +1,13 @@ -declare const NODE: string; -declare const INFERENCE_URL: string; -declare const TROUBLESHOOTING_URL: string; -declare const JAN_SERVER_INFERENCE_URL: string; +declare const NODE: string +declare const INFERENCE_URL: string +declare const TROUBLESHOOTING_URL: string +declare const JAN_SERVER_INFERENCE_URL: string /** * The response from the initModel function. * @property error - An error message if the model fails to load. */ interface ModelOperationResponse { - error?: any; - modelFile?: string; + error?: any + modelFile?: string } diff --git a/extensions/inference-nitro-extension/src/helpers/sse.ts b/extensions/inference-nitro-extension/src/helpers/sse.ts index aab260828f..06176c9b9d 100644 --- a/extensions/inference-nitro-extension/src/helpers/sse.ts +++ b/extensions/inference-nitro-extension/src/helpers/sse.ts @@ -1,5 +1,5 @@ -import { Model } from "@janhq/core"; -import { Observable } from "rxjs"; +import { Model } from '@janhq/core' +import { Observable } from 'rxjs' /** * Sends a request to the inference server to generate a response based on the recent messages. * @param recentMessages - An array of recent messages to use as context for the inference. @@ -17,50 +17,50 @@ export function requestInference( model: model.id, stream: true, ...model.parameters, - }); + }) fetch(inferenceUrl, { - method: "POST", + method: 'POST', headers: { - "Content-Type": "application/json", - "Access-Control-Allow-Origin": "*", - Accept: model.parameters.stream - ? "text/event-stream" - : "application/json", + 'Content-Type': 'application/json', + 'Access-Control-Allow-Origin': '*', + 'Accept': model.parameters.stream + ? 'text/event-stream' + : 'application/json', }, body: requestBody, signal: controller?.signal, }) .then(async (response) => { if (model.parameters.stream === false) { - const data = await response.json(); - subscriber.next(data.choices[0]?.message?.content ?? ""); + const data = await response.json() + subscriber.next(data.choices[0]?.message?.content ?? '') } else { - const stream = response.body; - const decoder = new TextDecoder("utf-8"); - const reader = stream?.getReader(); - let content = ""; + const stream = response.body + const decoder = new TextDecoder('utf-8') + const reader = stream?.getReader() + let content = '' while (true && reader) { - const { done, value } = await reader.read(); + const { done, value } = await reader.read() if (done) { - break; + break } - const text = decoder.decode(value); - const lines = text.trim().split("\n"); + const text = decoder.decode(value) + const lines = text.trim().split('\n') for (const line of lines) { - if (line.startsWith("data: ") && !line.includes("data: [DONE]")) { - const data = JSON.parse(line.replace("data: ", "")); - content += data.choices[0]?.delta?.content ?? ""; - if (content.startsWith("assistant: ")) { - content = content.replace("assistant: ", ""); + if (line.startsWith('data: ') && !line.includes('data: [DONE]')) { + const data = JSON.parse(line.replace('data: ', '')) + content += data.choices[0]?.delta?.content ?? '' + if (content.startsWith('assistant: ')) { + content = content.replace('assistant: ', '') } - subscriber.next(content); + subscriber.next(content) } } } } - subscriber.complete(); + subscriber.complete() }) - .catch((err) => subscriber.error(err)); - }); + .catch((err) => subscriber.error(err)) + }) } diff --git a/extensions/inference-nitro-extension/src/index.ts b/extensions/inference-nitro-extension/src/index.ts index b40edaaa73..b88501936c 100644 --- a/extensions/inference-nitro-extension/src/index.ts +++ b/extensions/inference-nitro-extension/src/index.ts @@ -27,9 +27,9 @@ import { InferenceEvent, ModelSettingParams, getJanDataFolderPath, -} from "@janhq/core"; -import { requestInference } from "./helpers/sse"; -import { ulid } from "ulid"; +} from '@janhq/core' +import { requestInference } from './helpers/sse' +import { ulid } from 'ulid' /** * A class that implements the InferenceExtension interface from the @janhq/core package. @@ -37,16 +37,16 @@ import { ulid } from "ulid"; * It also subscribes to events emitted by the @janhq/core package and handles new message requests. */ export default class JanInferenceNitroExtension extends InferenceExtension { - private static readonly _homeDir = "file://engines"; - private static readonly _settingsDir = "file://settings"; - private static readonly _engineMetadataFileName = "nitro.json"; + private static readonly _homeDir = 'file://engines' + private static readonly _settingsDir = 'file://settings' + private static readonly _engineMetadataFileName = 'nitro.json' /** * Checking the health for Nitro's process each 5 secs. */ - private static readonly _intervalHealthCheck = 5 * 1000; + private static readonly _intervalHealthCheck = 5 * 1000 - private _currentModel: Model | undefined; + private _currentModel: Model | undefined private _engineSettings: ModelSettingParams = { ctx_len: 2048, @@ -54,23 +54,22 @@ export default class JanInferenceNitroExtension extends InferenceExtension { cpu_threads: 1, cont_batching: false, embedding: true, - }; + } - controller = new AbortController(); - isCancelled = false; + controller = new AbortController() + isCancelled = false /** * The interval id for the health check. Used to stop the health check. */ - private getNitroProcesHealthIntervalId: NodeJS.Timeout | undefined = - undefined; + private getNitroProcesHealthIntervalId: NodeJS.Timeout | undefined = undefined /** * Tracking the current state of nitro process. */ - private nitroProcessInfo: any = undefined; + private nitroProcessInfo: any = undefined - private inferenceUrl = ""; + private inferenceUrl = '' /** * Subscribes to events emitted by the @janhq/core package. @@ -78,44 +77,40 @@ export default class JanInferenceNitroExtension extends InferenceExtension { async onLoad() { if (!(await fs.existsSync(JanInferenceNitroExtension._homeDir))) { try { - await fs.mkdirSync(JanInferenceNitroExtension._homeDir); + await fs.mkdirSync(JanInferenceNitroExtension._homeDir) } catch (e) { - console.debug(e); + console.debug(e) } } // init inference url // @ts-ignore - const electronApi = window?.electronAPI; - this.inferenceUrl = INFERENCE_URL; + const electronApi = window?.electronAPI + this.inferenceUrl = INFERENCE_URL if (!electronApi) { - this.inferenceUrl = JAN_SERVER_INFERENCE_URL; + this.inferenceUrl = JAN_SERVER_INFERENCE_URL } - console.debug("Inference url: ", this.inferenceUrl); + console.debug('Inference url: ', this.inferenceUrl) if (!(await fs.existsSync(JanInferenceNitroExtension._settingsDir))) - await fs.mkdirSync(JanInferenceNitroExtension._settingsDir); - this.writeDefaultEngineSettings(); + await fs.mkdirSync(JanInferenceNitroExtension._settingsDir) + this.writeDefaultEngineSettings() // Events subscription events.on(MessageEvent.OnMessageSent, (data: MessageRequest) => this.onMessageRequest(data) - ); + ) - events.on(ModelEvent.OnModelInit, (model: Model) => - this.onModelInit(model) - ); + events.on(ModelEvent.OnModelInit, (model: Model) => this.onModelInit(model)) - events.on(ModelEvent.OnModelStop, (model: Model) => - this.onModelStop(model) - ); + events.on(ModelEvent.OnModelStop, (model: Model) => this.onModelStop(model)) events.on(InferenceEvent.OnInferenceStopped, () => this.onInferenceStopped() - ); + ) // Attempt to fetch nvidia info - await executeOnMain(NODE, "updateNvidiaInfo", {}); + await executeOnMain(NODE, 'updateNvidiaInfo', {}) } /** @@ -128,59 +123,59 @@ export default class JanInferenceNitroExtension extends InferenceExtension { const engineFile = await joinPath([ JanInferenceNitroExtension._homeDir, JanInferenceNitroExtension._engineMetadataFileName, - ]); + ]) if (await fs.existsSync(engineFile)) { - const engine = await fs.readFileSync(engineFile, "utf-8"); + const engine = await fs.readFileSync(engineFile, 'utf-8') this._engineSettings = - typeof engine === "object" ? engine : JSON.parse(engine); + typeof engine === 'object' ? engine : JSON.parse(engine) } else { await fs.writeFileSync( engineFile, JSON.stringify(this._engineSettings, null, 2) - ); + ) } } catch (err) { - console.error(err); + console.error(err) } } private async onModelInit(model: Model) { - if (model.engine !== InferenceEngine.nitro) return; + if (model.engine !== InferenceEngine.nitro) return const modelFolder = await joinPath([ await getJanDataFolderPath(), - "models", + 'models', model.id, - ]); - this._currentModel = model; - const nitroInitResult = await executeOnMain(NODE, "runModel", { + ]) + this._currentModel = model + const nitroInitResult = await executeOnMain(NODE, 'runModel', { modelFolder, model, - }); + }) if (nitroInitResult?.error) { - events.emit(ModelEvent.OnModelFail, model); - return; + events.emit(ModelEvent.OnModelFail, model) + return } - events.emit(ModelEvent.OnModelReady, model); + events.emit(ModelEvent.OnModelReady, model) this.getNitroProcesHealthIntervalId = setInterval( () => this.periodicallyGetNitroHealth(), JanInferenceNitroExtension._intervalHealthCheck - ); + ) } private async onModelStop(model: Model) { - if (model.engine !== "nitro") return; + if (model.engine !== 'nitro') return - await executeOnMain(NODE, "stopModel"); - events.emit(ModelEvent.OnModelStopped, {}); + await executeOnMain(NODE, 'stopModel') + events.emit(ModelEvent.OnModelStopped, {}) // stop the periocally health check if (this.getNitroProcesHealthIntervalId) { - clearInterval(this.getNitroProcesHealthIntervalId); - this.getNitroProcesHealthIntervalId = undefined; + clearInterval(this.getNitroProcesHealthIntervalId) + this.getNitroProcesHealthIntervalId = undefined } } @@ -188,19 +183,19 @@ export default class JanInferenceNitroExtension extends InferenceExtension { * Periodically check for nitro process's health. */ private async periodicallyGetNitroHealth(): Promise { - const health = await executeOnMain(NODE, "getCurrentNitroProcessInfo"); + const health = await executeOnMain(NODE, 'getCurrentNitroProcessInfo') - const isRunning = this.nitroProcessInfo?.isRunning ?? false; + const isRunning = this.nitroProcessInfo?.isRunning ?? false if (isRunning && health.isRunning === false) { - console.debug("Nitro process is stopped"); - events.emit(ModelEvent.OnModelStopped, {}); + console.debug('Nitro process is stopped') + events.emit(ModelEvent.OnModelStopped, {}) } - this.nitroProcessInfo = health; + this.nitroProcessInfo = health } private async onInferenceStopped() { - this.isCancelled = true; - this.controller?.abort(); + this.isCancelled = true + this.controller?.abort() } /** @@ -209,20 +204,20 @@ export default class JanInferenceNitroExtension extends InferenceExtension { * @returns {Promise} A promise that resolves with the inference response. */ async inference(data: MessageRequest): Promise { - const timestamp = Date.now(); + const timestamp = Date.now() const message: ThreadMessage = { thread_id: data.threadId, created: timestamp, updated: timestamp, status: MessageStatus.Ready, - id: "", + id: '', role: ChatCompletionRole.Assistant, - object: "thread.message", + object: 'thread.message', content: [], - }; + } return new Promise(async (resolve, reject) => { - if (!this._currentModel) return Promise.reject("No model loaded"); + if (!this._currentModel) return Promise.reject('No model loaded') requestInference( this.inferenceUrl, @@ -231,13 +226,13 @@ export default class JanInferenceNitroExtension extends InferenceExtension { ).subscribe({ next: (_content: any) => {}, complete: async () => { - resolve(message); + resolve(message) }, error: async (err: any) => { - reject(err); + reject(err) }, - }); - }); + }) + }) } /** @@ -248,10 +243,10 @@ export default class JanInferenceNitroExtension extends InferenceExtension { */ private async onMessageRequest(data: MessageRequest) { if (data.model?.engine !== InferenceEngine.nitro || !this._currentModel) { - return; + return } - const timestamp = Date.now(); + const timestamp = Date.now() const message: ThreadMessage = { id: ulid(), thread_id: data.threadId, @@ -262,21 +257,21 @@ export default class JanInferenceNitroExtension extends InferenceExtension { status: MessageStatus.Pending, created: timestamp, updated: timestamp, - object: "thread.message", - }; + object: 'thread.message', + } if (data.type !== MessageRequestType.Summary) { - events.emit(MessageEvent.OnMessageResponse, message); + events.emit(MessageEvent.OnMessageResponse, message) } - this.isCancelled = false; - this.controller = new AbortController(); + this.isCancelled = false + this.controller = new AbortController() // @ts-ignore const model: Model = { ...(this._currentModel || {}), ...(data.model || {}), - }; + } requestInference( this.inferenceUrl, data.messages ?? [], @@ -290,26 +285,26 @@ export default class JanInferenceNitroExtension extends InferenceExtension { value: content.trim(), annotations: [], }, - }; - message.content = [messageContent]; - events.emit(MessageEvent.OnMessageUpdate, message); + } + message.content = [messageContent] + events.emit(MessageEvent.OnMessageUpdate, message) }, complete: async () => { message.status = message.content.length ? MessageStatus.Ready - : MessageStatus.Error; - events.emit(MessageEvent.OnMessageUpdate, message); + : MessageStatus.Error + events.emit(MessageEvent.OnMessageUpdate, message) }, error: async (err: any) => { if (this.isCancelled || message.content.length) { - message.status = MessageStatus.Stopped; - events.emit(MessageEvent.OnMessageUpdate, message); - return; + message.status = MessageStatus.Stopped + events.emit(MessageEvent.OnMessageUpdate, message) + return } - message.status = MessageStatus.Error; - events.emit(MessageEvent.OnMessageUpdate, message); - log(`[APP]::Error: ${err.message}`); + message.status = MessageStatus.Error + events.emit(MessageEvent.OnMessageUpdate, message) + log(`[APP]::Error: ${err.message}`) }, - }); + }) } } diff --git a/extensions/inference-nitro-extension/src/node/execute.ts b/extensions/inference-nitro-extension/src/node/execute.ts index 83b5226d41..795c83ded0 100644 --- a/extensions/inference-nitro-extension/src/node/execute.ts +++ b/extensions/inference-nitro-extension/src/node/execute.ts @@ -1,65 +1,65 @@ -import { readFileSync } from "fs"; -import * as path from "path"; -import { NVIDIA_INFO_FILE } from "./nvidia"; +import { readFileSync } from 'fs' +import * as path from 'path' +import { NVIDIA_INFO_FILE } from './nvidia' export interface NitroExecutableOptions { - executablePath: string; - cudaVisibleDevices: string; + executablePath: string + cudaVisibleDevices: string } /** * Find which executable file to run based on the current platform. * @returns The name of the executable file to run. */ export const executableNitroFile = (): NitroExecutableOptions => { - let binaryFolder = path.join(__dirname, "..", "bin"); // Current directory by default - let cudaVisibleDevices = ""; - let binaryName = "nitro"; + let binaryFolder = path.join(__dirname, '..', 'bin') // Current directory by default + let cudaVisibleDevices = '' + let binaryName = 'nitro' /** * The binary folder is different for each platform. */ - if (process.platform === "win32") { + if (process.platform === 'win32') { /** * For Windows: win-cpu, win-cuda-11-7, win-cuda-12-0 */ - let nvidiaInfo = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8")); - if (nvidiaInfo["run_mode"] === "cpu") { - binaryFolder = path.join(binaryFolder, "win-cpu"); + let nvidiaInfo = JSON.parse(readFileSync(NVIDIA_INFO_FILE, 'utf-8')) + if (nvidiaInfo['run_mode'] === 'cpu') { + binaryFolder = path.join(binaryFolder, 'win-cpu') } else { - if (nvidiaInfo["cuda"].version === "11") { - binaryFolder = path.join(binaryFolder, "win-cuda-11-7"); + if (nvidiaInfo['cuda'].version === '11') { + binaryFolder = path.join(binaryFolder, 'win-cuda-11-7') } else { - binaryFolder = path.join(binaryFolder, "win-cuda-12-0"); + binaryFolder = path.join(binaryFolder, 'win-cuda-12-0') } - cudaVisibleDevices = nvidiaInfo["gpus_in_use"].join(","); + cudaVisibleDevices = nvidiaInfo['gpus_in_use'].join(',') } - binaryName = "nitro.exe"; - } else if (process.platform === "darwin") { + binaryName = 'nitro.exe' + } else if (process.platform === 'darwin') { /** * For MacOS: mac-arm64 (Silicon), mac-x64 (InteL) */ - if (process.arch === "arm64") { - binaryFolder = path.join(binaryFolder, "mac-arm64"); + if (process.arch === 'arm64') { + binaryFolder = path.join(binaryFolder, 'mac-arm64') } else { - binaryFolder = path.join(binaryFolder, "mac-x64"); + binaryFolder = path.join(binaryFolder, 'mac-x64') } } else { /** * For Linux: linux-cpu, linux-cuda-11-7, linux-cuda-12-0 */ - let nvidiaInfo = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8")); - if (nvidiaInfo["run_mode"] === "cpu") { - binaryFolder = path.join(binaryFolder, "linux-cpu"); + let nvidiaInfo = JSON.parse(readFileSync(NVIDIA_INFO_FILE, 'utf-8')) + if (nvidiaInfo['run_mode'] === 'cpu') { + binaryFolder = path.join(binaryFolder, 'linux-cpu') } else { - if (nvidiaInfo["cuda"].version === "11") { - binaryFolder = path.join(binaryFolder, "linux-cuda-11-7"); + if (nvidiaInfo['cuda'].version === '11') { + binaryFolder = path.join(binaryFolder, 'linux-cuda-11-7') } else { - binaryFolder = path.join(binaryFolder, "linux-cuda-12-0"); + binaryFolder = path.join(binaryFolder, 'linux-cuda-12-0') } - cudaVisibleDevices = nvidiaInfo["gpus_in_use"].join(","); + cudaVisibleDevices = nvidiaInfo['gpus_in_use'].join(',') } } return { executablePath: path.join(binaryFolder, binaryName), cudaVisibleDevices, - }; -}; + } +} diff --git a/extensions/inference-nitro-extension/src/node/nvidia.ts b/extensions/inference-nitro-extension/src/node/nvidia.ts index bed2856a19..60468f0c9e 100644 --- a/extensions/inference-nitro-extension/src/node/nvidia.ts +++ b/extensions/inference-nitro-extension/src/node/nvidia.ts @@ -1,47 +1,47 @@ -import { writeFileSync, existsSync, readFileSync } from "fs"; -import { exec } from "child_process"; -import path from "path"; -import { getJanDataFolderPath } from "@janhq/core/node"; +import { writeFileSync, existsSync, readFileSync } from 'fs' +import { exec } from 'child_process' +import path from 'path' +import { getJanDataFolderPath } from '@janhq/core/node' /** * Default GPU settings **/ const DEFALT_SETTINGS = { notify: true, - run_mode: "cpu", + run_mode: 'cpu', nvidia_driver: { exist: false, - version: "", + version: '', }, cuda: { exist: false, - version: "", + version: '', }, gpus: [], - gpu_highest_vram: "", + gpu_highest_vram: '', gpus_in_use: [], is_initial: true, -}; +} /** * Path to the settings file **/ export const NVIDIA_INFO_FILE = path.join( getJanDataFolderPath(), - "settings", - "settings.json" -); + 'settings', + 'settings.json' +) /** * Current nitro process */ -let nitroProcessInfo: NitroProcessInfo | undefined = undefined; +let nitroProcessInfo: NitroProcessInfo | undefined = undefined /** * Nitro process info */ export interface NitroProcessInfo { - isRunning: boolean; + isRunning: boolean } /** @@ -49,16 +49,16 @@ export interface NitroProcessInfo { * Will be called when the extension is loaded to turn on GPU acceleration if supported */ export async function updateNvidiaInfo() { - if (process.platform !== "darwin") { - let data; + if (process.platform !== 'darwin') { + let data try { - data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8")); + data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, 'utf-8')) } catch (error) { - data = DEFALT_SETTINGS; - writeFileSync(NVIDIA_INFO_FILE, JSON.stringify(data, null, 2)); + data = DEFALT_SETTINGS + writeFileSync(NVIDIA_INFO_FILE, JSON.stringify(data, null, 2)) } - updateNvidiaDriverInfo(); - updateGpuInfo(); + updateNvidiaDriverInfo() + updateGpuInfo() } } @@ -68,31 +68,31 @@ export async function updateNvidiaInfo() { export const getNitroProcessInfo = (subprocess: any): NitroProcessInfo => { nitroProcessInfo = { isRunning: subprocess != null, - }; - return nitroProcessInfo; -}; + } + return nitroProcessInfo +} /** * Validate nvidia and cuda for linux and windows */ export async function updateNvidiaDriverInfo(): Promise { exec( - "nvidia-smi --query-gpu=driver_version --format=csv,noheader", + 'nvidia-smi --query-gpu=driver_version --format=csv,noheader', (error, stdout) => { - let data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8")); + let data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, 'utf-8')) if (!error) { - const firstLine = stdout.split("\n")[0].trim(); - data["nvidia_driver"].exist = true; - data["nvidia_driver"].version = firstLine; + const firstLine = stdout.split('\n')[0].trim() + data['nvidia_driver'].exist = true + data['nvidia_driver'].version = firstLine } else { - data["nvidia_driver"].exist = false; + data['nvidia_driver'].exist = false } - writeFileSync(NVIDIA_INFO_FILE, JSON.stringify(data, null, 2)); - Promise.resolve(); + writeFileSync(NVIDIA_INFO_FILE, JSON.stringify(data, null, 2)) + Promise.resolve() } - ); + ) } /** @@ -102,54 +102,56 @@ export function checkFileExistenceInPaths( file: string, paths: string[] ): boolean { - return paths.some((p) => existsSync(path.join(p, file))); + return paths.some((p) => existsSync(path.join(p, file))) } /** * Validate cuda for linux and windows */ -export function updateCudaExistence(data: Record = DEFALT_SETTINGS): Record { - let filesCuda12: string[]; - let filesCuda11: string[]; - let paths: string[]; - let cudaVersion: string = ""; - - if (process.platform === "win32") { - filesCuda12 = ["cublas64_12.dll", "cudart64_12.dll", "cublasLt64_12.dll"]; - filesCuda11 = ["cublas64_11.dll", "cudart64_11.dll", "cublasLt64_11.dll"]; - paths = process.env.PATH ? process.env.PATH.split(path.delimiter) : []; +export function updateCudaExistence( + data: Record = DEFALT_SETTINGS +): Record { + let filesCuda12: string[] + let filesCuda11: string[] + let paths: string[] + let cudaVersion: string = '' + + if (process.platform === 'win32') { + filesCuda12 = ['cublas64_12.dll', 'cudart64_12.dll', 'cublasLt64_12.dll'] + filesCuda11 = ['cublas64_11.dll', 'cudart64_11.dll', 'cublasLt64_11.dll'] + paths = process.env.PATH ? process.env.PATH.split(path.delimiter) : [] } else { - filesCuda12 = ["libcudart.so.12", "libcublas.so.12", "libcublasLt.so.12"]; - filesCuda11 = ["libcudart.so.11.0", "libcublas.so.11", "libcublasLt.so.11"]; + filesCuda12 = ['libcudart.so.12', 'libcublas.so.12', 'libcublasLt.so.12'] + filesCuda11 = ['libcudart.so.11.0', 'libcublas.so.11', 'libcublasLt.so.11'] paths = process.env.LD_LIBRARY_PATH ? process.env.LD_LIBRARY_PATH.split(path.delimiter) - : []; - paths.push("/usr/lib/x86_64-linux-gnu/"); + : [] + paths.push('/usr/lib/x86_64-linux-gnu/') } let cudaExists = filesCuda12.every( (file) => existsSync(file) || checkFileExistenceInPaths(file, paths) - ); + ) if (!cudaExists) { cudaExists = filesCuda11.every( (file) => existsSync(file) || checkFileExistenceInPaths(file, paths) - ); + ) if (cudaExists) { - cudaVersion = "11"; + cudaVersion = '11' } } else { - cudaVersion = "12"; + cudaVersion = '12' } - data["cuda"].exist = cudaExists; - data["cuda"].version = cudaVersion; - console.log(data["is_initial"], data["gpus_in_use"]); - if (cudaExists && data["is_initial"] && data["gpus_in_use"].length > 0) { - data.run_mode = "gpu"; + data['cuda'].exist = cudaExists + data['cuda'].version = cudaVersion + console.log(data['is_initial'], data['gpus_in_use']) + if (cudaExists && data['is_initial'] && data['gpus_in_use'].length > 0) { + data.run_mode = 'gpu' } - data.is_initial = false; - return data; + data.is_initial = false + return data } /** @@ -157,41 +159,41 @@ export function updateCudaExistence(data: Record = DEFALT_SETTINGS) */ export async function updateGpuInfo(): Promise { exec( - "nvidia-smi --query-gpu=index,memory.total,name --format=csv,noheader,nounits", + 'nvidia-smi --query-gpu=index,memory.total,name --format=csv,noheader,nounits', (error, stdout) => { - let data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8")); + let data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, 'utf-8')) if (!error) { // Get GPU info and gpu has higher memory first - let highestVram = 0; - let highestVramId = "0"; + let highestVram = 0 + let highestVramId = '0' let gpus = stdout .trim() - .split("\n") + .split('\n') .map((line) => { - let [id, vram, name] = line.split(", "); - vram = vram.replace(/\r/g, ""); + let [id, vram, name] = line.split(', ') + vram = vram.replace(/\r/g, '') if (parseFloat(vram) > highestVram) { - highestVram = parseFloat(vram); - highestVramId = id; + highestVram = parseFloat(vram) + highestVramId = id } - return { id, vram, name }; - }); + return { id, vram, name } + }) - data.gpus = gpus; - data.gpu_highest_vram = highestVramId; + data.gpus = gpus + data.gpu_highest_vram = highestVramId } else { - data.gpus = []; - data.gpu_highest_vram = ""; + data.gpus = [] + data.gpu_highest_vram = '' } - if (!data["gpus_in_use"] || data["gpus_in_use"].length === 0) { - data.gpus_in_use = [data["gpu_highest_vram"]]; + if (!data['gpus_in_use'] || data['gpus_in_use'].length === 0) { + data.gpus_in_use = [data['gpu_highest_vram']] } - data = updateCudaExistence(data); - writeFileSync(NVIDIA_INFO_FILE, JSON.stringify(data, null, 2)); - Promise.resolve(); + data = updateCudaExistence(data) + writeFileSync(NVIDIA_INFO_FILE, JSON.stringify(data, null, 2)) + Promise.resolve() } - ); + ) } diff --git a/extensions/inference-openai-extension/src/@types/global.d.ts b/extensions/inference-openai-extension/src/@types/global.d.ts index 84f86c1458..a49bb5a2f2 100644 --- a/extensions/inference-openai-extension/src/@types/global.d.ts +++ b/extensions/inference-openai-extension/src/@types/global.d.ts @@ -1,26 +1,26 @@ -declare const MODULE: string; -declare const OPENAI_DOMAIN: string; +declare const MODULE: string +declare const OPENAI_DOMAIN: string declare interface EngineSettings { - full_url?: string; - api_key?: string; + full_url?: string + api_key?: string } enum OpenAIChatCompletionModelName { - "gpt-3.5-turbo-instruct" = "gpt-3.5-turbo-instruct", - "gpt-3.5-turbo-instruct-0914" = "gpt-3.5-turbo-instruct-0914", - "gpt-4-1106-preview" = "gpt-4-1106-preview", - "gpt-3.5-turbo-0613" = "gpt-3.5-turbo-0613", - "gpt-3.5-turbo-0301" = "gpt-3.5-turbo-0301", - "gpt-3.5-turbo" = "gpt-3.5-turbo", - "gpt-3.5-turbo-16k-0613" = "gpt-3.5-turbo-16k-0613", - "gpt-3.5-turbo-1106" = "gpt-3.5-turbo-1106", - "gpt-4-vision-preview" = "gpt-4-vision-preview", - "gpt-4" = "gpt-4", - "gpt-4-0314" = "gpt-4-0314", - "gpt-4-0613" = "gpt-4-0613", + 'gpt-3.5-turbo-instruct' = 'gpt-3.5-turbo-instruct', + 'gpt-3.5-turbo-instruct-0914' = 'gpt-3.5-turbo-instruct-0914', + 'gpt-4-1106-preview' = 'gpt-4-1106-preview', + 'gpt-3.5-turbo-0613' = 'gpt-3.5-turbo-0613', + 'gpt-3.5-turbo-0301' = 'gpt-3.5-turbo-0301', + 'gpt-3.5-turbo' = 'gpt-3.5-turbo', + 'gpt-3.5-turbo-16k-0613' = 'gpt-3.5-turbo-16k-0613', + 'gpt-3.5-turbo-1106' = 'gpt-3.5-turbo-1106', + 'gpt-4-vision-preview' = 'gpt-4-vision-preview', + 'gpt-4' = 'gpt-4', + 'gpt-4-0314' = 'gpt-4-0314', + 'gpt-4-0613' = 'gpt-4-0613', } -declare type OpenAIModel = Omit & { - id: OpenAIChatCompletionModelName; -}; +declare type OpenAIModel = Omit & { + id: OpenAIChatCompletionModelName +} diff --git a/extensions/inference-openai-extension/src/helpers/sse.ts b/extensions/inference-openai-extension/src/helpers/sse.ts index 554d3b1f71..11db382827 100644 --- a/extensions/inference-openai-extension/src/helpers/sse.ts +++ b/extensions/inference-openai-extension/src/helpers/sse.ts @@ -1,4 +1,4 @@ -import { Observable } from "rxjs"; +import { Observable } from 'rxjs' /** * Sends a request to the inference server to generate a response based on the recent messages. @@ -14,26 +14,26 @@ export function requestInference( controller?: AbortController ): Observable { return new Observable((subscriber) => { - let model_id: string = model.id; + let model_id: string = model.id if (engine.full_url.includes(OPENAI_DOMAIN)) { - model_id = engine.full_url.split("/")[5]; + model_id = engine.full_url.split('/')[5] } const requestBody = JSON.stringify({ messages: recentMessages, stream: true, model: model_id, ...model.parameters, - }); + }) fetch(`${engine.full_url}`, { - method: "POST", + method: 'POST', headers: { - "Content-Type": "application/json", - Accept: model.parameters.stream - ? "text/event-stream" - : "application/json", - "Access-Control-Allow-Origin": "*", - Authorization: `Bearer ${engine.api_key}`, - "api-key": `${engine.api_key}`, + 'Content-Type': 'application/json', + 'Accept': model.parameters.stream + ? 'text/event-stream' + : 'application/json', + 'Access-Control-Allow-Origin': '*', + 'Authorization': `Bearer ${engine.api_key}`, + 'api-key': `${engine.api_key}`, }, body: requestBody, signal: controller?.signal, @@ -41,41 +41,41 @@ export function requestInference( .then(async (response) => { if (!response.ok) { subscriber.next( - (await response.json()).error?.message ?? "Error occurred." - ); - subscriber.complete(); - return; + (await response.json()).error?.message ?? 'Error occurred.' + ) + subscriber.complete() + return } if (model.parameters.stream === false) { - const data = await response.json(); - subscriber.next(data.choices[0]?.message?.content ?? ""); + const data = await response.json() + subscriber.next(data.choices[0]?.message?.content ?? '') } else { - const stream = response.body; - const decoder = new TextDecoder("utf-8"); - const reader = stream?.getReader(); - let content = ""; + const stream = response.body + const decoder = new TextDecoder('utf-8') + const reader = stream?.getReader() + let content = '' while (true && reader) { - const { done, value } = await reader.read(); + const { done, value } = await reader.read() if (done) { - break; + break } - const text = decoder.decode(value); - const lines = text.trim().split("\n"); + const text = decoder.decode(value) + const lines = text.trim().split('\n') for (const line of lines) { - if (line.startsWith("data: ") && !line.includes("data: [DONE]")) { - const data = JSON.parse(line.replace("data: ", "")); - content += data.choices[0]?.delta?.content ?? ""; - if (content.startsWith("assistant: ")) { - content = content.replace("assistant: ", ""); + if (line.startsWith('data: ') && !line.includes('data: [DONE]')) { + const data = JSON.parse(line.replace('data: ', '')) + content += data.choices[0]?.delta?.content ?? '' + if (content.startsWith('assistant: ')) { + content = content.replace('assistant: ', '') } - subscriber.next(content); + subscriber.next(content) } } } } - subscriber.complete(); + subscriber.complete() }) - .catch((err) => subscriber.error(err)); - }); + .catch((err) => subscriber.error(err)) + }) } diff --git a/extensions/inference-openai-extension/src/index.ts b/extensions/inference-openai-extension/src/index.ts index 23fd8983e7..4811717421 100644 --- a/extensions/inference-openai-extension/src/index.ts +++ b/extensions/inference-openai-extension/src/index.ts @@ -23,10 +23,10 @@ import { InferenceEvent, AppConfigurationEventName, joinPath, -} from "@janhq/core"; -import { requestInference } from "./helpers/sse"; -import { ulid } from "ulid"; -import { join } from "path"; +} from '@janhq/core' +import { requestInference } from './helpers/sse' +import { ulid } from 'ulid' +import { join } from 'path' /** * A class that implements the InferenceExtension interface from the @janhq/core package. @@ -34,18 +34,18 @@ import { join } from "path"; * It also subscribes to events emitted by the @janhq/core package and handles new message requests. */ export default class JanInferenceOpenAIExtension extends BaseExtension { - private static readonly _engineDir = "file://engines"; - private static readonly _engineMetadataFileName = "openai.json"; + private static readonly _engineDir = 'file://engines' + private static readonly _engineMetadataFileName = 'openai.json' - private static _currentModel: OpenAIModel; + private static _currentModel: OpenAIModel private static _engineSettings: EngineSettings = { - full_url: "https://api.openai.com/v1/chat/completions", - api_key: "sk-", - }; + full_url: 'https://api.openai.com/v1/chat/completions', + api_key: 'sk-', + } - controller = new AbortController(); - isCancelled = false; + controller = new AbortController() + isCancelled = false /** * Subscribes to events emitted by the @janhq/core package. @@ -54,40 +54,40 @@ export default class JanInferenceOpenAIExtension extends BaseExtension { if (!(await fs.existsSync(JanInferenceOpenAIExtension._engineDir))) { await fs .mkdirSync(JanInferenceOpenAIExtension._engineDir) - .catch((err) => console.debug(err)); + .catch((err) => console.debug(err)) } - JanInferenceOpenAIExtension.writeDefaultEngineSettings(); + JanInferenceOpenAIExtension.writeDefaultEngineSettings() // Events subscription events.on(MessageEvent.OnMessageSent, (data) => - JanInferenceOpenAIExtension.handleMessageRequest(data, this), - ); + JanInferenceOpenAIExtension.handleMessageRequest(data, this) + ) events.on(ModelEvent.OnModelInit, (model: OpenAIModel) => { - JanInferenceOpenAIExtension.handleModelInit(model); - }); + JanInferenceOpenAIExtension.handleModelInit(model) + }) events.on(ModelEvent.OnModelStop, (model: OpenAIModel) => { - JanInferenceOpenAIExtension.handleModelStop(model); - }); + JanInferenceOpenAIExtension.handleModelStop(model) + }) events.on(InferenceEvent.OnInferenceStopped, () => { - JanInferenceOpenAIExtension.handleInferenceStopped(this); - }); + JanInferenceOpenAIExtension.handleInferenceStopped(this) + }) const settingsFilePath = await joinPath([ JanInferenceOpenAIExtension._engineDir, JanInferenceOpenAIExtension._engineMetadataFileName, - ]); + ]) events.on( AppConfigurationEventName.OnConfigurationUpdate, (settingsKey: string) => { // Update settings on changes if (settingsKey === settingsFilePath) - JanInferenceOpenAIExtension.writeDefaultEngineSettings(); - }, - ); + JanInferenceOpenAIExtension.writeDefaultEngineSettings() + } + ) } /** @@ -99,45 +99,45 @@ export default class JanInferenceOpenAIExtension extends BaseExtension { try { const engineFile = join( JanInferenceOpenAIExtension._engineDir, - JanInferenceOpenAIExtension._engineMetadataFileName, - ); + JanInferenceOpenAIExtension._engineMetadataFileName + ) if (await fs.existsSync(engineFile)) { - const engine = await fs.readFileSync(engineFile, "utf-8"); + const engine = await fs.readFileSync(engineFile, 'utf-8') JanInferenceOpenAIExtension._engineSettings = - typeof engine === "object" ? engine : JSON.parse(engine); + typeof engine === 'object' ? engine : JSON.parse(engine) } else { await fs.writeFileSync( engineFile, - JSON.stringify(JanInferenceOpenAIExtension._engineSettings, null, 2), - ); + JSON.stringify(JanInferenceOpenAIExtension._engineSettings, null, 2) + ) } } catch (err) { - console.error(err); + console.error(err) } } private static async handleModelInit(model: OpenAIModel) { if (model.engine !== InferenceEngine.openai) { - return; + return } else { - JanInferenceOpenAIExtension._currentModel = model; - JanInferenceOpenAIExtension.writeDefaultEngineSettings(); + JanInferenceOpenAIExtension._currentModel = model + JanInferenceOpenAIExtension.writeDefaultEngineSettings() // Todo: Check model list with API key - events.emit(ModelEvent.OnModelReady, model); + events.emit(ModelEvent.OnModelReady, model) } } private static async handleModelStop(model: OpenAIModel) { - if (model.engine !== "openai") { - return; + if (model.engine !== 'openai') { + return } - events.emit(ModelEvent.OnModelStopped, model); + events.emit(ModelEvent.OnModelStopped, model) } private static async handleInferenceStopped( - instance: JanInferenceOpenAIExtension, + instance: JanInferenceOpenAIExtension ) { - instance.isCancelled = true; - instance.controller?.abort(); + instance.isCancelled = true + instance.controller?.abort() } /** @@ -148,13 +148,13 @@ export default class JanInferenceOpenAIExtension extends BaseExtension { */ private static async handleMessageRequest( data: MessageRequest, - instance: JanInferenceOpenAIExtension, + instance: JanInferenceOpenAIExtension ) { - if (data.model.engine !== "openai") { - return; + if (data.model.engine !== 'openai') { + return } - const timestamp = Date.now(); + const timestamp = Date.now() const message: ThreadMessage = { id: ulid(), thread_id: data.threadId, @@ -165,15 +165,15 @@ export default class JanInferenceOpenAIExtension extends BaseExtension { status: MessageStatus.Pending, created: timestamp, updated: timestamp, - object: "thread.message", - }; + object: 'thread.message', + } if (data.type !== MessageRequestType.Summary) { - events.emit(MessageEvent.OnMessageResponse, message); + events.emit(MessageEvent.OnMessageResponse, message) } - instance.isCancelled = false; - instance.controller = new AbortController(); + instance.isCancelled = false + instance.controller = new AbortController() requestInference( data?.messages ?? [], @@ -182,7 +182,7 @@ export default class JanInferenceOpenAIExtension extends BaseExtension { ...JanInferenceOpenAIExtension._currentModel, parameters: data.model.parameters, }, - instance.controller, + instance.controller ).subscribe({ next: (content) => { const messageContent: ThreadContent = { @@ -191,33 +191,33 @@ export default class JanInferenceOpenAIExtension extends BaseExtension { value: content.trim(), annotations: [], }, - }; - message.content = [messageContent]; - events.emit(MessageEvent.OnMessageUpdate, message); + } + message.content = [messageContent] + events.emit(MessageEvent.OnMessageUpdate, message) }, complete: async () => { message.status = message.content.length ? MessageStatus.Ready - : MessageStatus.Error; - events.emit(MessageEvent.OnMessageUpdate, message); + : MessageStatus.Error + events.emit(MessageEvent.OnMessageUpdate, message) }, error: async (err) => { if (instance.isCancelled || message.content.length > 0) { - message.status = MessageStatus.Stopped; - events.emit(MessageEvent.OnMessageUpdate, message); - return; + message.status = MessageStatus.Stopped + events.emit(MessageEvent.OnMessageUpdate, message) + return } const messageContent: ThreadContent = { type: ContentType.Text, text: { - value: "Error occurred: " + err.message, + value: 'Error occurred: ' + err.message, annotations: [], }, - }; - message.content = [messageContent]; - message.status = MessageStatus.Error; - events.emit(MessageEvent.OnMessageUpdate, message); + } + message.content = [messageContent] + message.status = MessageStatus.Error + events.emit(MessageEvent.OnMessageUpdate, message) }, - }); + }) } } diff --git a/extensions/inference-openai-extension/tsconfig.json b/extensions/inference-openai-extension/tsconfig.json index 7bfdd90096..2477d58ce5 100644 --- a/extensions/inference-openai-extension/tsconfig.json +++ b/extensions/inference-openai-extension/tsconfig.json @@ -8,7 +8,7 @@ "forceConsistentCasingInFileNames": true, "strict": false, "skipLibCheck": true, - "rootDir": "./src", + "rootDir": "./src" }, - "include": ["./src"], + "include": ["./src"] } diff --git a/extensions/inference-openai-extension/webpack.config.js b/extensions/inference-openai-extension/webpack.config.js index 72b7d90c10..ee2e3b6243 100644 --- a/extensions/inference-openai-extension/webpack.config.js +++ b/extensions/inference-openai-extension/webpack.config.js @@ -1,16 +1,16 @@ -const path = require("path"); -const webpack = require("webpack"); -const packageJson = require("./package.json"); +const path = require('path') +const webpack = require('webpack') +const packageJson = require('./package.json') module.exports = { experiments: { outputModule: true }, - entry: "./src/index.ts", // Adjust the entry point to match your project's main file - mode: "production", + entry: './src/index.ts', // Adjust the entry point to match your project's main file + mode: 'production', module: { rules: [ { test: /\.tsx?$/, - use: "ts-loader", + use: 'ts-loader', exclude: /node_modules/, }, ], @@ -18,22 +18,22 @@ module.exports = { plugins: [ new webpack.DefinePlugin({ MODULE: JSON.stringify(`${packageJson.name}/${packageJson.module}`), - OPENAI_DOMAIN: JSON.stringify("openai.azure.com"), + OPENAI_DOMAIN: JSON.stringify('openai.azure.com'), }), ], output: { - filename: "index.js", // Adjust the output file name as needed - path: path.resolve(__dirname, "dist"), - library: { type: "module" }, // Specify ESM output format + filename: 'index.js', // Adjust the output file name as needed + path: path.resolve(__dirname, 'dist'), + library: { type: 'module' }, // Specify ESM output format }, resolve: { - extensions: [".ts", ".js"], + extensions: ['.ts', '.js'], fallback: { - path: require.resolve("path-browserify"), + path: require.resolve('path-browserify'), }, }, optimization: { minimize: false, }, // Add loaders and other configuration as needed for your project -}; +} diff --git a/extensions/inference-triton-trtllm-extension/src/@types/global.d.ts b/extensions/inference-triton-trtllm-extension/src/@types/global.d.ts index 6224b8e68c..c834feba01 100644 --- a/extensions/inference-triton-trtllm-extension/src/@types/global.d.ts +++ b/extensions/inference-triton-trtllm-extension/src/@types/global.d.ts @@ -1,5 +1,5 @@ -import { Model } from "@janhq/core"; +import { Model } from '@janhq/core' declare interface EngineSettings { - base_url?: string; + base_url?: string } diff --git a/extensions/inference-triton-trtllm-extension/src/helpers/sse.ts b/extensions/inference-triton-trtllm-extension/src/helpers/sse.ts index da20fa32d9..9aff612654 100644 --- a/extensions/inference-triton-trtllm-extension/src/helpers/sse.ts +++ b/extensions/inference-triton-trtllm-extension/src/helpers/sse.ts @@ -1,6 +1,6 @@ -import { Observable } from "rxjs"; -import { EngineSettings } from "../@types/global"; -import { Model } from "@janhq/core"; +import { Observable } from 'rxjs' +import { EngineSettings } from '../@types/global' +import { Model } from '@janhq/core' /** * Sends a request to the inference server to generate a response based on the recent messages. @@ -16,48 +16,48 @@ export function requestInference( controller?: AbortController ): Observable { return new Observable((subscriber) => { - const text_input = recentMessages.map((message) => message.text).join("\n"); + const text_input = recentMessages.map((message) => message.text).join('\n') const requestBody = JSON.stringify({ text_input: text_input, max_tokens: 4096, temperature: 0, - bad_words: "", - stop_words: "[DONE]", - stream: true - }); + bad_words: '', + stop_words: '[DONE]', + stream: true, + }) fetch(`${engine.base_url}/v2/models/ensemble/generate_stream`, { - method: "POST", + method: 'POST', headers: { - "Content-Type": "application/json", - Accept: "text/event-stream", - "Access-Control-Allow-Origin": "*", + 'Content-Type': 'application/json', + 'Accept': 'text/event-stream', + 'Access-Control-Allow-Origin': '*', }, body: requestBody, signal: controller?.signal, }) .then(async (response) => { - const stream = response.body; - const decoder = new TextDecoder("utf-8"); - const reader = stream?.getReader(); - let content = ""; + const stream = response.body + const decoder = new TextDecoder('utf-8') + const reader = stream?.getReader() + let content = '' while (true && reader) { - const { done, value } = await reader.read(); + const { done, value } = await reader.read() if (done) { - break; + break } - const text = decoder.decode(value); - const lines = text.trim().split("\n"); + const text = decoder.decode(value) + const lines = text.trim().split('\n') for (const line of lines) { - if (line.startsWith("data: ") && !line.includes("data: [DONE]")) { - const data = JSON.parse(line.replace("data: ", "")); - content += data.choices[0]?.delta?.content ?? ""; - subscriber.next(content); + if (line.startsWith('data: ') && !line.includes('data: [DONE]')) { + const data = JSON.parse(line.replace('data: ', '')) + content += data.choices[0]?.delta?.content ?? '' + subscriber.next(content) } } } - subscriber.complete(); + subscriber.complete() }) - .catch((err) => subscriber.error(err)); - }); + .catch((err) => subscriber.error(err)) + }) } diff --git a/extensions/inference-triton-trtllm-extension/src/index.ts b/extensions/inference-triton-trtllm-extension/src/index.ts index 11ddf78933..f009a81e03 100644 --- a/extensions/inference-triton-trtllm-extension/src/index.ts +++ b/extensions/inference-triton-trtllm-extension/src/index.ts @@ -20,51 +20,49 @@ import { BaseExtension, MessageEvent, ModelEvent, -} from "@janhq/core"; -import { requestInference } from "./helpers/sse"; -import { ulid } from "ulid"; -import { join } from "path"; -import { EngineSettings } from "./@types/global"; +} from '@janhq/core' +import { requestInference } from './helpers/sse' +import { ulid } from 'ulid' +import { join } from 'path' +import { EngineSettings } from './@types/global' /** * A class that implements the InferenceExtension interface from the @janhq/core package. * The class provides methods for initializing and stopping a model, and for making inference requests. * It also subscribes to events emitted by the @janhq/core package and handles new message requests. */ -export default class JanInferenceTritonTrtLLMExtension - extends BaseExtension -{ - private static readonly _homeDir = "file://engines"; - private static readonly _engineMetadataFileName = "triton_trtllm.json"; +export default class JanInferenceTritonTrtLLMExtension extends BaseExtension { + private static readonly _homeDir = 'file://engines' + private static readonly _engineMetadataFileName = 'triton_trtllm.json' - static _currentModel: Model; + static _currentModel: Model static _engineSettings: EngineSettings = { - base_url: "", - }; + base_url: '', + } - controller = new AbortController(); - isCancelled = false; + controller = new AbortController() + isCancelled = false /** * Subscribes to events emitted by the @janhq/core package. */ async onLoad() { if (!(await fs.existsSync(JanInferenceTritonTrtLLMExtension._homeDir))) - JanInferenceTritonTrtLLMExtension.writeDefaultEngineSettings(); + JanInferenceTritonTrtLLMExtension.writeDefaultEngineSettings() // Events subscription events.on(MessageEvent.OnMessageSent, (data) => JanInferenceTritonTrtLLMExtension.handleMessageRequest(data, this) - ); + ) events.on(ModelEvent.OnModelInit, (model: Model) => { - JanInferenceTritonTrtLLMExtension.handleModelInit(model); - }); + JanInferenceTritonTrtLLMExtension.handleModelInit(model) + }) events.on(ModelEvent.OnModelStop, (model: Model) => { - JanInferenceTritonTrtLLMExtension.handleModelStop(model); - }); + JanInferenceTritonTrtLLMExtension.handleModelStop(model) + }) } /** @@ -81,7 +79,7 @@ export default class JanInferenceTritonTrtLLMExtension modelId: string, settings?: ModelSettingParams ): Promise { - return; + return } static async writeDefaultEngineSettings() { @@ -89,11 +87,11 @@ export default class JanInferenceTritonTrtLLMExtension const engine_json = join( JanInferenceTritonTrtLLMExtension._homeDir, JanInferenceTritonTrtLLMExtension._engineMetadataFileName - ); + ) if (await fs.existsSync(engine_json)) { - const engine = await fs.readFileSync(engine_json, "utf-8"); + const engine = await fs.readFileSync(engine_json, 'utf-8') JanInferenceTritonTrtLLMExtension._engineSettings = - typeof engine === "object" ? engine : JSON.parse(engine); + typeof engine === 'object' ? engine : JSON.parse(engine) } else { await fs.writeFileSync( engine_json, @@ -102,10 +100,10 @@ export default class JanInferenceTritonTrtLLMExtension null, 2 ) - ); + ) } } catch (err) { - console.error(err); + console.error(err) } } /** @@ -119,26 +117,26 @@ export default class JanInferenceTritonTrtLLMExtension * @returns {Promise} A promise that resolves when the streaming is stopped. */ async stopInference(): Promise { - this.isCancelled = true; - this.controller?.abort(); + this.isCancelled = true + this.controller?.abort() } private static async handleModelInit(model: Model) { - if (model.engine !== "triton_trtllm") { - return; + if (model.engine !== 'triton_trtllm') { + return } else { - JanInferenceTritonTrtLLMExtension._currentModel = model; - JanInferenceTritonTrtLLMExtension.writeDefaultEngineSettings(); + JanInferenceTritonTrtLLMExtension._currentModel = model + JanInferenceTritonTrtLLMExtension.writeDefaultEngineSettings() // Todo: Check model list with API key - events.emit(ModelEvent.OnModelReady, model); + events.emit(ModelEvent.OnModelReady, model) } } private static async handleModelStop(model: Model) { - if (model.engine !== "triton_trtllm") { - return; + if (model.engine !== 'triton_trtllm') { + return } - events.emit(ModelEvent.OnModelStopped, model); + events.emit(ModelEvent.OnModelStopped, model) } /** @@ -151,11 +149,11 @@ export default class JanInferenceTritonTrtLLMExtension data: MessageRequest, instance: JanInferenceTritonTrtLLMExtension ) { - if (data.model.engine !== "triton_trtllm") { - return; + if (data.model.engine !== 'triton_trtllm') { + return } - const timestamp = Date.now(); + const timestamp = Date.now() const message: ThreadMessage = { id: ulid(), thread_id: data.threadId, @@ -165,12 +163,12 @@ export default class JanInferenceTritonTrtLLMExtension status: MessageStatus.Pending, created: timestamp, updated: timestamp, - object: "thread.message", - }; - events.emit(MessageEvent.OnMessageResponse, message); + object: 'thread.message', + } + events.emit(MessageEvent.OnMessageResponse, message) - instance.isCancelled = false; - instance.controller = new AbortController(); + instance.isCancelled = false + instance.controller = new AbortController() requestInference( data?.messages ?? [], @@ -188,33 +186,33 @@ export default class JanInferenceTritonTrtLLMExtension value: content.trim(), annotations: [], }, - }; - message.content = [messageContent]; - events.emit(MessageEvent.OnMessageUpdate, message); + } + message.content = [messageContent] + events.emit(MessageEvent.OnMessageUpdate, message) }, complete: async () => { message.status = message.content.length ? MessageStatus.Ready - : MessageStatus.Error; - events.emit(MessageEvent.OnMessageUpdate, message); + : MessageStatus.Error + events.emit(MessageEvent.OnMessageUpdate, message) }, error: async (err) => { if (instance.isCancelled || message.content.length) { - message.status = MessageStatus.Error; - events.emit(MessageEvent.OnMessageUpdate, message); - return; + message.status = MessageStatus.Error + events.emit(MessageEvent.OnMessageUpdate, message) + return } const messageContent: ThreadContent = { type: ContentType.Text, text: { - value: "Error occurred: " + err.message, + value: 'Error occurred: ' + err.message, annotations: [], }, - }; - message.content = [messageContent]; - message.status = MessageStatus.Ready; - events.emit(MessageEvent.OnMessageUpdate, message); + } + message.content = [messageContent] + message.status = MessageStatus.Ready + events.emit(MessageEvent.OnMessageUpdate, message) }, - }); + }) } } diff --git a/extensions/inference-triton-trtllm-extension/tsconfig.json b/extensions/inference-triton-trtllm-extension/tsconfig.json index 7bfdd90096..2477d58ce5 100644 --- a/extensions/inference-triton-trtllm-extension/tsconfig.json +++ b/extensions/inference-triton-trtllm-extension/tsconfig.json @@ -8,7 +8,7 @@ "forceConsistentCasingInFileNames": true, "strict": false, "skipLibCheck": true, - "rootDir": "./src", + "rootDir": "./src" }, - "include": ["./src"], + "include": ["./src"] } diff --git a/extensions/inference-triton-trtllm-extension/webpack.config.js b/extensions/inference-triton-trtllm-extension/webpack.config.js index 57a0adb0a2..e83370a1ac 100644 --- a/extensions/inference-triton-trtllm-extension/webpack.config.js +++ b/extensions/inference-triton-trtllm-extension/webpack.config.js @@ -1,16 +1,16 @@ -const path = require("path"); -const webpack = require("webpack"); -const packageJson = require("./package.json"); +const path = require('path') +const webpack = require('webpack') +const packageJson = require('./package.json') module.exports = { experiments: { outputModule: true }, - entry: "./src/index.ts", // Adjust the entry point to match your project's main file - mode: "production", + entry: './src/index.ts', // Adjust the entry point to match your project's main file + mode: 'production', module: { rules: [ { test: /\.tsx?$/, - use: "ts-loader", + use: 'ts-loader', exclude: /node_modules/, }, ], @@ -21,18 +21,18 @@ module.exports = { }), ], output: { - filename: "index.js", // Adjust the output file name as needed - path: path.resolve(__dirname, "dist"), - library: { type: "module" }, // Specify ESM output format + filename: 'index.js', // Adjust the output file name as needed + path: path.resolve(__dirname, 'dist'), + library: { type: 'module' }, // Specify ESM output format }, resolve: { - extensions: [".ts", ".js"], + extensions: ['.ts', '.js'], fallback: { - path: require.resolve("path-browserify"), + path: require.resolve('path-browserify'), }, }, optimization: { minimize: false, }, // Add loaders and other configuration as needed for your project -}; +} diff --git a/extensions/model-extension/tsconfig.json b/extensions/model-extension/tsconfig.json index c175d94370..addd8e1274 100644 --- a/extensions/model-extension/tsconfig.json +++ b/extensions/model-extension/tsconfig.json @@ -8,7 +8,7 @@ "forceConsistentCasingInFileNames": true, "strict": false, "skipLibCheck": true, - "rootDir": "./src", + "rootDir": "./src" }, - "include": ["./src"], + "include": ["./src"] } diff --git a/extensions/model-extension/webpack.config.js b/extensions/model-extension/webpack.config.js index c67bf8dc05..347719f913 100644 --- a/extensions/model-extension/webpack.config.js +++ b/extensions/model-extension/webpack.config.js @@ -19,7 +19,7 @@ module.exports = { new webpack.DefinePlugin({ EXTENSION_NAME: JSON.stringify(packageJson.name), MODULE_PATH: JSON.stringify(`${packageJson.name}/${packageJson.module}`), - VERSION: JSON.stringify(packageJson.version) + VERSION: JSON.stringify(packageJson.version), }), ], output: { diff --git a/extensions/monitoring-extension/src/@types/global.d.ts b/extensions/monitoring-extension/src/@types/global.d.ts index 3b45ccc5ad..8106353cfc 100644 --- a/extensions/monitoring-extension/src/@types/global.d.ts +++ b/extensions/monitoring-extension/src/@types/global.d.ts @@ -1 +1 @@ -declare const MODULE: string; +declare const MODULE: string diff --git a/extensions/monitoring-extension/src/index.ts b/extensions/monitoring-extension/src/index.ts index 9297a770f5..ce9b2fc148 100644 --- a/extensions/monitoring-extension/src/index.ts +++ b/extensions/monitoring-extension/src/index.ts @@ -1,4 +1,4 @@ -import { MonitoringExtension, executeOnMain } from "@janhq/core"; +import { MonitoringExtension, executeOnMain } from '@janhq/core' /** * JanMonitoringExtension is a extension that provides system monitoring functionality. @@ -20,7 +20,7 @@ export default class JanMonitoringExtension extends MonitoringExtension { * @returns A Promise that resolves to an object containing information about the system resources. */ getResourcesInfo(): Promise { - return executeOnMain(MODULE, "getResourcesInfo"); + return executeOnMain(MODULE, 'getResourcesInfo') } /** @@ -28,6 +28,6 @@ export default class JanMonitoringExtension extends MonitoringExtension { * @returns A Promise that resolves to an object containing information about the current system load. */ getCurrentLoad(): Promise { - return executeOnMain(MODULE, "getCurrentLoad"); + return executeOnMain(MODULE, 'getCurrentLoad') } } diff --git a/extensions/monitoring-extension/src/module.ts b/extensions/monitoring-extension/src/module.ts index 2c1b143433..ea7319b474 100644 --- a/extensions/monitoring-extension/src/module.ts +++ b/extensions/monitoring-extension/src/module.ts @@ -1,73 +1,92 @@ -const nodeOsUtils = require("node-os-utils"); -const getJanDataFolderPath = require("@janhq/core/node").getJanDataFolderPath; -const path = require("path"); -const { readFileSync } = require("fs"); -const exec = require("child_process").exec; +const nodeOsUtils = require('node-os-utils') +const getJanDataFolderPath = require('@janhq/core/node').getJanDataFolderPath +const path = require('path') +const { readFileSync } = require('fs') +const exec = require('child_process').exec const NVIDIA_INFO_FILE = path.join( getJanDataFolderPath(), - "settings", - "settings.json" -); + 'settings', + 'settings.json' +) const getResourcesInfo = () => new Promise((resolve) => { nodeOsUtils.mem.used().then((ramUsedInfo) => { - const totalMemory = ramUsedInfo.totalMemMb * 1024 * 1024; - const usedMemory = ramUsedInfo.usedMemMb * 1024 * 1024; + const totalMemory = ramUsedInfo.totalMemMb * 1024 * 1024 + const usedMemory = ramUsedInfo.usedMemMb * 1024 * 1024 const response = { mem: { totalMemory, usedMemory, }, - }; - resolve(response); - }); - }); + } + resolve(response) + }) + }) const getCurrentLoad = () => new Promise((resolve, reject) => { nodeOsUtils.cpu.usage().then((cpuPercentage) => { let data = { - run_mode: "cpu", + run_mode: 'cpu', gpus_in_use: [], - }; - if (process.platform !== "darwin") { - data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8")); } - if (data.run_mode === "gpu" && data.gpus_in_use.length > 0) { - const gpuIds = data["gpus_in_use"].join(","); - if (gpuIds !== "") { + if (process.platform !== 'darwin') { + data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, 'utf-8')) + } + if (data.run_mode === 'gpu' && data.gpus_in_use.length > 0) { + const gpuIds = data['gpus_in_use'].join(',') + if (gpuIds !== '') { exec( `nvidia-smi --query-gpu=index,name,temperature.gpu,utilization.gpu,memory.total,memory.free,utilization.memory --format=csv,noheader,nounits --id=${gpuIds}`, (error, stdout, stderr) => { if (error) { - console.error(`exec error: ${error}`); - reject(error); - return; + console.error(`exec error: ${error}`) + reject(error) + return } - const gpuInfo = stdout.trim().split("\n").map((line) => { - const [id, name, temperature, utilization, memoryTotal, memoryFree, memoryUtilization] = line.split(", ").map(item => item.replace(/\r/g, "")); - return { id, name, temperature, utilization, memoryTotal, memoryFree, memoryUtilization }; - }); + const gpuInfo = stdout + .trim() + .split('\n') + .map((line) => { + const [ + id, + name, + temperature, + utilization, + memoryTotal, + memoryFree, + memoryUtilization, + ] = line.split(', ').map((item) => item.replace(/\r/g, '')) + return { + id, + name, + temperature, + utilization, + memoryTotal, + memoryFree, + memoryUtilization, + } + }) resolve({ cpu: { usage: cpuPercentage }, - gpu: gpuInfo - }); + gpu: gpuInfo, + }) } - ); + ) } else { // Handle the case where gpuIds is empty - resolve({ cpu: { usage: cpuPercentage }, gpu: [] }); + resolve({ cpu: { usage: cpuPercentage }, gpu: [] }) } } else { // Handle the case where run_mode is not 'gpu' or no GPUs are in use - resolve({ cpu: { usage: cpuPercentage }, gpu: [] }); + resolve({ cpu: { usage: cpuPercentage }, gpu: [] }) } - }); - }); + }) + }) module.exports = { getResourcesInfo, getCurrentLoad, -}; \ No newline at end of file +} diff --git a/extensions/monitoring-extension/webpack.config.js b/extensions/monitoring-extension/webpack.config.js index f54059222f..c8c3a34f79 100644 --- a/extensions/monitoring-extension/webpack.config.js +++ b/extensions/monitoring-extension/webpack.config.js @@ -1,24 +1,24 @@ -const path = require("path"); -const webpack = require("webpack"); -const packageJson = require("./package.json"); +const path = require('path') +const webpack = require('webpack') +const packageJson = require('./package.json') module.exports = { experiments: { outputModule: true }, - entry: "./src/index.ts", // Adjust the entry point to match your project's main file - mode: "production", + entry: './src/index.ts', // Adjust the entry point to match your project's main file + mode: 'production', module: { rules: [ { test: /\.tsx?$/, - use: "ts-loader", + use: 'ts-loader', exclude: /node_modules/, }, ], }, output: { - filename: "index.js", // Adjust the output file name as needed - path: path.resolve(__dirname, "dist"), - library: { type: "module" }, // Specify ESM output format + filename: 'index.js', // Adjust the output file name as needed + path: path.resolve(__dirname, 'dist'), + library: { type: 'module' }, // Specify ESM output format }, plugins: [ new webpack.DefinePlugin({ @@ -26,10 +26,10 @@ module.exports = { }), ], resolve: { - extensions: [".ts", ".js"], + extensions: ['.ts', '.js'], }, optimization: { minimize: false, }, // Add loaders and other configuration as needed for your project -}; +} diff --git a/server/helpers/setup.ts b/server/helpers/setup.ts index 51d8eebe5e..e6fab7a25e 100644 --- a/server/helpers/setup.ts +++ b/server/helpers/setup.ts @@ -1,47 +1,47 @@ -import { join, extname } from "path"; -import { existsSync, readdirSync, writeFileSync, mkdirSync } from "fs"; -import { init, installExtensions } from "@janhq/core/node"; +import { join, extname } from 'path' +import { existsSync, readdirSync, writeFileSync, mkdirSync } from 'fs' +import { init, installExtensions } from '@janhq/core/node' export async function setup() { /** * Setup Jan Data Directory */ - const appDir = process.env.JAN_DATA_DIRECTORY ?? join(__dirname, "..", "jan"); + const appDir = process.env.JAN_DATA_DIRECTORY ?? join(__dirname, '..', 'jan') - console.debug(`Create app data directory at ${appDir}...`); - if (!existsSync(appDir)) mkdirSync(appDir); + console.debug(`Create app data directory at ${appDir}...`) + if (!existsSync(appDir)) mkdirSync(appDir) //@ts-ignore global.core = { // Define appPath function for app to retrieve app path globaly appPath: () => appDir, - }; + } init({ - extensionsPath: join(appDir, "extensions"), - }); + extensionsPath: join(appDir, 'extensions'), + }) /** * Write app configurations. See #1619 */ - console.debug("Writing config file..."); + console.debug('Writing config file...') writeFileSync( - join(appDir, "settings.json"), + join(appDir, 'settings.json'), JSON.stringify({ data_folder: appDir, }), - "utf-8" - ); + 'utf-8' + ) /** * Install extensions */ - console.debug("Installing extensions..."); + console.debug('Installing extensions...') - const baseExtensionPath = join(__dirname, "../../..", "pre-install"); + const baseExtensionPath = join(__dirname, '../../..', 'pre-install') const extensions = readdirSync(baseExtensionPath) - .filter((file) => extname(file) === ".tgz") - .map((file) => join(baseExtensionPath, file)); + .filter((file) => extname(file) === '.tgz') + .map((file) => join(baseExtensionPath, file)) - await installExtensions(extensions); - console.debug("Extensions installed"); + await installExtensions(extensions) + console.debug('Extensions installed') } diff --git a/server/index.ts b/server/index.ts index 91349a81f4..dc518781fc 100644 --- a/server/index.ts +++ b/server/index.ts @@ -1,26 +1,26 @@ -import fastify from "fastify"; -import dotenv from "dotenv"; +import fastify from 'fastify' +import dotenv from 'dotenv' import { getServerLogPath, v1Router, logServer, getJanExtensionsPath, -} from "@janhq/core/node"; -import { join } from "path"; +} from '@janhq/core/node' +import { join } from 'path' // Load environment variables -dotenv.config(); +dotenv.config() // Define default settings -const JAN_API_HOST = process.env.JAN_API_HOST || "127.0.0.1"; -const JAN_API_PORT = Number.parseInt(process.env.JAN_API_PORT || "1337"); +const JAN_API_HOST = process.env.JAN_API_HOST || '127.0.0.1' +const JAN_API_PORT = Number.parseInt(process.env.JAN_API_PORT || '1337') // Initialize server settings -let server: any | undefined = undefined; -let hostSetting: string = JAN_API_HOST; -let portSetting: number = JAN_API_PORT; -let corsEnabled: boolean = true; -let isVerbose: boolean = true; +let server: any | undefined = undefined +let hostSetting: string = JAN_API_HOST +let portSetting: number = JAN_API_PORT +let corsEnabled: boolean = true +let isVerbose: boolean = true /** * Server configurations @@ -32,13 +32,13 @@ let isVerbose: boolean = true; * @param baseDir - Base directory for the OpenAPI schema file */ export interface ServerConfig { - host?: string; - port?: number; - isCorsEnabled?: boolean; - isVerboseEnabled?: boolean; - schemaPath?: string; - baseDir?: string; - storageAdataper?: any; + host?: string + port?: number + isCorsEnabled?: boolean + isVerboseEnabled?: boolean + schemaPath?: string + baseDir?: string + storageAdataper?: any } /** @@ -47,69 +47,69 @@ export interface ServerConfig { */ export const startServer = async (configs?: ServerConfig) => { // Update server settings - isVerbose = configs?.isVerboseEnabled ?? true; - hostSetting = configs?.host ?? JAN_API_HOST; - portSetting = configs?.port ?? JAN_API_PORT; - corsEnabled = configs?.isCorsEnabled ?? true; - const serverLogPath = getServerLogPath(); + isVerbose = configs?.isVerboseEnabled ?? true + hostSetting = configs?.host ?? JAN_API_HOST + portSetting = configs?.port ?? JAN_API_PORT + corsEnabled = configs?.isCorsEnabled ?? true + const serverLogPath = getServerLogPath() // Start the server try { // Log server start - if (isVerbose) logServer(`Debug: Starting JAN API server...`); + if (isVerbose) logServer(`Debug: Starting JAN API server...`) // Initialize Fastify server with logging server = fastify({ logger: { - level: "info", + level: 'info', file: serverLogPath, }, - }); + }) // Register CORS if enabled - if (corsEnabled) await server.register(require("@fastify/cors"), {}); + if (corsEnabled) await server.register(require('@fastify/cors'), {}) // Register Swagger for API documentation - await server.register(require("@fastify/swagger"), { - mode: "static", + await server.register(require('@fastify/swagger'), { + mode: 'static', specification: { - path: configs?.schemaPath ?? "./../docs/openapi/jan.yaml", - baseDir: configs?.baseDir ?? "./../docs/openapi", + path: configs?.schemaPath ?? './../docs/openapi/jan.yaml', + baseDir: configs?.baseDir ?? './../docs/openapi', }, - }); + }) // Register Swagger UI - await server.register(require("@fastify/swagger-ui"), { - routePrefix: "/", - baseDir: configs?.baseDir ?? join(__dirname, "../..", "./docs/openapi"), + await server.register(require('@fastify/swagger-ui'), { + routePrefix: '/', + baseDir: configs?.baseDir ?? join(__dirname, '../..', './docs/openapi'), uiConfig: { - docExpansion: "full", + docExpansion: 'full', deepLinking: false, }, staticCSP: false, transformSpecificationClone: true, - }); + }) // Register static file serving for extensions // TODO: Watch extension files changes and reload await server.register( (childContext: any, _: any, done: any) => { - childContext.register(require("@fastify/static"), { + childContext.register(require('@fastify/static'), { root: getJanExtensionsPath(), wildcard: false, - }); + }) - done(); + done() }, - { prefix: "extensions" } - ); + { prefix: 'extensions' } + ) // Register proxy middleware if (configs?.storageAdataper) - server.addHook("preHandler", configs.storageAdataper); + server.addHook('preHandler', configs.storageAdataper) // Register API routes - await server.register(v1Router, { prefix: "/v1" }); + await server.register(v1Router, { prefix: '/v1' }) // Start listening for requests await server .listen({ @@ -121,13 +121,13 @@ export const startServer = async (configs?: ServerConfig) => { if (isVerbose) logServer( `Debug: JAN API listening at: http://${hostSetting}:${portSetting}` - ); - }); + ) + }) } catch (e) { // Log any errors - if (isVerbose) logServer(`Error: ${e}`); + if (isVerbose) logServer(`Error: ${e}`) } -}; +} /** * Function to stop the server @@ -135,11 +135,11 @@ export const startServer = async (configs?: ServerConfig) => { export const stopServer = async () => { try { // Log server stop - if (isVerbose) logServer(`Debug: Server stopped`); + if (isVerbose) logServer(`Debug: Server stopped`) // Stop the server - await server.close(); + await server.close() } catch (e) { // Log any errors - if (isVerbose) logServer(`Error: ${e}`); + if (isVerbose) logServer(`Error: ${e}`) } -}; +} diff --git a/server/main.ts b/server/main.ts index 3be397e6ff..71fb111062 100644 --- a/server/main.ts +++ b/server/main.ts @@ -1,7 +1,7 @@ -import { s3 } from "./middleware/s3"; -import { setup } from "./helpers/setup"; -import { startServer as start } from "./index"; +import { s3 } from './middleware/s3' +import { setup } from './helpers/setup' +import { startServer as start } from './index' /** * Setup extensions and start the server */ -setup().then(() => start({ storageAdataper: s3 })); +setup().then(() => start({ storageAdataper: s3 })) diff --git a/server/middleware/s3.ts b/server/middleware/s3.ts index 6248652229..28971a42b4 100644 --- a/server/middleware/s3.ts +++ b/server/middleware/s3.ts @@ -1,4 +1,4 @@ -import { join } from "path"; +import { join } from 'path' // Middleware to intercept requests and proxy if certain conditions are met const config = { @@ -8,63 +8,63 @@ const config = { accessKeyId: process.env.AWS_ACCESS_KEY_ID, secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, }, -}; +} -const S3_BUCKET_NAME = process.env.S3_BUCKET_NAME; +const S3_BUCKET_NAME = process.env.S3_BUCKET_NAME -const fs = require("@cyclic.sh/s3fs")(S3_BUCKET_NAME, config); -const PROXY_PREFIX = "/v1/fs"; -const PROXY_ROUTES = ["/threads", "/messages"]; +const fs = require('@cyclic.sh/s3fs')(S3_BUCKET_NAME, config) +const PROXY_PREFIX = '/v1/fs' +const PROXY_ROUTES = ['/threads', '/messages'] export const s3 = (req: any, reply: any, done: any) => { // Proxy FS requests to S3 using S3FS if (req.url.startsWith(PROXY_PREFIX)) { - const route = req.url.split("/").pop(); - const args = parseRequestArgs(req); + const route = req.url.split('/').pop() + const args = parseRequestArgs(req) // Proxy matched requests to the s3fs module if (args.length && PROXY_ROUTES.some((route) => args[0].includes(route))) { try { // Handle customized route // S3FS does not handle appendFileSync - if (route === "appendFileSync") { - let result = handAppendFileSync(args); + if (route === 'appendFileSync') { + let result = handAppendFileSync(args) - reply.status(200).send(result); - return; + reply.status(200).send(result) + return } // Reroute the other requests to the s3fs module - const result = fs[route](...args); - reply.status(200).send(result); - return; + const result = fs[route](...args) + reply.status(200).send(result) + return } catch (ex) { - console.log(ex); + console.log(ex) } } } // Let other requests go through - done(); -}; + done() +} const parseRequestArgs = (req: Request) => { const { getJanDataFolderPath, normalizeFilePath, - } = require("@janhq/core/node"); + } = require('@janhq/core/node') return JSON.parse(req.body as any).map((arg: any) => - typeof arg === "string" && + typeof arg === 'string' && (arg.startsWith(`file:/`) || arg.startsWith(`file:\\`)) ? join(getJanDataFolderPath(), normalizeFilePath(arg)) : arg - ); -}; + ) +} const handAppendFileSync = (args: any[]) => { if (fs.existsSync(args[0])) { - const data = fs.readFileSync(args[0], "utf-8"); - return fs.writeFileSync(args[0], data + args[1]); + const data = fs.readFileSync(args[0], 'utf-8') + return fs.writeFileSync(args[0], data + args[1]) } else { - return fs.writeFileSync(args[0], args[1]); + return fs.writeFileSync(args[0], args[1]) } -}; +} diff --git a/uikit/types/declaration.d.ts b/uikit/types/declaration.d.ts index 85b1a7136f..f8e975fa54 100644 --- a/uikit/types/declaration.d.ts +++ b/uikit/types/declaration.d.ts @@ -1,4 +1,4 @@ declare module '*.scss' { - const content: Record; - export default content; -} \ No newline at end of file + const content: Record + export default content +} diff --git a/web/app/error.tsx b/web/app/error.tsx index f8993b8986..25b24b9ef5 100644 --- a/web/app/error.tsx +++ b/web/app/error.tsx @@ -4,7 +4,6 @@ import { useEffect, useState } from 'react' export default function Error({ error, - reset, }: { error: Error & { digest?: string } reset: () => void diff --git a/web/containers/Layout/BottomBar/index.tsx b/web/containers/Layout/BottomBar/index.tsx index 0c2df80fed..d34173ecbe 100644 --- a/web/containers/Layout/BottomBar/index.tsx +++ b/web/containers/Layout/BottomBar/index.tsx @@ -56,13 +56,6 @@ const BottomBar = () => { const setShowSelectModelModal = useSetAtom(showSelectModelModalAtom) const [serverEnabled] = useAtom(serverEnabledAtom) - const calculateGpuMemoryUsage = (gpu: Record) => { - const total = parseInt(gpu.memoryTotal) - const free = parseInt(gpu.memoryFree) - if (!total || !free) return 0 - return Math.round(((total - free) / total) * 100) - } - const calculateUtilization = () => { let sum = 0 const util = gpus.map((x) => { diff --git a/web/screens/Chat/index.tsx b/web/screens/Chat/index.tsx index 29f440cb69..44e2bd5005 100644 --- a/web/screens/Chat/index.tsx +++ b/web/screens/Chat/index.tsx @@ -13,7 +13,7 @@ import GenerateResponse from '@/containers/Loader/GenerateResponse' import ModelReload from '@/containers/Loader/ModelReload' import ModelStart from '@/containers/Loader/ModelStart' -import { currentPromptAtom, fileUploadAtom } from '@/containers/Providers/Jotai' +import { fileUploadAtom } from '@/containers/Providers/Jotai' import { showLeftSideBarAtom } from '@/containers/Providers/KeyListener' import { snackbar } from '@/containers/Toast' @@ -54,7 +54,6 @@ const renderError = (code: string) => { } const ChatScreen: React.FC = () => { - const setCurrentPrompt = useSetAtom(currentPromptAtom) const activeThread = useAtomValue(activeThreadAtom) const showLeftSideBar = useAtomValue(showLeftSideBarAtom) const engineParamsUpdate = useAtomValue(engineParamsUpdateAtom)