-
Notifications
You must be signed in to change notification settings - Fork 0
/
module.ts
118 lines (104 loc) · 3.36 KB
/
module.ts
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
const path = require("path");
const { app } = require("electron");
const { spawn } = require("child_process");
const fs = require("fs");
const tcpPortUsed = require("tcp-port-used");
const { killPortProcess } = require("kill-port-process");
const PORT = 3928;
let subprocess = null;
const initModel = (fileName) => {
return (
new Promise<void>(async (resolve, reject) => {
if (!fileName) {
reject("Model not found, please download again.");
}
resolve(fileName);
})
// Spawn Nitro subprocess to load model
.then(() => {
return tcpPortUsed.check(PORT, "127.0.0.1").then((inUse) => {
if (!inUse) {
let binaryFolder = path.join(__dirname, "nitro"); // Current directory by default
let binaryName;
if (process.platform === "win32") {
// Todo: Need to check for CUDA support to switch between CUDA and non-CUDA binaries
binaryName = "nitro_start_windows.bat";
} else if (process.platform === "darwin") {
// Mac OS platform
binaryName = process.arch === "arm64" ? "nitro_mac_arm64" : "nitro_mac_intel";
} else {
// Linux
// Todo: Need to check for CUDA support to switch between CUDA and non-CUDA binaries
binaryName = "nitro_start_linux.sh"; // For other platforms
}
const binaryPath = path.join(binaryFolder, binaryName);
// Execute the binary
subprocess = spawn(binaryPath,["0.0.0.0", PORT], { cwd: binaryFolder });
// Handle subprocess output
subprocess.stdout.on("data", (data) => {
console.log(`stdout: ${data}`);
});
subprocess.stderr.on("data", (data) => {
console.error(`stderr: ${data}`);
});
subprocess.on("close", (code) => {
console.log(`child process exited with code ${code}`);
subprocess = null;
});
}
});
})
.then(() => tcpPortUsed.waitUntilUsed(PORT, 300, 30000))
.then(() => {
const llama_model_path = path.join(appPath(), fileName);
const config = {
llama_model_path,
ctx_len: 2048,
ngl: 100,
embedding: true, // Always enable embedding mode on
};
// Load model config
return fetch(`http://127.0.0.1:${PORT}/inferences/llamacpp/loadmodel`, {
method: "POST",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify(config),
});
})
.then((res) => {
if (res.ok) {
return {};
}
throw new Error("Nitro: Model failed to load.");
})
.catch((err) => {
return { error: err };
})
);
};
function dispose() {
killSubprocess();
// clean other registered resources here
}
function killSubprocess() {
if (subprocess) {
subprocess.kill();
subprocess = null;
console.log("Subprocess terminated.");
} else {
killPortProcess(PORT);
console.error("No subprocess is currently running.");
}
}
function appPath() {
if (app) {
return app.getPath("userData");
}
return process.env.APPDATA || (process.platform == 'darwin' ? process.env.HOME + '/Library/Preferences' : process.env.HOME + "/.local/share");
}
module.exports = {
initModel,
killSubprocess,
dispose,
};