Skip to content

Commit

Permalink
chore: update convo summary (janhq#378)
Browse files Browse the repository at this point in the history 
* chore: update convo summary

* chore: conversation summary debouncing & retry attempt

* chore: convention
  • Loading branch information
@louis-jan
louis-jan authored Oct 18, 2023
1 parent ed4ea2e commit c623a9b
Show file tree
Hide file tree
Showing 4 changed files with 135 additions and 48 deletions.
135 changes: 91 additions & 44 deletions plugins/inference-plugin/index.ts
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { EventName, InferenceService, NewMessageRequest, PluginService, core, events, store } from "@janhq/core";
import { Observable } from "rxjs";

const inferenceUrl = "http://localhost:3928/llama/chat_completion";

Expand All @@ -8,11 +9,57 @@ const stopModel = () => {
core.invokePluginFunc(MODULE_PATH, "killSubprocess");
};

async function handleMessageRequest(data: NewMessageRequest) {
function requestInference(recentMessages: any[]): Observable<string> {
return new Observable((subscriber) => {
fetch(inferenceUrl, {
method: "POST",
headers: {
"Content-Type": "application/json",
Accept: "text/event-stream",
"Access-Control-Allow-Origin": "*",
},
body: JSON.stringify({
messages: recentMessages,
stream: true,
model: "gpt-3.5-turbo",
max_tokens: 500,
}),
})
.then(async (response) => {
const stream = response.body;
const decoder = new TextDecoder("utf-8");
const reader = stream?.getReader();
let content = "";

while (true && reader) {
const { done, value } = await reader.read();
if (done) {
console.log("SSE stream closed");
break;
}
const text = decoder.decode(value);
const lines = text.trim().split("\n");
for (const line of lines) {
if (line.startsWith("data: ") && !line.includes("data: [DONE]")) {
const data = JSON.parse(line.replace("data: ", ""));
content += data.choices[0]?.delta?.content ?? "";
if (content.startsWith("assistant: ")) {
content = content.replace("assistant: ", "");
}
subscriber.next(content);
}
}
}
subscriber.complete();
})
.catch(subscriber.error);
});
}

async function retrieveLastTenMessages(conversationId: string) {
// TODO: Common collections should be able to access via core functions instead of store
const messageHistory =
(await store.findMany("messages", { conversationId: data.conversationId }, [{ createdAt: "asc" }])) ?? [];
const recentMessages = messageHistory
const messageHistory = (await store.findMany("messages", { conversationId }, [{ createdAt: "asc" }])) ?? [];
return messageHistory
.filter((e) => e.message !== "" && (e.user === "user" || e.user === "assistant"))
.slice(-10)
.map((message) => {
Expand All @@ -21,7 +68,10 @@ async function handleMessageRequest(data: NewMessageRequest) {
role: message.user === "user" ? "user" : "assistant",
};
});
}

async function handleMessageRequest(data: NewMessageRequest) {
const recentMessages = await retrieveLastTenMessages(data.conversationId);
const message = {
...data,
message: "",
Expand All @@ -35,49 +85,45 @@ async function handleMessageRequest(data: NewMessageRequest) {
message._id = id;
events.emit(EventName.OnNewMessageResponse, message);

const response = await fetch(inferenceUrl, {
method: "POST",
headers: {
"Content-Type": "application/json",
Accept: "text/event-stream",
"Access-Control-Allow-Origi": "*",
requestInference(recentMessages).subscribe({
next: (content) => {
message.message = content;
events.emit(EventName.OnMessageResponseUpdate, message);
},
complete: async () => {
message.message = message.message.trim();
// TODO: Common collections should be able to access via core functions instead of store
await store.updateOne("messages", message._id, message);
},
error: async (err) => {
message.message = message.message.trim() + "\n" + "Error occurred: " + err;
// TODO: Common collections should be able to access via core functions instead of store
await store.updateOne("messages", message._id, message);
},
body: JSON.stringify({
messages: recentMessages,
stream: true,
model: "gpt-3.5-turbo",
max_tokens: 500,
}),
});
const stream = response.body;

const decoder = new TextDecoder("utf-8");
const reader = stream?.getReader();
let answer = "";
}

while (true && reader) {
const { done, value } = await reader.read();
if (done) {
console.log("SSE stream closed");
break;
}
const text = decoder.decode(value);
const lines = text.trim().split("\n");
for (const line of lines) {
if (line.startsWith("data: ") && !line.includes("data: [DONE]")) {
const data = JSON.parse(line.replace("data: ", ""));
answer += data.choices[0]?.delta?.content ?? "";
if (answer.startsWith("assistant: ")) {
answer = answer.replace("assistant: ", "");
}
message.message = answer;
events.emit(EventName.OnMessageResponseUpdate, message);
}
}
}
message.message = answer.trim();
// TODO: Common collections should be able to access via core functions instead of store
await store.updateOne("messages", message._id, message);
async function inferenceRequest(data: NewMessageRequest): Promise<any> {
const message = {
...data,
message: "",
user: "assistant",
createdAt: new Date().toISOString(),
};
return new Promise(async (resolve, reject) => {
const recentMessages = await retrieveLastTenMessages(data.conversationId);
requestInference([...recentMessages, { role: "user", content: data.message }]).subscribe({
next: (content) => {
message.message = content;
},
complete: async () => {
resolve(message);
},
error: async (err) => {
reject(err);
},
});
});
}

const registerListener = () => {
Expand All @@ -92,4 +138,5 @@ export function init({ register }) {
register(PluginService.OnStart, PLUGIN_NAME, onStart);
register(InferenceService.InitModel, initModel.name, initModel);
register(InferenceService.StopModel, stopModel.name, stopModel);
register(InferenceService.InferenceRequest, inferenceRequest.name, inferenceRequest);
}
3 changes: 2 additions & 1 deletion plugins/inference-plugin/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,9 @@
"webpack-cli": "^5.1.4"
},
"dependencies": {
"@janhq/core": "^0.1.1",
"@janhq/core": "^0.1.2",
"kill-port-process": "^3.2.0",
"rxjs": "^7.8.1",
"tcp-port-used": "^1.0.2",
"ts-loader": "^9.5.0"
},
Expand Down
43 changes: 41 additions & 2 deletions web/app/_hooks/useSendChatMessage.ts
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,44 @@
import { currentPromptAtom } from "@/_helpers/JotaiWrapper";
import { useAtom, useAtomValue, useSetAtom } from "jotai";
import { DataService, EventName, events } from "@janhq/core";
import { DataService, EventName, InferenceService, events, store } from "@janhq/core";
import { RawMessage, toChatMessage } from "@/_models/ChatMessage";
import { executeSerial } from "@/_services/pluginService";
import { addNewMessageAtom } from "@/_helpers/atoms/ChatMessage.atom";
import { currentConversationAtom } from "@/_helpers/atoms/Conversation.atom";
import { currentConversationAtom, updateConversationAtom } from "@/_helpers/atoms/Conversation.atom";

export default function useSendChatMessage() {
const currentConvo = useAtomValue(currentConversationAtom);
const addNewMessage = useSetAtom(addNewMessageAtom);
const updateConversation = useSetAtom(updateConversationAtom);

const [currentPrompt, setCurrentPrompt] = useAtom(currentPromptAtom);

let timeout: any | undefined = undefined;

function updateConvSummary(newMessage: any) {
if (timeout) {
clearTimeout(timeout);
}
timeout = setTimeout(() => {
const conv = currentConvo;
if (!currentConvo?.summary || currentConvo.summary === "" || currentConvo.summary.startsWith("User request:")) {
// Request convo summary
setTimeout(async () => {
newMessage.message = "summary this conversation in 5 words";
const result = await executeSerial(InferenceService.InferenceRequest, newMessage);
if (result?.message && result.message.split(" ").length <= 7 && conv?._id) {
const updatedConv = {
...conv,
summary: result.message,
};
updateConversation(updatedConv);
await executeSerial(DataService.UpdateConversation, updatedConv);
}
}, 1000);
}
}, 100);
}

const sendChatMessage = async () => {
setCurrentPrompt("");
const prompt = currentPrompt.trim();
Expand All @@ -26,7 +53,19 @@ export default function useSendChatMessage() {

const newChatMessage = toChatMessage(newMessage);
addNewMessage(newChatMessage);

events.emit(EventName.OnNewMessageRequest, newMessage);

if (!currentConvo?.summary && currentConvo) {
const updatedConv = {
...currentConvo,
summary: `Prompt: ${prompt}`,
};
updateConversation(updatedConv);
await executeSerial(DataService.UpdateConversation, updatedConv);
}

updateConvSummary(newMessage);
};

return {
Expand Down
2 changes: 1 addition & 1 deletion web/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
"dependencies": {
"@headlessui/react": "^1.7.15",
"@heroicons/react": "^2.0.18",
"@janhq/core": "^0.1.1",
"@janhq/core": "^0.1.2",
"@tailwindcss/typography": "^0.5.9",
"@types/react": "18.2.15",
"@types/react-dom": "18.2.7",
Expand Down

0 comments on commit c623a9b

Please sign in to comment.