forked from ourongxing/chatgpt-vercel
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
pref: use web worker to count token and render markdown
- Loading branch information
1 parent
8ff3811
commit d168a6d
Showing
10 changed files
with
191 additions
and
91 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,19 +1,17 @@ | ||
| import MarkdownIt from "markdown-it" | ||
| // @ts-ignore | ||
| import mdKatex from "markdown-it-katex" | ||
| import mdHighlight from "markdown-it-highlightjs" | ||
| import mdKbd from "markdown-it-kbd" | ||
| import preWrapperPlugin from "./preWrapper" | ||
|
|
||
| export async function mdFactory() { | ||
| // @ts-ignore | ||
| const { default: mdKatex } = await import("markdown-it-katex") | ||
| const { default: mdHighlight } = await import("markdown-it-highlightjs") | ||
| return MarkdownIt({ | ||
| linkify: true, | ||
| breaks: true | ||
| export const md = MarkdownIt({ | ||
| linkify: true, | ||
| breaks: true | ||
| }) | ||
| .use(mdKatex) | ||
| .use(mdHighlight, { | ||
| inline: true | ||
| }) | ||
| .use(mdKatex) | ||
| .use(mdHighlight, { | ||
| inline: true | ||
| }) | ||
| .use(mdKbd) | ||
| .use(preWrapperPlugin) | ||
| } | ||
| .use(mdKbd) | ||
| .use(preWrapperPlugin) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,3 @@ | ||
| /// <reference no-default-lib="true"/> | ||
| /// <reference lib="esnext" /> | ||
| /// <reference lib="webworker" /> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| import TokenWorker from "./tokens.worker?worker" | ||
| import MarkdownWorker from "./markdown.worker?worker" | ||
| import { generateId } from "~/utils" | ||
|
|
||
| const tokenWorker = new TokenWorker() | ||
| const markdownWorker = new MarkdownWorker() | ||
| export function countTokensInWorker(content: string): Promise<number> { | ||
| if (!content) return Promise.resolve(0) | ||
| const id = generateId() | ||
| tokenWorker.postMessage({ type: "token", id, payload: content }) | ||
| return new Promise(resolve => { | ||
| function handler(e: MessageEvent) { | ||
| if (e.data.type === "token-return" && e.data.id === id) { | ||
| tokenWorker.removeEventListener("message", handler) | ||
| resolve(e.data.payload as number) | ||
| } | ||
| } | ||
| tokenWorker.addEventListener("message", handler) | ||
| }) | ||
| } | ||
|
|
||
| export function renderMarkdown(content: string): Promise<string> { | ||
| if (!content) return Promise.resolve("") | ||
| const id = generateId() | ||
| markdownWorker.postMessage({ type: "markdown", id, payload: content }) | ||
| return new Promise(resolve => { | ||
| function handler(e: MessageEvent) { | ||
| if (e.data.type === "markdown-return" && e.data.id === id) { | ||
| markdownWorker.removeEventListener("message", handler) | ||
| resolve(e.data.payload as string) | ||
| } | ||
| } | ||
| markdownWorker.addEventListener("message", handler) | ||
| }) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,14 @@ | ||
| import { md } from "~/markdown-it" | ||
|
|
||
| const sw = self as unknown as ServiceWorkerGlobalScope & typeof globalThis | ||
|
|
||
| sw.addEventListener("message", event => { | ||
| if (event.data.type === "markdown" && event.data.payload) { | ||
| const renderd = md.render(event.data.payload) | ||
| sw.postMessage({ | ||
| type: "markdown-return", | ||
| payload: renderd, | ||
| id: event.data.id | ||
| }) | ||
| } | ||
| }) |
Oops, something went wrong.