feat: add a simple way to convert Hugging Face model to GGUF (janhq#1972

)

* chore: add react developer tools to electron

* feat: add small convert modal

* feat: separate modals and add hugging face extension

* feat: fully implement hugging face converter

* fix: forgot to uncomment this...

* fix: typo

* feat: try hf-to-gguf script first and then use convert.py

HF-to-GGUF has support for some unusual models
maybe using convert.py first would be better but we can change the usage order later

* fix: pre-install directory changed

* fix: sometimes exit code is undefined

* chore: download additional files for qwen

* fix: event handling changed

* chore: add one more necessary package

* feat: download gguf-py from llama.cpp

* fix: cannot interpret wildcards on GNU tar

Co-authored-by: hiento09 <[email protected]>

---------

Co-authored-by: hiento09 <[email protected]>

core/src/extension.ts

@@ -4,6 +4,7 @@ export enum ExtensionTypeEnum {
   Inference = 'inference',
   Model = 'model',
   SystemMonitoring = 'systemMonitoring',
+  HuggingFace = 'huggingFace',
 }
 
 export interface ExtensionType {

core/src/extensions/huggingface.ts

@@ -0,0 +1,30 @@
+import { BaseExtension, ExtensionTypeEnum } from '../extension'
+import { HuggingFaceInterface, HuggingFaceRepoData, Quantization } from '../types/huggingface'
+import { Model } from '../types/model'
+
+/**
+ * Hugging Face extension for converting HF models to GGUF.
+ */
+export abstract class HuggingFaceExtension extends BaseExtension implements HuggingFaceInterface {
+  interrupted = false
+  /**
+   * Hugging Face extension type.
+   */
+  type(): ExtensionTypeEnum | undefined {
+    return ExtensionTypeEnum.HuggingFace
+  }
+
+  abstract downloadModelFiles(
+    repoID: string,
+    repoData: HuggingFaceRepoData,
+    network?: { ignoreSSL?: boolean; proxy?: string }
+  ): Promise<void>
+  abstract convert(repoID: string): Promise<void>
+  abstract quantize(repoID: string, quantization: Quantization): Promise<void>
+  abstract generateMetadata(
+    repoID: string,
+    repoData: HuggingFaceRepoData,
+    quantization: Quantization
+  ): Promise<void>
+  abstract cancelConvert(repoID: string, repoData: HuggingFaceRepoData): Promise<void>
+}

core/src/extensions/index.ts

@@ -23,3 +23,8 @@ export { AssistantExtension } from './assistant'
  * Model extension for managing models.
  */
 export { ModelExtension } from './model'
+
+/**
+ * Hugging Face extension for converting HF models to GGUF.
+ */
+export { HuggingFaceExtension } from './huggingface'

core/src/types/huggingface/huggingfaceEntity.ts

@@ -0,0 +1,34 @@
+export interface HuggingFaceRepoData {
+  id: string
+  author: string
+  tags: Array<'transformers' | 'pytorch' | 'safetensors' | string>
+  siblings: {
+    rfilename: string
+  }[]
+  createdAt: string // ISO 8601 timestamp
+}
+
+/* eslint-disable @typescript-eslint/naming-convention */
+export enum Quantization {
+  Q3_K_S = 'Q3_K_S',
+  Q3_K_M = 'Q3_K_M', // eslint-disable-line @typescript-eslint/no-duplicate-enum-values
+  Q3_K_L = 'Q3_K_L',
+  Q4_K_S = 'Q4_K_S',
+  Q4_K_M = 'Q4_K_M', // eslint-disable-line @typescript-eslint/no-duplicate-enum-values
+  Q5_K_S = 'Q5_K_S',
+  Q5_K_M = 'Q5_K_M', // eslint-disable-line @typescript-eslint/no-duplicate-enum-values
+  Q4_0 = 'Q4_0',
+  Q4_1 = 'Q4_1',
+  Q5_0 = 'Q5_0',
+  Q5_1 = 'Q5_1',
+  IQ2_XXS = 'IQ2_XXS',
+  IQ2_XS = 'IQ2_XS',
+  Q2_K = 'Q2_K',
+  Q2_K_S = 'Q2_K_S',
+  Q6_K = 'Q6_K',
+  Q8_0 = 'Q8_0',
+  F16 = 'F16',
+  F32 = 'F32',
+  COPY = 'COPY',
+}
+/* eslint-enable @typescript-eslint/naming-convention */

core/src/types/huggingface/huggingfaceInterface.ts

@@ -0,0 +1,58 @@
+import { Model } from '../model'
+import { HuggingFaceRepoData, Quantization } from './huggingfaceEntity'
+
+/**
+ * Hugging Face extension for converting HF models to GGUF.
+ * @extends BaseExtension
+ */
+export interface HuggingFaceInterface {
+  interrupted: boolean
+  /**
+   * Downloads a Hugging Face model.
+   * @param repoID - The repo ID of the model to convert.
+   * @param repoData - The repo data of the model to convert.
+   * @param network - Optional object to specify proxy/whether to ignore SSL certificates.
+   * @returns A promise that resolves when the download is complete.
+   */
+  downloadModelFiles(
+    repoID: string,
+    repoData: HuggingFaceRepoData,
+    network?: { ignoreSSL?: boolean; proxy?: string }
+  ): Promise<void>
+
+  /**
+   * Converts a Hugging Face model to GGUF.
+   * @param repoID - The repo ID of the model to convert.
+   * @returns A promise that resolves when the conversion is complete.
+   */
+  convert(repoID: string): Promise<void>
+
+  /**
+   * Quantizes a GGUF model.
+   * @param repoID - The repo ID of the model to quantize.
+   * @param quantization - The quantization to use.
+   * @returns A promise that resolves when the quantization is complete.
+   */
+  quantize(repoID: string, quantization: Quantization): Promise<void>
+
+  /**
+   * Generates Jan model metadata from a Hugging Face model.
+   * @param repoID - The repo ID of the model to generate metadata for.
+   * @param repoData - The repo data of the model to generate metadata for.
+   * @param quantization - The quantization of the model.
+   * @returns A promise that resolves when the model metadata generation is complete.
+   */
+  generateMetadata(
+    repoID: string,
+    repoData: HuggingFaceRepoData,
+    quantization: Quantization
+  ): Promise<void>
+
+  /**
+   * Cancels the convert of current Hugging Face model.
+   * @param repoID - The repository ID to cancel.
+   * @param repoData - The repository data to cancel.
+   * @returns {Promise<void>} A promise that resolves when the download has been cancelled.
+   */
+  cancelConvert(repoID: string, repoData: HuggingFaceRepoData): Promise<void>
+}

core/src/types/huggingface/index.ts

@@ -0,0 +1,2 @@
+export * from './huggingfaceInterface'
+export * from './huggingfaceEntity'

core/src/types/index.ts

@@ -6,4 +6,5 @@ export * from './inference'
 export * from './monitoring'
 export * from './file'
 export * from './config'
+export * from './huggingface'
 export * from './miscellaneous'

electron/package.json

@@ -15,12 +15,14 @@
       "build/**/*.{js,map}",
       "pre-install",
       "models/**/*",
-      "docs/**/*"
+      "docs/**/*",
+      "scripts/**/*"
     ],
     "asarUnpack": [
       "pre-install",
       "models",
-      "docs"
+      "docs",
+      "scripts"
     ],
     "publish": [
       {

extensions/huggingface-extension/.gitignore

@@ -0,0 +1,3 @@
+bin
+scripts/convert*
+scripts/gguf-py

extensions/huggingface-extension/.prettierrc

@@ -0,0 +1,8 @@
+{
+  "semi": false,
+  "singleQuote": true,
+  "quoteProps": "consistent",
+  "trailingComma": "es5",
+  "endOfLine": "auto",
+  "plugins": ["prettier-plugin-tailwindcss"]
+}

extensions/huggingface-extension/README.md

@@ -0,0 +1,73 @@
+# Create a Jan Plugin using Typescript
+
+Use this template to bootstrap the creation of a TypeScript Jan plugin. 🚀
+
+## Create Your Own Plugin
+
+To create your own plugin, you can use this repository as a template! Just follow the below instructions:
+
+1. Click the Use this template button at the top of the repository
+2. Select Create a new repository
+3. Select an owner and name for your new repository
+4. Click Create repository
+5. Clone your new repository
+
+## Initial Setup
+
+After you've cloned the repository to your local machine or codespace, you'll need to perform some initial setup steps before you can develop your plugin.
+
+> [!NOTE]
+>
+> You'll need to have a reasonably modern version of
+> [Node.js](https://nodejs.org) handy. If you are using a version manager like
+> [`nodenv`](https://github.com/nodenv/nodenv) or
+> [`nvm`](https://github.com/nvm-sh/nvm), you can run `nodenv install` in the
+> root of your repository to install the version specified in
+> [`package.json`](./package.json). Otherwise, 20.x or later should work!
+
+1. :hammer_and_wrench: Install the dependencies
+
+   ```bash
+   npm install
+   ```
+
+1. :building_construction: Package the TypeScript for distribution
+
+   ```bash
+   npm run bundle
+   ```
+
+1. :white_check_mark: Check your artifact
+
+   There will be a tgz file in your plugin directory now
+
+## Update the Plugin Metadata
+
+The [`package.json`](package.json) file defines metadata about your plugin, such as
+plugin name, main entry, description and version.
+
+When you copy this repository, update `package.json` with the name, description for your plugin.
+
+## Update the Plugin Code
+
+The [`src/`](./src/) directory is the heart of your plugin! This contains the
+source code that will be run when your plugin extension functions are invoked. You can replace the
+contents of this directory with your own code.
+
+There are a few things to keep in mind when writing your plugin code:
+
+- Most Jan Plugin Extension functions are processed asynchronously.
+  In `index.ts`, you will see that the extension function will return a `Promise<any>`.
+
+  ```typescript
+  import { core } from "@janhq/core";
+
+  function onStart(): Promise<any> {
+    return core.invokePluginFunc(MODULE_PATH, "run", 0);
+  }
+  ```
+
+  For more information about the Jan Plugin Core module, see the
+  [documentation](https://github.com/janhq/jan/blob/main/core/README.md).
+
+So, what are you waiting for? Go ahead and start customizing your plugin!

extensions/huggingface-extension/download.bat

@@ -0,0 +1,3 @@
+@echo off
+set /p LLAMA_CPP_VERSION=<./scripts/version.txt
+.\node_modules\.bin\download https://github.com/ggerganov/llama.cpp/archive/refs/tags/%LLAMA_CPP_VERSION%.tar.gz -o . --filename ./scripts/llama.cpp.tar.gz && tar -xzf .\scripts\llama.cpp.tar.gz "llama.cpp-%LLAMA_CPP_VERSION%/convert.py" "llama.cpp-%LLAMA_CPP_VERSION%/convert-hf-to-gguf.py" "llama.cpp-%LLAMA_CPP_VERSION%/gguf-py" && cpx "./llama.cpp-%LLAMA_CPP_VERSION%/**" "scripts" && rimraf "./scripts/llama.cpp.tar.gz" && rimraf "./llama.cpp-%LLAMA_CPP_VERSION%"

extensions/huggingface-extension/package.json

@@ -0,0 +1,54 @@
+{
+  "name": "@janhq/huggingface-extension",
+  "version": "1.0.0",
+  "description": "Hugging Face extension for converting HF models to GGUF",
+  "main": "dist/index.js",
+  "node": "dist/node/index.cjs.js",
+  "author": "Jan <[email protected]>",
+  "license": "AGPL-3.0",
+  "scripts": {
+    "build": "tsc --module commonjs && rollup -c rollup.config.ts --configPlugin @rollup/plugin-typescript --bundleConfigAsCjs",
+    "download:llama": "run-script-os",
+    "download:llama:linux": "LLAMA_CPP_VERSION=$(cat ./scripts/version.txt) && download https://github.com/ggerganov/llama.cpp/archive/refs/tags/${LLAMA_CPP_VERSION}.tar.gz -o . --filename ./scripts/llama.cpp.tar.gz && tar -xzf ./scripts/llama.cpp.tar.gz --wildcards '*/convert.py' '*/convert-hf-to-gguf.py' '*/gguf-py' && cpx \"./llama.cpp-$LLAMA_CPP_VERSION/**\" \"scripts\" && rimraf \"./scripts/llama.cpp.tar.gz\" && rimraf \"./llama.cpp-$LLAMA_CPP_VERSION\"",
+    "download:llama:darwin": "LLAMA_CPP_VERSION=$(cat ./scripts/version.txt) && download https://github.com/ggerganov/llama.cpp/archive/refs/tags/${LLAMA_CPP_VERSION}.tar.gz -o . --filename ./scripts/llama.cpp.tar.gz && tar -xzf ./scripts/llama.cpp.tar.gz '*/convert.py' '*/convert-hf-to-gguf.py' '*/gguf-py' && cpx \"./llama.cpp-$LLAMA_CPP_VERSION/**\" \"scripts\" && rimraf \"./scripts/llama.cpp.tar.gz\" && rimraf \"./llama.cpp-$LLAMA_CPP_VERSION\"",
+    "download:llama:win32": "download.bat",
+    "build:publish": "rimraf *.tgz --glob && npm run build && npm run download:llama && cpx \"scripts/**\" \"dist/scripts\" && cpx \"bin/**\" \"dist/bin\" && npm pack && cpx *.tgz ../../pre-install"
+  },
+  "exports": {
+    ".": "./dist/index.js",
+    "./main": "./dist/node/index.cjs.js"
+  },
+  "devDependencies": {
+    "@rollup/plugin-commonjs": "^25.0.7",
+    "@rollup/plugin-json": "^6.1.0",
+    "@rollup/plugin-node-resolve": "^15.2.3",
+    "@rollup/plugin-replace": "^5.0.5",
+    "@rollup/plugin-typescript": "^11.1.6",
+    "@types/node": "^20.11.16",
+    "cpx": "^1.5.0",
+    "download-cli": "^1.1.1",
+    "rimraf": "^5.0.5",
+    "rollup": "^4.9.6",
+    "rollup-plugin-sourcemaps": "^0.6.3",
+    "rollup-plugin-typescript2": "^0.36.0",
+    "run-script-os": "^1.1.6",
+    "typescript": "^5.3.3"
+  },
+  "dependencies": {
+    "@janhq/core": "file:../../core",
+    "hyllama": "^0.1.2",
+    "python-shell": "^5.0.0",
+    "ts-loader": "^9.5.0"
+  },
+  "bundledDependencies": [
+    "python-shell"
+  ],
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "files": [
+    "dist/*",
+    "package.json",
+    "README.md"
+  ]
+}

extensions/huggingface-extension/rollup.config.ts

@@ -0,0 +1,72 @@
+import resolve from '@rollup/plugin-node-resolve'
+import commonjs from '@rollup/plugin-commonjs'
+import sourceMaps from 'rollup-plugin-sourcemaps'
+import typescript from 'rollup-plugin-typescript2'
+import json from '@rollup/plugin-json'
+import replace from '@rollup/plugin-replace'
+
+const packageJson = require('./package.json')
+
+export default [
+  {
+    input: `src/index.ts`,
+    output: [{ file: packageJson.main, format: 'es', sourcemap: true }],
+    // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash')
+    external: [],
+    watch: {
+      include: 'src/**',
+    },
+    plugins: [
+      replace({
+        EXTENSION_NAME: JSON.stringify(packageJson.name),
+        NODE_MODULE_PATH: JSON.stringify(
+          `${packageJson.name}/${packageJson.node}`
+        ),
+      }),
+      // Allow json resolution
+      json(),
+      //     Compile TypeScript files
+      typescript({ useTsconfigDeclarationDir: true }),
+      // Compile TypeScript files
+      // Allow bundling cjs modules (unlike webpack, rollup doesn't understand cjs)
+      commonjs(),
+      // Allow node_modules resolution, so you can use 'external' to control
+      // which external modules to include in the bundle
+      // https://github.com/rollup/rollup-plugin-node-resolve#usage
+      resolve({
+        extensions: ['.js', '.ts'],
+      }),
+
+      // Resolve source maps to the original source
+      sourceMaps(),
+    ],
+  },
+  {
+    input: `src/node/index.ts`,
+    output: [
+      { file: 'dist/node/index.cjs.js', format: 'cjs', sourcemap: true },
+    ],
+    // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash')
+    external: [],
+    watch: {
+      include: 'src/node/**',
+    },
+    plugins: [
+      // Allow json resolution
+      json(),
+      // Compile TypeScript files
+      typescript({ useTsconfigDeclarationDir: true }),
+      // Allow bundling cjs modules (unlike webpack, rollup doesn't understand cjs)
+      commonjs(),
+      // Allow node_modules resolution, so you can use 'external' to control
+      // which external modules to include in the bundle
+      // https://github.com/rollup/rollup-plugin-node-resolve#usage
+      resolve({
+        extensions: ['.ts', '.js', '.json'],
+      }),
+
+      // Resolve source maps to the original source
+      sourceMaps(),
+    ],
+  },
+]