typescript bindings maintenance (nomic-ai#2363)

* remove outdated comments Signed-off-by: limez <[email protected]> * simpler build from source Signed-off-by: limez <[email protected]> * update unix build script to create .so runtimes correctly Signed-off-by: limez <[email protected]> * configure ci build type, use RelWithDebInfo for dev build script Signed-off-by: limez <[email protected]> * add clean script Signed-off-by: limez <[email protected]> * fix streamed token decoding / emoji Signed-off-by: limez <[email protected]> * remove deprecated nCtx Signed-off-by: limez <[email protected]> * update typings Signed-off-by: jacob <[email protected]> update typings Signed-off-by: jacob <[email protected]> * readme,mspell Signed-off-by: jacob <[email protected]> * cuda/backend logic changes + name napi methods like their js counterparts Signed-off-by: limez <[email protected]> * convert llmodel example into a test, separate test suite that can run in ci Signed-off-by: limez <[email protected]> * update examples / naming Signed-off-by: limez <[email protected]> * update deps, remove the need for binding.ci.gyp, make node-gyp-build fallback easier testable Signed-off-by: limez <[email protected]> * make sure the assert-backend-sources.js script is published, but not the others Signed-off-by: limez <[email protected]> * build correctly on windows (regression on node-gyp-build) Signed-off-by: Jacob Nguyen <[email protected]> * codespell Signed-off-by: limez <[email protected]> * make sure dlhandle.cpp gets linked correctly Signed-off-by: limez <[email protected]> * add include for check_cxx_compiler_flag call during aarch64 builds Signed-off-by: limez <[email protected]> * x86 > arm64 cross compilation of runtimes and bindings Signed-off-by: limez <[email protected]> * default to cpu instead of kompute on arm64 Signed-off-by: limez <[email protected]> * formatting, more minimal example Signed-off-by: limez <[email protected]> --------- Signed-off-by: limez <[email protected]> Signed-off-by: jacob <[email protected]> Signed-off-by: Jacob Nguyen <[email protected]> Co-authored-by: Jacob Nguyen <[email protected]> Co-authored-by: jacob <[email protected]>
nanocm · Jun 3, 2024 · a602f7f · a602f7f
1 parent f001897
commit a602f7f
Show file tree

Hide file tree

Showing 30 changed files with 1,111 additions and 872 deletions.
diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml
@@ -570,22 +570,27 @@ jobs:
             wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
             sudo dpkg -i cuda-keyring_1.1-1_all.deb
             sudo apt-get update
-            sudo apt-get install -y cmake build-essential vulkan-sdk cuda-compiler-12-4 libcublas-dev-12-4 libnvidia-compute-550-server libmysqlclient21 libodbc2 libpq5
+            sudo apt-get install -y cmake build-essential g++-12-aarch64-linux-gnu gcc-12-aarch64-linux-gnu vulkan-sdk cuda-compiler-12-4 libcublas-dev-12-4 libnvidia-compute-550-server libmysqlclient21 libodbc2 libpq5
       - run:
           name: Build Libraries
           command: |
             export PATH=$PATH:/usr/local/cuda/bin
             cd gpt4all-backend
             mkdir -p runtimes/build
             cd runtimes/build
-            cmake ../..
-            cmake --build . --parallel --config Release
+            cmake ../.. -DCMAKE_BUILD_TYPE=Release
+            cmake --build . --parallel
             mkdir ../linux-x64
             cp -L *.so ../linux-x64 # otherwise persist_to_workspace seems to mess symlinks
+            cmake ../.. -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE="./toolchains/linux-arm64-toolchain.cmake"
+            cmake --build . --parallel
+            mkdir ../linux-arm64
+            cp -L *.so ../linux-arm64
       - persist_to_workspace:
           root: gpt4all-backend
           paths:
             - runtimes/linux-x64/*.so
+            - runtimes/linux-arm64/*.so
 
   build-bindings-backend-macos:
     macos:
@@ -896,6 +901,11 @@ jobs:
       - checkout
       - attach_workspace:
           at: /tmp/gpt4all-backend
+      - run:
+          name: Install dependencies
+          command: |
+            sudo apt-get update
+            sudo apt-get install -y g++-12-aarch64-linux-gnu gcc-12-aarch64-linux-gnu
       - node/install:
           install-yarn: true
           node-version: "18.16"
@@ -908,18 +918,24 @@ jobs:
       - run:
           command: | 
             cd gpt4all-bindings/typescript
-            yarn prebuildify -t 18.16.0 --napi
+            yarn build:prebuilds
       - run: 
           command: |
             mkdir -p gpt4all-backend/prebuilds/linux-x64
             mkdir -p gpt4all-backend/runtimes/linux-x64
             cp /tmp/gpt4all-backend/runtimes/linux-x64/*-*.so gpt4all-backend/runtimes/linux-x64
             cp gpt4all-bindings/typescript/prebuilds/linux-x64/*.node gpt4all-backend/prebuilds/linux-x64
+            mkdir -p gpt4all-backend/prebuilds/linux-arm64
+            mkdir -p gpt4all-backend/runtimes/linux-arm64
+            cp /tmp/gpt4all-backend/runtimes/linux-arm64/*-*.so gpt4all-backend/runtimes/linux-arm64
+            cp gpt4all-bindings/typescript/prebuilds/linux-arm64/*.node gpt4all-backend/prebuilds/linux-arm64
       - persist_to_workspace:
           root: gpt4all-backend
           paths:
             - prebuilds/linux-x64/*.node 
             - runtimes/linux-x64/*-*.so
+            - prebuilds/linux-arm64/*.node
+            - runtimes/linux-arm64/*-*.so
   build-nodejs-macos: 
     macos:
       xcode: "14.0.0"
@@ -1029,13 +1045,11 @@ jobs:
             cp /tmp/gpt4all-backend/runtimes/darwin/*-*.* runtimes/darwin/native/
 
             cp /tmp/gpt4all-backend/prebuilds/darwin-x64/*.node prebuilds/darwin-x64/    
-            
-            # Fallback build if user is not on above prebuilds
-            mv -f binding.ci.gyp binding.gyp
 
-            mkdir gpt4all-backend
+            # copy the backend source we depend on to make fallback builds work
+            mkdir backend
             cd ../../gpt4all-backend
-            mv llmodel.h llmodel.cpp llmodel_c.cpp llmodel_c.h sysinfo.h dlhandle.h ../gpt4all-bindings/typescript/gpt4all-backend/
+            mv llmodel.h llmodel.cpp llmodel_c.cpp llmodel_c.h sysinfo.h dlhandle.h ../gpt4all-bindings/typescript/backend/
             
       # Test install
       - node/install-packages:
@@ -1045,7 +1059,7 @@ jobs:
       - run: 
           command: | 
             cd gpt4all-bindings/typescript
-            yarn run test
+            yarn run test:ci
       - run:
           command: |
             cd gpt4all-bindings/typescript

diff --git a/gpt4all-backend/CMakeLists.txt b/gpt4all-backend/CMakeLists.txt
@@ -79,6 +79,7 @@ if (LLMODEL_ROCM)
 endif()
 
 set(CMAKE_VERBOSE_MAKEFILE ON)
+include(CheckCXXCompilerFlag)
 
 # Go through each build variant
 foreach(BUILD_VARIANT IN LISTS BUILD_VARIANTS)

diff --git a/gpt4all-backend/toolchains/linux-arm64-toolchain.cmake b/gpt4all-backend/toolchains/linux-arm64-toolchain.cmake
@@ -0,0 +1,11 @@
+# Toolchain to crosscompile runtimes for arm64 on jammy x86_64
+# You may have to `sudo apt-get install g++-12-aarch64-linux-gnu gcc-12-aarch64-linux-gnu`
+
+set(CMAKE_SYSTEM_NAME Linux)
+set(CMAKE_SYSTEM_PROCESSOR aarch64)
+set(CMAKE_C_COMPILER aarch64-linux-gnu-gcc-12)
+set(CMAKE_CXX_COMPILER aarch64-linux-gnu-g++-12)
+
+# Supported backends
+set(LLMODEL_CUDA off)
+set(LLMODEL_KOMPUTE off)
diff --git a/gpt4all-bindings/typescript/.gitignore b/gpt4all-bindings/typescript/.gitignore
@@ -8,4 +8,5 @@ prebuilds/
 !.yarn/sdks
 !.yarn/versions
 runtimes/
+backend/
 compile_flags.txt
diff --git a/gpt4all-bindings/typescript/.npmignore b/gpt4all-bindings/typescript/.npmignore
@@ -1,4 +1,5 @@
 test/
 spec/
-scripts/
+scripts/*
+!scripts/assert-backend-sources.js
 build
diff --git a/gpt4all-bindings/typescript/README.md b/gpt4all-bindings/typescript/README.md
@@ -188,6 +188,8 @@ model.dispose();
 *   python 3
 *   On Windows and Linux, building GPT4All requires the complete Vulkan SDK. You may download it from here: https://vulkan.lunarg.com/sdk/home
 *   macOS users do not need Vulkan, as GPT4All will use Metal instead.
+*   CUDA Toolkit >= 11.4  (you can bypass this with adding a custom flag to build step)
+    -  Windows: There is difficulty compiling with cuda if the Visual Studio IDE is NOT present.
 
 ### Build (from source)
 
@@ -196,23 +198,29 @@ git clone https://github.com/nomic-ai/gpt4all.git
 cd gpt4all-bindings/typescript
 ```
 
-*   The below shell commands assume the current working directory is `typescript`.
-
-*   To Build and Rebuild:
+llama.cpp git submodule for gpt4all can be possibly absent or outdated. Make sure to run
 
 ```sh
-node scripts/prebuild.js
+git submodule update --init --recursive
 ```
-*   llama.cpp git submodule for gpt4all can be possibly absent. If this is the case, make sure to run in llama.cpp parent directory
+
+The below shell commands assume the current working directory is `typescript`.
+
+Using yarn
 
 ```sh
-git submodule update --init --recursive
+yarn install
+yarn build
 ```
 
+Using npm
+
 ```sh
-yarn build:backend
+npm install
+npm run build
 ```
-This will build platform-dependent dynamic libraries, and will be located in runtimes/(platform)/native
+
+The `build:runtimes` script will create runtime libraries for your platform in `runtimes` and `build:prebuilds` will create the bindings in `prebuilds`. `build` is a shortcut for both.
 
 ### Test
 
@@ -259,7 +267,7 @@ yarn test
 
 This package has been stabilizing over time development, and breaking changes may happen until the api stabilizes. Here's what's the todo list:
 
-*   \[ ] Purely offline. Per the gui, which can be run completely offline, the bindings should be as well.
+*   \[x] [Purely offline](#Offline-usage). Per the gui, which can be run completely offline, the bindings should be as well. 
 *   \[ ] NPM bundle size reduction via optionalDependencies strategy (need help)
     *   Should include prebuilds to avoid painful node-gyp errors
 *   \[x] createChatSession ( the python equivalent to create\_chat\_session )
@@ -276,7 +284,7 @@ This package has been stabilizing over time development, and breaking changes ma
 This repository serves as the new bindings for nodejs users.
 - If you were a user of [these bindings](https://github.com/nomic-ai/gpt4all-ts), they are outdated.
 - Version 4 includes the follow breaking changes
-    * `createEmbedding` & `EmbeddingModel.embed()` returns an object, `EmbeddingResult`, instead of a float32array.
+    * `createEmbedding` & `EmbeddingModel.embed()` returns an object, `EmbeddingResult`, instead of a Float32Array.
     * Removed deprecated types `ModelType` and `ModelFile`
     * Removed deprecated initiation of model by string path only
 

diff --git a/gpt4all-bindings/typescript/binding.ci.gyp b/gpt4all-bindings/typescript/binding.ci.gyp
diff --git a/gpt4all-bindings/typescript/binding.gyp b/gpt4all-bindings/typescript/binding.gyp
@@ -1,19 +1,15 @@
 {
   "targets": [
     {
-      "target_name": "gpt4all", # gpt4all-ts will cause compile error
+      "target_name": "gpt4all",
       "include_dirs": [
         "<!@(node -p \"require('node-addon-api').include\")",
-        "../../gpt4all-backend",
+        "backend",
       ],
       "sources": [
-        # PREVIOUS VERSION: had to required the sources, but with newest changes do not need to
-        #"../../gpt4all-backend/llama.cpp/examples/common.cpp",
-        #"../../gpt4all-backend/llama.cpp/ggml.c",
-        #"../../gpt4all-backend/llama.cpp/llama.cpp",
-        # "../../gpt4all-backend/utils.cpp",
-        "../../gpt4all-backend/llmodel_c.cpp",
-        "../../gpt4all-backend/llmodel.cpp",
+        "backend/llmodel_c.cpp",
+        "backend/llmodel.cpp",
+        "backend/dlhandle.cpp",
         "prompt.cc",
         "index.cc",
        ],