diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index f980b9df73..db1eed38df 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,4 +1,4 @@ { - "name": "jan", - "image": "node:20" -} \ No newline at end of file + "name": "jan", + "image": "node:20" +} diff --git a/.github/workflows/clean-cloudflare-page-preview-url-and-r2.yml b/.github/workflows/clean-cloudflare-page-preview-url-and-r2.yml index 620f747144..de761ca69b 100644 --- a/.github/workflows/clean-cloudflare-page-preview-url-and-r2.yml +++ b/.github/workflows/clean-cloudflare-page-preview-url-and-r2.yml @@ -55,10 +55,10 @@ jobs: steps: - name: install-aws-cli-action uses: unfor19/install-aws-cli-action@v1 - - name: Delete object older than 7 days + - name: Delete object older than 10 days run: | # Get the list of objects in the 'latest' folder - OBJECTS=$(aws s3api list-objects --bucket ${{ secrets.CLOUDFLARE_R2_BUCKET_NAME }} --query 'Contents[?LastModified<`'$(date -d "$current_date -30 days" -u +"%Y-%m-%dT%H:%M:%SZ")'`].{Key: Key}' --endpoint-url https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com | jq -c .) + OBJECTS=$(aws s3api list-objects --bucket ${{ secrets.CLOUDFLARE_R2_BUCKET_NAME }} --query 'Contents[?LastModified<`'$(date -d "$current_date -10 days" -u +"%Y-%m-%dT%H:%M:%SZ")'`].{Key: Key}' --endpoint-url https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com | jq -c .) # Create a JSON file for the delete operation echo "{\"Objects\": $OBJECTS, \"Quiet\": false}" > delete.json diff --git a/.github/workflows/jan-electron-linter-and-test.yml b/.github/workflows/jan-electron-linter-and-test.yml index 6d5aaf150c..40085391f1 100644 --- a/.github/workflows/jan-electron-linter-and-test.yml +++ b/.github/workflows/jan-electron-linter-and-test.yml @@ -1,5 +1,6 @@ name: Jan Electron Linter & Test on: + workflow_dispatch: push: branches: - main diff --git a/.github/workflows/jan-server-build-nightly.yml b/.github/workflows/jan-server-build-nightly.yml new file mode 100644 index 0000000000..0d1bc3ca89 --- /dev/null +++ b/.github/workflows/jan-server-build-nightly.yml @@ -0,0 +1,40 @@ +name: Jan Build Docker Nightly or Manual + +on: + push: + branches: + - main + - feature/helmchart-and-ci-jan-server + paths-ignore: + - 'README.md' + - 'docs/**' + schedule: + - cron: '0 20 * * 1,2,3' # At 8 PM UTC on Monday, Tuesday, and Wednesday which is 3 AM UTC+7 Tuesday, Wednesday, and Thursday + workflow_dispatch: + +jobs: + # Job create Update app version based on latest release tag with build number and save to output + get-update-version: + uses: ./.github/workflows/template-get-update-version.yml + + build-cpu: + uses: ./.github/workflows/template-build-jan-server.yml + permissions: + packages: write + secrets: inherit + needs: [get-update-version] + with: + dockerfile_path: ./Dockerfile + docker_image_tag: "ghcr.io/janhq/jan-server:dev-cpu-latest,ghcr.io/janhq/jan-server:dev-cpu-${{ needs.get-update-version.outputs.new_version }}" + + build-gpu: + uses: ./.github/workflows/template-build-jan-server.yml + permissions: + packages: write + secrets: inherit + needs: [get-update-version] + with: + dockerfile_path: ./Dockerfile.gpu + docker_image_tag: "ghcr.io/janhq/jan-server:dev-cuda-12.2-latest,ghcr.io/janhq/jan-server:dev-cuda-12.2-${{ needs.get-update-version.outputs.new_version }}" + + diff --git a/.github/workflows/jan-server-build.yml b/.github/workflows/jan-server-build.yml new file mode 100644 index 0000000000..0665838d67 --- /dev/null +++ b/.github/workflows/jan-server-build.yml @@ -0,0 +1,30 @@ +name: Jan Build Docker + +on: + push: + tags: ["v[0-9]+.[0-9]+.[0-9]+"] + +jobs: + # Job create Update app version based on latest release tag with build number and save to output + get-update-version: + uses: ./.github/workflows/template-get-update-version.yml + + build-cpu: + permissions: + packages: write + uses: ./.github/workflows/template-build-jan-server.yml + secrets: inherit + needs: [get-update-version] + with: + dockerfile_path: ./Dockerfile + docker_image_tag: "ghcr.io/janhq/jan-server:cpu-latest,ghcr.io/janhq/jan-server:cpu-${{ needs.get-update-version.outputs.new_version }}" + + build-gpu: + permissions: + packages: write + uses: ./.github/workflows/template-build-jan-server.yml + secrets: inherit + needs: [get-update-version] + with: + dockerfile_path: ./Dockerfile.gpu + docker_image_tag: "ghcr.io/janhq/jan-server:cuda-12.2-latest,ghcr.io/janhq/jan-server:cuda-12.2-${{ needs.get-update-version.outputs.new_version }}" diff --git a/.github/workflows/template-build-jan-server.yml b/.github/workflows/template-build-jan-server.yml new file mode 100644 index 0000000000..9bb772605e --- /dev/null +++ b/.github/workflows/template-build-jan-server.yml @@ -0,0 +1,39 @@ +name: build-jan-server +on: + workflow_call: + inputs: + dockerfile_path: + required: false + type: string + default: './Dockerfile' + docker_image_tag: + required: true + type: string + default: 'ghcr.io/janhq/jan-server:dev-latest' + +jobs: + build: + runs-on: ubuntu-latest + env: + REGISTRY: ghcr.io + IMAGE_NAME: janhq/jan-server + permissions: + packages: write + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push Docker image + uses: docker/build-push-action@v3 + with: + context: . + file: ${{ inputs.dockerfile_path }} + push: true + tags: ${{ inputs.docker_image_tag }} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 4540e5c7ab..62878011e5 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ error.log node_modules *.tgz +!charts/server/charts/*.tgz yarn.lock dist build @@ -28,4 +29,5 @@ extensions/inference-nitro-extension/bin/*/*.exp extensions/inference-nitro-extension/bin/*/*.lib extensions/inference-nitro-extension/bin/saved-* extensions/inference-nitro-extension/bin/*.tar.gz - +extensions/inference-nitro-extension/bin/vulkaninfoSDK.exe +extensions/inference-nitro-extension/bin/vulkaninfo diff --git a/core/.prettierignore b/.prettierignore similarity index 100% rename from core/.prettierignore rename to .prettierignore diff --git a/core/.prettierrc b/.prettierrc similarity index 100% rename from core/.prettierrc rename to .prettierrc diff --git a/Dockerfile b/Dockerfile index 949a92673f..48b2d254fb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,39 +1,61 @@ -FROM node:20-bullseye AS base +FROM node:20-bookworm AS base # 1. Install dependencies only when needed -FROM base AS deps +FROM base AS builder + +# Install g++ 11 +RUN apt update && apt install -y gcc-11 g++-11 cpp-11 jq xsel && rm -rf /var/lib/apt/lists/* + WORKDIR /app # Install dependencies based on the preferred package manager -COPY package.json yarn.lock* package-lock.json* pnpm-lock.yaml* ./ -RUN yarn install +COPY . ./ -# # 2. Rebuild the source code only when needed -FROM base AS builder -WORKDIR /app -COPY --from=deps /app/node_modules ./node_modules -COPY . . -# This will do the trick, use the corresponding env file for each environment. -RUN yarn workspace server install -RUN yarn server:prod +RUN export NITRO_VERSION=$(cat extensions/inference-nitro-extension/bin/version.txt) && \ + jq --arg nitroVersion $NITRO_VERSION '(.scripts."downloadnitro:linux" | gsub("\\${NITRO_VERSION}"; $nitroVersion)) | gsub("\r"; "")' extensions/inference-nitro-extension/package.json > /tmp/newcommand.txt && export NEW_COMMAND=$(sed 's/^"//;s/"$//' /tmp/newcommand.txt) && jq --arg newCommand "$NEW_COMMAND" '.scripts."downloadnitro:linux" = $newCommand' extensions/inference-nitro-extension/package.json > /tmp/package.json && mv /tmp/package.json extensions/inference-nitro-extension/package.json +RUN make install-and-build -# 3. Production image, copy all the files and run next +# # 2. Rebuild the source code only when needed FROM base AS runner + +# Install g++ 11 +RUN apt update && apt install -y gcc-11 g++-11 cpp-11 jq xsel && rm -rf /var/lib/apt/lists/* + WORKDIR /app -ENV NODE_ENV=production +# Copy the package.json and yarn.lock of root yarn space to leverage Docker cache +COPY --from=builder /app/package.json ./package.json +COPY --from=builder /app/node_modules ./node_modules/ +COPY --from=builder /app/yarn.lock ./yarn.lock + +# Copy the package.json, yarn.lock, and build output of server yarn space to leverage Docker cache +COPY --from=builder /app/core ./core/ +COPY --from=builder /app/server ./server/ +RUN cd core && yarn install && yarn run build +RUN yarn workspace @janhq/server install && yarn workspace @janhq/server build +COPY --from=builder /app/docs/openapi ./docs/openapi/ + +# Copy pre-install dependencies +COPY --from=builder /app/pre-install ./pre-install/ + +# Copy the package.json, yarn.lock, and output of web yarn space to leverage Docker cache +COPY --from=builder /app/uikit ./uikit/ +COPY --from=builder /app/web ./web/ +COPY --from=builder /app/models ./models/ + +RUN yarn workspace @janhq/uikit install && yarn workspace @janhq/uikit build +RUN yarn workspace jan-web install + +RUN npm install -g serve@latest -# RUN addgroup -g 1001 -S nodejs; -COPY --from=builder /app/server/build ./ +EXPOSE 1337 3000 3928 -# Automatically leverage output traces to reduce image size -# https://nextjs.org/docs/advanced-features/output-file-tracing -COPY --from=builder /app/server/node_modules ./node_modules -COPY --from=builder /app/server/package.json ./package.json +ENV JAN_API_HOST 0.0.0.0 +ENV JAN_API_PORT 1337 -EXPOSE 4000 3928 +ENV API_BASE_URL http://localhost:1337 -ENV PORT 4000 -ENV APPDATA /app/data +CMD ["sh", "-c", "export NODE_ENV=production && yarn workspace jan-web build && cd web && npx serve out & cd server && node build/main.js"] -CMD ["node", "main.js"] \ No newline at end of file +# docker build -t jan . +# docker run -p 1337:1337 -p 3000:3000 -p 3928:3928 jan diff --git a/Dockerfile.gpu b/Dockerfile.gpu new file mode 100644 index 0000000000..832e2c18c5 --- /dev/null +++ b/Dockerfile.gpu @@ -0,0 +1,88 @@ +# Please change the base image to the appropriate CUDA version base on NVIDIA Driver Compatibility +# Run nvidia-smi to check the CUDA version and the corresponding driver version +# Then update the base image to the appropriate CUDA version refer https://catalog.ngc.nvidia.com/orgs/nvidia/containers/cuda/tags + +FROM nvidia/cuda:12.2.0-runtime-ubuntu22.04 AS base + +# 1. Install dependencies only when needed +FROM base AS builder + +# Install g++ 11 +RUN apt update && apt install -y gcc-11 g++-11 cpp-11 jq xsel curl gnupg make python3-dev && curl -sL https://deb.nodesource.com/setup_20.x | bash - && apt install nodejs -y && rm -rf /var/lib/apt/lists/* + +# Update alternatives for GCC and related tools +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 110 \ + --slave /usr/bin/g++ g++ /usr/bin/g++-11 \ + --slave /usr/bin/gcov gcov /usr/bin/gcov-11 \ + --slave /usr/bin/gcc-ar gcc-ar /usr/bin/gcc-ar-11 \ + --slave /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-11 && \ + update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-11 110 + +RUN npm install -g yarn + +WORKDIR /app + +# Install dependencies based on the preferred package manager +COPY . ./ + +RUN export NITRO_VERSION=$(cat extensions/inference-nitro-extension/bin/version.txt) && \ + jq --arg nitroVersion $NITRO_VERSION '(.scripts."downloadnitro:linux" | gsub("\\${NITRO_VERSION}"; $nitroVersion)) | gsub("\r"; "")' extensions/inference-nitro-extension/package.json > /tmp/newcommand.txt && export NEW_COMMAND=$(sed 's/^"//;s/"$//' /tmp/newcommand.txt) && jq --arg newCommand "$NEW_COMMAND" '.scripts."downloadnitro:linux" = $newCommand' extensions/inference-nitro-extension/package.json > /tmp/package.json && mv /tmp/package.json extensions/inference-nitro-extension/package.json +RUN make install-and-build + +# # 2. Rebuild the source code only when needed +FROM base AS runner + +# Install g++ 11 +RUN apt update && apt install -y gcc-11 g++-11 cpp-11 jq xsel curl gnupg make python3-dev && curl -sL https://deb.nodesource.com/setup_20.x | bash - && apt-get install nodejs -y && rm -rf /var/lib/apt/lists/* + +# Update alternatives for GCC and related tools +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 110 \ + --slave /usr/bin/g++ g++ /usr/bin/g++-11 \ + --slave /usr/bin/gcov gcov /usr/bin/gcov-11 \ + --slave /usr/bin/gcc-ar gcc-ar /usr/bin/gcc-ar-11 \ + --slave /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-11 && \ + update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-11 110 + +RUN npm install -g yarn + +WORKDIR /app + +# Copy the package.json and yarn.lock of root yarn space to leverage Docker cache +COPY --from=builder /app/package.json ./package.json +COPY --from=builder /app/node_modules ./node_modules/ +COPY --from=builder /app/yarn.lock ./yarn.lock + +# Copy the package.json, yarn.lock, and build output of server yarn space to leverage Docker cache +COPY --from=builder /app/core ./core/ +COPY --from=builder /app/server ./server/ +RUN cd core && yarn install && yarn run build +RUN yarn workspace @janhq/server install && yarn workspace @janhq/server build +COPY --from=builder /app/docs/openapi ./docs/openapi/ + +# Copy pre-install dependencies +COPY --from=builder /app/pre-install ./pre-install/ + +# Copy the package.json, yarn.lock, and output of web yarn space to leverage Docker cache +COPY --from=builder /app/uikit ./uikit/ +COPY --from=builder /app/web ./web/ +COPY --from=builder /app/models ./models/ + +RUN yarn workspace @janhq/uikit install && yarn workspace @janhq/uikit build +RUN yarn workspace jan-web install + +RUN npm install -g serve@latest + +EXPOSE 1337 3000 3928 + +ENV LD_LIBRARY_PATH=/usr/local/cuda/targets/x86_64-linux/lib:/usr/local/cuda-12.0/compat${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} + +ENV JAN_API_HOST 0.0.0.0 +ENV JAN_API_PORT 1337 + +ENV API_BASE_URL http://localhost:1337 + +CMD ["sh", "-c", "export NODE_ENV=production && yarn workspace jan-web build && cd web && npx serve out & cd server && node build/main.js"] + +# pre-requisites: nvidia-docker +# docker build -t jan-gpu . -f Dockerfile.gpu +# docker run -p 1337:1337 -p 3000:3000 -p 3928:3928 --gpus all jan-gpu diff --git a/Makefile b/Makefile index 905a68321d..a45477b294 100644 --- a/Makefile +++ b/Makefile @@ -24,9 +24,9 @@ endif check-file-counts: install-and-build ifeq ($(OS),Windows_NT) - powershell -Command "if ((Get-ChildItem -Path electron/pre-install -Filter *.tgz | Measure-Object | Select-Object -ExpandProperty Count) -ne (Get-ChildItem -Path extensions -Directory | Measure-Object | Select-Object -ExpandProperty Count)) { Write-Host 'Number of .tgz files in electron/pre-install does not match the number of subdirectories in extension'; exit 1 } else { Write-Host 'Extension build successful' }" + powershell -Command "if ((Get-ChildItem -Path pre-install -Filter *.tgz | Measure-Object | Select-Object -ExpandProperty Count) -ne (Get-ChildItem -Path extensions -Directory | Measure-Object | Select-Object -ExpandProperty Count)) { Write-Host 'Number of .tgz files in pre-install does not match the number of subdirectories in extension'; exit 1 } else { Write-Host 'Extension build successful' }" else - @tgz_count=$$(find electron/pre-install -type f -name "*.tgz" | wc -l); dir_count=$$(find extensions -mindepth 1 -maxdepth 1 -type d | wc -l); if [ $$tgz_count -ne $$dir_count ]; then echo "Number of .tgz files in electron/pre-install ($$tgz_count) does not match the number of subdirectories in extension ($$dir_count)"; exit 1; else echo "Extension build successful"; fi + @tgz_count=$$(find pre-install -type f -name "*.tgz" | wc -l); dir_count=$$(find extensions -mindepth 1 -maxdepth 1 -type d | wc -l); if [ $$tgz_count -ne $$dir_count ]; then echo "Number of .tgz files in pre-install ($$tgz_count) does not match the number of subdirectories in extension ($$dir_count)"; exit 1; else echo "Extension build successful"; fi endif dev: check-file-counts @@ -52,18 +52,28 @@ build: check-file-counts clean: ifeq ($(OS),Windows_NT) - powershell -Command "Get-ChildItem -Path . -Include node_modules, .next, dist -Recurse -Directory | Remove-Item -Recurse -Force" + powershell -Command "Get-ChildItem -Path . -Include node_modules, .next, dist, build, out -Recurse -Directory | Remove-Item -Recurse -Force" + powershell -Command "Remove-Item -Recurse -Force ./pre-install/*.tgz" + powershell -Command "Remove-Item -Recurse -Force ./electron/pre-install/*.tgz" rmdir /s /q "%USERPROFILE%\jan\extensions" else ifeq ($(shell uname -s),Linux) find . -name "node_modules" -type d -prune -exec rm -rf '{}' + find . -name ".next" -type d -exec rm -rf '{}' + find . -name "dist" -type d -exec rm -rf '{}' + + find . -name "build" -type d -exec rm -rf '{}' + + find . -name "out" -type d -exec rm -rf '{}' + + rm -rf ./pre-install/*.tgz + rm -rf ./electron/pre-install/*.tgz rm -rf "~/jan/extensions" rm -rf "~/.cache/jan*" else find . -name "node_modules" -type d -prune -exec rm -rf '{}' + find . -name ".next" -type d -exec rm -rf '{}' + find . -name "dist" -type d -exec rm -rf '{}' + + find . -name "build" -type d -exec rm -rf '{}' + + find . -name "out" -type d -exec rm -rf '{}' + + rm -rf ./pre-install/*.tgz + rm -rf ./electron/pre-install/*.tgz rm -rf ~/jan/extensions rm -rf ~/Library/Caches/jan* endif diff --git a/README.md b/README.md index 34eecc9f35..715625080b 100644 --- a/README.md +++ b/README.md @@ -43,31 +43,31 @@ Jan is an open-source ChatGPT alternative that runs 100% offline on your compute Stable (Recommended) - + jan.exe - + Intel - + M1/M2 - + jan.deb - + jan.AppImage @@ -76,31 +76,31 @@ Jan is an open-source ChatGPT alternative that runs 100% offline on your compute Experimental (Nightly Build) - + jan.exe - + Intel - + M1/M2 - + jan.deb - + jan.AppImage @@ -167,6 +167,7 @@ To reset your installation: - Clear Application cache in `~/Library/Caches/jan` ## Requirements for running Jan + - MacOS: 13 or higher - Windows: - Windows 10 or higher @@ -194,17 +195,17 @@ Contributions are welcome! Please read the [CONTRIBUTING.md](CONTRIBUTING.md) fi 1. **Clone the repository and prepare:** - ```bash - git clone https://github.com/janhq/jan - cd jan - git checkout -b DESIRED_BRANCH - ``` + ```bash + git clone https://github.com/janhq/jan + cd jan + git checkout -b DESIRED_BRANCH + ``` 2. **Run development and use Jan Desktop** - ```bash - make dev - ``` + ```bash + make dev + ``` This will start the development server and open the desktop app. @@ -218,6 +219,101 @@ make build This will build the app MacOS m1/m2 for production (with code signing already done) and put the result in `dist` folder. +### Docker mode + +- Supported OS: Linux, WSL2 Docker +- Pre-requisites: + + - Docker Engine and Docker Compose are required to run Jan in Docker mode. Follow the [instructions](https://docs.docker.com/engine/install/ubuntu/) below to get started with Docker Engine on Ubuntu. + + ```bash + curl -fsSL https://get.docker.com -o get-docker.sh + sudo sh ./get-docker.sh --dry-run + ``` + + - If you intend to run Jan in GPU mode, you need to install `nvidia-driver` and `nvidia-docker2`. Follow the instruction [here](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html) for installation. + +- Run Jan in Docker mode + +| Docker compose Profile | Description | +| ---------------------- | -------------------------------------------- | +| `cpu-fs` | Run Jan in CPU mode with default file system | +| `cpu-s3fs` | Run Jan in CPU mode with S3 file system | +| `gpu-fs` | Run Jan in GPU mode with default file system | +| `gpu-s3fs` | Run Jan in GPU mode with S3 file system | + +| Environment Variable | Description | +| ----------------------- | ------------------------------------------------------------------------------------------------------- | +| `S3_BUCKET_NAME` | S3 bucket name - leave blank for default file system | +| `AWS_ACCESS_KEY_ID` | AWS access key ID - leave blank for default file system | +| `AWS_SECRET_ACCESS_KEY` | AWS secret access key - leave blank for default file system | +| `AWS_ENDPOINT` | AWS endpoint URL - leave blank for default file system | +| `AWS_REGION` | AWS region - leave blank for default file system | +| `API_BASE_URL` | Jan Server URL, please modify it as your public ip address or domain name default http://localhost:1377 | + +- **Option 1**: Run Jan in CPU mode + + ```bash + # cpu mode with default file system + docker compose --profile cpu-fs up -d + + # cpu mode with S3 file system + docker compose --profile cpu-s3fs up -d + ``` + +- **Option 2**: Run Jan in GPU mode + + - **Step 1**: Check CUDA compatibility with your NVIDIA driver by running `nvidia-smi` and check the CUDA version in the output + + ```bash + nvidia-smi + + # Output + +---------------------------------------------------------------------------------------+ + | NVIDIA-SMI 531.18 Driver Version: 531.18 CUDA Version: 12.1 | + |-----------------------------------------+----------------------+----------------------+ + | GPU Name TCC/WDDM | Bus-Id Disp.A | Volatile Uncorr. ECC | + | Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. | + | | | MIG M. | + |=========================================+======================+======================| + | 0 NVIDIA GeForce RTX 4070 Ti WDDM | 00000000:01:00.0 On | N/A | + | 0% 44C P8 16W / 285W| 1481MiB / 12282MiB | 2% Default | + | | | N/A | + +-----------------------------------------+----------------------+----------------------+ + | 1 NVIDIA GeForce GTX 1660 Ti WDDM | 00000000:02:00.0 Off | N/A | + | 0% 49C P8 14W / 120W| 0MiB / 6144MiB | 0% Default | + | | | N/A | + +-----------------------------------------+----------------------+----------------------+ + | 2 NVIDIA GeForce GTX 1660 Ti WDDM | 00000000:05:00.0 Off | N/A | + | 29% 38C P8 11W / 120W| 0MiB / 6144MiB | 0% Default | + | | | N/A | + +-----------------------------------------+----------------------+----------------------+ + + +---------------------------------------------------------------------------------------+ + | Processes: | + | GPU GI CI PID Type Process name GPU Memory | + | ID ID Usage | + |=======================================================================================| + ``` + + - **Step 2**: Visit [NVIDIA NGC Catalog ](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/cuda/tags) and find the smallest minor version of image tag that matches your CUDA version (e.g., 12.1 -> 12.1.0) + + - **Step 3**: Update the `Dockerfile.gpu` line number 5 with the latest minor version of the image tag from step 2 (e.g. change `FROM nvidia/cuda:12.2.0-runtime-ubuntu22.04 AS base` to `FROM nvidia/cuda:12.1.0-runtime-ubuntu22.04 AS base`) + + - **Step 4**: Run command to start Jan in GPU mode + + ```bash + # GPU mode with default file system + docker compose --profile gpu up -d + + # GPU mode with S3 file system + docker compose --profile gpu-s3fs up -d + ``` + +This will start the web server and you can access Jan at `http://localhost:3000`. + +> Note: RAG feature is not supported in Docker mode with s3fs yet. + ## Acknowledgements Jan builds on top of other open-source projects: diff --git a/charts/server/Chart.lock b/charts/server/Chart.lock new file mode 100644 index 0000000000..915788d617 --- /dev/null +++ b/charts/server/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: common + repository: oci://ghcr.io/janhq/charts + version: 0.1.2 +digest: sha256:35e98bde174130787755b0f8ea2359b7b6790d965a7157c2f7cabf1bc8c04471 +generated: "2024-02-20T16:20:37.6530108+07:00" diff --git a/charts/server/Chart.yaml b/charts/server/Chart.yaml new file mode 100644 index 0000000000..fb2e1c91bd --- /dev/null +++ b/charts/server/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v2 +name: jan-server +description: A Helm chart for Kubernetes +type: application +version: 0.1.0 +appVersion: '1.0.0' +dependencies: + - name: common + version: 0.1.2 # common-chart-version + repository: oci://ghcr.io/janhq/charts diff --git a/charts/server/charts/common-0.1.2.tgz b/charts/server/charts/common-0.1.2.tgz new file mode 100644 index 0000000000..946617eabb Binary files /dev/null and b/charts/server/charts/common-0.1.2.tgz differ diff --git a/charts/server/config.json b/charts/server/config.json new file mode 100644 index 0000000000..62e9682fa6 --- /dev/null +++ b/charts/server/config.json @@ -0,0 +1,4 @@ +{ + "image-list": "server=ghcr.io/janhq/jan-server", + "platforms": "linux/amd64" +} \ No newline at end of file diff --git a/charts/server/values.yaml b/charts/server/values.yaml new file mode 100644 index 0000000000..70f4631746 --- /dev/null +++ b/charts/server/values.yaml @@ -0,0 +1,256 @@ +common: + imageTag: v0.4.6-cpu + # DO NOT CHANGE THE LINE ABOVE. MAKE ALL CHANGES BELOW + + # Global pvc for all workload + pvc: + enabled: false + name: 'janroot' + accessModes: 'ReadWriteOnce' + storageClassName: '' + capacity: '50Gi' + + # Global image pull secret + imagePullSecrets: [] + + externalSecret: + create: false + name: '' + annotations: {} + + nameOverride: 'jan-server' + fullnameOverride: 'jan-server' + + serviceAccount: + create: true + annotations: {} + name: 'jan-server-service-account' + + podDisruptionBudget: + create: false + minAvailable: 1 + + workloads: + - name: server + image: + repository: ghcr.io/janhq/jan-server + pullPolicy: Always + + command: ['/bin/sh', '-c'] + args: ['cd server && node build/main.js'] + + replicaCount: 1 + ports: + containerPort: 1337 + + strategy: + canary: + steps: + - setWeight: 50 + - pause: { duration: 1m } + + ingress: + enabled: true + className: 'nginx' + annotations: + nginx.ingress.kubernetes.io/proxy-body-size: '100m' + nginx.ingress.kubernetes.io/proxy-read-timeout: '1800' + nginx.ingress.kubernetes.io/proxy-send-timeout: '1800' + # cert-manager.io/cluster-issuer: 'jan-ai-dns01-cluster-issuer' + # nginx.ingress.kubernetes.io/force-ssl-redirect: 'true' + nginx.ingress.kubernetes.io/backend-protocol: HTTP + hosts: + - host: server.local + paths: + - path: / + pathType: Prefix + tls: + [] + # - hosts: + # - server-dev.jan.ai + # secretName: jan-server-prod-tls-v2 + + instrumentation: + enabled: false + podAnnotations: {} + + podSecurityContext: {} + + securityContext: {} + + service: + extenalLabel: {} + type: ClusterIP + port: 1337 + targetPort: 1337 + + # If you want to use GPU, please uncomment the following lines and change imageTag to the one with GPU support + resources: + # limits: + # nvidia.com/gpu: 1 + requests: + cpu: 2000m + memory: 8192M + + # If you want to use pv, please uncomment the following lines and enable pvc.enabled + volumes: + [] + # - name: janroot + # persistentVolumeClaim: + # claimName: janroot + + volumeMounts: + [] + # - name: janroot + # mountPath: /app/server/build/jan + + # AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, S3_BUCKET_NAME, AWS_ENDPOINT, AWS_REGION should mount as a secret env instead of plain text here + # Change API_BASE_URL to your server's public domain + env: + - name: API_BASE_URL + value: 'http://server.local' + + lifecycle: {} + autoscaling: + enabled: false + minReplicas: 2 + maxReplicas: 3 + targetCPUUtilizationPercentage: 95 + targetMemoryUtilizationPercentage: 95 + + kedaScaling: + enabled: false # ignore if autoscaling.enable = true + cooldownPeriod: 30 + pollingInterval: 2 + minReplicas: 1 + maxReplicas: 5 + metricName: celery_queue_length + query: celery_queue_length{queue_name="myqueue"} # change queue_name here + serverAddress: http://prometheus-prod-kube-prome-prometheus.monitoring.svc:9090 + threshold: '3' + + nodeSelector: {} + + tolerations: [] + + podSecurityGroup: + enabled: false + securitygroupid: [] + + # Reloader Option + reloader: 'false' + vpa: + enabled: false + + - name: web + image: + repository: ghcr.io/janhq/jan-server + pullPolicy: Always + + command: ['/bin/sh', '-c'] + args: + [ + 'export NODE_ENV=production && yarn workspace jan-web build && cd web && npx serve out', + ] + + replicaCount: 1 + ports: + containerPort: 3000 + + strategy: + canary: + steps: + - setWeight: 50 + - pause: { duration: 1m } + + ingress: + enabled: true + className: 'nginx' + annotations: + nginx.ingress.kubernetes.io/proxy-body-size: '100m' + nginx.ingress.kubernetes.io/proxy-read-timeout: '1800' + nginx.ingress.kubernetes.io/proxy-send-timeout: '1800' + # cert-manager.io/cluster-issuer: 'jan-ai-dns01-cluster-issuer' + # nginx.ingress.kubernetes.io/force-ssl-redirect: 'true' + nginx.ingress.kubernetes.io/backend-protocol: HTTP + hosts: + - host: web.local + paths: + - path: / + pathType: Prefix + tls: + [] + # - hosts: + # - server-dev.jan.ai + # secretName: jan-server-prod-tls-v2 + + instrumentation: + enabled: false + podAnnotations: {} + + podSecurityContext: {} + + securityContext: {} + + service: + extenalLabel: {} + type: ClusterIP + port: 3000 + targetPort: 3000 + + resources: + limits: + cpu: 1000m + memory: 2048M + requests: + cpu: 50m + memory: 500M + + volumes: + [] + # - name: janroot + # persistentVolumeClaim: + # claimName: janroot + + volumeMounts: + [] + # - name: janroot + # mountPath: /app/server/build/jan + + # AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, S3_BUCKET_NAME, AWS_ENDPOINT, AWS_REGION should mount as a secret env instead of plain text here + # Change API_BASE_URL to your server's public domain + env: + - name: API_BASE_URL + value: 'http://server.local' + + lifecycle: {} + autoscaling: + enabled: true + minReplicas: 1 + maxReplicas: 3 + targetCPUUtilizationPercentage: 95 + targetMemoryUtilizationPercentage: 95 + + kedaScaling: + enabled: false # ignore if autoscaling.enable = true + cooldownPeriod: 30 + pollingInterval: 2 + minReplicas: 1 + maxReplicas: 5 + metricName: celery_queue_length + query: celery_queue_length{queue_name="myqueue"} # change queue_name here + serverAddress: http://prometheus-prod-kube-prome-prometheus.monitoring.svc:9090 + threshold: '3' + + nodeSelector: {} + + tolerations: [] + + podSecurityGroup: + enabled: false + securitygroupid: [] + + # Reloader Option + reloader: 'false' + vpa: + enabled: false diff --git a/core/jest.config.js b/core/jest.config.js index fb03768fec..c18f550916 100644 --- a/core/jest.config.js +++ b/core/jest.config.js @@ -4,4 +4,4 @@ module.exports = { moduleNameMapper: { '@/(.*)': '/src/$1', }, -} \ No newline at end of file +} diff --git a/core/package.json b/core/package.json index 437e6d0a61..c3abe2d568 100644 --- a/core/package.json +++ b/core/package.json @@ -57,6 +57,7 @@ "rollup-plugin-typescript2": "^0.36.0", "ts-jest": "^26.1.1", "tslib": "^2.6.2", - "typescript": "^5.2.2" + "typescript": "^5.2.2", + "rimraf": "^3.0.2" } } diff --git a/core/rollup.config.ts b/core/rollup.config.ts index d78130a4de..ebea8e2377 100644 --- a/core/rollup.config.ts +++ b/core/rollup.config.ts @@ -54,7 +54,8 @@ export default [ 'url', 'http', 'os', - 'util' + 'util', + 'child_process', ], watch: { include: 'src/node/**', diff --git a/core/src/api/index.ts b/core/src/api/index.ts index 0d7cc51f75..6760207580 100644 --- a/core/src/api/index.ts +++ b/core/src/api/index.ts @@ -1,15 +1,22 @@ /** - * App Route APIs + * Native Route APIs * @description Enum of all the routes exposed by the app */ -export enum AppRoute { +export enum NativeRoute { openExternalUrl = 'openExternalUrl', openAppDirectory = 'openAppDirectory', openFileExplore = 'openFileExplorer', selectDirectory = 'selectDirectory', + relaunch = 'relaunch', +} + +/** + * App Route APIs + * @description Enum of all the routes exposed by the app + */ +export enum AppRoute { getAppConfigurations = 'getAppConfigurations', updateAppConfiguration = 'updateAppConfiguration', - relaunch = 'relaunch', joinPath = 'joinPath', isSubdirectory = 'isSubdirectory', baseName = 'baseName', @@ -30,6 +37,7 @@ export enum DownloadRoute { downloadFile = 'downloadFile', pauseDownload = 'pauseDownload', resumeDownload = 'resumeDownload', + getDownloadProgress = 'getDownloadProgress', } export enum DownloadEvent { @@ -68,6 +76,10 @@ export enum FileManagerRoute { export type ApiFunction = (...args: any[]) => any +export type NativeRouteFunctions = { + [K in NativeRoute]: ApiFunction +} + export type AppRouteFunctions = { [K in AppRoute]: ApiFunction } @@ -96,7 +108,8 @@ export type FileManagerRouteFunctions = { [K in FileManagerRoute]: ApiFunction } -export type APIFunctions = AppRouteFunctions & +export type APIFunctions = NativeRouteFunctions & + AppRouteFunctions & AppEventFunctions & DownloadRouteFunctions & DownloadEventFunctions & @@ -104,11 +117,13 @@ export type APIFunctions = AppRouteFunctions & FileSystemRouteFunctions & FileManagerRoute -export const APIRoutes = [ +export const CoreRoutes = [ ...Object.values(AppRoute), ...Object.values(DownloadRoute), ...Object.values(ExtensionRoute), ...Object.values(FileSystemRoute), ...Object.values(FileManagerRoute), ] + +export const APIRoutes = [...CoreRoutes, ...Object.values(NativeRoute)] export const APIEvents = [...Object.values(AppEvent), ...Object.values(DownloadEvent)] diff --git a/core/src/extension.ts b/core/src/extension.ts index 0b7f9b7fc1..3528f581cc 100644 --- a/core/src/extension.ts +++ b/core/src/extension.ts @@ -1,13 +1,13 @@ export enum ExtensionTypeEnum { - Assistant = "assistant", - Conversational = "conversational", - Inference = "inference", - Model = "model", - SystemMonitoring = "systemMonitoring", + Assistant = 'assistant', + Conversational = 'conversational', + Inference = 'inference', + Model = 'model', + SystemMonitoring = 'systemMonitoring', } export interface ExtensionType { - type(): ExtensionTypeEnum | undefined; + type(): ExtensionTypeEnum | undefined } /** * Represents a base extension. @@ -20,16 +20,16 @@ export abstract class BaseExtension implements ExtensionType { * Undefined means its not extending any known extension by the application. */ type(): ExtensionTypeEnum | undefined { - return undefined; + return undefined } /** * Called when the extension is loaded. * Any initialization logic for the extension should be put here. */ - abstract onLoad(): void; + abstract onLoad(): void /** * Called when the extension is unloaded. * Any cleanup logic for the extension should be put here. */ - abstract onUnload(): void; + abstract onUnload(): void } diff --git a/core/src/extensions/assistant.ts b/core/src/extensions/assistant.ts index ba345711ae..5c3114f41b 100644 --- a/core/src/extensions/assistant.ts +++ b/core/src/extensions/assistant.ts @@ -1,5 +1,5 @@ -import { Assistant, AssistantInterface } from "../index"; -import { BaseExtension, ExtensionTypeEnum } from "../extension"; +import { Assistant, AssistantInterface } from '../index' +import { BaseExtension, ExtensionTypeEnum } from '../extension' /** * Assistant extension for managing assistants. @@ -10,10 +10,10 @@ export abstract class AssistantExtension extends BaseExtension implements Assist * Assistant extension type. */ type(): ExtensionTypeEnum | undefined { - return ExtensionTypeEnum.Assistant; + return ExtensionTypeEnum.Assistant } - abstract createAssistant(assistant: Assistant): Promise; - abstract deleteAssistant(assistant: Assistant): Promise; - abstract getAssistants(): Promise; + abstract createAssistant(assistant: Assistant): Promise + abstract deleteAssistant(assistant: Assistant): Promise + abstract getAssistants(): Promise } diff --git a/core/src/extensions/conversational.ts b/core/src/extensions/conversational.ts index 4319784c35..a49a4e6895 100644 --- a/core/src/extensions/conversational.ts +++ b/core/src/extensions/conversational.ts @@ -14,7 +14,7 @@ export abstract class ConversationalExtension * Conversation extension type. */ type(): ExtensionTypeEnum | undefined { - return ExtensionTypeEnum.Conversational; + return ExtensionTypeEnum.Conversational } abstract getThreads(): Promise diff --git a/core/src/extensions/index.ts b/core/src/extensions/index.ts index 1796c16187..5223345489 100644 --- a/core/src/extensions/index.ts +++ b/core/src/extensions/index.ts @@ -2,24 +2,24 @@ * Conversational extension. Persists and retrieves conversations. * @module */ -export { ConversationalExtension } from "./conversational"; +export { ConversationalExtension } from './conversational' /** * Inference extension. Start, stop and inference models. */ -export { InferenceExtension } from "./inference"; +export { InferenceExtension } from './inference' /** * Monitoring extension for system monitoring. */ -export { MonitoringExtension } from "./monitoring"; +export { MonitoringExtension } from './monitoring' /** * Assistant extension for managing assistants. */ -export { AssistantExtension } from "./assistant"; +export { AssistantExtension } from './assistant' /** * Model extension for managing models. */ -export { ModelExtension } from "./model"; +export { ModelExtension } from './model' diff --git a/core/src/extensions/inference.ts b/core/src/extensions/inference.ts index c551d108f5..e8e51f9eb9 100644 --- a/core/src/extensions/inference.ts +++ b/core/src/extensions/inference.ts @@ -1,5 +1,5 @@ -import { InferenceInterface, MessageRequest, ThreadMessage } from "../index"; -import { BaseExtension, ExtensionTypeEnum } from "../extension"; +import { InferenceInterface, MessageRequest, ThreadMessage } from '../index' +import { BaseExtension, ExtensionTypeEnum } from '../extension' /** * Inference extension. Start, stop and inference models. @@ -9,8 +9,8 @@ export abstract class InferenceExtension extends BaseExtension implements Infere * Inference extension type. */ type(): ExtensionTypeEnum | undefined { - return ExtensionTypeEnum.Inference; + return ExtensionTypeEnum.Inference } - abstract inference(data: MessageRequest): Promise; + abstract inference(data: MessageRequest): Promise } diff --git a/core/src/extensions/model.ts b/core/src/extensions/model.ts index 30aa5b6ba2..df7d14f421 100644 --- a/core/src/extensions/model.ts +++ b/core/src/extensions/model.ts @@ -1,5 +1,5 @@ -import { BaseExtension, ExtensionTypeEnum } from "../extension"; -import { Model, ModelInterface } from "../index"; +import { BaseExtension, ExtensionTypeEnum } from '../extension' +import { Model, ModelInterface } from '../index' /** * Model extension for managing models. @@ -9,16 +9,16 @@ export abstract class ModelExtension extends BaseExtension implements ModelInter * Model extension type. */ type(): ExtensionTypeEnum | undefined { - return ExtensionTypeEnum.Model; + return ExtensionTypeEnum.Model } abstract downloadModel( model: Model, - network?: { proxy: string; ignoreSSL?: boolean }, - ): Promise; - abstract cancelModelDownload(modelId: string): Promise; - abstract deleteModel(modelId: string): Promise; - abstract saveModel(model: Model): Promise; - abstract getDownloadedModels(): Promise; - abstract getConfiguredModels(): Promise; + network?: { proxy: string; ignoreSSL?: boolean } + ): Promise + abstract cancelModelDownload(modelId: string): Promise + abstract deleteModel(modelId: string): Promise + abstract saveModel(model: Model): Promise + abstract getDownloadedModels(): Promise + abstract getConfiguredModels(): Promise } diff --git a/core/src/extensions/monitoring.ts b/core/src/extensions/monitoring.ts index 2de9b9ae56..ba193f0f4d 100644 --- a/core/src/extensions/monitoring.ts +++ b/core/src/extensions/monitoring.ts @@ -1,5 +1,5 @@ -import { BaseExtension, ExtensionTypeEnum } from "../extension"; -import { MonitoringInterface } from "../index"; +import { BaseExtension, ExtensionTypeEnum } from '../extension' +import { MonitoringInterface } from '../index' /** * Monitoring extension for system monitoring. @@ -10,9 +10,9 @@ export abstract class MonitoringExtension extends BaseExtension implements Monit * Monitoring extension type. */ type(): ExtensionTypeEnum | undefined { - return ExtensionTypeEnum.SystemMonitoring; + return ExtensionTypeEnum.SystemMonitoring } - abstract getResourcesInfo(): Promise; - abstract getCurrentLoad(): Promise; + abstract getResourcesInfo(): Promise + abstract getCurrentLoad(): Promise } diff --git a/core/src/index.ts b/core/src/index.ts index a56b6f0e13..3505797b19 100644 --- a/core/src/index.ts +++ b/core/src/index.ts @@ -38,3 +38,10 @@ export * from './extension' * @module */ export * from './extensions/index' + +/** + * Declare global object + */ +declare global { + var core: any | undefined +} diff --git a/core/src/node/api/common/adapter.ts b/core/src/node/api/common/adapter.ts new file mode 100644 index 0000000000..56f4cedb35 --- /dev/null +++ b/core/src/node/api/common/adapter.ts @@ -0,0 +1,43 @@ +import { + AppRoute, + DownloadRoute, + ExtensionRoute, + FileManagerRoute, + FileSystemRoute, +} from '../../../api' +import { Downloader } from '../processors/download' +import { FileSystem } from '../processors/fs' +import { Extension } from '../processors/extension' +import { FSExt } from '../processors/fsExt' +import { App } from '../processors/app' + +export class RequestAdapter { + downloader: Downloader + fileSystem: FileSystem + extension: Extension + fsExt: FSExt + app: App + + constructor(observer?: Function) { + this.downloader = new Downloader(observer) + this.fileSystem = new FileSystem() + this.extension = new Extension() + this.fsExt = new FSExt() + this.app = new App() + } + + // TODO: Clearer Factory pattern here + process(route: string, ...args: any) { + if (route in DownloadRoute) { + return this.downloader.process(route, ...args) + } else if (route in FileSystemRoute) { + return this.fileSystem.process(route, ...args) + } else if (route in ExtensionRoute) { + return this.extension.process(route, ...args) + } else if (route in FileManagerRoute) { + return this.fsExt.process(route, ...args) + } else if (route in AppRoute) { + return this.app.process(route, ...args) + } + } +} diff --git a/core/src/node/api/common/handler.ts b/core/src/node/api/common/handler.ts new file mode 100644 index 0000000000..4a39ae52a6 --- /dev/null +++ b/core/src/node/api/common/handler.ts @@ -0,0 +1,23 @@ +import { CoreRoutes } from '../../../api' +import { RequestAdapter } from './adapter' + +export type Handler = (route: string, args: any) => any + +export class RequestHandler { + handler: Handler + adataper: RequestAdapter + + constructor(handler: Handler, observer?: Function) { + this.handler = handler + this.adataper = new RequestAdapter(observer) + } + + handle() { + CoreRoutes.map((route) => { + this.handler(route, async (...args: any[]) => { + const values = await this.adataper.process(route, ...args) + return values + }) + }) + } +} diff --git a/core/src/node/api/index.ts b/core/src/node/api/index.ts index 4c3041ba3f..ab0c516569 100644 --- a/core/src/node/api/index.ts +++ b/core/src/node/api/index.ts @@ -1,2 +1,3 @@ export * from './HttpServer' -export * from './routes' +export * from './restful/v1' +export * from './common/handler' diff --git a/core/src/node/api/processors/Processor.ts b/core/src/node/api/processors/Processor.ts new file mode 100644 index 0000000000..8ef0c6e191 --- /dev/null +++ b/core/src/node/api/processors/Processor.ts @@ -0,0 +1,3 @@ +export abstract class Processor { + abstract process(key: string, ...args: any[]): any +} diff --git a/core/src/node/api/processors/app.ts b/core/src/node/api/processors/app.ts new file mode 100644 index 0000000000..c62b5011d8 --- /dev/null +++ b/core/src/node/api/processors/app.ts @@ -0,0 +1,96 @@ +import { basename, isAbsolute, join, relative } from 'path' + +import { Processor } from './Processor' +import { getAppConfigurations as appConfiguration, updateAppConfiguration } from '../../helper' +import { log as writeLog, logServer as writeServerLog } from '../../helper/log' +import { appResourcePath } from '../../helper/path' + +export class App implements Processor { + observer?: Function + + constructor(observer?: Function) { + this.observer = observer + } + + process(key: string, ...args: any[]): any { + const instance = this as any + const func = instance[key] + return func(...args) + } + + /** + * Joins multiple paths together, respect to the current OS. + */ + joinPath(args: any[]) { + return join(...args) + } + + /** + * Checks if the given path is a subdirectory of the given directory. + * + * @param _event - The IPC event object. + * @param from - The path to check. + * @param to - The directory to check against. + * + * @returns {Promise} - A promise that resolves with the result. + */ + isSubdirectory(from: any, to: any) { + const rel = relative(from, to) + const isSubdir = rel && !rel.startsWith('..') && !isAbsolute(rel) + + if (isSubdir === '') return false + else return isSubdir + } + + /** + * Retrieve basename from given path, respect to the current OS. + */ + baseName(args: any) { + return basename(args) + } + + /** + * Log message to log file. + */ + log(args: any) { + writeLog(args) + } + + /** + * Log message to log file. + */ + logServer(args: any) { + writeServerLog(args) + } + + getAppConfigurations() { + return appConfiguration() + } + + async updateAppConfiguration(args: any) { + await updateAppConfiguration(args) + } + + /** + * Start Jan API Server. + */ + async startServer(args?: any) { + const { startServer } = require('@janhq/server') + return startServer({ + host: args?.host, + port: args?.port, + isCorsEnabled: args?.isCorsEnabled, + isVerboseEnabled: args?.isVerboseEnabled, + schemaPath: join(await appResourcePath(), 'docs', 'openapi', 'jan.yaml'), + baseDir: join(await appResourcePath(), 'docs', 'openapi'), + }) + } + + /** + * Stop Jan API Server. + */ + stopServer() { + const { stopServer } = require('@janhq/server') + return stopServer() + } +} diff --git a/core/src/node/api/processors/download.ts b/core/src/node/api/processors/download.ts new file mode 100644 index 0000000000..686ba58a1e --- /dev/null +++ b/core/src/node/api/processors/download.ts @@ -0,0 +1,106 @@ +import { resolve, sep } from 'path' +import { DownloadEvent } from '../../../api' +import { normalizeFilePath } from '../../helper/path' +import { getJanDataFolderPath } from '../../helper' +import { DownloadManager } from '../../helper/download' +import { createWriteStream, renameSync } from 'fs' +import { Processor } from './Processor' +import { DownloadState } from '../../../types' + +export class Downloader implements Processor { + observer?: Function + + constructor(observer?: Function) { + this.observer = observer + } + + process(key: string, ...args: any[]): any { + const instance = this as any + const func = instance[key] + return func(this.observer, ...args) + } + + downloadFile(observer: any, url: string, localPath: string, network: any) { + const request = require('request') + const progress = require('request-progress') + + const strictSSL = !network?.ignoreSSL + const proxy = network?.proxy?.startsWith('http') ? network.proxy : undefined + if (typeof localPath === 'string') { + localPath = normalizeFilePath(localPath) + } + const array = localPath.split(sep) + const fileName = array.pop() ?? '' + const modelId = array.pop() ?? '' + + const destination = resolve(getJanDataFolderPath(), localPath) + const rq = request({ url, strictSSL, proxy }) + + // Put request to download manager instance + DownloadManager.instance.setRequest(localPath, rq) + + // Downloading file to a temp file first + const downloadingTempFile = `${destination}.download` + + progress(rq, {}) + .on('progress', (state: any) => { + const downloadState: DownloadState = { + ...state, + modelId, + fileName, + downloadState: 'downloading', + } + console.log('progress: ', downloadState) + observer?.(DownloadEvent.onFileDownloadUpdate, downloadState) + DownloadManager.instance.downloadProgressMap[modelId] = downloadState + }) + .on('error', (error: Error) => { + const currentDownloadState = DownloadManager.instance.downloadProgressMap[modelId] + const downloadState: DownloadState = { + ...currentDownloadState, + error: error.message, + downloadState: 'error', + } + if (currentDownloadState) { + DownloadManager.instance.downloadProgressMap[modelId] = downloadState + } + + observer?.(DownloadEvent.onFileDownloadError, downloadState) + }) + .on('end', () => { + const currentDownloadState = DownloadManager.instance.downloadProgressMap[modelId] + if (currentDownloadState && DownloadManager.instance.networkRequests[localPath]) { + // Finished downloading, rename temp file to actual file + renameSync(downloadingTempFile, destination) + const downloadState: DownloadState = { + ...currentDownloadState, + downloadState: 'end', + } + observer?.(DownloadEvent.onFileDownloadSuccess, downloadState) + DownloadManager.instance.downloadProgressMap[modelId] = downloadState + } + }) + .pipe(createWriteStream(downloadingTempFile)) + } + + abortDownload(observer: any, fileName: string) { + const rq = DownloadManager.instance.networkRequests[fileName] + if (rq) { + DownloadManager.instance.networkRequests[fileName] = undefined + rq?.abort() + } else { + observer?.(DownloadEvent.onFileDownloadError, { + fileName, + error: 'aborted', + }) + } + } + + resumeDownload(observer: any, fileName: any) { + DownloadManager.instance.networkRequests[fileName]?.resume() + } + + pauseDownload(observer: any, fileName: any) { + DownloadManager.instance.networkRequests[fileName]?.pause() + } +} diff --git a/core/src/node/api/processors/extension.ts b/core/src/node/api/processors/extension.ts new file mode 100644 index 0000000000..df5d2d945c --- /dev/null +++ b/core/src/node/api/processors/extension.ts @@ -0,0 +1,88 @@ +import { readdirSync } from 'fs' +import { join, extname } from 'path' + +import { Processor } from './Processor' +import { ModuleManager } from '../../helper/module' +import { getJanExtensionsPath as getPath } from '../../helper' +import { + getActiveExtensions as getExtensions, + getExtension, + removeExtension, + installExtensions, +} from '../../extension/store' +import { appResourcePath } from '../../helper/path' + +export class Extension implements Processor { + observer?: Function + + constructor(observer?: Function) { + this.observer = observer + } + + process(key: string, ...args: any[]): any { + const instance = this as any + const func = instance[key] + return func(...args) + } + + invokeExtensionFunc(modulePath: string, method: string, ...params: any[]) { + const module = require(join(getPath(), modulePath)) + ModuleManager.instance.setModule(modulePath, module) + + if (typeof module[method] === 'function') { + return module[method](...params) + } else { + console.debug(module[method]) + console.error(`Function "${method}" does not exist in the module.`) + } + } + + /** + * Returns the paths of the base extensions. + * @returns An array of paths to the base extensions. + */ + async baseExtensions() { + const baseExtensionPath = join(await appResourcePath(), 'pre-install') + return readdirSync(baseExtensionPath) + .filter((file) => extname(file) === '.tgz') + .map((file) => join(baseExtensionPath, file)) + } + + /**MARK: Extension Manager handlers */ + async installExtension(extensions: any) { + // Install and activate all provided extensions + const installed = await installExtensions(extensions) + return JSON.parse(JSON.stringify(installed)) + } + + // Register IPC route to uninstall a extension + async uninstallExtension(extensions: any) { + // Uninstall all provided extensions + for (const ext of extensions) { + const extension = getExtension(ext) + await extension.uninstall() + if (extension.name) removeExtension(extension.name) + } + + // Reload all renderer pages if needed + return true + } + + // Register IPC route to update a extension + async updateExtension(extensions: any) { + // Update all provided extensions + const updated: any[] = [] + for (const ext of extensions) { + const extension = getExtension(ext) + const res = await extension.update() + if (res) updated.push(extension) + } + + // Reload all renderer pages if needed + return JSON.parse(JSON.stringify(updated)) + } + + getActiveExtensions() { + return JSON.parse(JSON.stringify(getExtensions())) + } +} diff --git a/core/src/node/api/processors/fs.ts b/core/src/node/api/processors/fs.ts new file mode 100644 index 0000000000..93a5f19057 --- /dev/null +++ b/core/src/node/api/processors/fs.ts @@ -0,0 +1,25 @@ +import { join } from 'path' +import { normalizeFilePath } from '../../helper/path' +import { getJanDataFolderPath } from '../../helper' +import { Processor } from './Processor' + +export class FileSystem implements Processor { + observer?: Function + private static moduleName = 'fs' + + constructor(observer?: Function) { + this.observer = observer + } + + process(route: string, ...args: any[]): any { + return import(FileSystem.moduleName).then((mdl) => + mdl[route]( + ...args.map((arg: any) => + typeof arg === 'string' && (arg.startsWith(`file:/`) || arg.startsWith(`file:\\`)) + ? join(getJanDataFolderPath(), normalizeFilePath(arg)) + : arg + ) + ) + ) + } +} diff --git a/core/src/node/api/processors/fsExt.ts b/core/src/node/api/processors/fsExt.ts new file mode 100644 index 0000000000..71e07ae57b --- /dev/null +++ b/core/src/node/api/processors/fsExt.ts @@ -0,0 +1,78 @@ +import { join } from 'path' +import fs from 'fs' +import { FileManagerRoute } from '../../../api' +import { appResourcePath, normalizeFilePath } from '../../helper/path' +import { getJanDataFolderPath, getJanDataFolderPath as getPath } from '../../helper' +import { Processor } from './Processor' +import { FileStat } from '../../../types' + +export class FSExt implements Processor { + observer?: Function + + constructor(observer?: Function) { + this.observer = observer + } + + process(key: string, ...args: any): any { + const instance = this as any + const func = instance[key] + return func(...args) + } + + // Handles the 'syncFile' IPC event. This event is triggered to synchronize a file from a source path to a destination path. + syncFile(src: string, dest: string) { + const reflect = require('@alumna/reflect') + return reflect({ + src, + dest, + recursive: true, + delete: false, + overwrite: true, + errorOnExist: false, + }) + } + + // Handles the 'getJanDataFolderPath' IPC event. This event is triggered to get the user space path. + getJanDataFolderPath() { + return Promise.resolve(getPath()) + } + + // Handles the 'getResourcePath' IPC event. This event is triggered to get the resource path. + getResourcePath() { + return appResourcePath() + } + + // Handles the 'getUserHomePath' IPC event. This event is triggered to get the user home path. + getUserHomePath() { + return process.env[process.platform == 'win32' ? 'USERPROFILE' : 'HOME'] + } + + // handle fs is directory here + fileStat(path: string) { + const normalizedPath = normalizeFilePath(path) + + const fullPath = join(getJanDataFolderPath(), normalizedPath) + const isExist = fs.existsSync(fullPath) + if (!isExist) return undefined + + const isDirectory = fs.lstatSync(fullPath).isDirectory() + const size = fs.statSync(fullPath).size + + const fileStat: FileStat = { + isDirectory, + size, + } + + return fileStat + } + + writeBlob(path: string, data: any) { + try { + const normalizedPath = normalizeFilePath(path) + const dataBuffer = Buffer.from(data, 'base64') + fs.writeFileSync(join(getJanDataFolderPath(), normalizedPath), dataBuffer) + } catch (err) { + console.error(`writeFile ${path} result: ${err}`) + } + } +} diff --git a/core/src/node/api/restful/app/download.ts b/core/src/node/api/restful/app/download.ts new file mode 100644 index 0000000000..b5919659b1 --- /dev/null +++ b/core/src/node/api/restful/app/download.ts @@ -0,0 +1,23 @@ +import { DownloadRoute } from '../../../../api' +import { DownloadManager } from '../../../helper/download' +import { HttpServer } from '../../HttpServer' + +export const downloadRouter = async (app: HttpServer) => { + app.get(`/download/${DownloadRoute.getDownloadProgress}/:modelId`, async (req, res) => { + const modelId = req.params.modelId + + console.debug(`Getting download progress for model ${modelId}`) + console.debug( + `All Download progress: ${JSON.stringify(DownloadManager.instance.downloadProgressMap)}` + ) + + // check if null DownloadManager.instance.downloadProgressMap + if (!DownloadManager.instance.downloadProgressMap[modelId]) { + return res.status(404).send({ + message: 'Download progress not found', + }) + } else { + return res.status(200).send(DownloadManager.instance.downloadProgressMap[modelId]) + } + }) +} diff --git a/core/src/node/api/restful/app/handlers.ts b/core/src/node/api/restful/app/handlers.ts new file mode 100644 index 0000000000..43c3f7add9 --- /dev/null +++ b/core/src/node/api/restful/app/handlers.ts @@ -0,0 +1,13 @@ +import { HttpServer } from '../../HttpServer' +import { Handler, RequestHandler } from '../../common/handler' + +export function handleRequests(app: HttpServer) { + const restWrapper: Handler = (route: string, listener: (...args: any[]) => any) => { + app.post(`/app/${route}`, async (request: any, reply: any) => { + const args = JSON.parse(request.body) as any[] + reply.send(JSON.stringify(await listener(...args))) + }) + } + const handler = new RequestHandler(restWrapper) + handler.handle() +} diff --git a/core/src/node/api/routes/common.ts b/core/src/node/api/restful/common.ts similarity index 53% rename from core/src/node/api/routes/common.ts rename to core/src/node/api/restful/common.ts index 27385e5619..4336329890 100644 --- a/core/src/node/api/routes/common.ts +++ b/core/src/node/api/restful/common.ts @@ -1,22 +1,34 @@ -import { AppRoute } from '../../../api' import { HttpServer } from '../HttpServer' -import { basename, join } from 'path' import { chatCompletions, deleteBuilder, downloadModel, getBuilder, retrieveBuilder, -} from '../common/builder' + createMessage, + createThread, + getMessages, + retrieveMessage, + updateThread, +} from './helper/builder' -import { JanApiRouteConfiguration } from '../common/configuration' -import { startModel, stopModel } from '../common/startStopModel' +import { JanApiRouteConfiguration } from './helper/configuration' +import { startModel, stopModel } from './helper/startStopModel' import { ModelSettingParams } from '../../../types' export const commonRouter = async (app: HttpServer) => { + const normalizeData = (data: any) => { + return { + object: 'list', + data, + } + } // Common Routes + // Read & Delete :: Threads | Models | Assistants Object.keys(JanApiRouteConfiguration).forEach((key) => { - app.get(`/${key}`, async (_request) => getBuilder(JanApiRouteConfiguration[key])) + app.get(`/${key}`, async (_request) => + getBuilder(JanApiRouteConfiguration[key]).then(normalizeData) + ) app.get(`/${key}/:id`, async (request: any) => retrieveBuilder(JanApiRouteConfiguration[key], request.params.id) @@ -27,7 +39,26 @@ export const commonRouter = async (app: HttpServer) => { ) }) - // Download Model Routes + // Threads + app.post(`/threads/`, async (req, res) => createThread(req.body)) + + app.get(`/threads/:threadId/messages`, async (req, res) => + getMessages(req.params.threadId).then(normalizeData) + ) + + app.get(`/threads/:threadId/messages/:messageId`, async (req, res) => + retrieveMessage(req.params.threadId, req.params.messageId) + ) + + app.post(`/threads/:threadId/messages`, async (req, res) => + createMessage(req.params.threadId as any, req.body as any) + ) + + app.patch(`/threads/:threadId`, async (request: any) => + updateThread(request.params.threadId, request.body) + ) + + // Models app.get(`/models/download/:modelId`, async (request: any) => downloadModel(request.params.modelId, { ignoreSSL: request.query.ignoreSSL === 'true', @@ -46,17 +77,6 @@ export const commonRouter = async (app: HttpServer) => { app.put(`/models/:modelId/stop`, async (request: any) => stopModel(request.params.modelId)) - // Chat Completion Routes + // Chat Completion app.post(`/chat/completions`, async (request: any, reply: any) => chatCompletions(request, reply)) - - // App Routes - app.post(`/app/${AppRoute.joinPath}`, async (request: any, reply: any) => { - const args = JSON.parse(request.body) as any[] - reply.send(JSON.stringify(join(...args[0]))) - }) - - app.post(`/app/${AppRoute.baseName}`, async (request: any, reply: any) => { - const args = JSON.parse(request.body) as any[] - reply.send(JSON.stringify(basename(args[0]))) - }) } diff --git a/core/src/node/api/common/builder.ts b/core/src/node/api/restful/helper/builder.ts similarity index 96% rename from core/src/node/api/common/builder.ts rename to core/src/node/api/restful/helper/builder.ts index 5c99cf4d8e..7001c0c769 100644 --- a/core/src/node/api/common/builder.ts +++ b/core/src/node/api/restful/helper/builder.ts @@ -1,10 +1,11 @@ import fs from 'fs' import { JanApiRouteConfiguration, RouteConfiguration } from './configuration' import { join } from 'path' -import { ContentType, MessageStatus, Model, ThreadMessage } from './../../../index' -import { getEngineConfiguration, getJanDataFolderPath } from '../../utils' +import { ContentType, MessageStatus, Model, ThreadMessage } from '../../../../index' +import { getEngineConfiguration, getJanDataFolderPath } from '../../../helper' import { DEFAULT_CHAT_COMPLETION_URL } from './consts' +// TODO: Refactor these export const getBuilder = async (configuration: RouteConfiguration) => { const directoryPath = join(getJanDataFolderPath(), configuration.dirName) try { @@ -124,7 +125,7 @@ export const getMessages = async (threadId: string): Promise => } } -export const retrieveMesasge = async (threadId: string, messageId: string) => { +export const retrieveMessage = async (threadId: string, messageId: string) => { const messages = await getMessages(threadId) const filteredMessages = messages.filter((m) => m.id === messageId) if (!filteredMessages || filteredMessages.length === 0) { @@ -317,13 +318,6 @@ export const chatCompletions = async (request: any, reply: any) => { apiUrl = engineConfiguration.full_url } - reply.raw.writeHead(200, { - 'Content-Type': 'text/event-stream', - 'Cache-Control': 'no-cache', - 'Connection': 'keep-alive', - 'Access-Control-Allow-Origin': '*', - }) - const headers: Record = { 'Content-Type': 'application/json', } @@ -342,8 +336,14 @@ export const chatCompletions = async (request: any, reply: any) => { }) if (response.status !== 200) { console.error(response) - return + reply.code(400).send(response) } else { + reply.raw.writeHead(200, { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + 'Connection': 'keep-alive', + 'Access-Control-Allow-Origin': '*', + }) response.body.pipe(reply.raw) } } diff --git a/core/src/node/api/common/configuration.ts b/core/src/node/api/restful/helper/configuration.ts similarity index 100% rename from core/src/node/api/common/configuration.ts rename to core/src/node/api/restful/helper/configuration.ts diff --git a/core/src/node/api/common/consts.ts b/core/src/node/api/restful/helper/consts.ts similarity index 100% rename from core/src/node/api/common/consts.ts rename to core/src/node/api/restful/helper/consts.ts diff --git a/core/src/node/api/common/startStopModel.ts b/core/src/node/api/restful/helper/startStopModel.ts similarity index 99% rename from core/src/node/api/common/startStopModel.ts rename to core/src/node/api/restful/helper/startStopModel.ts index 0d4934e1c0..0e6972b0bf 100644 --- a/core/src/node/api/common/startStopModel.ts +++ b/core/src/node/api/restful/helper/startStopModel.ts @@ -1,9 +1,9 @@ import fs from 'fs' import { join } from 'path' -import { getJanDataFolderPath, getJanExtensionsPath, getSystemResourceInfo } from '../../utils' -import { logServer } from '../../log' +import { getJanDataFolderPath, getJanExtensionsPath, getSystemResourceInfo } from '../../../helper' +import { logServer } from '../../../helper/log' import { ChildProcessWithoutNullStreams, spawn } from 'child_process' -import { Model, ModelSettingParams, PromptTemplate } from '../../../types' +import { Model, ModelSettingParams, PromptTemplate } from '../../../../types' import { LOCAL_HOST, NITRO_DEFAULT_PORT, diff --git a/core/src/node/api/restful/v1.ts b/core/src/node/api/restful/v1.ts new file mode 100644 index 0000000000..5eb8f50679 --- /dev/null +++ b/core/src/node/api/restful/v1.ts @@ -0,0 +1,16 @@ +import { HttpServer } from '../HttpServer' +import { commonRouter } from './common' +import { downloadRouter } from './app/download' +import { handleRequests } from './app/handlers' + +export const v1Router = async (app: HttpServer) => { + // MARK: Public API Routes + app.register(commonRouter) + + // MARK: Internal Application Routes + handleRequests(app) + + // Expanded route for tracking download progress + // TODO: Replace by Observer Wrapper (ZeroMQ / Vanilla Websocket) + app.register(downloadRouter) +} diff --git a/core/src/node/api/routes/download.ts b/core/src/node/api/routes/download.ts deleted file mode 100644 index b4e11f9578..0000000000 --- a/core/src/node/api/routes/download.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { DownloadRoute } from '../../../api' -import { join } from 'path' -import { DownloadManager } from '../../download' -import { HttpServer } from '../HttpServer' -import { createWriteStream } from 'fs' -import { getJanDataFolderPath } from '../../utils' -import { normalizeFilePath } from "../../path"; - -export const downloadRouter = async (app: HttpServer) => { - app.post(`/${DownloadRoute.downloadFile}`, async (req, res) => { - const strictSSL = !(req.query.ignoreSSL === "true"); - const proxy = req.query.proxy?.startsWith("http") ? req.query.proxy : undefined; - const body = JSON.parse(req.body as any); - const normalizedArgs = body.map((arg: any) => { - if (typeof arg === "string") { - return join(getJanDataFolderPath(), normalizeFilePath(arg)); - } - return arg; - }); - - const localPath = normalizedArgs[1]; - const fileName = localPath.split("/").pop() ?? ""; - - const request = require("request"); - const progress = require("request-progress"); - - const rq = request({ url: normalizedArgs[0], strictSSL, proxy }); - progress(rq, {}) - .on("progress", function (state: any) { - console.log("download onProgress", state); - }) - .on("error", function (err: Error) { - console.log("download onError", err); - }) - .on("end", function () { - console.log("download onEnd"); - }) - .pipe(createWriteStream(normalizedArgs[1])); - - DownloadManager.instance.setRequest(fileName, rq); - }); - - app.post(`/${DownloadRoute.abortDownload}`, async (req, res) => { - const body = JSON.parse(req.body as any); - const normalizedArgs = body.map((arg: any) => { - if (typeof arg === "string") { - return join(getJanDataFolderPath(), normalizeFilePath(arg)); - } - return arg; - }); - - const localPath = normalizedArgs[0]; - const fileName = localPath.split("/").pop() ?? ""; - const rq = DownloadManager.instance.networkRequests[fileName]; - DownloadManager.instance.networkRequests[fileName] = undefined; - rq?.abort(); - }); -}; diff --git a/core/src/node/api/routes/extension.ts b/core/src/node/api/routes/extension.ts deleted file mode 100644 index 02bc54eb37..0000000000 --- a/core/src/node/api/routes/extension.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { join, extname } from 'path' -import { ExtensionRoute } from '../../../api/index' -import { ModuleManager } from '../../module' -import { getActiveExtensions, installExtensions } from '../../extension/store' -import { HttpServer } from '../HttpServer' - -import { readdirSync } from 'fs' -import { getJanExtensionsPath } from '../../utils' - -export const extensionRouter = async (app: HttpServer) => { - // TODO: Share code between node projects - app.post(`/${ExtensionRoute.getActiveExtensions}`, async (_req, res) => { - const activeExtensions = await getActiveExtensions() - res.status(200).send(activeExtensions) - }) - - app.post(`/${ExtensionRoute.baseExtensions}`, async (_req, res) => { - const baseExtensionPath = join(__dirname, '..', '..', '..', 'pre-install') - const extensions = readdirSync(baseExtensionPath) - .filter((file) => extname(file) === '.tgz') - .map((file) => join(baseExtensionPath, file)) - - res.status(200).send(extensions) - }) - - app.post(`/${ExtensionRoute.installExtension}`, async (req) => { - const extensions = req.body as any - const installed = await installExtensions(JSON.parse(extensions)[0]) - return JSON.parse(JSON.stringify(installed)) - }) - - app.post(`/${ExtensionRoute.invokeExtensionFunc}`, async (req, res) => { - const args = JSON.parse(req.body as any) - console.debug(args) - const module = await import(join(getJanExtensionsPath(), args[0])) - - ModuleManager.instance.setModule(args[0], module) - const method = args[1] - if (typeof module[method] === 'function') { - // remove first item from args - const newArgs = args.slice(2) - console.log(newArgs) - return module[method](...args.slice(2)) - } else { - console.debug(module[method]) - console.error(`Function "${method}" does not exist in the module.`) - } - }) -} diff --git a/core/src/node/api/routes/fileManager.ts b/core/src/node/api/routes/fileManager.ts deleted file mode 100644 index 66056444e0..0000000000 --- a/core/src/node/api/routes/fileManager.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { FileManagerRoute } from '../../../api' -import { HttpServer } from '../../index' - -export const fsRouter = async (app: HttpServer) => { - app.post(`/app/${FileManagerRoute.syncFile}`, async (request: any, reply: any) => {}) - - app.post(`/app/${FileManagerRoute.getJanDataFolderPath}`, async (request: any, reply: any) => {}) - - app.post(`/app/${FileManagerRoute.getResourcePath}`, async (request: any, reply: any) => {}) - - app.post(`/app/${FileManagerRoute.getUserHomePath}`, async (request: any, reply: any) => {}) - - app.post(`/app/${FileManagerRoute.fileStat}`, async (request: any, reply: any) => {}) -} diff --git a/core/src/node/api/routes/fs.ts b/core/src/node/api/routes/fs.ts deleted file mode 100644 index c5404ccce9..0000000000 --- a/core/src/node/api/routes/fs.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { FileSystemRoute } from '../../../api' -import { join } from 'path' -import { HttpServer } from '../HttpServer' -import { getJanDataFolderPath } from '../../utils' -import { normalizeFilePath } from '../../path' - -export const fsRouter = async (app: HttpServer) => { - const moduleName = 'fs' - // Generate handlers for each fs route - Object.values(FileSystemRoute).forEach((route) => { - app.post(`/${route}`, async (req, res) => { - const body = JSON.parse(req.body as any) - try { - const result = await import(moduleName).then((mdl) => { - return mdl[route]( - ...body.map((arg: any) => - typeof arg === 'string' && (arg.startsWith(`file:/`) || arg.startsWith(`file:\\`)) - ? join(getJanDataFolderPath(), normalizeFilePath(arg)) - : arg - ) - ) - }) - res.status(200).send(result) - } catch (ex) { - console.log(ex) - } - }) - }) -} diff --git a/core/src/node/api/routes/index.ts b/core/src/node/api/routes/index.ts deleted file mode 100644 index e6edc62f7c..0000000000 --- a/core/src/node/api/routes/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -export * from './download' -export * from './extension' -export * from './fs' -export * from './thread' -export * from './common' -export * from './v1' diff --git a/core/src/node/api/routes/thread.ts b/core/src/node/api/routes/thread.ts deleted file mode 100644 index 4066d27165..0000000000 --- a/core/src/node/api/routes/thread.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { HttpServer } from '../HttpServer' -import { - createMessage, - createThread, - getMessages, - retrieveMesasge, - updateThread, -} from '../common/builder' - -export const threadRouter = async (app: HttpServer) => { - // create thread - app.post(`/`, async (req, res) => createThread(req.body)) - - app.get(`/:threadId/messages`, async (req, res) => getMessages(req.params.threadId)) - - // retrieve message - app.get(`/:threadId/messages/:messageId`, async (req, res) => - retrieveMesasge(req.params.threadId, req.params.messageId), - ) - - // create message - app.post(`/:threadId/messages`, async (req, res) => - createMessage(req.params.threadId as any, req.body as any), - ) - - // modify thread - app.patch(`/:threadId`, async (request: any) => - updateThread(request.params.threadId, request.body), - ) -} diff --git a/core/src/node/api/routes/v1.ts b/core/src/node/api/routes/v1.ts deleted file mode 100644 index a2a48cd8b6..0000000000 --- a/core/src/node/api/routes/v1.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { HttpServer } from '../HttpServer' -import { commonRouter } from './common' -import { threadRouter } from './thread' -import { fsRouter } from './fs' -import { extensionRouter } from './extension' -import { downloadRouter } from './download' - -export const v1Router = async (app: HttpServer) => { - // MARK: External Routes - app.register(commonRouter) - app.register(threadRouter, { - prefix: '/threads', - }) - - // MARK: Internal Application Routes - app.register(fsRouter, { - prefix: '/fs', - }) - app.register(extensionRouter, { - prefix: '/extension', - }) - app.register(downloadRouter, { - prefix: '/download', - }) -} diff --git a/core/src/node/extension/extension.ts b/core/src/node/extension/extension.ts index aeb0277c0b..1f8dfa3ec2 100644 --- a/core/src/node/extension/extension.ts +++ b/core/src/node/extension/extension.ts @@ -104,7 +104,7 @@ export default class Extension { await pacote.extract( this.specifier, join(ExtensionManager.instance.getExtensionsPath() ?? '', this.name ?? ''), - this.installOptions, + this.installOptions ) // Set the url using the custom extensions protocol diff --git a/core/src/node/extension/index.ts b/core/src/node/extension/index.ts index ed8544773a..994fc97f2f 100644 --- a/core/src/node/extension/index.ts +++ b/core/src/node/extension/index.ts @@ -41,8 +41,8 @@ async function registerExtensionProtocol() { console.error('Electron is not available') } const extensionPath = ExtensionManager.instance.getExtensionsPath() - if (electron) { - return electron.protocol.registerFileProtocol('extension', (request: any, callback: any) => { + if (electron && electron.protocol) { + return electron.protocol?.registerFileProtocol('extension', (request: any, callback: any) => { const entry = request.url.substr('extension://'.length - 1) const url = normalize(extensionPath + entry) @@ -69,7 +69,7 @@ export function useExtensions(extensionsPath: string) { // Read extension list from extensions folder const extensions = JSON.parse( - readFileSync(ExtensionManager.instance.getExtensionsFile(), 'utf-8'), + readFileSync(ExtensionManager.instance.getExtensionsFile(), 'utf-8') ) try { // Create and store a Extension instance for each extension in list @@ -82,7 +82,7 @@ export function useExtensions(extensionsPath: string) { throw new Error( 'Could not successfully rebuild list of installed extensions.\n' + error + - '\nPlease check the extensions.json file in the extensions folder.', + '\nPlease check the extensions.json file in the extensions folder.' ) } @@ -122,7 +122,7 @@ function loadExtension(ext: any) { export function getStore() { if (!ExtensionManager.instance.getExtensionsFile()) { throw new Error( - 'The extension path has not yet been set up. Please run useExtensions before accessing the store', + 'The extension path has not yet been set up. Please run useExtensions before accessing the store' ) } diff --git a/core/src/node/extension/store.ts b/core/src/node/extension/store.ts index 84b1f9caf3..93b1aeb2b6 100644 --- a/core/src/node/extension/store.ts +++ b/core/src/node/extension/store.ts @@ -1,6 +1,6 @@ -import { writeFileSync } from "fs"; -import Extension from "./extension"; -import { ExtensionManager } from "./manager"; +import { writeFileSync } from 'fs' +import Extension from './extension' +import { ExtensionManager } from './manager' /** * @module store @@ -11,7 +11,7 @@ import { ExtensionManager } from "./manager"; * Register of installed extensions * @type {Object.} extension - List of installed extensions */ -const extensions: Record = {}; +const extensions: Record = {} /** * Get a extension from the stored extensions. @@ -21,10 +21,10 @@ const extensions: Record = {}; */ export function getExtension(name: string) { if (!Object.prototype.hasOwnProperty.call(extensions, name)) { - throw new Error(`Extension ${name} does not exist`); + throw new Error(`Extension ${name} does not exist`) } - return extensions[name]; + return extensions[name] } /** @@ -33,7 +33,7 @@ export function getExtension(name: string) { * @alias extensionManager.getAllExtensions */ export function getAllExtensions() { - return Object.values(extensions); + return Object.values(extensions) } /** @@ -42,7 +42,7 @@ export function getAllExtensions() { * @alias extensionManager.getActiveExtensions */ export function getActiveExtensions() { - return Object.values(extensions).filter((extension) => extension.active); + return Object.values(extensions).filter((extension) => extension.active) } /** @@ -53,9 +53,9 @@ export function getActiveExtensions() { * @alias extensionManager.removeExtension */ export function removeExtension(name: string, persist = true) { - const del = delete extensions[name]; - if (persist) persistExtensions(); - return del; + const del = delete extensions[name] + if (persist) persistExtensions() + return del } /** @@ -65,10 +65,10 @@ export function removeExtension(name: string, persist = true) { * @returns {void} */ export function addExtension(extension: Extension, persist = true) { - if (extension.name) extensions[extension.name] = extension; + if (extension.name) extensions[extension.name] = extension if (persist) { - persistExtensions(); - extension.subscribe("pe-persist", persistExtensions); + persistExtensions() + extension.subscribe('pe-persist', persistExtensions) } } @@ -77,14 +77,11 @@ export function addExtension(extension: Extension, persist = true) { * @returns {void} */ export function persistExtensions() { - const persistData: Record = {}; + const persistData: Record = {} for (const name in extensions) { - persistData[name] = extensions[name]; + persistData[name] = extensions[name] } - writeFileSync( - ExtensionManager.instance.getExtensionsFile(), - JSON.stringify(persistData), - ); + writeFileSync(ExtensionManager.instance.getExtensionsFile(), JSON.stringify(persistData)) } /** @@ -94,26 +91,29 @@ export function persistExtensions() { * @returns {Promise.>} New extension * @alias extensionManager.installExtensions */ -export async function installExtensions(extensions: any, store = true) { - const installed: Extension[] = []; +export async function installExtensions(extensions: any) { + const installed: Extension[] = [] for (const ext of extensions) { // Set install options and activation based on input type - const isObject = typeof ext === "object"; - const spec = isObject ? [ext.specifier, ext] : [ext]; - const activate = isObject ? ext.activate !== false : true; + const isObject = typeof ext === 'object' + const spec = isObject ? [ext.specifier, ext] : [ext] + const activate = isObject ? ext.activate !== false : true // Install and possibly activate extension - const extension = new Extension(...spec); - await extension._install(); - if (activate) extension.setActive(true); + const extension = new Extension(...spec) + if (!extension.origin) { + continue + } + await extension._install() + if (activate) extension.setActive(true) // Add extension to store if needed - if (store) addExtension(extension); - installed.push(extension); + addExtension(extension) + installed.push(extension) } // Return list of all installed extensions - return installed; + return installed } /** diff --git a/core/src/node/utils/index.ts b/core/src/node/helper/config.ts similarity index 91% rename from core/src/node/utils/index.ts rename to core/src/node/helper/config.ts index 4bcbf13b17..71e7215780 100644 --- a/core/src/node/utils/index.ts +++ b/core/src/node/helper/config.ts @@ -1,8 +1,7 @@ -import { AppConfiguration, SystemResourceInfo } from '../../types' +import { AppConfiguration } from '../../types' import { join } from 'path' import fs from 'fs' import os from 'os' -import { log, logServer } from '../log' import childProcess from 'child_process' // TODO: move this to core @@ -56,34 +55,6 @@ export const updateAppConfiguration = (configuration: AppConfiguration): Promise return Promise.resolve() } -/** - * Utility function to get server log path - * - * @returns {string} The log path. - */ -export const getServerLogPath = (): string => { - const appConfigurations = getAppConfigurations() - const logFolderPath = join(appConfigurations.data_folder, 'logs') - if (!fs.existsSync(logFolderPath)) { - fs.mkdirSync(logFolderPath, { recursive: true }) - } - return join(logFolderPath, 'server.log') -} - -/** - * Utility function to get app log path - * - * @returns {string} The log path. - */ -export const getAppLogPath = (): string => { - const appConfigurations = getAppConfigurations() - const logFolderPath = join(appConfigurations.data_folder, 'logs') - if (!fs.existsSync(logFolderPath)) { - fs.mkdirSync(logFolderPath, { recursive: true }) - } - return join(logFolderPath, 'app.log') -} - /** * Utility function to get data folder path * @@ -146,18 +117,6 @@ const exec = async (command: string): Promise => { }) } -export const getSystemResourceInfo = async (): Promise => { - const cpu = await physicalCpuCount() - const message = `[NITRO]::CPU informations - ${cpu}` - log(message) - logServer(message) - - return { - numCpuPhysicalCore: cpu, - memAvailable: 0, // TODO: this should not be 0 - } -} - export const getEngineConfiguration = async (engineId: string) => { if (engineId !== 'openai') { return undefined @@ -167,3 +126,31 @@ export const getEngineConfiguration = async (engineId: string) => { const data = fs.readFileSync(filePath, 'utf-8') return JSON.parse(data) } + +/** + * Utility function to get server log path + * + * @returns {string} The log path. + */ +export const getServerLogPath = (): string => { + const appConfigurations = getAppConfigurations() + const logFolderPath = join(appConfigurations.data_folder, 'logs') + if (!fs.existsSync(logFolderPath)) { + fs.mkdirSync(logFolderPath, { recursive: true }) + } + return join(logFolderPath, 'server.log') +} + +/** + * Utility function to get app log path + * + * @returns {string} The log path. + */ +export const getAppLogPath = (): string => { + const appConfigurations = getAppConfigurations() + const logFolderPath = join(appConfigurations.data_folder, 'logs') + if (!fs.existsSync(logFolderPath)) { + fs.mkdirSync(logFolderPath, { recursive: true }) + } + return join(logFolderPath, 'app.log') +} diff --git a/core/src/node/download.ts b/core/src/node/helper/download.ts similarity index 67% rename from core/src/node/download.ts rename to core/src/node/helper/download.ts index 6d15fc3445..b9fb88bb5c 100644 --- a/core/src/node/download.ts +++ b/core/src/node/helper/download.ts @@ -1,15 +1,18 @@ +import { DownloadState } from '../../types' /** * Manages file downloads and network requests. */ export class DownloadManager { - public networkRequests: Record = {}; + public networkRequests: Record = {} - public static instance: DownloadManager = new DownloadManager(); + public static instance: DownloadManager = new DownloadManager() + + public downloadProgressMap: Record = {} constructor() { if (DownloadManager.instance) { - return DownloadManager.instance; + return DownloadManager.instance } } /** @@ -18,6 +21,6 @@ export class DownloadManager { * @param {Request | undefined} request - The network request to set, or undefined to clear the request. */ setRequest(fileName: string, request: any | undefined) { - this.networkRequests[fileName] = request; + this.networkRequests[fileName] = request } } diff --git a/core/src/node/helper/index.ts b/core/src/node/helper/index.ts new file mode 100644 index 0000000000..6fc54fc6b1 --- /dev/null +++ b/core/src/node/helper/index.ts @@ -0,0 +1,6 @@ +export * from './config' +export * from './download' +export * from './log' +export * from './module' +export * from './path' +export * from './resource' diff --git a/core/src/node/log.ts b/core/src/node/helper/log.ts similarity index 93% rename from core/src/node/log.ts rename to core/src/node/helper/log.ts index 6f2c2f80f3..8ff1969434 100644 --- a/core/src/node/log.ts +++ b/core/src/node/helper/log.ts @@ -1,6 +1,6 @@ import fs from 'fs' import util from 'util' -import { getAppLogPath, getServerLogPath } from './utils' +import { getAppLogPath, getServerLogPath } from './config' export const log = (message: string) => { const path = getAppLogPath() diff --git a/core/src/node/module.ts b/core/src/node/helper/module.ts similarity index 100% rename from core/src/node/module.ts rename to core/src/node/helper/module.ts diff --git a/core/src/node/helper/path.ts b/core/src/node/helper/path.ts new file mode 100644 index 0000000000..c20889f4c9 --- /dev/null +++ b/core/src/node/helper/path.ts @@ -0,0 +1,35 @@ +import { join } from 'path' + +/** + * Normalize file path + * Remove all file protocol prefix + * @param path + * @returns + */ +export function normalizeFilePath(path: string): string { + return path.replace(/^(file:[\\/]+)([^:\s]+)$/, '$2') +} + +export async function appResourcePath(): Promise { + let electron: any = undefined + + try { + const moduleName = 'electron' + electron = await import(moduleName) + } catch (err) { + console.error('Electron is not available') + } + + // electron + if (electron && electron.protocol) { + let appPath = join(electron.app.getAppPath(), '..', 'app.asar.unpacked') + + if (!electron.app.isPackaged) { + // for development mode + appPath = join(electron.app.getAppPath()) + } + return appPath + } + // server + return join(global.core.appPath(), '../../..') +} diff --git a/core/src/node/helper/resource.ts b/core/src/node/helper/resource.ts new file mode 100644 index 0000000000..c79a63688b --- /dev/null +++ b/core/src/node/helper/resource.ts @@ -0,0 +1,14 @@ +import { SystemResourceInfo } from '../../types' +import { physicalCpuCount } from './config' +import { log, logServer } from './log' + +export const getSystemResourceInfo = async (): Promise => { + const cpu = await physicalCpuCount() + const message = `[NITRO]::CPU informations - ${cpu}` + log(message) + + return { + numCpuPhysicalCore: cpu, + memAvailable: 0, // TODO: this should not be 0 + } +} diff --git a/core/src/node/index.ts b/core/src/node/index.ts index 10385ecfcc..31f2f076e9 100644 --- a/core/src/node/index.ts +++ b/core/src/node/index.ts @@ -2,9 +2,5 @@ export * from './extension/index' export * from './extension/extension' export * from './extension/manager' export * from './extension/store' -export * from './download' -export * from './module' export * from './api' -export * from './log' -export * from './utils' -export * from './path' +export * from './helper' diff --git a/core/src/node/path.ts b/core/src/node/path.ts deleted file mode 100644 index adbc38c6c1..0000000000 --- a/core/src/node/path.ts +++ /dev/null @@ -1,9 +0,0 @@ -/** - * Normalize file path - * Remove all file protocol prefix - * @param path - * @returns - */ -export function normalizeFilePath(path: string): string { - return path.replace(/^(file:[\\/]+)([^:\s]+)$/, "$2"); -} diff --git a/core/src/types/assistant/assistantEvent.ts b/core/src/types/assistant/assistantEvent.ts new file mode 100644 index 0000000000..8c32f5d37a --- /dev/null +++ b/core/src/types/assistant/assistantEvent.ts @@ -0,0 +1,7 @@ +/** + * The `EventName` enumeration contains the names of all the available events in the Jan platform. + */ +export enum AssistantEvent { + /** The `OnAssistantsUpdate` event is emitted when the assistant list is updated. */ + OnAssistantsUpdate = 'OnAssistantsUpdate', +} diff --git a/core/src/types/assistant/index.ts b/core/src/types/assistant/index.ts index 83ea73f856..e18589551a 100644 --- a/core/src/types/assistant/index.ts +++ b/core/src/types/assistant/index.ts @@ -1,2 +1,3 @@ export * from './assistantEntity' +export * from './assistantEvent' export * from './assistantInterface' diff --git a/core/src/types/file/index.ts b/core/src/types/file/index.ts index 6526cfc6d4..cc7274a28f 100644 --- a/core/src/types/file/index.ts +++ b/core/src/types/file/index.ts @@ -2,3 +2,26 @@ export type FileStat = { isDirectory: boolean size: number } + +export type DownloadState = { + modelId: string + fileName: string + time: DownloadTime + speed: number + percent: number + + size: DownloadSize + children?: DownloadState[] + error?: string + downloadState: 'downloading' | 'error' | 'end' +} + +type DownloadTime = { + elapsed: number + remaining: number +} + +type DownloadSize = { + total: number + transferred: number +} diff --git a/core/src/types/message/index.ts b/core/src/types/message/index.ts index e8d78deda4..ebb4c363d8 100644 --- a/core/src/types/message/index.ts +++ b/core/src/types/message/index.ts @@ -1,3 +1,4 @@ export * from './messageEntity' export * from './messageInterface' export * from './messageEvent' +export * from './messageRequestType' diff --git a/core/src/types/message/messageEntity.ts b/core/src/types/message/messageEntity.ts index 87e4b1997e..e9211d5508 100644 --- a/core/src/types/message/messageEntity.ts +++ b/core/src/types/message/messageEntity.ts @@ -27,6 +27,8 @@ export type ThreadMessage = { updated: number /** The additional metadata of this message. **/ metadata?: Record + + type?: string } /** @@ -56,6 +58,8 @@ export type MessageRequest = { /** The thread of this message is belong to. **/ // TODO: deprecate threadId field thread?: Thread + + type?: string } /** diff --git a/core/src/types/message/messageRequestType.ts b/core/src/types/message/messageRequestType.ts new file mode 100644 index 0000000000..cbb4cf4217 --- /dev/null +++ b/core/src/types/message/messageRequestType.ts @@ -0,0 +1,5 @@ +export enum MessageRequestType { + Thread = 'Thread', + Assistant = 'Assistant', + Summary = 'Summary', +} diff --git a/core/src/types/model/modelEvent.ts b/core/src/types/model/modelEvent.ts index 978a487249..443f3a34fb 100644 --- a/core/src/types/model/modelEvent.ts +++ b/core/src/types/model/modelEvent.ts @@ -12,4 +12,6 @@ export enum ModelEvent { OnModelStop = 'OnModelStop', /** The `OnModelStopped` event is emitted when a model stopped ok. */ OnModelStopped = 'OnModelStopped', + /** The `OnModelUpdate` event is emitted when the model list is updated. */ + OnModelsUpdate = 'OnModelsUpdate', } diff --git a/core/src/types/model/modelInterface.ts b/core/src/types/model/modelInterface.ts index 74a479f3cb..93d5867eeb 100644 --- a/core/src/types/model/modelInterface.ts +++ b/core/src/types/model/modelInterface.ts @@ -10,7 +10,7 @@ export interface ModelInterface { * @param network - Optional object to specify proxy/whether to ignore SSL certificates. * @returns A Promise that resolves when the model has been downloaded. */ - downloadModel(model: Model, network?: { ignoreSSL?: boolean, proxy?: string }): Promise + downloadModel(model: Model, network?: { ignoreSSL?: boolean; proxy?: string }): Promise /** * Cancels the download of a specific model. diff --git a/core/tests/node/path.test.ts b/core/tests/node/path.test.ts index 9f8a557bb0..5390df1193 100644 --- a/core/tests/node/path.test.ts +++ b/core/tests/node/path.test.ts @@ -1,4 +1,4 @@ -import { normalizeFilePath } from "../../src/node/path"; +import { normalizeFilePath } from "../../src/node/helper/path"; describe("Test file normalize", () => { test("returns no file protocol prefix on Unix", async () => { diff --git a/core/tslint.json b/core/tslint.json index 398a416704..6543a641a1 100644 --- a/core/tslint.json +++ b/core/tslint.json @@ -1,6 +1,3 @@ { - "extends": [ - "tslint-config-standard", - "tslint-config-prettier" - ] -} \ No newline at end of file + "extends": ["tslint-config-standard", "tslint-config-prettier"] +} diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000000..1691a841a0 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,172 @@ +# Docker Compose file for setting up Minio, createbuckets, app_cpu, and app_gpu services + +version: '3.7' + +services: + # Minio service for object storage + minio: + image: minio/minio + volumes: + - minio_data:/data + ports: + - "9000:9000" + - "9001:9001" + environment: + # Set the root user and password for Minio + MINIO_ROOT_USER: minioadmin # This acts as AWS_ACCESS_KEY + MINIO_ROOT_PASSWORD: minioadmin # This acts as AWS_SECRET_ACCESS_KEY + command: server --console-address ":9001" /data + restart: always + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 30s + timeout: 20s + retries: 3 + networks: + vpcbr: + ipv4_address: 10.5.0.2 + + # createbuckets service to create a bucket and set its policy + createbuckets: + image: minio/mc + depends_on: + - minio + entrypoint: > + /bin/sh -c " + /usr/bin/mc alias set myminio http://minio:9000 minioadmin minioadmin; + /usr/bin/mc mb myminio/mybucket; + /usr/bin/mc policy set public myminio/mybucket; + exit 0; + " + networks: + vpcbr: + + # app_cpu service for running the CPU version of the application + app_cpu_s3fs: + image: jan:latest + volumes: + - app_data_cpu_s3fs:/app/server/build/jan + build: + context: . + dockerfile: Dockerfile + environment: + # Set the AWS access key, secret access key, bucket name, endpoint, and region for app_cpu + AWS_ACCESS_KEY_ID: minioadmin + AWS_SECRET_ACCESS_KEY: minioadmin + S3_BUCKET_NAME: mybucket + AWS_ENDPOINT: http://10.5.0.2:9000 + AWS_REGION: us-east-1 + API_BASE_URL: http://localhost:1337 + restart: always + profiles: + - cpu-s3fs + ports: + - "3000:3000" + - "1337:1337" + - "3928:3928" + networks: + vpcbr: + ipv4_address: 10.5.0.3 + + # app_gpu service for running the GPU version of the application + app_gpu_s3fs: + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + image: jan-gpu:latest + volumes: + - app_data_gpu_s3fs:/app/server/build/jan + build: + context: . + dockerfile: Dockerfile.gpu + restart: always + environment: + # Set the AWS access key, secret access key, bucket name, endpoint, and region for app_gpu + AWS_ACCESS_KEY_ID: minioadmin + AWS_SECRET_ACCESS_KEY: minioadmin + S3_BUCKET_NAME: mybucket + AWS_ENDPOINT: http://10.5.0.2:9000 + AWS_REGION: us-east-1 + API_BASE_URL: http://localhost:1337 + profiles: + - gpu-s3fs + ports: + - "3000:3000" + - "1337:1337" + - "3928:3928" + networks: + vpcbr: + ipv4_address: 10.5.0.4 + + app_cpu_fs: + image: jan:latest + volumes: + - app_data_cpu_fs:/app/server/build/jan + build: + context: . + dockerfile: Dockerfile + environment: + API_BASE_URL: http://localhost:1337 + restart: always + profiles: + - cpu-fs + ports: + - "3000:3000" + - "1337:1337" + - "3928:3928" + networks: + vpcbr: + ipv4_address: 10.5.0.5 + + # app_gpu service for running the GPU version of the application + app_gpu_fs: + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + image: jan-gpu:latest + volumes: + - app_data_gpu_fs:/app/server/build/jan + build: + context: . + dockerfile: Dockerfile.gpu + restart: always + environment: + API_BASE_URL: http://localhost:1337 + profiles: + - gpu-fs + ports: + - "3000:3000" + - "1337:1337" + - "3928:3928" + networks: + vpcbr: + ipv4_address: 10.5.0.6 + +volumes: + minio_data: + app_data_cpu_s3fs: + app_data_gpu_s3fs: + app_data_cpu_fs: + app_data_gpu_fs: + +networks: + vpcbr: + driver: bridge + ipam: + config: + - subnet: 10.5.0.0/16 + gateway: 10.5.0.1 + +# Usage: +# - Run 'docker compose --profile cpu-s3fs up -d' to start the app_cpu service +# - Run 'docker compose --profile gpu-s3fs up -d' to start the app_gpu service +# - Run 'docker compose --profile cpu-fs up -d' to start the app_cpu service +# - Run 'docker compose --profile gpu-fs up -d' to start the app_gpu service diff --git a/docs/blog/authors.yml b/docs/blog/authors.yml index f30d4610dd..ec58002e48 100644 --- a/docs/blog/authors.yml +++ b/docs/blog/authors.yml @@ -1,6 +1,76 @@ dan-jan: name: Daniel Onggunhao title: Co-Founder - url: https://github.com/dan-jan + url: https://github.com/dan-jan image_url: https://avatars.githubusercontent.com/u/101145494?v=4 - email: daniel@jan.ai \ No newline at end of file + email: daniel@jan.ai + +namchuai: + name: Nam Nguyen + title: Developer + url: https://github.com/namchuai + image_url: https://avatars.githubusercontent.com/u/10397206?v=4 + email: james@jan.ai + +hiro-v: + name: Hiro Vuong + title: MLE + url: https://github.com/hiro-v + image_url: https://avatars.githubusercontent.com/u/22463238?v=4 + email: hiro@jan.ai + +ashley-jan: + name: Ashley Tran + title: Product Designer + url: https://github.com/imtuyethan + image_url: https://avatars.githubusercontent.com/u/89722390?v=4 + email: ashley@jan.ai + +hientominh: + name: Hien To + title: DevOps Engineer + url: https://github.com/hientominh + image_url: https://avatars.githubusercontent.com/u/37921427?v=4 + email: hien@jan.ai + +Van-QA: + name: Van Pham + title: QA & Release Manager + url: https://github.com/Van-QA + image_url: https://avatars.githubusercontent.com/u/64197333?v=4 + email: van@jan.ai + +louis-jan: + name: Louis Le + title: Software Engineer + url: https://github.com/louis-jan + image_url: https://avatars.githubusercontent.com/u/133622055?v=4 + email: louis@jan.ai + +hahuyhoang411: + name: Rex Ha + title: LLM Researcher & Content Writer + url: https://github.com/hahuyhoang411 + image_url: https://avatars.githubusercontent.com/u/64120343?v=4 + email: rex@jan.ai + +automaticcat: + name: Alan Dao + title: AI Engineer + url: https://github.com/tikikun + image_url: https://avatars.githubusercontent.com/u/22268502?v=4 + email: alan@jan.ai + +hieu-jan: + name: Henry Ho + title: Software Engineer + url: https://github.com/hieu-jan + image_url: https://avatars.githubusercontent.com/u/150573299?v=4 + email: hieu@jan.ai + +0xsage: + name: Nicole Zhu + title: Co-Founder + url: https://github.com/0xsage + image_url: https://avatars.githubusercontent.com/u/69952136?v=4 + email: nicole@jan.ai diff --git a/docs/docs/about/01-README.md b/docs/docs/about/01-README.md index 3b27595135..d5d3b8dc22 100644 --- a/docs/docs/about/01-README.md +++ b/docs/docs/about/01-README.md @@ -110,9 +110,10 @@ Adhering to Jan's privacy preserving philosophy, our analytics philosophy is to #### What is tracked -1. By default, Github tracks downloads and device metadata for all public Github repos. This helps us troubleshoot & ensure cross platform support. -1. We use Posthog to track a single `app.opened` event without additional user metadata, in order to understand retention. -1. Additionally, we plan to enable a `Settings` feature for users to turn off all tracking. +1. By default, Github tracks downloads and device metadata for all public GitHub repositories. This helps us troubleshoot & ensure cross-platform support. +2. We use [Umami](https://umami.is/) to collect, analyze, and understand application data while maintaining visitor privacy and data ownership. We are using the Umami Cloud in Europe to ensure GDPR compliance. Please see [Umami Privacy Policy](https://umami.is/privacy) for more details. +3. We use Umami to track a single `app.opened` event without additional user metadata, in order to understand retention. In addition, we track `app.event` to understand app version usage. +4. Additionally, we plan to enable a `Settings` feature for users to turn off all tracking. #### Request for help diff --git a/docs/docs/developer/01-overview/04-install-and-prerequisites.md b/docs/docs/developer/01-overview/04-install-and-prerequisites.md new file mode 100644 index 0000000000..110f62e361 --- /dev/null +++ b/docs/docs/developer/01-overview/04-install-and-prerequisites.md @@ -0,0 +1,79 @@ +--- +title: Installation and Prerequisites +slug: /developer/prereq +description: Guide to install and setup Jan for development. +keywords: + [ + Jan AI, + Jan, + ChatGPT alternative, + local AI, + private AI, + conversational AI, + no-subscription fee, + large language model, + installation, + prerequisites, + developer setup, + ] +--- + +## Requirements + +### Hardware Requirements + +Ensure your system meets the following specifications to guarantee a smooth development experience: + +- [Hardware Requirements](../../guides/02-installation/06-hardware.md) + +### System Requirements + +Make sure your operating system meets the specific requirements for Jan development: + +- [Windows](../../install/windows/#system-requirements) +- [MacOS](../../install/mac/#system-requirements) +- [Linux](../../install/linux/#system-requirements) + +## Prerequisites + +- [Node.js](https://nodejs.org/en/) (version 20.0.0 or higher) +- [yarn](https://yarnpkg.com/) (version 1.22.0 or higher) +- [make](https://www.gnu.org/software/make/) (version 3.81 or higher) + +## Instructions + +1. **Clone the Repository:** + +```bash +git clone https://github.com/janhq/jan +cd jan +git checkout -b DESIRED_BRANCH +``` + +2. **Install Dependencies** + +```bash +yarn install +``` + +3. **Run Development and Use Jan Desktop** + +```bash +make dev +``` + +This command starts the development server and opens the Jan Desktop app. + +## For Production Build + +```bash +# Do steps 1 and 2 in the previous section +# Build the app +make build +``` + +This will build the app MacOS (M1/M2/M3) for production (with code signing already done) and place the result in `/electron/dist` folder. + +## Troubleshooting + +If you run into any issues due to a broken build, please check the [Stuck on a Broken Build](../../troubleshooting/stuck-on-broken-build) guide. diff --git a/docs/docs/guides/02-installation/01-mac.md b/docs/docs/guides/02-installation/01-mac.md index 8e67b5bed2..7a39613843 100644 --- a/docs/docs/guides/02-installation/01-mac.md +++ b/docs/docs/guides/02-installation/01-mac.md @@ -12,11 +12,16 @@ keywords: conversational AI, no-subscription fee, large language model, + installation guide, ] --- # Installing Jan on MacOS +## System Requirements + +Ensure that your MacOS version is 13 or higher to run Jan. + ## Installation Jan is available for download via our homepage, [https://jan.ai/](https://jan.ai/). diff --git a/docs/docs/guides/02-installation/02-windows.md b/docs/docs/guides/02-installation/02-windows.md index b200554d22..d60ab86f7f 100644 --- a/docs/docs/guides/02-installation/02-windows.md +++ b/docs/docs/guides/02-installation/02-windows.md @@ -12,11 +12,23 @@ keywords: conversational AI, no-subscription fee, large language model, + installation guide, ] --- # Installing Jan on Windows +## System Requirements + +Ensure that your system meets the following requirements: + +- Windows 10 or higher is required to run Jan. + +To enable GPU support, you will need: + +- NVIDIA GPU with CUDA Toolkit 11.7 or higher +- NVIDIA driver 470.63.01 or higher + ## Installation Jan is available for download via our homepage, [https://jan.ai](https://jan.ai/). @@ -59,13 +71,3 @@ To remove all user data associated with Jan, you can delete the `/jan` directory cd C:\Users\%USERNAME%\AppData\Roaming rmdir /S jan ``` - -## Troubleshooting - -### Microsoft Defender - -**Error: "Microsoft Defender SmartScreen prevented an unrecognized app from starting"** - -Windows Defender may display the above warning when running the Jan Installer, as a standard security measure. - -To proceed, select the "More info" option and select the "Run Anyway" option to continue with the installation. diff --git a/docs/docs/guides/02-installation/03-linux.md b/docs/docs/guides/02-installation/03-linux.md index 21dfac1a96..0ec7fea605 100644 --- a/docs/docs/guides/02-installation/03-linux.md +++ b/docs/docs/guides/02-installation/03-linux.md @@ -12,11 +12,24 @@ keywords: conversational AI, no-subscription fee, large language model, + installation guide, ] --- # Installing Jan on Linux +## System Requirements + +Ensure that your system meets the following requirements: + +- glibc 2.27 or higher (check with `ldd --version`) +- gcc 11, g++ 11, cpp 11, or higher, refer to this [link](https://jan.ai/guides/troubleshooting/gpu-not-used/#specific-requirements-for-linux) for more information. + +To enable GPU support, you will need: + +- NVIDIA GPU with CUDA Toolkit 11.7 or higher +- NVIDIA driver 470.63.01 or higher + ## Installation Jan is available for download via our homepage, [https://jan.ai](https://jan.ai/). @@ -66,7 +79,6 @@ jan-linux-amd64-{version}.deb # AppImage jan-linux-x86_64-{version}.AppImage ``` -``` ## Uninstall Jan diff --git a/docs/docs/guides/02-installation/05-docker.md b/docs/docs/guides/02-installation/05-docker.md new file mode 100644 index 0000000000..6236ed92e4 --- /dev/null +++ b/docs/docs/guides/02-installation/05-docker.md @@ -0,0 +1,102 @@ +--- +title: Docker +slug: /install/docker +description: Jan is a ChatGPT-alternative that runs on your own computer, with a local API server. +keywords: + [ + Jan AI, + Jan, + ChatGPT alternative, + local AI, + private AI, + conversational AI, + no-subscription fee, + large language model, + docker installation, + ] +--- + +# Installing Jan using Docker + +## Installation + +### Pre-requisites + +:::note + +**Supported OS**: Linux, WSL2 Docker + +::: + +- Docker Engine and Docker Compose are required to run Jan in Docker mode. Follow the [instructions](https://docs.docker.com/engine/install/ubuntu/) below to get started with Docker Engine on Ubuntu. + +```bash +curl -fsSL https://get.docker.com -o get-docker.sh +sudo sh ./get-docker.sh --dry-run +``` + +- If you intend to run Jan in GPU mode, you need to install `nvidia-driver` and `nvidia-docker2`. Follow the instruction [here](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html) for installation. + +### Instructions + +- Run Jan in Docker mode + + - **Option 1**: Run Jan in CPU mode + + ```bash + docker compose --profile cpu up -d + ``` + + - **Option 2**: Run Jan in GPU mode + + - **Step 1**: Check CUDA compatibility with your NVIDIA driver by running `nvidia-smi` and check the CUDA version in the output + + ```bash + nvidia-smi + + # Output + +---------------------------------------------------------------------------------------+ + | NVIDIA-SMI 531.18 Driver Version: 531.18 CUDA Version: 12.1 | + |-----------------------------------------+----------------------+----------------------+ + | GPU Name TCC/WDDM | Bus-Id Disp.A | Volatile Uncorr. ECC | + | Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. | + | | | MIG M. | + |=========================================+======================+======================| + | 0 NVIDIA GeForce RTX 4070 Ti WDDM | 00000000:01:00.0 On | N/A | + | 0% 44C P8 16W / 285W| 1481MiB / 12282MiB | 2% Default | + | | | N/A | + +-----------------------------------------+----------------------+----------------------+ + | 1 NVIDIA GeForce GTX 1660 Ti WDDM | 00000000:02:00.0 Off | N/A | + | 0% 49C P8 14W / 120W| 0MiB / 6144MiB | 0% Default | + | | | N/A | + +-----------------------------------------+----------------------+----------------------+ + | 2 NVIDIA GeForce GTX 1660 Ti WDDM | 00000000:05:00.0 Off | N/A | + | 29% 38C P8 11W / 120W| 0MiB / 6144MiB | 0% Default | + | | | N/A | + +-----------------------------------------+----------------------+----------------------+ + + +---------------------------------------------------------------------------------------+ + | Processes: | + | GPU GI CI PID Type Process name GPU Memory | + | ID ID Usage | + |=======================================================================================| + ``` + + - **Step 2**: Visit [NVIDIA NGC Catalog ](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/cuda/tags) and find the smallest minor version of image tag that matches your CUDA version (e.g., 12.1 -> 12.1.0) + + - **Step 3**: Update the `Dockerfile.gpu` line number 5 with the latest minor version of the image tag from step 2 (e.g. change `FROM nvidia/cuda:12.2.0-runtime-ubuntu22.04 AS base` to `FROM nvidia/cuda:12.1.0-runtime-ubuntu22.04 AS base`) + + - **Step 4**: Run command to start Jan in GPU mode + + ```bash + # GPU mode + docker compose --profile gpu up -d + ``` + + This will start the web server and you can access Jan at `http://localhost:3000`. + +:::warning + +- Docker mode is currently only suitable for development and localhost. Production is not supported yet, and the RAG feature is not available in Docker mode. + +::: diff --git a/docs/docs/guides/02-installation/05-nightly-build.md b/docs/docs/guides/02-installation/07-nightly-build.md similarity index 100% rename from docs/docs/guides/02-installation/05-nightly-build.md rename to docs/docs/guides/02-installation/07-nightly-build.md diff --git a/docs/docs/guides/02-installation/07-antivirus-compatibility-testing.md b/docs/docs/guides/02-installation/08-antivirus-compatibility-testing.md similarity index 100% rename from docs/docs/guides/02-installation/07-antivirus-compatibility-testing.md rename to docs/docs/guides/02-installation/08-antivirus-compatibility-testing.md diff --git a/docs/docs/guides/04-using-models/02-import-manually.mdx b/docs/docs/guides/04-using-models/02-import-manually.mdx index 68142a8af0..7c446ea1c9 100644 --- a/docs/docs/guides/04-using-models/02-import-manually.mdx +++ b/docs/docs/guides/04-using-models/02-import-manually.mdx @@ -29,6 +29,10 @@ In this section, we will show you how to import a GGUF model from [HuggingFace]( > We are fast shipping a UI to make this easier, but it's a bit manual for now. Apologies. +## Import Models Using Absolute Filepath (version 0.4.7) + +Starting from version 0.4.7, Jan has introduced the capability to import models using an absolute file path. It allows you to import models from any directory on your computer. Please check the [import models using absolute filepath](../import-models-using-absolute-filepath) guide for more information. + ## Manually Importing a Downloaded Model (nightly versions and v0.4.4+) ### 1. Create a Model Folder @@ -186,7 +190,6 @@ This means that you can easily reconfigure your models, export them, and share y Edit `model.json` and include the following configurations: -- Ensure the filename must be `model.json`. - Ensure the `id` property matches the folder name you created. - Ensure the GGUF filename should match the `id` property exactly. - Ensure the `source.url` property is the direct binary download link ending in `.gguf`. In HuggingFace, you can find the direct links in the `Files and versions` tab. diff --git a/docs/docs/guides/04-using-models/03-import-models-using-absolute-filepath.mdx b/docs/docs/guides/04-using-models/03-import-models-using-absolute-filepath.mdx new file mode 100644 index 0000000000..490f68cd67 --- /dev/null +++ b/docs/docs/guides/04-using-models/03-import-models-using-absolute-filepath.mdx @@ -0,0 +1,84 @@ +--- +title: Import Models Using Absolute Filepath +slug: /guides/using-models/import-models-using-absolute-filepath +description: Guide to import model using absolute filepath in Jan. +keywords: + [ + Jan AI, + Jan, + ChatGPT alternative, + local AI, + private AI, + conversational AI, + no-subscription fee, + large language model, + import-models-manually, + absolute-filepath, + ] +--- + +In this guide, we will walk you through the process of importing a model using an absolute filepath in Jan, using our latest model, [Trinity](https://huggingface.co/janhq/trinity-v1-GGUF), as an example. + +### 1. Get the Absolute Filepath of the Model + +After downloading .gguf model, you can get the absolute filepath of the model file. + +### 2. Configure the Model JSON + +1. Navigate to the `~/jan/models` folder. +2. Create a folder named ``, for example, `tinyllama`. +3. Create a `model.json` file inside the folder, including the following configurations: + +- Ensure the `id` property matches the folder name you created. +- Ensure the `url` property is the direct binary download link ending in `.gguf`. Now, you can use the absolute filepath of the model file. +- Ensure the `engine` property is set to `nitro`. + +```json +{ + "sources": [ + { + "filename": "tinyllama.gguf", + // highlight-next-line + "url": "" + } + ], + "id": "tinyllama-1.1b", + "object": "model", + "name": "(Absolute Path) TinyLlama Chat 1.1B Q4", + "version": "1.0", + "description": "TinyLlama is a tiny model with only 1.1B. It's a good model for less powerful computers.", + "format": "gguf", + "settings": { + "ctx_len": 4096, + "prompt_template": "<|system|>\n{system_message}<|user|>\n{prompt}<|assistant|>", + "llama_model_path": "tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf" + }, + "parameters": { + "temperature": 0.7, + "top_p": 0.95, + "stream": true, + "max_tokens": 2048, + "stop": [], + "frequency_penalty": 0, + "presence_penalty": 0 + }, + "metadata": { + "author": "TinyLlama", + "tags": ["Tiny", "Foundation Model"], + "size": 669000000 + }, + "engine": "nitro" +} +``` + +:::warning + +- If you are using Windows, you need to use double backslashes in the url property, for example: `C:\\Users\\username\\filename.gguf`. + +::: + +### 3. Start the Model + +Restart Jan and navigate to the Hub. Locate your model and click the Use button. + +![Demo](assets/03-demo-absolute-filepath.gif) \ No newline at end of file diff --git a/docs/docs/guides/04-using-models/03-integrate-with-remote-server.mdx b/docs/docs/guides/04-using-models/04-integrate-with-remote-server.mdx similarity index 90% rename from docs/docs/guides/04-using-models/03-integrate-with-remote-server.mdx rename to docs/docs/guides/04-using-models/04-integrate-with-remote-server.mdx index 533797fcaa..3632a40b02 100644 --- a/docs/docs/guides/04-using-models/03-integrate-with-remote-server.mdx +++ b/docs/docs/guides/04-using-models/04-integrate-with-remote-server.mdx @@ -65,6 +65,13 @@ Navigate to the `~/jan/models` folder. Create a folder named `gpt-3.5-turbo-16k` } ``` +:::tip + +- You can find the list of available models in the [OpenAI Platform](https://platform.openai.com/docs/models/overview). +- Please note that the `id` property need to match the model name in the list. For example, if you want to use the [GPT-4 Turbo](https://platform.openai.com/docs/models/gpt-4-and-gpt-4-turbo), you need to set the `id` property as `gpt-4-1106-preview`. + +::: + ### 2. Configure OpenAI API Keys You can find your API keys in the [OpenAI Platform](https://platform.openai.com/api-keys) and set the OpenAI API keys in `~/jan/engines/openai.json` file. @@ -81,7 +88,7 @@ You can find your API keys in the [OpenAI Platform](https://platform.openai.com/ Restart Jan and navigate to the Hub. Then, select your configured model and start the model. -![image-01](assets/03-openai-platform-configuration.png) +![image-01](assets/04-openai-platform-configuration.png) ## Engines with OAI Compatible Configuration @@ -152,7 +159,7 @@ Navigate to the `~/jan/models` folder. Create a folder named `mistral-ins-7b-q4` Restart Jan and navigate to the Hub. Locate your model and click the Use button. -![image-02](assets/03-oai-compatible-configuration.png) +![image-02](assets/04-oai-compatible-configuration.png) ## Assistance and Support diff --git a/docs/docs/guides/04-using-models/04-customize-engine-settings.mdx b/docs/docs/guides/04-using-models/05-customize-engine-settings.mdx similarity index 100% rename from docs/docs/guides/04-using-models/04-customize-engine-settings.mdx rename to docs/docs/guides/04-using-models/05-customize-engine-settings.mdx diff --git a/docs/docs/guides/04-using-models/assets/03-demo-absolute-filepath.gif b/docs/docs/guides/04-using-models/assets/03-demo-absolute-filepath.gif new file mode 100644 index 0000000000..24dcc251a7 Binary files /dev/null and b/docs/docs/guides/04-using-models/assets/03-demo-absolute-filepath.gif differ diff --git a/docs/docs/guides/04-using-models/assets/03-oai-compatible-configuration.png b/docs/docs/guides/04-using-models/assets/04-oai-compatible-configuration.png similarity index 100% rename from docs/docs/guides/04-using-models/assets/03-oai-compatible-configuration.png rename to docs/docs/guides/04-using-models/assets/04-oai-compatible-configuration.png diff --git a/docs/docs/guides/04-using-models/assets/03-openai-platform-configuration.png b/docs/docs/guides/04-using-models/assets/04-openai-platform-configuration.png similarity index 100% rename from docs/docs/guides/04-using-models/assets/03-openai-platform-configuration.png rename to docs/docs/guides/04-using-models/assets/04-openai-platform-configuration.png diff --git a/docs/docs/guides/05-using-server/01-server.md b/docs/docs/guides/05-using-server/01-server.md deleted file mode 100644 index 952b7399fa..0000000000 --- a/docs/docs/guides/05-using-server/01-server.md +++ /dev/null @@ -1,33 +0,0 @@ ---- -title: Connect to Server -description: Connect to Jan's built-in API server. -keywords: - [ - Jan AI, - Jan, - ChatGPT alternative, - local AI, - private AI, - conversational AI, - no-subscription fee, - large language model, - ] ---- - -:::warning - -This page is under construction. - -::: - -Jan ships with a built-in API server, that can be used as a drop-in, local replacement for OpenAI's API. - -Jan runs on port `1337` by default, but this can (soon) be changed in Settings. - -1. Go to Settings > Advanced > Enable API Server - -2. Go to http://localhost:1337 for the API docs. - -3. In terminal, simply CURL... - -Note: Some UI states may be broken when in Server Mode. diff --git a/docs/docs/guides/05-using-server/01-start-server.md b/docs/docs/guides/05-using-server/01-start-server.md new file mode 100644 index 0000000000..2433fd80a1 --- /dev/null +++ b/docs/docs/guides/05-using-server/01-start-server.md @@ -0,0 +1,72 @@ +--- +title: Start Local Server +slug: /guides/using-server/start-server +description: How to run Jan's built-in API server. +keywords: + [ + Jan AI, + Jan, + ChatGPT alternative, + local AI, + private AI, + conversational AI, + no-subscription fee, + large language model, + local server, + api server, + ] +--- + +Jan ships with a built-in API server that can be used as a drop-in, local replacement for OpenAI's API. You can run your server by following these simple steps. + +## Open Local API Server View + +Navigate to the Local API Server view by clicking the corresponding icon on the left side of the screen. + +

+ +![01-local-api-view](./assets/01-local-api-view.gif) + +## Choosing a Model + +On the top right of your screen under `Model Settings`, set the LLM that your local server will be running. You can choose from any of the models already installed, or pick a new model by clicking `Explore the Hub`. + +

+ +![01-choose-model](./assets/01-choose-model.png) + +## Server Options + +On the left side of your screen, you can set custom server options. + +

+ +![01-server-settings](./assets/01-server-options.png) + +### Local Server Address + +By default, Jan will be accessible only on localhost `127.0.0.1`. This means a local server can only be accessed on the same machine where the server is being run. + +You can make the local server more accessible by clicking on the address and choosing `0.0.0.0` instead, which allows the server to be accessed from other devices on the local network. This is less secure than choosing localhost, and should be done with caution. + +### Port + +Jan runs on port `1337` by default. You can change the port to any other port number if needed. + +### Cross-Origin Resource Sharing (CORS) + +Cross-Origin Resource Sharing (CORS) manages resource access on the local server from external domains. Enabled for security by default, it can be disabled if needed. + +### Verbose Server Logs + +The center of the screen displays the server logs as the local server runs. This option provides extensive details about server activities. + +## Start Server + +Click the `Start Server` button on the top left of your screen. You will see the server log display a message such as `Server listening at http://127.0.0.1:1337`, and the `Start Server` button will change to a red `Stop Server` button. + +

+ +![01-running-server](./assets/01-running-server.gif) + +You server is now running and you can use the server address and port to make requests to the local server. diff --git a/docs/docs/guides/05-using-server/02-using-server.md b/docs/docs/guides/05-using-server/02-using-server.md new file mode 100644 index 0000000000..3d4b004a1f --- /dev/null +++ b/docs/docs/guides/05-using-server/02-using-server.md @@ -0,0 +1,102 @@ +--- +title: Using Jan's Built-in API Server +description: How to use Jan's built-in API server. +keywords: + [ + Jan AI, + Jan, + ChatGPT alternative, + local AI, + private AI, + conversational AI, + no-subscription fee, + large language model, + local server, + api server, + ] +--- + +Jan's built-in API server is compatible with [OpenAI's API](https://platform.openai.com/docs/api-reference) and can be used as a drop-in, local replacement. Follow these steps to use the API server. + +## Open the API Reference + +Jan contains a comprehensive API reference. This reference displays all the API endpoints available, gives you examples requests and responses, and allows you to execute them in browser. + +On the top left of your screen below the red `Stop Server` button is the blue `API Reference`. Clicking this will open the reference in your browser. + +

+ +![02-api-reference](./assets/02-api-reference.png) + +Scroll through the various available endpoints to learn what options are available and try them out by executing the example requests. In addition, you can also use the [Jan API Reference](https://jan.ai/api-reference/) on the Jan website. + +### Chat + +In the Chat section of the API reference, you will see an example JSON request body. + +

+ +![02-chat-example](./assets/02-chat-example.png) + +With your local server running, you can click the `Try it out` button on the top left, then the blue `Execute` button below the JSON. The browser will send the example request to your server, and display the response body below. + +Use the API endpoints, request and response body examples as models for your own application. + +### cURL Request Example + +Here is an example curl request with a local server running `tinyllama-1.1b`: + +

+ +```json +{ + "messages": [ + { + "content": "You are a helpful assistant.", + "role": "system" + }, + { + "content": "Hello!", + "role": "user" + } + ], + "model": "tinyllama-1.1b", + "stream": true, + "max_tokens": 2048, + "stop": [ + "hello" + ], + "frequency_penalty": 0, + "presence_penalty": 0, + "temperature": 0.7, + "top_p": 0.95 +} +' +``` + +### Response Body Example + +```json +{ + "choices": [ + { + "finish_reason": null, + "index": 0, + "message": { + "content": "Hello user. What can I help you with?", + "role": "assistant" + } + } + ], + "created": 1700193928, + "id": "ebwd2niJvJB1Q2Whyvkz", + "model": "_", + "object": "chat.completion", + "system_fingerprint": "_", + "usage": { + "completion_tokens": 500, + "prompt_tokens": 33, + "total_tokens": 533 + } +} +``` diff --git a/docs/docs/guides/05-using-server/assets/01-choose-model.png b/docs/docs/guides/05-using-server/assets/01-choose-model.png new file mode 100644 index 0000000000..9062a1e951 Binary files /dev/null and b/docs/docs/guides/05-using-server/assets/01-choose-model.png differ diff --git a/docs/docs/guides/05-using-server/assets/01-local-api-view.gif b/docs/docs/guides/05-using-server/assets/01-local-api-view.gif new file mode 100644 index 0000000000..cb221fce45 Binary files /dev/null and b/docs/docs/guides/05-using-server/assets/01-local-api-view.gif differ diff --git a/docs/docs/guides/05-using-server/assets/01-running-server.gif b/docs/docs/guides/05-using-server/assets/01-running-server.gif new file mode 100644 index 0000000000..a4225f3cb0 Binary files /dev/null and b/docs/docs/guides/05-using-server/assets/01-running-server.gif differ diff --git a/docs/docs/guides/05-using-server/assets/01-server-options.png b/docs/docs/guides/05-using-server/assets/01-server-options.png new file mode 100644 index 0000000000..c48844e405 Binary files /dev/null and b/docs/docs/guides/05-using-server/assets/01-server-options.png differ diff --git a/docs/docs/guides/05-using-server/assets/02-api-reference.png b/docs/docs/guides/05-using-server/assets/02-api-reference.png new file mode 100644 index 0000000000..154d9dfc90 Binary files /dev/null and b/docs/docs/guides/05-using-server/assets/02-api-reference.png differ diff --git a/docs/docs/guides/05-using-server/assets/02-chat-example.png b/docs/docs/guides/05-using-server/assets/02-chat-example.png new file mode 100644 index 0000000000..bd7e33a6ae Binary files /dev/null and b/docs/docs/guides/05-using-server/assets/02-chat-example.png differ diff --git a/docs/docs/guides/07-integrations/01-integrate-continue.mdx b/docs/docs/guides/07-integrations/01-integrate-continue.mdx index 3a0e9f282b..1fa0397e29 100644 --- a/docs/docs/guides/07-integrations/01-integrate-continue.mdx +++ b/docs/docs/guides/07-integrations/01-integrate-continue.mdx @@ -35,7 +35,7 @@ To get started with Continue in VS Code, please follow this [guide to install Co ### 2. Enable Jan API Server -To configure the Continue to use Jan's Local Server, you need to enable Jan API Server with your preferred model, please follow this [guide to enable Jan API Server](../05-using-server/01-server.md) +To configure the Continue to use Jan's Local Server, you need to enable Jan API Server with your preferred model, please follow this [guide to enable Jan API Server](/guides/using-server/start-server). ### 3. Configure Continue to Use Jan's Local Server @@ -77,7 +77,7 @@ Edit the `config.json` file and include the following configuration. // highlight-start "model": "mistral-ins-7b-q4", "apiKey": "EMPTY", - "apiBase": "http://localhost:1337" + "apiBase": "http://localhost:1337/v1" // highlight-end } ] @@ -86,7 +86,7 @@ Edit the `config.json` file and include the following configuration. - Ensure that the `provider` is `openai`. - Ensure that the `model` is the same as the one you enabled in the Jan API Server. -- Ensure that the `apiBase` is `http://localhost:1337`. +- Ensure that the `apiBase` is `http://localhost:1337/v1`. - Ensure that the `apiKey` is `EMPTY`. ### 4. Ensure the Using Model Is Activated in Jan diff --git a/docs/docs/guides/07-integrations/04-integrate-mistral-ai.mdx b/docs/docs/guides/07-integrations/04-integrate-mistral-ai.mdx new file mode 100644 index 0000000000..14ddeaa750 --- /dev/null +++ b/docs/docs/guides/07-integrations/04-integrate-mistral-ai.mdx @@ -0,0 +1,89 @@ +--- +title: Integrate Mistral AI with Jan +slug: /guides/integrations/mistral-ai +description: Guide to integrate Mistral AI with Jan +keywords: + [ + Jan AI, + Jan, + ChatGPT alternative, + local AI, + private AI, + conversational AI, + no-subscription fee, + large language model, + Mistral integration, + ] +--- + +## Quick Introduction + +[Mistral AI](https://docs.mistral.ai/) currently provides two ways of accessing their Large Language Models (LLM) - via their API or via open source models available on Hugging Face. In this guide, we will show you how to integrate Mistral AI with Jan using the API method. + +## Steps to Integrate Mistral AI with Jan + +### 1. Configure Mistral API key + +You can find your API keys in the [Mistral API Key](https://console.mistral.ai/user/api-keys/) and set the Mistral AI API key in `~/jan/engines/openai.json` file. + +```json title="~/jan/engines/openai.json" +{ + // highlight-start + "full_url": "https://api.mistral.ai/v1/chat/completions", + "api_key": "" + // highlight-end +} +``` + +### 2. Modify a Model JSON + +Navigate to the `~/jan/models` folder. Create a folder named ``, for example, `mistral-tiny` and create a `model.json` file inside the folder including the following configurations: + +- Ensure the filename must be `model.json`. +- Ensure the `id` property is set to the model id from Mistral AI. +- Ensure the `format` property is set to `api`. +- Ensure the `engine` property is set to `openai`. +- Ensure the `state` property is set to `ready`. + +```json title="~/jan/models/mistral-tiny/model.json" +{ + "sources": [ + { + "filename": "mistral-tiny", + "url": "https://mistral.ai/" + } + ], + "id": "mistral-tiny", + "object": "model", + "name": "Mistral-7B-v0.2 (Tiny Endpoint)", + "version": "1.0", + "description": "Currently powered by Mistral-7B-v0.2, a better fine-tuning of the initial Mistral-7B released, inspired by the fantastic work of the community.", + // highlight-next-line + "format": "api", + "settings": {}, + "parameters": {}, + "metadata": { + "author": "Mistral AI", + "tags": ["General", "Big Context Length"] + }, + // highlight-start + "engine": "openai" + // highlight-end +} +``` + +:::tip + +Mistral AI provides different endpoints. Please check out their [endpoint documentation](https://docs.mistral.ai/platform/endpoints/) to find the one that suits your needs. In this example, we will use the `mistral-tiny` model. + +::: + +### 3. Start the Model + +Restart Jan and navigate to the Hub. Locate your model and click the Use button. + +![Mitral AI Tiny Model](assets/04-mistral-ai-tiny-hub.png) + +### 4. Try Out the Integration of Jan and Mistral AI + +![Mistral AI Integration Demo](assets/04-mistral-ai-integration-demo.gif) diff --git a/docs/docs/guides/07-integrations/05-integrate-lmstudio.mdx b/docs/docs/guides/07-integrations/05-integrate-lmstudio.mdx new file mode 100644 index 0000000000..58e2f0be97 --- /dev/null +++ b/docs/docs/guides/07-integrations/05-integrate-lmstudio.mdx @@ -0,0 +1,184 @@ +--- +title: Integrate LM Studio with Jan +slug: /guides/integrations/lmstudio +description: Guide to integrate LM Studio with Jan +keywords: + [ + Jan AI, + Jan, + ChatGPT alternative, + local AI, + private AI, + conversational AI, + no-subscription fee, + large language model, + LM Studio integration, + ] +--- + +## Quick Introduction + +With [LM Studio](https://lmstudio.ai/), you can discover, download, and run local Large Language Models (LLMs). In this guide, we will show you how to integrate and use your current models on LM Studio with Jan using 2 methods. The first method is integrating LM Studio server with Jan UI. The second method is migrating your downloaded model from LM Studio to Jan. We will use the [Phi 2 - GGUF](https://huggingface.co/TheBloke/phi-2-GGUF) model on Hugging Face as an example. + +## Steps to Integrate LM Studio Server with Jan UI + +### 1. Start the LM Studio Server + +1. Navigate to the `Local Inference Server` on the LM Studio application. +2. Select the model you want to use. +3. Start the server after configuring the server port and options. + +![LM Studio Server](assets/05-setting-lmstudio-server.gif) + +

+ +Modify the `openai.json` file in the `~/jan/engines` folder to include the full URL of the LM Studio server. + +```json title="~/jan/engines/openai.json" +{ + "full_url": "http://localhost:/v1/chat/completions" +} +``` + +:::tip + +- Replace `` with the port number you set in the LM Studio server. The default port is `1234`. + +::: + +### 2. Modify a Model JSON + +Navigate to the `~/jan/models` folder. Create a folder named ``, for example, `lmstudio-phi-2` and create a `model.json` file inside the folder including the following configurations: + +- Set the `format` property to `api`. +- Set the `engine` property to `openai`. +- Set the `state` property to `ready`. + +```json title="~/jan/models/lmstudio-phi-2/model.json" +{ + "sources": [ + { + "filename": "phi-2-GGUF", + "url": "https://huggingface.co/TheBloke/phi-2-GGUF" + } + ], + "id": "lmstudio-phi-2", + "object": "model", + "name": "LM Studio - Phi 2 - GGUF", + "version": "1.0", + "description": "TheBloke/phi-2-GGUF", + // highlight-next-line + "format": "api", + "settings": {}, + "parameters": {}, + "metadata": { + "author": "Microsoft", + "tags": ["General", "Big Context Length"] + }, + // highlight-start + "engine": "openai" + // highlight-end +} +``` + +### 3. Start the Model + +1. Restart Jan and navigate to the **Hub**. +2. Locate your model and click the **Use** button. + +![LM Studio Model](assets/05-lmstudio-run.png) + +### 4. Try Out the Integration of Jan and LM Studio + +![LM Studio Integration Demo](assets/05-lmstudio-integration-demo.gif) + +## Steps to Migrate Your Downloaded Model from LM Studio to Jan (version 0.4.6 and older) + +### 1. Migrate Your Downloaded Model + +1. Navigate to `My Models` in the LM Studio application and reveal the model folder. + +![Reveal-model-folder-lmstudio](assets/05-reveal-model-folder-lmstudio.gif) + +2. Copy the model folder that you want to migrate to `~/jan/models` folder. + +3. Ensure the folder name property is the same as the model name of `.gguf` filename by changing the folder name if necessary. For example, in this case, we changed foldername from `TheBloke` to `phi-2.Q4_K_S`. + +### 2. Start the Model + +1. Restart Jan and navigate to the **Hub**. Jan will automatically detect the model and display it in the **Hub**. +2. Locate your model and click the **Use** button to try the migrating model. + +![Demo](assets/05-demo-migrating-model.gif) + +## Steps to Pointing to the Downloaded Model of LM Studio from Jan (version 0.4.7+) + +Starting from version 0.4.7, Jan supports importing models using an absolute filepath, so you can directly use the model from the LM Studio folder. + +### 1. Reveal the Model Absolute Path + +Navigate to `My Models` in the LM Studio application and reveal the model folder. Then, you can get the absolute path of your model. + +![Reveal-model-folder-lmstudio](assets/05-reveal-model-folder-lmstudio.gif) + +### 2. Modify a Model JSON + +Navigate to the `~/jan/models` folder. Create a folder named ``, for example, `phi-2.Q4_K_S` and create a `model.json` file inside the folder including the following configurations: + +- Ensure the `id` property matches the folder name you created. +- Ensure the `url` property is the direct binary download link ending in `.gguf`. Now, you can use the absolute filepath of the model file. In this example, the absolute filepath is `/Users//.cache/lm-studio/models/TheBloke/phi-2-GGUF/phi-2.Q4_K_S.gguf`. +- Ensure the `engine` property is set to `nitro`. + +```json +{ + "object": "model", + "version": 1, + "format": "gguf", + "sources": [ + { + "filename": "phi-2.Q4_K_S.gguf", + "url": "" + } + ], + "id": "phi-2.Q4_K_S", + "name": "phi-2.Q4_K_S", + "created": 1708308111506, + "description": "phi-2.Q4_K_S - user self import model", + "settings": { + "ctx_len": 4096, + "embedding": false, + "prompt_template": "{system_message}\n### Instruction: {prompt}\n### Response:", + "llama_model_path": "phi-2.Q4_K_S.gguf" + }, + "parameters": { + "temperature": 0.7, + "top_p": 0.95, + "stream": true, + "max_tokens": 2048, + "stop": [""], + "frequency_penalty": 0, + "presence_penalty": 0 + }, + "metadata": { + "size": 1615568736, + "author": "User", + "tags": [] + }, + "engine": "nitro" +} +``` + +:::warning + +- If you are using Windows, you need to use double backslashes in the url property, for example: `C:\\Users\\username\\filename.gguf`. + +::: + + +### 3. Start the Model + +1. Restart Jan and navigate to the **Hub**. +2. Jan will automatically detect the model and display it in the **Hub**. +3. Locate your model and click the **Use** button to try the migrating model. + +![Demo](assets/05-demo-pointing-model.gif) diff --git a/docs/docs/guides/07-integrations/06-integrate-ollama.mdx b/docs/docs/guides/07-integrations/06-integrate-ollama.mdx new file mode 100644 index 0000000000..e55c3e49f7 --- /dev/null +++ b/docs/docs/guides/07-integrations/06-integrate-ollama.mdx @@ -0,0 +1,90 @@ +--- +title: Integrate Ollama with Jan +slug: /guides/integrations/ollama +description: Guide to integrate Ollama with Jan +keywords: + [ + Jan AI, + Jan, + ChatGPT alternative, + local AI, + private AI, + conversational AI, + no-subscription fee, + large language model, + Ollama integration, + ] +--- + +## Quick Introduction + +With [Ollama](https://ollama.com/), you can run large language models locally. In this guide, we will show you how to integrate and use your current models on Ollama with Jan using 2 methods. The first method is integrating Ollama server with Jan UI. The second method is migrating your downloaded model from Ollama to Jan. We will use the [llama2](https://ollama.com/library/llama2) model as an example. + +## Steps to Integrate Ollama Server with Jan UI + +### 1. Start the Ollama Server + +1. Select the model you want to use from the [Ollama library](https://ollama.com/library). +2. Run your model by using the following command: + +```bash +ollama run +``` + +3. According to the [Ollama documentation on OpenAI compatibility](https://github.com/ollama/ollama/blob/main/docs/openai.md), you can use the `http://localhost:11434/v1/chat/completions` endpoint to interact with the Ollama server. Thus, modify the `openai.json` file in the `~/jan/engines` folder to include the full URL of the Ollama server. + +```json title="~/jan/engines/openai.json" +{ + "full_url": "http://localhost:11434/v1/chat/completions" +} +``` + +### 2. Modify a Model JSON + +1. Navigate to the `~/jan/models` folder. +2. Create a folder named ``, for example, `lmstudio-phi-2`. +3. Create a `model.json` file inside the folder including the following configurations: + +- Set the `id` property to the model name as Ollama model name. +- Set the `format` property to `api`. +- Set the `engine` property to `openai`. +- Set the `state` property to `ready`. + +```json title="~/jan/models/llama2/model.json" +{ + "sources": [ + { + "filename": "llama2", + "url": "https://ollama.com/library/llama2" + } + ], + // highlight-next-line + "id": "llama2", + "object": "model", + "name": "Ollama - Llama2", + "version": "1.0", + "description": "Llama 2 is a collection of foundation language models ranging from 7B to 70B parameters.", + // highlight-next-line + "format": "api", + "settings": {}, + "parameters": {}, + "metadata": { + "author": "Meta", + "tags": ["General", "Big Context Length"] + }, + // highlight-next-line + "engine": "openai" +} +``` + +### 3. Start the Model + +1. Restart Jan and navigate to the **Hub**. +2. Locate your model and click the **Use** button. + +![Ollama Model](assets/06-ollama-run.png) + +### 4. Try Out the Integration of Jan and Ollama + +![Ollama Integration Demo](assets/06-ollama-integration-demo.gif) + diff --git a/docs/docs/guides/07-integrations/assets/04-mistral-ai-integration-demo.gif b/docs/docs/guides/07-integrations/assets/04-mistral-ai-integration-demo.gif new file mode 100644 index 0000000000..015167e2ab Binary files /dev/null and b/docs/docs/guides/07-integrations/assets/04-mistral-ai-integration-demo.gif differ diff --git a/docs/docs/guides/07-integrations/assets/04-mistral-ai-tiny-hub.png b/docs/docs/guides/07-integrations/assets/04-mistral-ai-tiny-hub.png new file mode 100644 index 0000000000..1ae377d709 Binary files /dev/null and b/docs/docs/guides/07-integrations/assets/04-mistral-ai-tiny-hub.png differ diff --git a/docs/docs/guides/07-integrations/assets/05-demo-migrating-model.gif b/docs/docs/guides/07-integrations/assets/05-demo-migrating-model.gif new file mode 100644 index 0000000000..985755e47c Binary files /dev/null and b/docs/docs/guides/07-integrations/assets/05-demo-migrating-model.gif differ diff --git a/docs/docs/guides/07-integrations/assets/05-demo-pointing-model.gif b/docs/docs/guides/07-integrations/assets/05-demo-pointing-model.gif new file mode 100644 index 0000000000..137fb955ac Binary files /dev/null and b/docs/docs/guides/07-integrations/assets/05-demo-pointing-model.gif differ diff --git a/docs/docs/guides/07-integrations/assets/05-lmstudio-integration-demo.gif b/docs/docs/guides/07-integrations/assets/05-lmstudio-integration-demo.gif new file mode 100644 index 0000000000..445ea3416a Binary files /dev/null and b/docs/docs/guides/07-integrations/assets/05-lmstudio-integration-demo.gif differ diff --git a/docs/docs/guides/07-integrations/assets/05-lmstudio-run.png b/docs/docs/guides/07-integrations/assets/05-lmstudio-run.png new file mode 100644 index 0000000000..721581f72e Binary files /dev/null and b/docs/docs/guides/07-integrations/assets/05-lmstudio-run.png differ diff --git a/docs/docs/guides/07-integrations/assets/05-reveal-model-folder-lmstudio.gif b/docs/docs/guides/07-integrations/assets/05-reveal-model-folder-lmstudio.gif new file mode 100644 index 0000000000..4c1ee85fc3 Binary files /dev/null and b/docs/docs/guides/07-integrations/assets/05-reveal-model-folder-lmstudio.gif differ diff --git a/docs/docs/guides/07-integrations/assets/05-setting-lmstudio-server.gif b/docs/docs/guides/07-integrations/assets/05-setting-lmstudio-server.gif new file mode 100644 index 0000000000..63084be01d Binary files /dev/null and b/docs/docs/guides/07-integrations/assets/05-setting-lmstudio-server.gif differ diff --git a/docs/docs/guides/07-integrations/assets/06-ollama-integration-demo.gif b/docs/docs/guides/07-integrations/assets/06-ollama-integration-demo.gif new file mode 100644 index 0000000000..708f2058a7 Binary files /dev/null and b/docs/docs/guides/07-integrations/assets/06-ollama-integration-demo.gif differ diff --git a/docs/docs/guides/07-integrations/assets/06-ollama-run.png b/docs/docs/guides/07-integrations/assets/06-ollama-run.png new file mode 100644 index 0000000000..7f18e1b15b Binary files /dev/null and b/docs/docs/guides/07-integrations/assets/06-ollama-run.png differ diff --git a/docs/docs/guides/08-troubleshooting/02-somethings-amiss.mdx b/docs/docs/guides/08-troubleshooting/02-somethings-amiss.mdx index a5669e36dd..4e16e362a5 100644 --- a/docs/docs/guides/08-troubleshooting/02-somethings-amiss.mdx +++ b/docs/docs/guides/08-troubleshooting/02-somethings-amiss.mdx @@ -45,7 +45,9 @@ This may occur due to several reasons. Please follow these steps to resolve it: 5. If you are on Nvidia GPUs, please download [Cuda](https://developer.nvidia.com/cuda-downloads). -6. When [checking app logs](https://jan.ai/troubleshooting/how-to-get-error-logs/), if you encounter the error log `Bind address failed at 127.0.0.1:3928`, it indicates that the port used by Nitro might already be in use. Use the following commands to check the port status: +6. If you're using Linux, please ensure that your system meets the following requirements gcc 11, g++ 11, cpp 11, or higher, refer to this [link](https://jan.ai/guides/troubleshooting/gpu-not-used/#specific-requirements-for-linux) for more information. + +7. When [checking app logs](https://jan.ai/troubleshooting/how-to-get-error-logs/), if you encounter the error log `Bind address failed at 127.0.0.1:3928`, it indicates that the port used by Nitro might already be in use. Use the following commands to check the port status: diff --git a/docs/docs/guides/08-troubleshooting/03-gpu-not-used.mdx b/docs/docs/guides/08-troubleshooting/03-gpu-not-used.mdx index d35993ab6a..53638027b0 100644 --- a/docs/docs/guides/08-troubleshooting/03-gpu-not-used.mdx +++ b/docs/docs/guides/08-troubleshooting/03-gpu-not-used.mdx @@ -188,4 +188,6 @@ Troubleshooting tips: 2. If the issue persists, ensure your (V)RAM is accessible by the application. Some folks have virtual RAM and need additional configuration. -3. Get help in [Jan Discord](https://discord.gg/mY69SZaMaC). +3. If you are facing issues with the installation of RTX issues, please update the NVIDIA driver that supports CUDA 11.7 or higher. Ensure that the CUDA path is added to the environment variable. + +4. Get help in [Jan Discord](https://discord.gg/mY69SZaMaC). diff --git a/docs/docs/guides/08-troubleshooting/06-unexpected-token.mdx b/docs/docs/guides/08-troubleshooting/06-unexpected-token.mdx index 973001f1b0..1de609ffa4 100644 --- a/docs/docs/guides/08-troubleshooting/06-unexpected-token.mdx +++ b/docs/docs/guides/08-troubleshooting/06-unexpected-token.mdx @@ -17,4 +17,8 @@ keywords: ] --- -1. You may receive an error response `Error occurred: Unexpected token '<', "/nitro` and run the nitro manually and see if you get any error messages. +3. Resolve the error messages you get from the nitro and see if the issue persists. +4. Reopen the Jan app and see if the issue is resolved. +5. If the issue persists, please share with us the [app logs](https://jan.ai/troubleshooting/how-to-get-error-logs/) via [Jan Discord](https://discord.gg/mY69SZaMaC). diff --git a/docs/docs/guides/09-advanced-settings/01-https-proxy.mdx b/docs/docs/guides/09-advanced-settings/01-https-proxy.mdx new file mode 100644 index 0000000000..35f4c30f90 --- /dev/null +++ b/docs/docs/guides/09-advanced-settings/01-https-proxy.mdx @@ -0,0 +1,101 @@ +--- +title: HTTPS Proxy +slug: /guides/advanced-settings/https-proxy +description: Jan is a ChatGPT-alternative that runs on your own computer, with a local API server. +keywords: + [ + Jan AI, + Jan, + ChatGPT alternative, + local AI, + private AI, + conversational AI, + no-subscription fee, + large language model, + advanced-settings, + https-proxy, + ] +--- + +In this guide, we will show you how to set up your own HTTPS proxy server and configure Jan to use it. + +## Why HTTPS Proxy? +An HTTPS proxy helps you to maintain your privacy and security while still being able to browser the internet circumventing geographical restrictions. + +## Setting Up Your Own HTTPS Proxy Server +In this section, we will show you a high-level overview of how to set up your own HTTPS proxy server. This guide focus on using Squid as a popular and open-source proxy server software, but there are other software options you might consider based on your needs and preferences. + +### Step 1: Choosing a Server +Firstly, you need to choose a server to host your proxy server. We recommend using a cloud provider like Amazon AWS, Google Cloud, Microsoft Azure, Digital Ocean, etc. Ensure that your server has a public IP address and is accessible from the internet. + +### Step 2: Installing Squid +```bash +sudo apt-get update +sudo apt-get install squid +``` + +### Step 3: Configure Squid for HTTPS + +To enable HTTPS, you will need to configure Squid with SSL support. + +- Generate SSL certificate + +Squid requires an SSL certificate to be able to handle HTTPS traffic. You can generate a self-signed certificate or obtain one from a Certificate Authority (CA). For a self-signed certificate, you can use OpenSSL: + +```bash +openssl req -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout squid-proxy.pem -out squid-proxy.pem +``` + +- Configure Squid to use the SSL certificate: Edit the Squid configuration file `/etc/squid/squid.conf` to include the path to your SSL certificate and enable the HTTPS port: + +```bash +http_port 3128 ssl-bump cert=/path/to/your/squid-proxy.pem +ssl_bump server-first all +ssl_bump bump all +``` + +- Enable SSL Bumping: To intercept HTTPS traffic, Squid uses a process called SSL Bumping. This process allows Squid to decrypt and re-encrypt HTTPS traffic. To enable SSL Bumping, ensure the `ssl_bump` directives are configured correctly in your `squid.conf` file. + +### Step 4 (Optional): Configure ACLs and Authentication + +- Access Control Lists (ACLs): You can define rules to control who can access your proxy. This is done by editing the squid.conf file and defining ACLs: + +```bash +acl allowed_ips src "/etc/squid/allowed_ips.txt" +http_access allow allowed_ips +``` + +- Authentication: If you want to add an authentication layer, Squid supports several authentication schemes. Basic authentication setup might look like this: + +```bash +auth_param basic program /usr/lib/squid/basic_ncsa_auth /etc/squid/passwords +acl authenticated proxy_auth REQUIRED +http_access allow authenticated +``` + +### Step 5: Restart and Test Your Proxy + +After configuring, restart Squid to apply the changes: + +```bash +sudo systemctl restart squid +``` + +To test, configure your browser or another client to use the proxy server with its IP address and port (default is 3128). Check if you can access the internet through your proxy. + +:::tip + +Tips for Secure Your Proxy: +- Firewall rules: Ensure that only intended users or IP addresses can connect to your proxy server. This can be achieved by setting up appropriate firewall rules. +- Regular updates: Keep your server and proxy software updated to ensure that you are protected against known vulnerabilities. +- Monitoring and logging: Monitor your proxy server for unusual activity and enable logging to keep track of the traffic passing through your proxy. + +::: + +## Setting Up Jan to Use Your HTTPS Proxy + +Once you have your HTTPS proxy server set up, you can configure Jan to use it. Navigate to `Settings` > `Advanced Settings` and specify the HTTPS proxy (proxy auto-configuration and SOCKS not supported). + +You can turn on the feature `Ignore SSL Certificates` if you are using a self-signed certificate. This feature allows self-signed or unverified certificates. + +![01-https-proxy-jan-configure](./assets/01-https-proxy-jan-configure.png) \ No newline at end of file diff --git a/docs/docs/guides/09-advanced-settings/README.mdx b/docs/docs/guides/09-advanced-settings/README.mdx new file mode 100644 index 0000000000..ba3da9bb18 --- /dev/null +++ b/docs/docs/guides/09-advanced-settings/README.mdx @@ -0,0 +1,65 @@ +--- +title: Advanced Settings +slug: /guides/advanced-settings/ +description: Jan is a ChatGPT-alternative that runs on your own computer, with a local API server. +keywords: + [ + Jan AI, + Jan, + ChatGPT alternative, + local AI, + private AI, + conversational AI, + no-subscription fee, + large language model, + advanced-settings, + ] +--- + +This guide will show you how to use the advanced settings in Jan. + +## Keyboard Shortcuts + +Keyboard shortcuts are a great way to speed up your workflow. Here are some of the keyboard shortcuts that you can use in Jan. + +| Combination | Description | +| --------------- | -------------------------------------------------- | +| `⌘ E` | Show list your models | +| `⌘ K` | Show list navigation pages | +| `⌘ B` | Toggle collapsible left panel | +| `⌘ ,` | Navigate to setting page | +| `Enter` | Send a message | +| `Shift + Enter` | Insert new line in input box | +| `Arrow Up` | Navigate to previous option (within search dialog) | +| `Arrow Down` | Navigate to next option (within search dialog) | + +

+ +:::note +`⌘` is the command key on macOS, and `Ctrl` on Windows. +::: + +## Experimental Mode + +Experimental mode allows you to enable experimental features that may be unstable tested. + +## Jan Data Folder + +The Jan data folder is the location where messages, model configurations, and other user data are placed. You can change the location of the data folder to a different location. + +![00-changing-folder](./assets/00-changing-folder.gif) + +## HTTPS Proxy & Ignore SSL Certificate + +HTTPS Proxy allows you to use a proxy server to connect to the internet. You can also ignore SSL certificates if you are using a self-signed certificate. +Please check out the guide on [how to set up your own HTTPS proxy server and configure Jan to use it](../advanced-settings/https-proxy) for more information. + +## Clear Logs + +Clear logs will remove all logs from the Jan application. + +## Reset To Factory Default + +Reset the application to its original state, deleting all your usage data, including model customizations and conversation history. This action is irreversible and recommended only if the application is in a corrupted state. + +![00-reset-factory-settings](./assets/00-reset-factory-settings.gif) diff --git a/docs/docs/guides/09-advanced-settings/assets/00-changing-folder.gif b/docs/docs/guides/09-advanced-settings/assets/00-changing-folder.gif new file mode 100644 index 0000000000..ac280a5c34 Binary files /dev/null and b/docs/docs/guides/09-advanced-settings/assets/00-changing-folder.gif differ diff --git a/docs/docs/guides/09-advanced-settings/assets/00-reset-factory-settings.gif b/docs/docs/guides/09-advanced-settings/assets/00-reset-factory-settings.gif new file mode 100644 index 0000000000..81760848d3 Binary files /dev/null and b/docs/docs/guides/09-advanced-settings/assets/00-reset-factory-settings.gif differ diff --git a/docs/docs/guides/09-advanced-settings/assets/01-https-proxy-jan-configure.png b/docs/docs/guides/09-advanced-settings/assets/01-https-proxy-jan-configure.png new file mode 100644 index 0000000000..25e0f76601 Binary files /dev/null and b/docs/docs/guides/09-advanced-settings/assets/01-https-proxy-jan-configure.png differ diff --git a/docs/docs/template/QA_script.md b/docs/docs/template/QA_script.md index 05dbed2b41..bba667bcdc 100644 --- a/docs/docs/template/QA_script.md +++ b/docs/docs/template/QA_script.md @@ -1,6 +1,6 @@ # [Release Version] QA Script -**Release Version:** +**Release Version:** v0.4.6 **Operating System:** @@ -25,10 +25,10 @@ ### 3. Users uninstall app -- [ ] :key: Check that the uninstallation process removes all components of the app from the system. +- [ ] :key::warning: Check that the uninstallation process removes the app successfully from the system. - [ ] Clean the Jan root directory and open the app to check if it creates all the necessary folders, especially models and extensions. - [ ] When updating the app, check if the `/models` directory has any JSON files that change according to the update. -- [ ] Verify if updating the app also updates extensions correctly (test functionality changes; support notifications for necessary tests with each version related to extensions update). +- [ ] Verify if updating the app also updates extensions correctly (test functionality changes, support notifications for necessary tests with each version related to extensions update). ### 4. Users close app @@ -60,49 +60,45 @@ - [ ] :key: Ensure that the conversation thread is maintained without any loss of data upon sending multiple messages. - [ ] Test for the ability to send different types of messages (e.g., text, emojis, code blocks). - [ ] :key: Validate the scroll functionality in the chat window for lengthy conversations. -- [ ] Check if the user can renew responses multiple times. - [ ] Check if the user can copy the response. - [ ] Check if the user can delete responses. -- [ ] :warning: Test if the user deletes the message midway, then the assistant stops that response. - [ ] :key: Check the `clear message` button works. - [ ] :key: Check the `delete entire chat` works. -- [ ] :warning: Check if deleting all the chat retains the system prompt. +- [ ] Check if deleting all the chat retains the system prompt. - [ ] Check the output format of the AI (code blocks, JSON, markdown, ...). - [ ] :key: Validate that there is appropriate error handling and messaging if the assistant fails to respond. - [ ] Test assistant's ability to maintain context over multiple exchanges. - [ ] :key: Check the `create new chat` button works correctly - [ ] Confirm that by changing `models` mid-thread the app can still handle it. -- [ ] Check that by changing `instructions` mid-thread the app can still handle it. -- [ ] Check the `regenerate` button renews the response. -- [ ] Check the `Instructions` update correctly after the user updates it midway. +- [ ] Check the `regenerate` button renews the response (single / multiple times). +- [ ] Check the `Instructions` update correctly after the user updates it midway (mid-thread). ### 2. Users can customize chat settings like model parameters via both the GUI & thread.json -- [ ] :key: Confirm that the chat settings options are accessible via the GUI. +- [ ] :key: Confirm that the Threads settings options are accessible. - [ ] Test the functionality to adjust model parameters (e.g., Temperature, Top K, Top P) from the GUI and verify they are reflected in the chat behavior. - [ ] :key: Ensure that changes can be saved and persisted between sessions. - [ ] Validate that users can access and modify the thread.json file. - [ ] :key: Check that changes made in thread.json are correctly applied to the chat session upon reload or restart. -- [ ] Verify if there is a revert option to go back to previous settings after changes are made. -- [ ] Test for user feedback or confirmation after saving changes to settings. - [ ] Check the maximum and minimum limits of the adjustable parameters and how they affect the assistant's responses. - [ ] :key: Validate user permissions for those who can change settings and persist them. - [ ] :key: Ensure that users switch between threads with different models, the app can handle it. -### 3. Users can click on a history thread +### 3. Model dropdown +- [ ] :key: Model list should highlight recommended based on user RAM +- [ ] Model size should display (for both installed and imported models) +### 4. Users can click on a history thread - [ ] Test the ability to click on any thread in the history panel. - [ ] :key: Verify that clicking a thread brings up the past conversation in the main chat window. - [ ] :key: Ensure that the selected thread is highlighted or otherwise indicated in the history panel. - [ ] Confirm that the chat window displays the entire conversation from the selected history thread without any missing messages. - [ ] :key: Check the performance and accuracy of the history feature when dealing with a large number of threads. - [ ] Validate that historical threads reflect the exact state of the chat at that time, including settings. -- [ ] :key: :warning: Test the search functionality within the history panel for quick navigation. - [ ] :key: Verify the ability to delete or clean old threads. - [ ] :key: Confirm that changing the title of the thread updates correctly. -### 4. Users can config instructions for the assistant. - +### 5. Users can config instructions for the assistant. - [ ] Ensure there is a clear interface to input or change instructions for the assistant. - [ ] Test if the instructions set by the user are being followed by the assistant in subsequent conversations. - [ ] :key: Validate that changes to instructions are updated in real time and do not require a restart of the application or session. @@ -112,6 +108,8 @@ - [ ] Validate that instructions can be saved with descriptive names for easy retrieval. - [ ] :key: Check if the assistant can handle conflicting instructions and how it resolves them. - [ ] Ensure that instruction configurations are documented for user reference. +- [ ] :key: RAG - Users can import documents and the system should process queries about the uploaded file, providing accurate and appropriate responses in the conversation thread. + ## D. Hub @@ -125,8 +123,7 @@ - [ ] Display the best model for their RAM at the top. - [ ] :key: Ensure that models are labeled with RAM requirements and compatibility. -- [ ] :key: Validate that the download function is disabled for models that exceed the user's system capabilities. -- [ ] Test that the platform provides alternative recommendations for models not suitable due to RAM limitations. +- [ ] :warning: Test that the platform provides alternative recommendations for models not suitable due to RAM limitations. - [ ] :key: Check the download model functionality and validate if the cancel download feature works correctly. ### 3. Users can download models via a HuggingFace URL (coming soon) @@ -139,7 +136,7 @@ - [ ] :key: Have clear instructions so users can do their own. - [ ] :key: Ensure the new model updates after restarting the app. -- [ ] Ensure it raises clear errors for users to fix the problem while adding a new model. +- [ ] :warning:Ensure it raises clear errors for users to fix the problem while adding a new model. ### 5. Users can use the model as they want @@ -149,9 +146,13 @@ - [ ] Check if starting another model stops the other model entirely. - [ ] Check the `Explore models` navigate correctly to the model panel. - [ ] :key: Check when deleting a model it will delete all the files on the user's computer. -- [ ] The recommended tags should present right for the user's hardware. +- [ ] :warning:The recommended tags should present right for the user's hardware. - [ ] Assess that the descriptions of models are accurate and informative. +### 6. Users can Integrate With a Remote Server +- [ ] :key: Import openAI GPT model https://jan.ai/guides/using-models/integrate-with-remote-server/ and the model displayed in Hub / Thread dropdown +- [ ] Users can use the remote model properly + ## E. System Monitor ### 1. Users can see disk and RAM utilization @@ -181,7 +182,7 @@ - [ ] Confirm that the application saves the theme preference and persists it across sessions. - [ ] Validate that all elements of the UI are compatible with the theme changes and maintain legibility and contrast. -### 2. Users change the extensions +### 2. Users change the extensions [TBU] - [ ] Confirm that the `Extensions` tab lists all available plugins. - [ ] :key: Test the toggle switch for each plugin to ensure it enables or disables the plugin correctly. @@ -208,3 +209,19 @@ - [ ] :key: Test that the application prevents the installation of incompatible or corrupt plugin files. - [ ] :key: Check that the user can uninstall or disable custom plugins as easily as pre-installed ones. - [ ] Verify that the application's performance remains stable after the installation of custom plugins. + +### 5. Advanced Settings +- [ ] Attemp to test downloading model from hub using **HTTP Proxy** [guideline](https://github.com/janhq/jan/pull/1562) +- [ ] Users can move **Jan data folder** +- [ ] Users can click on Reset button to **factory reset** app settings to its original state & delete all usage data. + +## G. Local API server + +### 1. Local Server Usage with Server Options +- [ ] :key: Explore API Reference: Swagger API for sending/receiving requests + - [ ] Use default server option + - [ ] Configure and use custom server options +- [ ] Test starting/stopping the local API server with different Model/Model settings +- [ ] Server logs captured with correct Server Options provided +- [ ] Verify functionality of Open logs/Clear feature +- [ ] Ensure that threads and other functions impacting the model are disabled while the local server is running diff --git a/docs/openapi/jan.yaml b/docs/openapi/jan.yaml index bfff0ad738..864c80fdf6 100644 --- a/docs/openapi/jan.yaml +++ b/docs/openapi/jan.yaml @@ -67,20 +67,31 @@ paths: x-codeSamples: - lang: cURL source: | - curl http://localhost:1337/v1/chat/completions \ - -H "Content-Type: application/json" \ + curl -X 'POST' \ + 'http://localhost:1337/v1/chat/completions' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ -d '{ - "model": "tinyllama-1.1b", "messages": [ { - "role": "system", - "content": "You are a helpful assistant." + "content": "You are a helpful assistant.", + "role": "system" }, { - "role": "user", - "content": "Hello!" + "content": "Hello!", + "role": "user" } - ] + ], + "model": "tinyllama-1.1b", + "stream": true, + "max_tokens": 2048, + "stop": [ + "hello" + ], + "frequency_penalty": 0, + "presence_penalty": 0, + "temperature": 0.7, + "top_p": 0.95 }' /models: get: @@ -103,7 +114,9 @@ paths: x-codeSamples: - lang: cURL source: | - curl http://localhost:1337/v1/models + curl -X 'GET' \ + 'http://localhost:1337/v1/models' \ + -H 'accept: application/json' "/models/download/{model_id}": get: operationId: downloadModel @@ -131,7 +144,9 @@ paths: x-codeSamples: - lang: cURL source: | - curl -X POST http://localhost:1337/v1/models/download/{model_id} + curl -X 'GET' \ + 'http://localhost:1337/v1/models/download/{model_id}' \ + -H 'accept: application/json' "/models/{model_id}": get: operationId: retrieveModel @@ -162,7 +177,9 @@ paths: x-codeSamples: - lang: cURL source: | - curl http://localhost:1337/v1/models/{model_id} + curl -X 'GET' \ + 'http://localhost:1337/v1/models/{model_id}' \ + -H 'accept: application/json' delete: operationId: deleteModel tags: @@ -191,7 +208,9 @@ paths: x-codeSamples: - lang: cURL source: | - curl -X DELETE http://localhost:1337/v1/models/{model_id} + curl -X 'DELETE' \ + 'http://localhost:1337/v1/models/{model_id}' \ + -H 'accept: application/json' /threads: post: operationId: createThread diff --git a/docs/openapi/specs/assistants.yaml b/docs/openapi/specs/assistants.yaml index d784c315a6..5db1f6a976 100644 --- a/docs/openapi/specs/assistants.yaml +++ b/docs/openapi/specs/assistants.yaml @@ -316,4 +316,4 @@ components: deleted: type: boolean description: Indicates whether the assistant was successfully deleted. - example: true \ No newline at end of file + example: true diff --git a/docs/openapi/specs/chat.yaml b/docs/openapi/specs/chat.yaml index b324501a86..cfa3915982 100644 --- a/docs/openapi/specs/chat.yaml +++ b/docs/openapi/specs/chat.yaml @@ -188,4 +188,4 @@ components: total_tokens: type: integer example: 533 - description: Total number of tokens used \ No newline at end of file + description: Total number of tokens used diff --git a/docs/openapi/specs/messages.yaml b/docs/openapi/specs/messages.yaml index d9d7d87a40..6f5fe1a58f 100644 --- a/docs/openapi/specs/messages.yaml +++ b/docs/openapi/specs/messages.yaml @@ -1,3 +1,4 @@ +--- components: schemas: MessageObject: @@ -75,7 +76,7 @@ components: example: msg_abc123 object: type: string - description: "Type of the object, indicating it's a thread message." + description: Type of the object, indicating it's a thread message. default: thread.message created_at: type: integer @@ -88,7 +89,7 @@ components: example: thread_abc123 role: type: string - description: "Role of the sender, either 'user' or 'assistant'." + description: Role of the sender, either 'user' or 'assistant'. example: user content: type: array @@ -97,7 +98,7 @@ components: properties: type: type: string - description: "Type of content, e.g., 'text'." + description: Type of content, e.g., 'text'. example: text text: type: object @@ -110,21 +111,21 @@ components: type: array items: type: string - description: "Annotations for the text content, if any." + description: Annotations for the text content, if any. example: [] file_ids: type: array items: type: string - description: "Array of file IDs associated with the message, if any." + description: Array of file IDs associated with the message, if any. example: [] assistant_id: type: string - description: "Identifier of the assistant involved in the message, if applicable." + description: Identifier of the assistant involved in the message, if applicable. example: null run_id: type: string - description: "Run ID associated with the message, if applicable." + description: Run ID associated with the message, if applicable. example: null metadata: type: object @@ -139,7 +140,7 @@ components: example: msg_abc123 object: type: string - description: "Type of the object, indicating it's a thread message." + description: Type of the object, indicating it's a thread message. example: thread.message created_at: type: integer @@ -152,7 +153,7 @@ components: example: thread_abc123 role: type: string - description: "Role of the sender, either 'user' or 'assistant'." + description: Role of the sender, either 'user' or 'assistant'. example: user content: type: array @@ -161,7 +162,7 @@ components: properties: type: type: string - description: "Type of content, e.g., 'text'." + description: Type of content, e.g., 'text'. example: text text: type: object @@ -174,21 +175,21 @@ components: type: array items: type: string - description: "Annotations for the text content, if any." + description: Annotations for the text content, if any. example: [] file_ids: type: array items: type: string - description: "Array of file IDs associated with the message, if any." + description: Array of file IDs associated with the message, if any. example: [] assistant_id: type: string - description: "Identifier of the assistant involved in the message, if applicable." + description: Identifier of the assistant involved in the message, if applicable. example: null run_id: type: string - description: "Run ID associated with the message, if applicable." + description: Run ID associated with the message, if applicable. example: null metadata: type: object @@ -199,7 +200,7 @@ components: properties: object: type: string - description: "Type of the object, indicating it's a list." + description: Type of the object, indicating it's a list. default: list data: type: array @@ -226,7 +227,7 @@ components: example: msg_abc123 object: type: string - description: "Type of the object, indicating it's a thread message." + description: Type of the object, indicating it's a thread message. example: thread.message created_at: type: integer @@ -239,7 +240,7 @@ components: example: thread_abc123 role: type: string - description: "Role of the sender, either 'user' or 'assistant'." + description: Role of the sender, either 'user' or 'assistant'. example: user content: type: array @@ -248,7 +249,7 @@ components: properties: type: type: string - description: "Type of content, e.g., 'text'." + description: Type of content, e.g., 'text'. text: type: object properties: @@ -260,20 +261,20 @@ components: type: array items: type: string - description: "Annotations for the text content, if any." + description: Annotations for the text content, if any. file_ids: type: array items: type: string - description: "Array of file IDs associated with the message, if any." + description: Array of file IDs associated with the message, if any. example: [] assistant_id: type: string - description: "Identifier of the assistant involved in the message, if applicable." + description: Identifier of the assistant involved in the message, if applicable. example: null run_id: type: string - description: "Run ID associated with the message, if applicable." + description: Run ID associated with the message, if applicable. example: null metadata: type: object @@ -309,4 +310,4 @@ components: data: type: array items: - $ref: "#/components/schemas/MessageFileObject" \ No newline at end of file + $ref: "#/components/schemas/MessageFileObject" diff --git a/docs/openapi/specs/models.yaml b/docs/openapi/specs/models.yaml index 8113f3ab80..40e6abaaff 100644 --- a/docs/openapi/specs/models.yaml +++ b/docs/openapi/specs/models.yaml @@ -18,114 +18,82 @@ components: Model: type: object properties: - type: + source_url: type: string - default: model - description: The type of the object. - version: - type: string - default: "1" - description: The version number of the model. + format: uri + description: URL to the source of the model. + example: https://huggingface.co/janhq/trinity-v1.2-GGUF/resolve/main/trinity-v1.2.Q4_K_M.gguf id: type: string - description: Unique identifier used in chat-completions model_name, matches + description: + Unique identifier used in chat-completions model_name, matches folder name. - example: zephyr-7b + example: trinity-v1.2-7b + object: + type: string + example: model name: type: string description: Name of the model. - example: Zephyr 7B - owned_by: + example: Trinity-v1.2 7B Q4 + version: type: string - description: Compatibility field for OpenAI. - default: "" - created: - type: integer - format: int64 - description: Unix timestamp representing the creation time. + default: "1.0" + description: The version number of the model. description: type: string description: Description of the model. - state: - type: string - enum: - - null - - downloading - - ready - - starting - - stopping - description: Current state of the model. + example: + Trinity is an experimental model merge using the Slerp method. + Recommended for daily assistance purposes. format: type: string description: State format of the model, distinct from the engine. - example: ggufv3 - source: - type: array - items: - type: object - properties: - url: - format: uri - description: URL to the source of the model. - example: https://huggingface.co/TheBloke/zephyr-7B-beta-GGUF/blob/main/zephyr-7b-beta.Q4_K_M.gguf - filename: - type: string - description: Filename of the model. - example: zephyr-7b-beta.Q4_K_M.gguf + example: gguf settings: type: object properties: ctx_len: - type: string + type: integer description: Context length. - example: "4096" - ngl: - type: string - description: Number of layers. - example: "100" - embedding: - type: string - description: Indicates if embedding is enabled. - example: "true" - n_parallel: + example: 4096 + prompt_template: type: string - description: Number of parallel processes. - example: "4" + example: "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant" additionalProperties: false parameters: type: object properties: temperature: - type: string - description: Temperature setting for the model. - example: "0.7" - token_limit: - type: string - description: Token limit for the model. - example: "4096" - top_k: - type: string - description: Top-k setting for the model. - example: "0" + example: 0.7 top_p: - type: string - description: Top-p setting for the model. - example: "1" + example: 0.95 stream: - type: string - description: Indicates if streaming is enabled. - example: "true" + example: true + max_tokens: + example: 4096 + stop: + example: [] + frequency_penalty: + example: 0 + presence_penalty: + example: 0 additionalProperties: false metadata: - type: object - description: Additional metadata. - assets: - type: array - items: + author: type: string - description: List of assets related to the model. - required: - - source + example: Jan + tags: + example: + - 7B + - Merged + - Featured + size: + example: 4370000000, + cover: + example: https://raw.githubusercontent.com/janhq/jan/main/models/trinity-v1.2-7b/cover.png + engine: + example: nitro ModelObject: type: object properties: @@ -133,7 +101,7 @@ components: type: string description: | The identifier of the model. - example: zephyr-7b + example: trinity-v1.2-7b object: type: string description: | @@ -153,197 +121,89 @@ components: GetModelResponse: type: object properties: + source_url: + type: string + format: uri + description: URL to the source of the model. + example: https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.2-GGUF/resolve/main/mistral-7b-instruct-v0.2.Q4_K_M.gguf id: type: string - description: The identifier of the model. - example: zephyr-7b + description: + Unique identifier used in chat-completions model_name, matches + folder name. + example: mistral-ins-7b-q4 object: type: string - description: Type of the object, indicating it's a model. - default: model - created: - type: integer - format: int64 - description: Unix timestamp representing the creation time of the model. - owned_by: + example: model + name: type: string - description: The entity that owns the model. - example: _ - state: + description: Name of the model. + example: Mistral Instruct 7B Q4 + version: type: string - enum: - - not_downloaded - - downloaded - - running - - stopped - description: The current state of the model. - source: - type: array - items: - type: object - properties: - url: - format: uri - description: URL to the source of the model. - example: https://huggingface.co/TheBloke/zephyr-7B-beta-GGUF/blob/main/zephyr-7b-beta.Q4_K_M.gguf - filename: - type: string - description: Filename of the model. - example: zephyr-7b-beta.Q4_K_M.gguf - engine_parameters: + default: "1.0" + description: The version number of the model. + description: + type: string + description: Description of the model. + example: + Trinity is an experimental model merge using the Slerp method. + Recommended for daily assistance purposes. + format: + type: string + description: State format of the model, distinct from the engine. + example: gguf + settings: type: object properties: - pre_prompt: - type: string - description: Predefined prompt used for setting up internal configurations. - default: "" - example: Initial setup complete. - system_prompt: - type: string - description: Prefix used for system-level prompts. - default: "SYSTEM: " - user_prompt: - type: string - description: Prefix used for user prompts. - default: "USER: " - ai_prompt: - type: string - description: Prefix used for assistant prompts. - default: "ASSISTANT: " - ngl: - type: integer - description: Number of neural network layers loaded onto the GPU for - acceleration. - minimum: 0 - maximum: 100 - default: 100 - example: 100 ctx_len: type: integer - description: Context length for model operations, varies based on the specific - model. - minimum: 128 - maximum: 4096 - default: 4096 + description: Context length. example: 4096 - n_parallel: - type: integer - description: Number of parallel operations, relevant when continuous batching is - enabled. - minimum: 1 - maximum: 10 - default: 1 - example: 4 - cont_batching: - type: boolean - description: Indicates if continuous batching is used for processing. - default: false - example: false - cpu_threads: - type: integer - description: Number of threads allocated for CPU-based inference. - minimum: 1 - example: 8 - embedding: - type: boolean - description: Indicates if embedding layers are enabled in the model. - default: true - example: true - model_parameters: + prompt_template: + type: string + example: "[INST] {prompt} [/INST]" + additionalProperties: false + parameters: type: object properties: - ctx_len: - type: integer - description: Maximum context length the model can handle. - minimum: 0 - maximum: 4096 - default: 4096 - example: 4096 - ngl: - type: integer - description: Number of layers in the neural network. - minimum: 1 - maximum: 100 - default: 100 - example: 100 - embedding: - type: boolean - description: Indicates if embedding layers are used. - default: true - example: true - n_parallel: - type: integer - description: Number of parallel processes the model can run. - minimum: 1 - maximum: 10 - default: 1 - example: 4 temperature: - type: number - description: Controls randomness in model's responses. Higher values lead to - more random responses. - minimum: 0 - maximum: 2 - default: 0.7 example: 0.7 - token_limit: - type: integer - description: Maximum number of tokens the model can generate in a single - response. - minimum: 1 - maximum: 4096 - default: 4096 + top_p: + example: 0.95 + stream: + example: true + max_tokens: example: 4096 - top_k: - type: integer - description: Limits the model to consider only the top k most likely next tokens - at each step. - minimum: 0 - maximum: 100 - default: 0 + stop: + example: [] + frequency_penalty: example: 0 - top_p: - type: number - description: Nucleus sampling parameter. The model considers the smallest set of - tokens whose cumulative probability exceeds the top_p value. - minimum: 0 - maximum: 1 - default: 1 - example: 1 + presence_penalty: + example: 0 + additionalProperties: false metadata: - type: object - properties: - engine: - type: string - description: The engine used by the model. - enum: - - nitro - - openai - - hf_inference - quantization: - type: string - description: Quantization parameter of the model. - example: Q3_K_L - size: - type: string - description: Size of the model. - example: 7B - required: - - id - - object - - created - - owned_by - - state - - source - - parameters - - metadata + author: + type: string + example: MistralAI + tags: + example: + - 7B + - Featured + - Foundation Model + size: + example: 4370000000, + cover: + example: https://raw.githubusercontent.com/janhq/jan/main/models/mistral-ins-7b-q4/cover.png + engine: + example: nitro DeleteModelResponse: type: object properties: id: type: string description: The identifier of the model that was deleted. - example: model-zephyr-7B + example: mistral-ins-7b-q4 object: type: string description: Type of the object, indicating it's a model. diff --git a/docs/openapi/specs/threads.yaml b/docs/openapi/specs/threads.yaml index fe00f75884..40b2463fa5 100644 --- a/docs/openapi/specs/threads.yaml +++ b/docs/openapi/specs/threads.yaml @@ -142,7 +142,7 @@ components: example: Jan instructions: type: string - description: | + description: > The instruction of assistant, defaults to "Be my grammar corrector" model: type: object @@ -224,4 +224,4 @@ components: deleted: type: boolean description: Indicates whether the thread was successfully deleted. - example: true \ No newline at end of file + example: true diff --git a/electron/.prettierrc b/electron/.prettierrc deleted file mode 100644 index 46f1abcb02..0000000000 --- a/electron/.prettierrc +++ /dev/null @@ -1,8 +0,0 @@ -{ - "semi": false, - "singleQuote": true, - "quoteProps": "consistent", - "trailingComma": "es5", - "endOfLine": "auto", - "plugins": ["prettier-plugin-tailwindcss"] -} diff --git a/electron/handlers/app.ts b/electron/handlers/app.ts deleted file mode 100644 index c1f431ef3c..0000000000 --- a/electron/handlers/app.ts +++ /dev/null @@ -1,173 +0,0 @@ -import { app, ipcMain, dialog, shell } from 'electron' -import { join, basename, relative as getRelative, isAbsolute } from 'path' -import { WindowManager } from './../managers/window' -import { getResourcePath } from './../utils/path' -import { AppRoute, AppConfiguration } from '@janhq/core' -import { ServerConfig, startServer, stopServer } from '@janhq/server' -import { - ModuleManager, - getJanDataFolderPath, - getJanExtensionsPath, - init, - log, - logServer, - getAppConfigurations, - updateAppConfiguration, -} from '@janhq/core/node' - -export function handleAppIPCs() { - /** - * Handles the "openAppDirectory" IPC message by opening the app's user data directory. - * The `shell.openPath` method is used to open the directory in the user's default file explorer. - * @param _event - The IPC event object. - */ - ipcMain.handle(AppRoute.openAppDirectory, async (_event) => { - shell.openPath(getJanDataFolderPath()) - }) - - /** - * Opens a URL in the user's default browser. - * @param _event - The IPC event object. - * @param url - The URL to open. - */ - ipcMain.handle(AppRoute.openExternalUrl, async (_event, url) => { - shell.openExternal(url) - }) - - /** - * Opens a URL in the user's default browser. - * @param _event - The IPC event object. - * @param url - The URL to open. - */ - ipcMain.handle(AppRoute.openFileExplore, async (_event, url) => { - shell.openPath(url) - }) - - /** - * Joins multiple paths together, respect to the current OS. - */ - ipcMain.handle(AppRoute.joinPath, async (_event, paths: string[]) => - join(...paths) - ) - - /** - * Checks if the given path is a subdirectory of the given directory. - * - * @param _event - The IPC event object. - * @param from - The path to check. - * @param to - The directory to check against. - * - * @returns {Promise} - A promise that resolves with the result. - */ - ipcMain.handle( - AppRoute.isSubdirectory, - async (_event, from: string, to: string) => { - const relative = getRelative(from, to) - const isSubdir = - relative && !relative.startsWith('..') && !isAbsolute(relative) - - if (isSubdir === '') return false - else return isSubdir - } - ) - - /** - * Retrieve basename from given path, respect to the current OS. - */ - ipcMain.handle(AppRoute.baseName, async (_event, path: string) => - basename(path) - ) - - /** - * Start Jan API Server. - */ - ipcMain.handle(AppRoute.startServer, async (_event, configs?: ServerConfig) => - startServer({ - host: configs?.host, - port: configs?.port, - isCorsEnabled: configs?.isCorsEnabled, - isVerboseEnabled: configs?.isVerboseEnabled, - schemaPath: app.isPackaged - ? join(getResourcePath(), 'docs', 'openapi', 'jan.yaml') - : undefined, - baseDir: app.isPackaged - ? join(getResourcePath(), 'docs', 'openapi') - : undefined, - }) - ) - - /** - * Stop Jan API Server. - */ - ipcMain.handle(AppRoute.stopServer, stopServer) - - /** - * Relaunches the app in production - reload window in development. - * @param _event - The IPC event object. - * @param url - The URL to reload. - */ - ipcMain.handle(AppRoute.relaunch, async (_event) => { - ModuleManager.instance.clearImportedModules() - - if (app.isPackaged) { - app.relaunch() - app.exit() - } else { - for (const modulePath in ModuleManager.instance.requiredModules) { - delete require.cache[ - require.resolve(join(getJanExtensionsPath(), modulePath)) - ] - } - init({ - // Function to check from the main process that user wants to install a extension - confirmInstall: async (_extensions: string[]) => { - return true - }, - // Path to install extension to - extensionsPath: getJanExtensionsPath(), - }) - WindowManager.instance.currentWindow?.reload() - } - }) - - /** - * Log message to log file. - */ - ipcMain.handle(AppRoute.log, async (_event, message) => log(message)) - - /** - * Log message to log file. - */ - ipcMain.handle(AppRoute.logServer, async (_event, message) => - logServer(message) - ) - - ipcMain.handle(AppRoute.selectDirectory, async () => { - const mainWindow = WindowManager.instance.currentWindow - if (!mainWindow) { - console.error('No main window found') - return - } - const { canceled, filePaths } = await dialog.showOpenDialog(mainWindow, { - title: 'Select a folder', - buttonLabel: 'Select Folder', - properties: ['openDirectory', 'createDirectory'], - }) - if (canceled) { - return - } else { - return filePaths[0] - } - }) - - ipcMain.handle(AppRoute.getAppConfigurations, async () => - getAppConfigurations() - ) - - ipcMain.handle( - AppRoute.updateAppConfiguration, - async (_event, appConfiguration: AppConfiguration) => { - await updateAppConfiguration(appConfiguration) - } - ) -} diff --git a/electron/handlers/common.ts b/electron/handlers/common.ts new file mode 100644 index 0000000000..5a54a92bdf --- /dev/null +++ b/electron/handlers/common.ts @@ -0,0 +1,25 @@ +import { Handler, RequestHandler } from '@janhq/core/node' +import { ipcMain } from 'electron' +import { WindowManager } from '../managers/window' + +export function injectHandler() { + const ipcWrapper: Handler = ( + route: string, + listener: (...args: any[]) => any + ) => { + return ipcMain.handle(route, async (event, ...args: any[]) => { + return listener(...args) + }) + } + + const handler = new RequestHandler( + ipcWrapper, + (channel: string, args: any) => { + return WindowManager.instance.currentWindow?.webContents.send( + channel, + args + ) + } + ) + handler.handle() +} diff --git a/electron/handlers/download.ts b/electron/handlers/download.ts deleted file mode 100644 index f63e56f6bb..0000000000 --- a/electron/handlers/download.ts +++ /dev/null @@ -1,121 +0,0 @@ -import { ipcMain } from 'electron' -import { resolve } from 'path' -import { WindowManager } from './../managers/window' -import request from 'request' -import { createWriteStream, renameSync } from 'fs' -import { DownloadEvent, DownloadRoute } from '@janhq/core' -const progress = require('request-progress') -import { DownloadManager, getJanDataFolderPath, normalizeFilePath } from '@janhq/core/node' - -export function handleDownloaderIPCs() { - /** - * Handles the "pauseDownload" IPC message by pausing the download associated with the provided fileName. - * @param _event - The IPC event object. - * @param fileName - The name of the file being downloaded. - */ - ipcMain.handle(DownloadRoute.pauseDownload, async (_event, fileName) => { - DownloadManager.instance.networkRequests[fileName]?.pause() - }) - - /** - * Handles the "resumeDownload" IPC message by resuming the download associated with the provided fileName. - * @param _event - The IPC event object. - * @param fileName - The name of the file being downloaded. - */ - ipcMain.handle(DownloadRoute.resumeDownload, async (_event, fileName) => { - DownloadManager.instance.networkRequests[fileName]?.resume() - }) - - /** - * Handles the "abortDownload" IPC message by aborting the download associated with the provided fileName. - * The network request associated with the fileName is then removed from the networkRequests object. - * @param _event - The IPC event object. - * @param fileName - The name of the file being downloaded. - */ - ipcMain.handle(DownloadRoute.abortDownload, async (_event, fileName) => { - const rq = DownloadManager.instance.networkRequests[fileName] - if (rq) { - DownloadManager.instance.networkRequests[fileName] = undefined - rq?.abort() - } else { - WindowManager?.instance.currentWindow?.webContents.send( - DownloadEvent.onFileDownloadError, - { - fileName, - err: { message: 'aborted' }, - } - ) - } - }) - - /** - * Downloads a file from a given URL. - * @param _event - The IPC event object. - * @param url - The URL to download the file from. - * @param fileName - The name to give the downloaded file. - */ - ipcMain.handle( - DownloadRoute.downloadFile, - async (_event, url, fileName, network) => { - const strictSSL = !network?.ignoreSSL - const proxy = network?.proxy?.startsWith('http') - ? network.proxy - : undefined - - if (typeof fileName === 'string') { - fileName = normalizeFilePath(fileName) - } - const destination = resolve(getJanDataFolderPath(), fileName) - const rq = request({ url, strictSSL, proxy }) - - // Put request to download manager instance - DownloadManager.instance.setRequest(fileName, rq) - - // Downloading file to a temp file first - const downloadingTempFile = `${destination}.download` - - progress(rq, {}) - .on('progress', function (state: any) { - WindowManager?.instance.currentWindow?.webContents.send( - DownloadEvent.onFileDownloadUpdate, - { - ...state, - fileName, - } - ) - }) - .on('error', function (err: Error) { - WindowManager?.instance.currentWindow?.webContents.send( - DownloadEvent.onFileDownloadError, - { - fileName, - err, - } - ) - }) - .on('end', function () { - if (DownloadManager.instance.networkRequests[fileName]) { - // Finished downloading, rename temp file to actual file - renameSync(downloadingTempFile, destination) - - WindowManager?.instance.currentWindow?.webContents.send( - DownloadEvent.onFileDownloadSuccess, - { - fileName, - } - ) - DownloadManager.instance.setRequest(fileName, undefined) - } else { - WindowManager?.instance.currentWindow?.webContents.send( - DownloadEvent.onFileDownloadError, - { - fileName, - err: { message: 'aborted' }, - } - ) - } - }) - .pipe(createWriteStream(downloadingTempFile)) - } - ) -} diff --git a/electron/handlers/extension.ts b/electron/handlers/extension.ts deleted file mode 100644 index 763c4cdecb..0000000000 --- a/electron/handlers/extension.ts +++ /dev/null @@ -1,104 +0,0 @@ -import { ipcMain, webContents } from 'electron' -import { readdirSync } from 'fs' -import { join, extname } from 'path' - -import { - installExtensions, - getExtension, - removeExtension, - getActiveExtensions, - ModuleManager, - getJanExtensionsPath, -} from '@janhq/core/node' - -import { getResourcePath } from './../utils/path' -import { ExtensionRoute } from '@janhq/core' - -export function handleExtensionIPCs() { - /**MARK: General handlers */ - /** - * Invokes a function from a extension module in main node process. - * @param _event - The IPC event object. - * @param modulePath - The path to the extension module. - * @param method - The name of the function to invoke. - * @param args - The arguments to pass to the function. - * @returns The result of the invoked function. - */ - ipcMain.handle( - ExtensionRoute.invokeExtensionFunc, - async (_event, modulePath, method, ...args) => { - const module = require( - /* webpackIgnore: true */ join(getJanExtensionsPath(), modulePath) - ) - ModuleManager.instance.setModule(modulePath, module) - - if (typeof module[method] === 'function') { - return module[method](...args) - } else { - console.debug(module[method]) - console.error(`Function "${method}" does not exist in the module.`) - } - } - ) - - /** - * Returns the paths of the base extensions. - * @param _event - The IPC event object. - * @returns An array of paths to the base extensions. - */ - ipcMain.handle(ExtensionRoute.baseExtensions, async (_event) => { - const baseExtensionPath = join(getResourcePath(), 'pre-install') - return readdirSync(baseExtensionPath) - .filter((file) => extname(file) === '.tgz') - .map((file) => join(baseExtensionPath, file)) - }) - - /**MARK: Extension Manager handlers */ - ipcMain.handle(ExtensionRoute.installExtension, async (e, extensions) => { - // Install and activate all provided extensions - const installed = await installExtensions(extensions) - return JSON.parse(JSON.stringify(installed)) - }) - - // Register IPC route to uninstall a extension - ipcMain.handle( - ExtensionRoute.uninstallExtension, - async (e, extensions, reload) => { - // Uninstall all provided extensions - for (const ext of extensions) { - const extension = getExtension(ext) - await extension.uninstall() - if (extension.name) removeExtension(extension.name) - } - - // Reload all renderer pages if needed - reload && webContents.getAllWebContents().forEach((wc) => wc.reload()) - return true - } - ) - - // Register IPC route to update a extension - ipcMain.handle( - ExtensionRoute.updateExtension, - async (e, extensions, reload) => { - // Update all provided extensions - const updated: any[] = [] - for (const ext of extensions) { - const extension = getExtension(ext) - const res = await extension.update() - if (res) updated.push(extension) - } - - // Reload all renderer pages if needed - if (updated.length && reload) - webContents.getAllWebContents().forEach((wc) => wc.reload()) - - return JSON.parse(JSON.stringify(updated)) - } - ) - - // Register IPC route to get the list of active extensions - ipcMain.handle(ExtensionRoute.getActiveExtensions, () => { - return JSON.parse(JSON.stringify(getActiveExtensions())) - }) -} diff --git a/electron/handlers/fileManager.ts b/electron/handlers/fileManager.ts deleted file mode 100644 index e328cb53b7..0000000000 --- a/electron/handlers/fileManager.ts +++ /dev/null @@ -1,82 +0,0 @@ -import { ipcMain, app } from 'electron' -// @ts-ignore -import reflect from '@alumna/reflect' - -import { FileManagerRoute, FileStat } from '@janhq/core' -import { getResourcePath } from './../utils/path' -import fs from 'fs' -import { join } from 'path' -import { getJanDataFolderPath, normalizeFilePath } from '@janhq/core/node' - -/** - * Handles file system extensions operations. - */ -export function handleFileMangerIPCs() { - // Handles the 'syncFile' IPC event. This event is triggered to synchronize a file from a source path to a destination path. - ipcMain.handle( - FileManagerRoute.syncFile, - async (_event, src: string, dest: string) => { - return reflect({ - src, - dest, - recursive: true, - delete: false, - overwrite: true, - errorOnExist: false, - }) - } - ) - - // Handles the 'getJanDataFolderPath' IPC event. This event is triggered to get the user space path. - ipcMain.handle( - FileManagerRoute.getJanDataFolderPath, - (): Promise => Promise.resolve(getJanDataFolderPath()) - ) - - // Handles the 'getResourcePath' IPC event. This event is triggered to get the resource path. - ipcMain.handle(FileManagerRoute.getResourcePath, async (_event) => - getResourcePath() - ) - - ipcMain.handle(FileManagerRoute.getUserHomePath, async (_event) => - app.getPath('home') - ) - - // handle fs is directory here - ipcMain.handle( - FileManagerRoute.fileStat, - async (_event, path: string): Promise => { - const normalizedPath = normalizeFilePath(path) - - const fullPath = join(getJanDataFolderPath(), normalizedPath) - const isExist = fs.existsSync(fullPath) - if (!isExist) return undefined - - const isDirectory = fs.lstatSync(fullPath).isDirectory() - const size = fs.statSync(fullPath).size - - const fileStat: FileStat = { - isDirectory, - size, - } - - return fileStat - } - ) - - ipcMain.handle( - FileManagerRoute.writeBlob, - async (_event, path: string, data: string): Promise => { - try { - const normalizedPath = normalizeFilePath(path) - const dataBuffer = Buffer.from(data, 'base64') - fs.writeFileSync( - join(getJanDataFolderPath(), normalizedPath), - dataBuffer - ) - } catch (err) { - console.error(`writeFile ${path} result: ${err}`) - } - } - ) -} diff --git a/electron/handlers/fs.ts b/electron/handlers/fs.ts deleted file mode 100644 index 34026b9409..0000000000 --- a/electron/handlers/fs.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { ipcMain } from 'electron' - -import { getJanDataFolderPath, normalizeFilePath } from '@janhq/core/node' -import fs from 'fs' -import { FileManagerRoute, FileSystemRoute } from '@janhq/core' -import { join } from 'path' -/** - * Handles file system operations. - */ -export function handleFsIPCs() { - const moduleName = 'fs' - Object.values(FileSystemRoute).forEach((route) => { - ipcMain.handle(route, async (event, ...args) => { - return import(moduleName).then((mdl) => - mdl[route]( - ...args.map((arg) => - typeof arg === 'string' && - (arg.startsWith(`file:/`) || arg.startsWith(`file:\\`)) - ? join(getJanDataFolderPath(), normalizeFilePath(arg)) - : arg - ) - ) - ) - }) - }) -} diff --git a/electron/handlers/native.ts b/electron/handlers/native.ts new file mode 100644 index 0000000000..14ead07bd3 --- /dev/null +++ b/electron/handlers/native.ts @@ -0,0 +1,86 @@ +import { app, ipcMain, dialog, shell } from 'electron' +import { join } from 'path' +import { WindowManager } from '../managers/window' +import { + ModuleManager, + getJanDataFolderPath, + getJanExtensionsPath, + init, +} from '@janhq/core/node' +import { NativeRoute } from '@janhq/core' + +export function handleAppIPCs() { + /** + * Handles the "openAppDirectory" IPC message by opening the app's user data directory. + * The `shell.openPath` method is used to open the directory in the user's default file explorer. + * @param _event - The IPC event object. + */ + ipcMain.handle(NativeRoute.openAppDirectory, async (_event) => { + shell.openPath(getJanDataFolderPath()) + }) + + /** + * Opens a URL in the user's default browser. + * @param _event - The IPC event object. + * @param url - The URL to open. + */ + ipcMain.handle(NativeRoute.openExternalUrl, async (_event, url) => { + shell.openExternal(url) + }) + + /** + * Opens a URL in the user's default browser. + * @param _event - The IPC event object. + * @param url - The URL to open. + */ + ipcMain.handle(NativeRoute.openFileExplore, async (_event, url) => { + shell.openPath(url) + }) + + /** + * Relaunches the app in production - reload window in development. + * @param _event - The IPC event object. + * @param url - The URL to reload. + */ + ipcMain.handle(NativeRoute.relaunch, async (_event) => { + ModuleManager.instance.clearImportedModules() + + if (app.isPackaged) { + app.relaunch() + app.exit() + } else { + for (const modulePath in ModuleManager.instance.requiredModules) { + delete require.cache[ + require.resolve(join(getJanExtensionsPath(), modulePath)) + ] + } + init({ + // Function to check from the main process that user wants to install a extension + confirmInstall: async (_extensions: string[]) => { + return true + }, + // Path to install extension to + extensionsPath: getJanExtensionsPath(), + }) + WindowManager.instance.currentWindow?.reload() + } + }) + + ipcMain.handle(NativeRoute.selectDirectory, async () => { + const mainWindow = WindowManager.instance.currentWindow + if (!mainWindow) { + console.error('No main window found') + return + } + const { canceled, filePaths } = await dialog.showOpenDialog(mainWindow, { + title: 'Select a folder', + buttonLabel: 'Select Folder', + properties: ['openDirectory', 'createDirectory'], + }) + if (canceled) { + return + } else { + return filePaths[0] + } + }) +} diff --git a/electron/handlers/update.ts b/electron/handlers/update.ts index cbb34c22b5..0d8cc4cc07 100644 --- a/electron/handlers/update.ts +++ b/electron/handlers/update.ts @@ -11,7 +11,8 @@ export function handleAppUpdates() { /* New Update Available */ autoUpdater.on('update-available', async (_info: any) => { const action = await dialog.showMessageBox({ - message: `Update available. Do you want to download the latest update?`, + title: 'Update Available', + message: 'Would you like to download and install it now?', buttons: ['Download', 'Later'], }) if (action.response === 0) await autoUpdater.downloadUpdate() @@ -36,7 +37,7 @@ export function handleAppUpdates() { autoUpdater.on('error', (info: any) => { WindowManager.instance.currentWindow?.webContents.send( AppEvent.onAppUpdateDownloadError, - {} + info ) }) diff --git a/electron/main.ts b/electron/main.ts index 5d7e59c0f3..de18b8f9de 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -1,4 +1,4 @@ -import { app, BrowserWindow } from 'electron' +import { app, BrowserWindow, shell } from 'electron' import { join } from 'path' /** * Managers @@ -9,12 +9,9 @@ import { log } from '@janhq/core/node' /** * IPC Handlers **/ -import { handleDownloaderIPCs } from './handlers/download' -import { handleExtensionIPCs } from './handlers/extension' -import { handleFileMangerIPCs } from './handlers/fileManager' -import { handleAppIPCs } from './handlers/app' +import { injectHandler } from './handlers/common' import { handleAppUpdates } from './handlers/update' -import { handleFsIPCs } from './handlers/fs' +import { handleAppIPCs } from './handlers/native' /** * Utils @@ -25,25 +22,12 @@ import { migrateExtensions } from './utils/migration' import { cleanUpAndQuit } from './utils/clean' import { setupExtensions } from './utils/extension' import { setupCore } from './utils/setup' +import { setupReactDevTool } from './utils/dev' +import { cleanLogs } from './utils/log' app .whenReady() - .then(async () => { - if (!app.isPackaged) { - // Which means you're running from source code - const { default: installExtension, REACT_DEVELOPER_TOOLS } = await import( - 'electron-devtools-installer' - ) // Don't use import on top level, since the installer package is dev-only - try { - const name = installExtension(REACT_DEVELOPER_TOOLS) - console.log(`Added Extension: ${name}`) - } catch (err) { - console.log('An error occurred while installing devtools:') - console.error(err) - // Only log the error and don't throw it because it's not critical - } - } - }) + .then(setupReactDevTool) .then(setupCore) .then(createUserSpace) .then(migrateExtensions) @@ -59,6 +43,7 @@ app } }) }) + .then(() => cleanLogs()) app.once('window-all-closed', () => { cleanUpAndQuit() @@ -92,23 +77,24 @@ function createMainWindow() { /* Open external links in the default browser */ mainWindow.webContents.setWindowOpenHandler(({ url }) => { - require('electron').shell.openExternal(url) + shell.openExternal(url) return { action: 'deny' } }) /* Enable dev tools for development */ if (!app.isPackaged) mainWindow.webContents.openDevTools() + log(`Version: ${app.getVersion()}`) } /** * Handles various IPC messages from the renderer process. */ function handleIPCs() { - handleFsIPCs() - handleDownloaderIPCs() - handleExtensionIPCs() + // Inject core handlers for IPCs + injectHandler() + + // Handle native IPCs handleAppIPCs() - handleFileMangerIPCs() } /* diff --git a/electron/merge-latest-ymls.js b/electron/merge-latest-ymls.js index 8172a31768..ee8caf825d 100644 --- a/electron/merge-latest-ymls.js +++ b/electron/merge-latest-ymls.js @@ -9,7 +9,9 @@ const file3 = args[2] // check that all arguments are present and throw error instead if (!file1 || !file2 || !file3) { - throw new Error('Please provide 3 file paths as arguments: path to file1, to file2 and destination path') + throw new Error( + 'Please provide 3 file paths as arguments: path to file1, to file2 and destination path' + ) } const doc1 = yaml.load(fs.readFileSync(file1, 'utf8')) diff --git a/electron/package.json b/electron/package.json index 08f15b2626..a89803077c 100644 --- a/electron/package.json +++ b/electron/package.json @@ -4,6 +4,7 @@ "main": "./build/main.js", "author": "Jan ", "license": "MIT", + "productName": "Jan", "homepage": "https://github.com/janhq/jan/tree/main/electron", "description": "Use offline LLMs with your own data. Run open source models like Llama2 or Falcon on your internal computers/servers.", "build": { @@ -11,7 +12,6 @@ "productName": "Jan", "files": [ "renderer/**/*", - "build/*.{js,map}", "build/**/*.{js,map}", "pre-install", "models/**/*", @@ -57,16 +57,17 @@ "scripts": { "lint": "eslint . --ext \".js,.jsx,.ts,.tsx\"", "test:e2e": "playwright test --workers=1", - "dev": "tsc -p . && electron .", - "build": "run-script-os", - "build:test": "run-script-os", + "copy:assets": "rimraf --glob \"./pre-install/*.tgz\" && cpx \"../pre-install/*.tgz\" \"./pre-install\"", + "dev": "yarn copy:assets && tsc -p . && electron .", + "build": "yarn copy:assets && run-script-os", + "build:test": "yarn copy:assets && run-script-os", "build:test:darwin": "tsc -p . && electron-builder -p never -m --dir", "build:test:win32": "tsc -p . && electron-builder -p never -w --dir", "build:test:linux": "tsc -p . && electron-builder -p never -l --dir", "build:darwin": "tsc -p . && electron-builder -p never -m", "build:win32": "tsc -p . && electron-builder -p never -w", "build:linux": "tsc -p . && electron-builder -p never -l deb -l AppImage", - "build:publish": "run-script-os", + "build:publish": "yarn copy:assets && run-script-os", "build:publish:darwin": "tsc -p . && electron-builder -p always -m", "build:publish:win32": "tsc -p . && electron-builder -p always -w", "build:publish:linux": "tsc -p . && electron-builder -p always -l deb -l AppImage" @@ -76,7 +77,6 @@ "@janhq/core": "link:./core", "@janhq/server": "link:./server", "@npmcli/arborist": "^7.1.0", - "@types/request": "^2.48.12", "@uiball/loaders": "^1.3.0", "electron-store": "^8.1.0", "electron-updater": "^6.1.7", @@ -85,8 +85,6 @@ "pacote": "^17.0.4", "request": "^2.88.2", "request-progress": "^3.0.0", - "rimraf": "^5.0.5", - "typescript": "^5.2.2", "ulid": "^2.3.0", "use-debounce": "^9.0.4" }, @@ -95,6 +93,7 @@ "@playwright/test": "^1.38.1", "@types/npmcli__arborist": "^5.6.4", "@types/pacote": "^11.1.7", + "@types/request": "^2.48.12", "@typescript-eslint/eslint-plugin": "^6.7.3", "@typescript-eslint/parser": "^6.7.3", "electron": "28.0.0", @@ -102,7 +101,9 @@ "electron-devtools-installer": "^3.2.0", "electron-playwright-helpers": "^1.6.0", "eslint-plugin-react": "^7.33.2", - "run-script-os": "^1.1.6" + "rimraf": "^5.0.5", + "run-script-os": "^1.1.6", + "typescript": "^5.2.2" }, "installConfig": { "hoistingLimits": "workspaces" diff --git a/electron/playwright.config.ts b/electron/playwright.config.ts index 1fa3313f27..d3dff40c6a 100644 --- a/electron/playwright.config.ts +++ b/electron/playwright.config.ts @@ -1,9 +1,14 @@ import { PlaywrightTestConfig } from '@playwright/test' const config: PlaywrightTestConfig = { - testDir: './tests', + testDir: './tests/e2e', retries: 0, - globalTimeout: 300000, + globalTimeout: 350000, + use: { + screenshot: 'only-on-failure', + video: 'retain-on-failure', + trace: 'retain-on-failure', + }, + reporter: [['html', { outputFolder: './playwright-report' }]], } - export default config diff --git a/electron/sign.js b/electron/sign.js index 6e973eb6e3..73afedc4ef 100644 --- a/electron/sign.js +++ b/electron/sign.js @@ -1,44 +1,48 @@ -const { exec } = require('child_process'); - - -function sign({ path, name, certUrl, clientId, tenantId, clientSecret, certName, timestampServer, version }) { - return new Promise((resolve, reject) => { - - const command = `azuresigntool.exe sign -kvu "${certUrl}" -kvi "${clientId}" -kvt "${tenantId}" -kvs "${clientSecret}" -kvc "${certName}" -tr "${timestampServer}" -v "${path}"`; - - - exec(command, (error, stdout, stderr) => { - if (error) { - console.error(`Error: ${error}`); - return reject(error); - } - console.log(`stdout: ${stdout}`); - console.error(`stderr: ${stderr}`); - resolve(); - }); - }); +const { exec } = require('child_process') + +function sign({ + path, + name, + certUrl, + clientId, + tenantId, + clientSecret, + certName, + timestampServer, + version, +}) { + return new Promise((resolve, reject) => { + const command = `azuresigntool.exe sign -kvu "${certUrl}" -kvi "${clientId}" -kvt "${tenantId}" -kvs "${clientSecret}" -kvc "${certName}" -tr "${timestampServer}" -v "${path}"` + + exec(command, (error, stdout, stderr) => { + if (error) { + console.error(`Error: ${error}`) + return reject(error) + } + console.log(`stdout: ${stdout}`) + console.error(`stderr: ${stderr}`) + resolve() + }) + }) } - -exports.default = async function(options) { - - const certUrl = process.env.AZURE_KEY_VAULT_URI; - const clientId = process.env.AZURE_CLIENT_ID; - const tenantId = process.env.AZURE_TENANT_ID; - const clientSecret = process.env.AZURE_CLIENT_SECRET; - const certName = process.env.AZURE_CERT_NAME; - const timestampServer = 'http://timestamp.globalsign.com/tsa/r6advanced1'; - - - await sign({ - path: options.path, - name: "jan-win-x64", - certUrl, - clientId, - tenantId, - clientSecret, - certName, - timestampServer, - version: options.version - }); -}; +exports.default = async function (options) { + const certUrl = process.env.AZURE_KEY_VAULT_URI + const clientId = process.env.AZURE_CLIENT_ID + const tenantId = process.env.AZURE_TENANT_ID + const clientSecret = process.env.AZURE_CLIENT_SECRET + const certName = process.env.AZURE_CERT_NAME + const timestampServer = 'http://timestamp.globalsign.com/tsa/r6advanced1' + + await sign({ + path: options.path, + name: 'jan-win-x64', + certUrl, + clientId, + tenantId, + clientSecret, + certName, + timestampServer, + version: options.version, + }) +} diff --git a/electron/tests/config/constants.ts b/electron/tests/config/constants.ts new file mode 100644 index 0000000000..7039ad58c3 --- /dev/null +++ b/electron/tests/config/constants.ts @@ -0,0 +1,4 @@ +export const Constants = { + VIDEO_DIR: './playwright-video', + TIMEOUT: '300000', +} diff --git a/electron/tests/config/fixtures.ts b/electron/tests/config/fixtures.ts new file mode 100644 index 0000000000..680b097853 --- /dev/null +++ b/electron/tests/config/fixtures.ts @@ -0,0 +1,119 @@ +import { + _electron as electron, + BrowserContext, + ElectronApplication, + expect, + Page, + test as base, +} from '@playwright/test' +import { + ElectronAppInfo, + findLatestBuild, + parseElectronApp, + stubDialog, +} from 'electron-playwright-helpers' +import { Constants } from './constants' +import { HubPage } from '../pages/hubPage' +import { CommonActions } from '../pages/commonActions' + +export let electronApp: ElectronApplication +export let page: Page +export let appInfo: ElectronAppInfo +export const TIMEOUT = parseInt(process.env.TEST_TIMEOUT || Constants.TIMEOUT) + +export async function setupElectron() { + process.env.CI = 'e2e' + + const latestBuild = findLatestBuild('dist') + expect(latestBuild).toBeTruthy() + + // parse the packaged Electron app and find paths and other info + appInfo = parseElectronApp(latestBuild) + expect(appInfo).toBeTruthy() + + electronApp = await electron.launch({ + args: [appInfo.main], // main file from package.json + executablePath: appInfo.executable, // path to the Electron executable + // recordVideo: { dir: Constants.VIDEO_DIR }, // Specify the directory for video recordings + }) + await stubDialog(electronApp, 'showMessageBox', { response: 1 }) + + page = await electronApp.firstWindow({ + timeout: TIMEOUT, + }) +} + +export async function teardownElectron() { + await page.close() + await electronApp.close() +} + +/** + * this fixture is needed to record and attach videos / screenshot on failed tests when + * tests are run in serial mode (i.e. browser is not closed between tests) + */ +export const test = base.extend< + { + commonActions: CommonActions + hubPage: HubPage + attachVideoPage: Page + attachScreenshotsToReport: void + }, + { createVideoContext: BrowserContext } +>({ + commonActions: async ({ request }, use, testInfo) => { + await use(new CommonActions(page, testInfo)) + }, + hubPage: async ({ commonActions }, use) => { + await use(new HubPage(page, commonActions)) + }, + createVideoContext: [ + async ({ playwright }, use) => { + const context = electronApp.context() + await use(context) + }, + { scope: 'worker' }, + ], + + attachVideoPage: [ + async ({ createVideoContext }, use, testInfo) => { + await use(page) + + if (testInfo.status !== testInfo.expectedStatus) { + const path = await createVideoContext.pages()[0].video()?.path() + await createVideoContext.close() + await testInfo.attach('video', { + path: path, + }) + } + }, + { scope: 'test', auto: true }, + ], + + attachScreenshotsToReport: [ + async ({ commonActions }, use, testInfo) => { + await use() + + // After the test, we can check whether the test passed or failed. + if (testInfo.status !== testInfo.expectedStatus) { + await commonActions.takeScreenshot('') + } + }, + { auto: true }, + ], +}) + +test.setTimeout(TIMEOUT) + +test.beforeAll(async () => { + await setupElectron() + await page.waitForSelector('img[alt="Jan - Logo"]', { + state: 'visible', + timeout: TIMEOUT, + }) +}) + +test.afterAll(async () => { + // temporally disabling this due to the config for parallel testing WIP + // teardownElectron() +}) diff --git a/electron/tests/e2e/hub.e2e.spec.ts b/electron/tests/e2e/hub.e2e.spec.ts new file mode 100644 index 0000000000..d968e76419 --- /dev/null +++ b/electron/tests/e2e/hub.e2e.spec.ts @@ -0,0 +1,19 @@ +import { test, appInfo } from '../config/fixtures' +import { expect } from '@playwright/test' + +test.beforeAll(async () => { + expect(appInfo).toMatchObject({ + asar: true, + executable: expect.anything(), + main: expect.anything(), + name: 'jan', + packageJson: expect.objectContaining({ name: 'jan' }), + platform: process.platform, + resourcesDir: expect.anything(), + }) +}) + +test('explores hub', async ({ hubPage }) => { + await hubPage.navigateByMenu() + await hubPage.verifyContainerVisible() +}) diff --git a/electron/tests/e2e/navigation.e2e.spec.ts b/electron/tests/e2e/navigation.e2e.spec.ts new file mode 100644 index 0000000000..66924ce786 --- /dev/null +++ b/electron/tests/e2e/navigation.e2e.spec.ts @@ -0,0 +1,24 @@ +import { expect } from '@playwright/test' +import { page, test, TIMEOUT } from '../config/fixtures' + +test('renders left navigation panel', async () => { + const systemMonitorBtn = await page + .getByTestId('System Monitor') + .first() + .isEnabled({ + timeout: TIMEOUT, + }) + const settingsBtn = await page + .getByTestId('Thread') + .first() + .isEnabled({ timeout: TIMEOUT }) + expect([systemMonitorBtn, settingsBtn].filter((e) => !e).length).toBe(0) + // Chat section should be there + await page.getByTestId('Local API Server').first().click({ + timeout: TIMEOUT, + }) + const localServer = page.getByTestId('local-server-testid').first() + await expect(localServer).toBeVisible({ + timeout: TIMEOUT, + }) +}) diff --git a/electron/tests/e2e/settings.e2e.spec.ts b/electron/tests/e2e/settings.e2e.spec.ts new file mode 100644 index 0000000000..06b4d1accf --- /dev/null +++ b/electron/tests/e2e/settings.e2e.spec.ts @@ -0,0 +1,11 @@ +import { expect } from '@playwright/test' + +import { test, page, TIMEOUT } from '../config/fixtures' + +test('shows settings', async () => { + await page.getByTestId('Settings').first().click({ + timeout: TIMEOUT, + }) + const settingDescription = page.getByTestId('testid-setting-description') + await expect(settingDescription).toBeVisible({ timeout: TIMEOUT }) +}) diff --git a/electron/tests/hub.e2e.spec.ts b/electron/tests/hub.e2e.spec.ts deleted file mode 100644 index cc72e037ea..0000000000 --- a/electron/tests/hub.e2e.spec.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { _electron as electron } from 'playwright' -import { ElectronApplication, Page, expect, test } from '@playwright/test' - -import { - findLatestBuild, - parseElectronApp, - stubDialog, -} from 'electron-playwright-helpers' - -let electronApp: ElectronApplication -let page: Page -const TIMEOUT: number = parseInt(process.env.TEST_TIMEOUT || '300000') - -test.beforeAll(async () => { - process.env.CI = 'e2e' - - const latestBuild = findLatestBuild('dist') - expect(latestBuild).toBeTruthy() - - // parse the packaged Electron app and find paths and other info - const appInfo = parseElectronApp(latestBuild) - expect(appInfo).toBeTruthy() - - electronApp = await electron.launch({ - args: [appInfo.main], // main file from package.json - executablePath: appInfo.executable, // path to the Electron executable - }) - await stubDialog(electronApp, 'showMessageBox', { response: 1 }) - - page = await electronApp.firstWindow({ - timeout: TIMEOUT, - }) -}) - -test.afterAll(async () => { - await electronApp.close() - await page.close() -}) - -test('explores hub', async () => { - test.setTimeout(TIMEOUT) - await page.getByTestId('Hub').first().click({ - timeout: TIMEOUT, - }) - await page.getByTestId('hub-container-test-id').isVisible({ - timeout: TIMEOUT, - }) -}) diff --git a/electron/tests/navigation.e2e.spec.ts b/electron/tests/navigation.e2e.spec.ts deleted file mode 100644 index 5c8721c2fa..0000000000 --- a/electron/tests/navigation.e2e.spec.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { _electron as electron } from 'playwright' -import { ElectronApplication, Page, expect, test } from '@playwright/test' - -import { - findLatestBuild, - parseElectronApp, - stubDialog, -} from 'electron-playwright-helpers' - -let electronApp: ElectronApplication -let page: Page -const TIMEOUT: number = parseInt(process.env.TEST_TIMEOUT || '300000') - -test.beforeAll(async () => { - process.env.CI = 'e2e' - - const latestBuild = findLatestBuild('dist') - expect(latestBuild).toBeTruthy() - - // parse the packaged Electron app and find paths and other info - const appInfo = parseElectronApp(latestBuild) - expect(appInfo).toBeTruthy() - - electronApp = await electron.launch({ - args: [appInfo.main], // main file from package.json - executablePath: appInfo.executable, // path to the Electron executable - }) - await stubDialog(electronApp, 'showMessageBox', { response: 1 }) - - page = await electronApp.firstWindow({ - timeout: TIMEOUT, - }) -}) - -test.afterAll(async () => { - await electronApp.close() - await page.close() -}) - -test('renders left navigation panel', async () => { - test.setTimeout(TIMEOUT) - const systemMonitorBtn = await page - .getByTestId('System Monitor') - .first() - .isEnabled({ - timeout: TIMEOUT, - }) - const settingsBtn = await page - .getByTestId('Thread') - .first() - .isEnabled({ timeout: TIMEOUT }) - expect([systemMonitorBtn, settingsBtn].filter((e) => !e).length).toBe(0) - // Chat section should be there - await page.getByTestId('Local API Server').first().click({ - timeout: TIMEOUT, - }) - const localServer = await page.getByTestId('local-server-testid').first() - await expect(localServer).toBeVisible({ - timeout: TIMEOUT, - }) -}) diff --git a/electron/tests/pages/basePage.ts b/electron/tests/pages/basePage.ts new file mode 100644 index 0000000000..4e16a3c232 --- /dev/null +++ b/electron/tests/pages/basePage.ts @@ -0,0 +1,49 @@ +import { Page, expect } from '@playwright/test' +import { CommonActions } from './commonActions' +import { TIMEOUT } from '../config/fixtures' + +export class BasePage { + menuId: string + + constructor( + protected readonly page: Page, + readonly action: CommonActions, + protected containerId: string + ) {} + + public getValue(key: string) { + return this.action.getValue(key) + } + + public setValue(key: string, value: string) { + this.action.setValue(key, value) + } + + async takeScreenshot(name: string = '') { + await this.action.takeScreenshot(name) + } + + async navigateByMenu() { + await this.page.getByTestId(this.menuId).first().click() + } + + async verifyContainerVisible() { + const container = this.page.getByTestId(this.containerId) + expect(container.isVisible()).toBeTruthy() + } + + async waitUpdateLoader() { + await this.isElementVisible('img[alt="Jan - Logo"]') + } + + //wait and find a specific element with it's selector and return Visible + async isElementVisible(selector: any) { + let isVisible = true + await this.page + .waitForSelector(selector, { state: 'visible', timeout: TIMEOUT }) + .catch(() => { + isVisible = false + }) + return isVisible + } +} diff --git a/electron/tests/pages/commonActions.ts b/electron/tests/pages/commonActions.ts new file mode 100644 index 0000000000..08ea15f92a --- /dev/null +++ b/electron/tests/pages/commonActions.ts @@ -0,0 +1,34 @@ +import { Page, TestInfo } from '@playwright/test' +import { page } from '../config/fixtures' + +export class CommonActions { + private testData = new Map() + + constructor( + public page: Page, + public testInfo: TestInfo + ) {} + + async takeScreenshot(name: string) { + const screenshot = await page.screenshot({ + fullPage: true, + }) + const attachmentName = `${this.testInfo.title}_${name || new Date().toISOString().slice(5, 19).replace(/[-:]/g, '').replace('T', '_')}` + await this.testInfo.attach(attachmentName.replace(/\s+/g, ''), { + body: screenshot, + contentType: 'image/png', + }) + } + + async hooks() { + console.log('hook from the scenario page') + } + + setValue(key: string, value: string) { + this.testData.set(key, value) + } + + getValue(key: string) { + return this.testData.get(key) + } +} diff --git a/electron/tests/pages/hubPage.ts b/electron/tests/pages/hubPage.ts new file mode 100644 index 0000000000..0299ab15d4 --- /dev/null +++ b/electron/tests/pages/hubPage.ts @@ -0,0 +1,15 @@ +import { Page } from '@playwright/test' +import { BasePage } from './basePage' +import { CommonActions } from './commonActions' + +export class HubPage extends BasePage { + readonly menuId: string = 'Hub' + static readonly containerId: string = 'hub-container-test-id' + + constructor( + public page: Page, + readonly action: CommonActions + ) { + super(page, action, HubPage.containerId) + } +} diff --git a/electron/tests/settings.e2e.spec.ts b/electron/tests/settings.e2e.spec.ts deleted file mode 100644 index ad2d7b4a49..0000000000 --- a/electron/tests/settings.e2e.spec.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { _electron as electron } from 'playwright' -import { ElectronApplication, Page, expect, test } from '@playwright/test' - -import { - findLatestBuild, - parseElectronApp, - stubDialog, -} from 'electron-playwright-helpers' - -let electronApp: ElectronApplication -let page: Page -const TIMEOUT: number = parseInt(process.env.TEST_TIMEOUT || '300000') - -test.beforeAll(async () => { - process.env.CI = 'e2e' - - const latestBuild = findLatestBuild('dist') - expect(latestBuild).toBeTruthy() - - // parse the packaged Electron app and find paths and other info - const appInfo = parseElectronApp(latestBuild) - expect(appInfo).toBeTruthy() - - electronApp = await electron.launch({ - args: [appInfo.main], // main file from package.json - executablePath: appInfo.executable, // path to the Electron executable - }) - await stubDialog(electronApp, 'showMessageBox', { response: 1 }) - - page = await electronApp.firstWindow({ - timeout: TIMEOUT, - }) -}) - -test.afterAll(async () => { - await electronApp.close() - await page.close() -}) - -test('shows settings', async () => { - test.setTimeout(TIMEOUT) - await page.getByTestId('Settings').first().click({ timeout: TIMEOUT }) - const settingDescription = page.getByTestId('testid-setting-description') - await expect(settingDescription).toBeVisible({ timeout: TIMEOUT }) -}) diff --git a/electron/utils/dev.ts b/electron/utils/dev.ts new file mode 100644 index 0000000000..b2a4928866 --- /dev/null +++ b/electron/utils/dev.ts @@ -0,0 +1,18 @@ +import { app } from 'electron' + +export const setupReactDevTool = async () => { + if (!app.isPackaged) { + // Which means you're running from source code + const { default: installExtension, REACT_DEVELOPER_TOOLS } = await import( + 'electron-devtools-installer' + ) // Don't use import on top level, since the installer package is dev-only + try { + const name = await installExtension(REACT_DEVELOPER_TOOLS) + console.log(`Added Extension: ${name}`) + } catch (err) { + console.log('An error occurred while installing devtools:') + console.error(err) + // Only log the error and don't throw it because it's not critical + } + } +} diff --git a/electron/utils/disposable.ts b/electron/utils/disposable.ts index 462f7e3e51..59018a7751 100644 --- a/electron/utils/disposable.ts +++ b/electron/utils/disposable.ts @@ -1,8 +1,8 @@ export function dispose(requiredModules: Record) { for (const key in requiredModules) { - const module = requiredModules[key]; - if (typeof module["dispose"] === "function") { - module["dispose"](); + const module = requiredModules[key] + if (typeof module['dispose'] === 'function') { + module['dispose']() } } } diff --git a/electron/utils/log.ts b/electron/utils/log.ts new file mode 100644 index 0000000000..84c185d754 --- /dev/null +++ b/electron/utils/log.ts @@ -0,0 +1,67 @@ +import { getJanDataFolderPath } from '@janhq/core/node' +import * as fs from 'fs' +import * as path from 'path' + +export function cleanLogs( + maxFileSizeBytes?: number | undefined, + daysToKeep?: number | undefined, + delayMs?: number | undefined +): void { + const size = maxFileSizeBytes ?? 1 * 1024 * 1024 // 1 MB + const days = daysToKeep ?? 7 // 7 days + const delays = delayMs ?? 10000 // 10 seconds + const logDirectory = path.join(getJanDataFolderPath(), 'logs') + + // Perform log cleaning + const currentDate = new Date() + fs.readdir(logDirectory, (err, files) => { + if (err) { + console.error('Error reading log directory:', err) + return + } + + files.forEach((file) => { + const filePath = path.join(logDirectory, file) + fs.stat(filePath, (err, stats) => { + if (err) { + console.error('Error getting file stats:', err) + return + } + + // Check size + if (stats.size > size) { + fs.unlink(filePath, (err) => { + if (err) { + console.error('Error deleting log file:', err) + return + } + console.log( + `Deleted log file due to exceeding size limit: ${filePath}` + ) + }) + } else { + // Check age + const creationDate = new Date(stats.ctime) + const daysDifference = Math.floor( + (currentDate.getTime() - creationDate.getTime()) / + (1000 * 3600 * 24) + ) + if (daysDifference > days) { + fs.unlink(filePath, (err) => { + if (err) { + console.error('Error deleting log file:', err) + return + } + console.log(`Deleted old log file: ${filePath}`) + }) + } + } + }) + }) + }) + + // Schedule the next execution with doubled delays + setTimeout(() => { + cleanLogs(maxFileSizeBytes, daysToKeep, delays * 2) + }, delays) +} diff --git a/electron/utils/menu.ts b/electron/utils/menu.ts index 7721b7c78b..893907c48a 100644 --- a/electron/utils/menu.ts +++ b/electron/utils/menu.ts @@ -1,19 +1,41 @@ // @ts-nocheck -import { app, Menu, dialog, shell } from 'electron' -const isMac = process.platform === 'darwin' +import { app, Menu, shell, dialog } from 'electron' import { autoUpdater } from 'electron-updater' -import { compareSemanticVersions } from './versionDiff' +import { log } from '@janhq/core/node' +const isMac = process.platform === 'darwin' const template: (Electron.MenuItemConstructorOptions | Electron.MenuItem)[] = [ { label: app.name, submenu: [ - { role: 'about' }, + { + label: `About ${app.name}`, + click: () => + dialog.showMessageBox({ + title: `Jan`, + message: `Jan Version v${app.getVersion()}\n\nCopyright © 2024 Jan`, + }), + }, { label: 'Check for Updates...', click: () => // Check for updates and notify user if there are any - autoUpdater.checkForUpdatesAndNotify(), + autoUpdater + .checkForUpdatesAndNotify() + .then((updateCheckResult) => { + if ( + !updateCheckResult?.updateInfo || + updateCheckResult?.updateInfo.version === app.getVersion() + ) { + dialog.showMessageBox({ + message: `No updates available.`, + }) + return + } + }) + .catch((error) => { + log('Error checking for updates:' + JSON.stringify(error)) + }), }, { type: 'separator' }, { role: 'services' }, diff --git a/electron/utils/path.ts b/electron/utils/path.ts index 4e47cc312b..4438156bcb 100644 --- a/electron/utils/path.ts +++ b/electron/utils/path.ts @@ -1,5 +1,3 @@ -import { join } from 'path' -import { app } from 'electron' import { mkdir } from 'fs-extra' import { existsSync } from 'fs' import { getJanDataFolderPath } from '@janhq/core/node' @@ -16,13 +14,3 @@ export async function createUserSpace(): Promise { } } } - -export function getResourcePath() { - let appPath = join(app.getAppPath(), '..', 'app.asar.unpacked') - - if (!app.isPackaged) { - // for development mode - appPath = join(__dirname, '..', '..') - } - return appPath -} diff --git a/electron/utils/setup.ts b/electron/utils/setup.ts index 887c3c2b7a..01b0b31da2 100644 --- a/electron/utils/setup.ts +++ b/electron/utils/setup.ts @@ -1,9 +1,9 @@ import { app } from 'electron' export const setupCore = async () => { - // Setup core api for main process - global.core = { - // Define appPath function for app to retrieve app path globaly - appPath: () => app.getPath('userData') - } -} \ No newline at end of file + // Setup core api for main process + global.core = { + // Define appPath function for app to retrieve app path globaly + appPath: () => app.getPath('userData'), + } +} diff --git a/electron/utils/versionDiff.ts b/electron/utils/versionDiff.ts deleted file mode 100644 index 25934e87f0..0000000000 --- a/electron/utils/versionDiff.ts +++ /dev/null @@ -1,21 +0,0 @@ -export const compareSemanticVersions = (a: string, b: string) => { - - // 1. Split the strings into their parts. - const a1 = a.split('.'); - const b1 = b.split('.'); - // 2. Contingency in case there's a 4th or 5th version - const len = Math.min(a1.length, b1.length); - // 3. Look through each version number and compare. - for (let i = 0; i < len; i++) { - const a2 = +a1[ i ] || 0; - const b2 = +b1[ i ] || 0; - - if (a2 !== b2) { - return a2 > b2 ? 1 : -1; - } - } - - // 4. We hit this if the all checked versions so far are equal - // - return b1.length - a1.length; -}; \ No newline at end of file diff --git a/extensions/assistant-extension/package.json b/extensions/assistant-extension/package.json index 84bcdf47e2..baa8586557 100644 --- a/extensions/assistant-extension/package.json +++ b/extensions/assistant-extension/package.json @@ -1,16 +1,17 @@ { "name": "@janhq/assistant-extension", - "version": "1.0.0", + "version": "1.0.1", "description": "This extension enables assistants, including Jan, a default assistant that can call all downloaded models", "main": "dist/index.js", "node": "dist/node/index.js", "author": "Jan ", "license": "AGPL-3.0", "scripts": { - "build": "tsc --module commonjs && rollup -c rollup.config.ts", - "build:publish:linux": "rimraf *.tgz --glob && npm run build && npm pack && cpx *.tgz ../../electron/pre-install", - "build:publish:darwin": "rimraf *.tgz --glob && npm run build && ../../.github/scripts/auto-sign.sh && npm pack && cpx *.tgz ../../electron/pre-install", - "build:publish:win32": "rimraf *.tgz --glob && npm run build && npm pack && cpx *.tgz ../../electron/pre-install", + "clean:modules": "rimraf node_modules/pdf-parse/test && cd node_modules/pdf-parse/lib/pdf.js && rimraf v1.9.426 v1.10.88 v2.0.550", + "build": "yarn clean:modules && tsc --module commonjs && rollup -c rollup.config.ts", + "build:publish:linux": "rimraf *.tgz --glob && npm run build && npm pack && cpx *.tgz ../../pre-install", + "build:publish:darwin": "rimraf *.tgz --glob && npm run build && ../../.github/scripts/auto-sign.sh && npm pack && cpx *.tgz ../../pre-install", + "build:publish:win32": "rimraf *.tgz --glob && npm run build && npm pack && cpx *.tgz ../../pre-install", "build:publish": "run-script-os" }, "devDependencies": { @@ -25,7 +26,7 @@ "rollup-plugin-define": "^1.0.1", "rollup-plugin-sourcemaps": "^0.6.3", "rollup-plugin-typescript2": "^0.36.0", - "typescript": "^5.3.3", + "typescript": "^5.2.2", "run-script-os": "^1.1.6" }, "dependencies": { @@ -44,9 +45,6 @@ ], "bundleDependencies": [ "@janhq/core", - "@langchain/community", - "hnswlib-node", - "langchain", - "pdf-parse" + "hnswlib-node" ] } diff --git a/extensions/assistant-extension/rollup.config.ts b/extensions/assistant-extension/rollup.config.ts index 7916ef9c82..d3c39cab2d 100644 --- a/extensions/assistant-extension/rollup.config.ts +++ b/extensions/assistant-extension/rollup.config.ts @@ -1,22 +1,22 @@ -import resolve from "@rollup/plugin-node-resolve"; -import commonjs from "@rollup/plugin-commonjs"; -import sourceMaps from "rollup-plugin-sourcemaps"; -import typescript from "rollup-plugin-typescript2"; -import json from "@rollup/plugin-json"; -import replace from "@rollup/plugin-replace"; +import resolve from '@rollup/plugin-node-resolve' +import commonjs from '@rollup/plugin-commonjs' +import sourceMaps from 'rollup-plugin-sourcemaps' +import typescript from 'rollup-plugin-typescript2' +import json from '@rollup/plugin-json' +import replace from '@rollup/plugin-replace' -const packageJson = require("./package.json"); +const packageJson = require('./package.json') -const pkg = require("./package.json"); +const pkg = require('./package.json') export default [ { input: `src/index.ts`, - output: [{ file: pkg.main, format: "es", sourcemap: true }], + output: [{ file: pkg.main, format: 'es', sourcemap: true }], // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') external: [], watch: { - include: "src/**", + include: 'src/**', }, plugins: [ replace({ @@ -35,7 +35,7 @@ export default [ // which external modules to include in the bundle // https://github.com/rollup/rollup-plugin-node-resolve#usage resolve({ - extensions: [".js", ".ts", ".svelte"], + extensions: ['.js', '.ts', '.svelte'], }), // Resolve source maps to the original source @@ -44,18 +44,11 @@ export default [ }, { input: `src/node/index.ts`, - output: [{ dir: "dist/node", format: "cjs", sourcemap: false }], + output: [{ dir: 'dist/node', format: 'cjs', sourcemap: false }], // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') - external: [ - "@janhq/core/node", - "@langchain/community", - "langchain", - "langsmith", - "path", - "hnswlib-node", - ], + external: ['@janhq/core/node', 'path', 'hnswlib-node'], watch: { - include: "src/node/**", + include: 'src/node/**', }, // inlineDynamicImports: true, plugins: [ @@ -71,11 +64,11 @@ export default [ // which external modules to include in the bundle // https://github.com/rollup/rollup-plugin-node-resolve#usage resolve({ - extensions: [".ts", ".js", ".json"], + extensions: ['.ts', '.js', '.json'], }), // Resolve source maps to the original source // sourceMaps(), ], }, -]; +] diff --git a/extensions/assistant-extension/src/@types/global.d.ts b/extensions/assistant-extension/src/@types/global.d.ts index dc11709a4f..bc97157cdf 100644 --- a/extensions/assistant-extension/src/@types/global.d.ts +++ b/extensions/assistant-extension/src/@types/global.d.ts @@ -1,3 +1,3 @@ -declare const NODE: string; -declare const EXTENSION_NAME: string; -declare const VERSION: string; +declare const NODE: string +declare const EXTENSION_NAME: string +declare const VERSION: string diff --git a/extensions/assistant-extension/src/index.ts b/extensions/assistant-extension/src/index.ts index 6495ea7869..0a5319c8a7 100644 --- a/extensions/assistant-extension/src/index.ts +++ b/extensions/assistant-extension/src/index.ts @@ -9,143 +9,169 @@ import { joinPath, executeOnMain, AssistantExtension, -} from "@janhq/core"; + AssistantEvent, +} from '@janhq/core' export default class JanAssistantExtension extends AssistantExtension { - private static readonly _homeDir = "file://assistants"; + private static readonly _homeDir = 'file://assistants' + private static readonly _threadDir = 'file://threads' - controller = new AbortController(); - isCancelled = false; - retrievalThreadId: string | undefined = undefined; + controller = new AbortController() + isCancelled = false + retrievalThreadId: string | undefined = undefined async onLoad() { // making the assistant directory const assistantDirExist = await fs.existsSync( - JanAssistantExtension._homeDir, - ); + JanAssistantExtension._homeDir + ) if ( localStorage.getItem(`${EXTENSION_NAME}-version`) !== VERSION || !assistantDirExist ) { - if (!assistantDirExist) - await fs.mkdirSync(JanAssistantExtension._homeDir); + if (!assistantDirExist) await fs.mkdirSync(JanAssistantExtension._homeDir) // Write assistant metadata - this.createJanAssistant(); + await this.createJanAssistant() // Finished migration - localStorage.setItem(`${EXTENSION_NAME}-version`, VERSION); + localStorage.setItem(`${EXTENSION_NAME}-version`, VERSION) + // Update the assistant list + events.emit(AssistantEvent.OnAssistantsUpdate, {}) } // Events subscription events.on(MessageEvent.OnMessageSent, (data: MessageRequest) => - JanAssistantExtension.handleMessageRequest(data, this), - ); + JanAssistantExtension.handleMessageRequest(data, this) + ) events.on(InferenceEvent.OnInferenceStopped, () => { - JanAssistantExtension.handleInferenceStopped(this); - }); + JanAssistantExtension.handleInferenceStopped(this) + }) } private static async handleInferenceStopped(instance: JanAssistantExtension) { - instance.isCancelled = true; - instance.controller?.abort(); + instance.isCancelled = true + instance.controller?.abort() } private static async handleMessageRequest( data: MessageRequest, - instance: JanAssistantExtension, + instance: JanAssistantExtension ) { - instance.isCancelled = false; - instance.controller = new AbortController(); + instance.isCancelled = false + instance.controller = new AbortController() if ( data.model?.engine !== InferenceEngine.tool_retrieval_enabled || !data.messages || + // TODO: Since the engine is defined, its unsafe to assume that assistant tools are defined + // That could lead to an issue where thread stuck at generating response !data.thread?.assistants[0]?.tools ) { - return; + return } - const latestMessage = data.messages[data.messages.length - 1]; + const latestMessage = data.messages[data.messages.length - 1] - // Ingest the document if needed + // 1. Ingest the document if needed if ( latestMessage && latestMessage.content && - typeof latestMessage.content !== "string" + typeof latestMessage.content !== 'string' && + latestMessage.content.length > 1 ) { - const docFile = latestMessage.content[1]?.doc_url?.url; + const docFile = latestMessage.content[1]?.doc_url?.url if (docFile) { await executeOnMain( NODE, - "toolRetrievalIngestNewDocument", + 'toolRetrievalIngestNewDocument', docFile, - data.model?.proxyEngine, - ); + data.model?.proxyEngine + ) + } + } else if ( + // Check whether we need to ingest document or not + // Otherwise wrong context will be sent + !(await fs.existsSync( + await joinPath([ + JanAssistantExtension._threadDir, + data.threadId, + 'memory', + ]) + )) + ) { + // No document ingested, reroute the result to inference engine + const output = { + ...data, + model: { + ...data.model, + engine: data.model.proxyEngine, + }, } + events.emit(MessageEvent.OnMessageSent, output) + return } - - // Load agent on thread changed + // 2. Load agent on thread changed if (instance.retrievalThreadId !== data.threadId) { - await executeOnMain(NODE, "toolRetrievalLoadThreadMemory", data.threadId); + await executeOnMain(NODE, 'toolRetrievalLoadThreadMemory', data.threadId) - instance.retrievalThreadId = data.threadId; + instance.retrievalThreadId = data.threadId // Update the text splitter await executeOnMain( NODE, - "toolRetrievalUpdateTextSplitter", + 'toolRetrievalUpdateTextSplitter', data.thread.assistants[0].tools[0]?.settings?.chunk_size ?? 4000, - data.thread.assistants[0].tools[0]?.settings?.chunk_overlap ?? 200, - ); + data.thread.assistants[0].tools[0]?.settings?.chunk_overlap ?? 200 + ) } + // 3. Using the retrieval template with the result and query if (latestMessage.content) { const prompt = - typeof latestMessage.content === "string" + typeof latestMessage.content === 'string' ? latestMessage.content - : latestMessage.content[0].text; + : latestMessage.content[0].text // Retrieve the result - console.debug("toolRetrievalQuery", latestMessage.content); const retrievalResult = await executeOnMain( NODE, - "toolRetrievalQueryResult", - prompt, - ); + 'toolRetrievalQueryResult', + prompt + ) + console.debug('toolRetrievalQueryResult', retrievalResult) - // Update the message content - // Using the retrieval template with the result and query - if (data.thread?.assistants[0].tools) + // Update message content + if (data.thread?.assistants[0]?.tools && retrievalResult) data.messages[data.messages.length - 1].content = data.thread.assistants[0].tools[0].settings?.retrieval_template - ?.replace("{CONTEXT}", retrievalResult) - .replace("{QUESTION}", prompt); + ?.replace('{CONTEXT}', retrievalResult) + .replace('{QUESTION}', prompt) } // Filter out all the messages that are not text data.messages = data.messages.map((message) => { if ( message.content && - typeof message.content !== "string" && + typeof message.content !== 'string' && (message.content.length ?? 0) > 0 ) { return { ...message, content: [message.content[0]], - }; + } } - return message; - }); + return message + }) - // Reroute the result to inference engine + // 4. Reroute the result to inference engine const output = { ...data, model: { ...data.model, engine: data.model.proxyEngine, }, - }; - events.emit(MessageEvent.OnMessageSent, output); + } + events.emit(MessageEvent.OnMessageSent, output) } /** @@ -157,107 +183,107 @@ export default class JanAssistantExtension extends AssistantExtension { const assistantDir = await joinPath([ JanAssistantExtension._homeDir, assistant.id, - ]); - if (!(await fs.existsSync(assistantDir))) await fs.mkdirSync(assistantDir); + ]) + if (!(await fs.existsSync(assistantDir))) await fs.mkdirSync(assistantDir) // store the assistant metadata json const assistantMetadataPath = await joinPath([ assistantDir, - "assistant.json", - ]); + 'assistant.json', + ]) try { await fs.writeFileSync( assistantMetadataPath, - JSON.stringify(assistant, null, 2), - ); + JSON.stringify(assistant, null, 2) + ) } catch (err) { - console.error(err); + console.error(err) } } async getAssistants(): Promise { // get all the assistant directories // get all the assistant metadata json - const results: Assistant[] = []; + const results: Assistant[] = [] const allFileName: string[] = await fs.readdirSync( - JanAssistantExtension._homeDir, - ); + JanAssistantExtension._homeDir + ) for (const fileName of allFileName) { const filePath = await joinPath([ JanAssistantExtension._homeDir, fileName, - ]); + ]) - if (filePath.includes(".DS_Store")) continue; + if (filePath.includes('.DS_Store')) continue const jsonFiles: string[] = (await fs.readdirSync(filePath)).filter( - (file: string) => file === "assistant.json", - ); + (file: string) => file === 'assistant.json' + ) if (jsonFiles.length !== 1) { // has more than one assistant file -> ignore - continue; + continue } const content = await fs.readFileSync( await joinPath([filePath, jsonFiles[0]]), - "utf-8", - ); + 'utf-8' + ) const assistant: Assistant = - typeof content === "object" ? content : JSON.parse(content); + typeof content === 'object' ? content : JSON.parse(content) - results.push(assistant); + results.push(assistant) } - return results; + return results } async deleteAssistant(assistant: Assistant): Promise { - if (assistant.id === "jan") { - return Promise.reject("Cannot delete Jan Assistant"); + if (assistant.id === 'jan') { + return Promise.reject('Cannot delete Jan Assistant') } // remove the directory const assistantDir = await joinPath([ JanAssistantExtension._homeDir, assistant.id, - ]); - await fs.rmdirSync(assistantDir); - return Promise.resolve(); + ]) + await fs.rmdirSync(assistantDir) + return Promise.resolve() } private async createJanAssistant(): Promise { const janAssistant: Assistant = { - avatar: "", + avatar: '', thread_location: undefined, - id: "jan", - object: "assistant", + id: 'jan', + object: 'assistant', created_at: Date.now(), - name: "Jan", - description: "A default assistant that can use all downloaded models", - model: "*", - instructions: "", + name: 'Jan', + description: 'A default assistant that can use all downloaded models', + model: '*', + instructions: '', tools: [ { - type: "retrieval", + type: 'retrieval', enabled: false, settings: { top_k: 2, chunk_size: 1024, chunk_overlap: 64, retrieval_template: `Use the following pieces of context to answer the question at the end. If you don't know the answer, just say that you don't know, don't try to make up an answer. - ---------------- - CONTEXT: {CONTEXT} - ---------------- - QUESTION: {QUESTION} - ---------------- - Helpful Answer:`, +---------------- +CONTEXT: {CONTEXT} +---------------- +QUESTION: {QUESTION} +---------------- +Helpful Answer:`, }, }, ], file_ids: [], metadata: undefined, - }; + } - await this.createAssistant(janAssistant); + await this.createAssistant(janAssistant) } } diff --git a/extensions/assistant-extension/src/node/engine.ts b/extensions/assistant-extension/src/node/engine.ts index 54b2a6ba16..70d02af1f5 100644 --- a/extensions/assistant-extension/src/node/engine.ts +++ b/extensions/assistant-extension/src/node/engine.ts @@ -1,13 +1,13 @@ -import fs from "fs"; -import path from "path"; -import { getJanDataFolderPath } from "@janhq/core/node"; +import fs from 'fs' +import path from 'path' +import { getJanDataFolderPath } from '@janhq/core/node' // Sec: Do not send engine settings over requests // Read it manually instead export const readEmbeddingEngine = (engineName: string) => { const engineSettings = fs.readFileSync( - path.join(getJanDataFolderPath(), "engines", `${engineName}.json`), - "utf-8", - ); - return JSON.parse(engineSettings); -}; + path.join(getJanDataFolderPath(), 'engines', `${engineName}.json`), + 'utf-8' + ) + return JSON.parse(engineSettings) +} diff --git a/extensions/assistant-extension/src/node/index.ts b/extensions/assistant-extension/src/node/index.ts index 95a7243a43..d52a4b23e3 100644 --- a/extensions/assistant-extension/src/node/index.ts +++ b/extensions/assistant-extension/src/node/index.ts @@ -1,39 +1,39 @@ -import { getJanDataFolderPath, normalizeFilePath } from "@janhq/core/node"; -import { Retrieval } from "./tools/retrieval"; -import path from "path"; +import { getJanDataFolderPath, normalizeFilePath } from '@janhq/core/node' +import { retrieval } from './tools/retrieval' +import path from 'path' -const retrieval = new Retrieval(); - -export async function toolRetrievalUpdateTextSplitter( +export function toolRetrievalUpdateTextSplitter( chunkSize: number, - chunkOverlap: number, + chunkOverlap: number ) { - retrieval.updateTextSplitter(chunkSize, chunkOverlap); - return Promise.resolve(); + retrieval.updateTextSplitter(chunkSize, chunkOverlap) } export async function toolRetrievalIngestNewDocument( file: string, - engine: string, + engine: string ) { - const filePath = path.join(getJanDataFolderPath(), normalizeFilePath(file)); - const threadPath = path.dirname(filePath.replace("files", "")); - retrieval.updateEmbeddingEngine(engine); - await retrieval.ingestAgentKnowledge(filePath, `${threadPath}/memory`); - return Promise.resolve(); + const filePath = path.join(getJanDataFolderPath(), normalizeFilePath(file)) + const threadPath = path.dirname(filePath.replace('files', '')) + retrieval.updateEmbeddingEngine(engine) + return retrieval + .ingestAgentKnowledge(filePath, `${threadPath}/memory`) + .catch((err) => { + console.error(err) + }) } export async function toolRetrievalLoadThreadMemory(threadId: string) { - try { - await retrieval.loadRetrievalAgent( - path.join(getJanDataFolderPath(), "threads", threadId, "memory"), - ); - return Promise.resolve(); - } catch (err) { - console.debug(err); - } + return retrieval + .loadRetrievalAgent( + path.join(getJanDataFolderPath(), 'threads', threadId, 'memory') + ) + .catch((err) => { + console.error(err) + }) } export async function toolRetrievalQueryResult(query: string) { - const res = await retrieval.generateResult(query); - return Promise.resolve(res); + return retrieval.generateResult(query).catch((err) => { + console.error(err) + }) } diff --git a/extensions/assistant-extension/src/node/tools/retrieval/index.ts b/extensions/assistant-extension/src/node/tools/retrieval/index.ts index 8c7a6aa2bc..e58ec0c46c 100644 --- a/extensions/assistant-extension/src/node/tools/retrieval/index.ts +++ b/extensions/assistant-extension/src/node/tools/retrieval/index.ts @@ -1,77 +1,80 @@ -import { RecursiveCharacterTextSplitter } from "langchain/text_splitter"; -import { formatDocumentsAsString } from "langchain/util/document"; -import { PDFLoader } from "langchain/document_loaders/fs/pdf"; +import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter' +import { formatDocumentsAsString } from 'langchain/util/document' +import { PDFLoader } from 'langchain/document_loaders/fs/pdf' -import { HNSWLib } from "langchain/vectorstores/hnswlib"; +import { HNSWLib } from 'langchain/vectorstores/hnswlib' -import { OpenAIEmbeddings } from "langchain/embeddings/openai"; -import { readEmbeddingEngine } from "../../engine"; +import { OpenAIEmbeddings } from 'langchain/embeddings/openai' +import { readEmbeddingEngine } from '../../engine' export class Retrieval { - public chunkSize: number = 100; - public chunkOverlap?: number = 0; - private retriever: any; + public chunkSize: number = 100 + public chunkOverlap?: number = 0 + private retriever: any - private embeddingModel?: OpenAIEmbeddings = undefined; - private textSplitter?: RecursiveCharacterTextSplitter; + private embeddingModel?: OpenAIEmbeddings = undefined + private textSplitter?: RecursiveCharacterTextSplitter constructor(chunkSize: number = 4000, chunkOverlap: number = 200) { - this.updateTextSplitter(chunkSize, chunkOverlap); + this.updateTextSplitter(chunkSize, chunkOverlap) } public updateTextSplitter(chunkSize: number, chunkOverlap: number): void { - this.chunkSize = chunkSize; - this.chunkOverlap = chunkOverlap; + this.chunkSize = chunkSize + this.chunkOverlap = chunkOverlap this.textSplitter = new RecursiveCharacterTextSplitter({ chunkSize: chunkSize, chunkOverlap: chunkOverlap, - }); + }) } public updateEmbeddingEngine(engine: string): void { // Engine settings are not compatible with the current embedding model params // Switch case manually for now - const settings = readEmbeddingEngine(engine); - if (engine === "nitro") { + const settings = readEmbeddingEngine(engine) + if (engine === 'nitro') { this.embeddingModel = new OpenAIEmbeddings( - { openAIApiKey: "nitro-embedding" }, - { basePath: "http://127.0.0.1:3928/v1" }, - ); + { openAIApiKey: 'nitro-embedding' }, + // TODO: Raw settings + { basePath: 'http://127.0.0.1:3928/v1' } + ) } else { // Fallback to OpenAI Settings this.embeddingModel = new OpenAIEmbeddings({ openAIApiKey: settings.api_key, - }); + }) } } public ingestAgentKnowledge = async ( filePath: string, - memoryPath: string, + memoryPath: string ): Promise => { const loader = new PDFLoader(filePath, { splitPages: true, - }); - if (!this.embeddingModel) return Promise.reject(); - const doc = await loader.load(); - const docs = await this.textSplitter!.splitDocuments(doc); - const vectorStore = await HNSWLib.fromDocuments(docs, this.embeddingModel); - return vectorStore.save(memoryPath); - }; + }) + if (!this.embeddingModel) return Promise.reject() + const doc = await loader.load() + const docs = await this.textSplitter!.splitDocuments(doc) + const vectorStore = await HNSWLib.fromDocuments(docs, this.embeddingModel) + return vectorStore.save(memoryPath) + } public loadRetrievalAgent = async (memoryPath: string): Promise => { - if (!this.embeddingModel) return Promise.reject(); - const vectorStore = await HNSWLib.load(memoryPath, this.embeddingModel); - this.retriever = vectorStore.asRetriever(2); - return Promise.resolve(); - }; + if (!this.embeddingModel) return Promise.reject() + const vectorStore = await HNSWLib.load(memoryPath, this.embeddingModel) + this.retriever = vectorStore.asRetriever(2) + return Promise.resolve() + } public generateResult = async (query: string): Promise => { if (!this.retriever) { - return Promise.resolve(" "); + return Promise.resolve(' ') } - const relevantDocs = await this.retriever.getRelevantDocuments(query); - const serializedDoc = formatDocumentsAsString(relevantDocs); - return Promise.resolve(serializedDoc); - }; + const relevantDocs = await this.retriever.getRelevantDocuments(query) + const serializedDoc = formatDocumentsAsString(relevantDocs) + return Promise.resolve(serializedDoc) + } } + +export const retrieval = new Retrieval() diff --git a/extensions/assistant-extension/tsconfig.json b/extensions/assistant-extension/tsconfig.json index d3794cace9..e425358c35 100644 --- a/extensions/assistant-extension/tsconfig.json +++ b/extensions/assistant-extension/tsconfig.json @@ -14,7 +14,7 @@ "outDir": "dist", "importHelpers": true, "typeRoots": ["node_modules/@types"], - "skipLibCheck": true, + "skipLibCheck": true }, - "include": ["src"], + "include": ["src"] } diff --git a/extensions/conversational-extension/.prettierrc b/extensions/conversational-extension/.prettierrc deleted file mode 100644 index 46f1abcb02..0000000000 --- a/extensions/conversational-extension/.prettierrc +++ /dev/null @@ -1,8 +0,0 @@ -{ - "semi": false, - "singleQuote": true, - "quoteProps": "consistent", - "trailingComma": "es5", - "endOfLine": "auto", - "plugins": ["prettier-plugin-tailwindcss"] -} diff --git a/extensions/conversational-extension/package.json b/extensions/conversational-extension/package.json index a60c12339f..8a6da14e50 100644 --- a/extensions/conversational-extension/package.json +++ b/extensions/conversational-extension/package.json @@ -7,7 +7,7 @@ "license": "MIT", "scripts": { "build": "tsc -b . && webpack --config webpack.config.js", - "build:publish": "rimraf *.tgz --glob && npm run build && npm pack && cpx *.tgz ../../electron/pre-install" + "build:publish": "rimraf *.tgz --glob && npm run build && npm pack && cpx *.tgz ../../pre-install" }, "exports": { ".": "./dist/index.js", @@ -17,12 +17,12 @@ "cpx": "^1.5.0", "rimraf": "^3.0.2", "webpack": "^5.88.2", - "webpack-cli": "^5.1.4" + "webpack-cli": "^5.1.4", + "ts-loader": "^9.5.0" }, "dependencies": { "@janhq/core": "file:../../core", - "path-browserify": "^1.0.1", - "ts-loader": "^9.5.0" + "path-browserify": "^1.0.1" }, "engines": { "node": ">=18.0.0" diff --git a/extensions/conversational-extension/src/index.ts b/extensions/conversational-extension/src/index.ts index 3d28a9c1d5..bf8c213add 100644 --- a/extensions/conversational-extension/src/index.ts +++ b/extensions/conversational-extension/src/index.ts @@ -12,7 +12,7 @@ import { * functionality for managing threads. */ export default class JSONConversationalExtension extends ConversationalExtension { - private static readonly _homeDir = 'file://threads' + private static readonly _threadFolder = 'file://threads' private static readonly _threadInfoFileName = 'thread.json' private static readonly _threadMessagesFileName = 'messages.jsonl' @@ -20,8 +20,8 @@ export default class JSONConversationalExtension extends ConversationalExtension * Called when the extension is loaded. */ async onLoad() { - if (!(await fs.existsSync(JSONConversationalExtension._homeDir))) - await fs.mkdirSync(JSONConversationalExtension._homeDir) + if (!(await fs.existsSync(JSONConversationalExtension._threadFolder))) + await fs.mkdirSync(JSONConversationalExtension._threadFolder) console.debug('JSONConversationalExtension loaded') } @@ -68,7 +68,7 @@ export default class JSONConversationalExtension extends ConversationalExtension async saveThread(thread: Thread): Promise { try { const threadDirPath = await joinPath([ - JSONConversationalExtension._homeDir, + JSONConversationalExtension._threadFolder, thread.id, ]) const threadJsonPath = await joinPath([ @@ -92,7 +92,7 @@ export default class JSONConversationalExtension extends ConversationalExtension */ async deleteThread(threadId: string): Promise { const path = await joinPath([ - JSONConversationalExtension._homeDir, + JSONConversationalExtension._threadFolder, `${threadId}`, ]) try { @@ -109,7 +109,7 @@ export default class JSONConversationalExtension extends ConversationalExtension async addNewMessage(message: ThreadMessage): Promise { try { const threadDirPath = await joinPath([ - JSONConversationalExtension._homeDir, + JSONConversationalExtension._threadFolder, message.thread_id, ]) const threadMessagePath = await joinPath([ @@ -177,7 +177,7 @@ export default class JSONConversationalExtension extends ConversationalExtension ): Promise { try { const threadDirPath = await joinPath([ - JSONConversationalExtension._homeDir, + JSONConversationalExtension._threadFolder, threadId, ]) const threadMessagePath = await joinPath([ @@ -205,7 +205,7 @@ export default class JSONConversationalExtension extends ConversationalExtension private async readThread(threadDirName: string): Promise { return fs.readFileSync( await joinPath([ - JSONConversationalExtension._homeDir, + JSONConversationalExtension._threadFolder, threadDirName, JSONConversationalExtension._threadInfoFileName, ]), @@ -219,14 +219,14 @@ export default class JSONConversationalExtension extends ConversationalExtension */ private async getValidThreadDirs(): Promise { const fileInsideThread: string[] = await fs.readdirSync( - JSONConversationalExtension._homeDir + JSONConversationalExtension._threadFolder ) const threadDirs: string[] = [] for (let i = 0; i < fileInsideThread.length; i++) { if (fileInsideThread[i].includes('.DS_Store')) continue const path = await joinPath([ - JSONConversationalExtension._homeDir, + JSONConversationalExtension._threadFolder, fileInsideThread[i], ]) @@ -246,7 +246,7 @@ export default class JSONConversationalExtension extends ConversationalExtension async getAllMessages(threadId: string): Promise { try { const threadDirPath = await joinPath([ - JSONConversationalExtension._homeDir, + JSONConversationalExtension._threadFolder, threadId, ]) @@ -263,22 +263,17 @@ export default class JSONConversationalExtension extends ConversationalExtension JSONConversationalExtension._threadMessagesFileName, ]) - const result = await fs - .readFileSync(messageFilePath, 'utf-8') - .then((content) => - content - .toString() - .split('\n') - .filter((line) => line !== '') - ) + let readResult = await fs.readFileSync(messageFilePath, 'utf-8') + + if (typeof readResult === 'object') { + readResult = JSON.stringify(readResult) + } + + const result = readResult.split('\n').filter((line) => line !== '') const messages: ThreadMessage[] = [] result.forEach((line: string) => { - try { - messages.push(JSON.parse(line) as ThreadMessage) - } catch (err) { - console.error(err) - } + messages.push(JSON.parse(line)) }) return messages } catch (err) { diff --git a/extensions/conversational-extension/webpack.config.js b/extensions/conversational-extension/webpack.config.js index 36e3382953..a3eb873d71 100644 --- a/extensions/conversational-extension/webpack.config.js +++ b/extensions/conversational-extension/webpack.config.js @@ -1,27 +1,27 @@ -const path = require("path"); -const webpack = require("webpack"); +const path = require('path') +const webpack = require('webpack') module.exports = { experiments: { outputModule: true }, - entry: "./src/index.ts", // Adjust the entry point to match your project's main file - mode: "production", + entry: './src/index.ts', // Adjust the entry point to match your project's main file + mode: 'production', module: { rules: [ { test: /\.tsx?$/, - use: "ts-loader", + use: 'ts-loader', exclude: /node_modules/, }, ], }, output: { - filename: "index.js", // Adjust the output file name as needed - path: path.resolve(__dirname, "dist"), - library: { type: "module" }, // Specify ESM output format + filename: 'index.js', // Adjust the output file name as needed + path: path.resolve(__dirname, 'dist'), + library: { type: 'module' }, // Specify ESM output format }, plugins: [new webpack.DefinePlugin({})], resolve: { - extensions: [".ts", ".js"], + extensions: ['.ts', '.js'], fallback: { path: require.resolve('path-browserify'), }, @@ -31,4 +31,4 @@ module.exports = { minimize: false, }, // Add loaders and other configuration as needed for your project -}; +} diff --git a/extensions/inference-nitro-extension/README.md b/extensions/inference-nitro-extension/README.md index 455783efb1..f499e0b9c5 100644 --- a/extensions/inference-nitro-extension/README.md +++ b/extensions/inference-nitro-extension/README.md @@ -64,10 +64,10 @@ There are a few things to keep in mind when writing your plugin code: In `index.ts`, you will see that the extension function will return a `Promise`. ```typescript - import { core } from "@janhq/core"; + import { core } from '@janhq/core' function onStart(): Promise { - return core.invokePluginFunc(MODULE_PATH, "run", 0); + return core.invokePluginFunc(MODULE_PATH, 'run', 0) } ``` @@ -75,4 +75,3 @@ There are a few things to keep in mind when writing your plugin code: [documentation](https://github.com/janhq/jan/blob/main/core/README.md). So, what are you waiting for? Go ahead and start customizing your plugin! - diff --git a/extensions/inference-nitro-extension/bin/version.txt b/extensions/inference-nitro-extension/bin/version.txt index c2c0004f0e..0b9c019963 100644 --- a/extensions/inference-nitro-extension/bin/version.txt +++ b/extensions/inference-nitro-extension/bin/version.txt @@ -1 +1 @@ -0.3.5 +0.3.12 diff --git a/extensions/inference-nitro-extension/download.bat b/extensions/inference-nitro-extension/download.bat index 22e1c85b35..2ef3165c16 100644 --- a/extensions/inference-nitro-extension/download.bat +++ b/extensions/inference-nitro-extension/download.bat @@ -1,3 +1,3 @@ @echo off set /p NITRO_VERSION=<./bin/version.txt -.\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-cuda-12-0.tar.gz -e --strip 1 -o ./bin/win-cuda-12-0 && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-cuda-11-7.tar.gz -e --strip 1 -o ./bin/win-cuda-11-7 && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64.tar.gz -e --strip 1 -o ./bin/win-cpu +.\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-cuda-12-0.tar.gz -e --strip 1 -o ./bin/win-cuda-12-0 && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-cuda-11-7.tar.gz -e --strip 1 -o ./bin/win-cuda-11-7 && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64.tar.gz -e --strip 1 -o ./bin/win-cpu && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-vulkan.tar.gz -e --strip 1 -o ./bin/win-vulkan && .\node_modules\.bin\download https://delta.jan.ai/vulkaninfoSDK.exe -o ./bin diff --git a/extensions/inference-nitro-extension/package.json b/extensions/inference-nitro-extension/package.json index 8ad516ad98..ba6b473ebf 100644 --- a/extensions/inference-nitro-extension/package.json +++ b/extensions/inference-nitro-extension/package.json @@ -8,13 +8,13 @@ "license": "AGPL-3.0", "scripts": { "build": "tsc --module commonjs && rollup -c rollup.config.ts", - "downloadnitro:linux": "NITRO_VERSION=$(cat ./bin/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64.tar.gz -e --strip 1 -o ./bin/linux-cpu && chmod +x ./bin/linux-cpu/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda-12-0.tar.gz -e --strip 1 -o ./bin/linux-cuda-12-0 && chmod +x ./bin/linux-cuda-12-0/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda-11-7.tar.gz -e --strip 1 -o ./bin/linux-cuda-11-7 && chmod +x ./bin/linux-cuda-11-7/nitro", + "downloadnitro:linux": "NITRO_VERSION=$(cat ./bin/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64.tar.gz -e --strip 1 -o ./bin/linux-cpu && chmod +x ./bin/linux-cpu/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda-12-0.tar.gz -e --strip 1 -o ./bin/linux-cuda-12-0 && chmod +x ./bin/linux-cuda-12-0/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda-11-7.tar.gz -e --strip 1 -o ./bin/linux-cuda-11-7 && chmod +x ./bin/linux-cuda-11-7/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-vulkan.tar.gz -e --strip 1 -o ./bin/linux-vulkan && chmod +x ./bin/linux-vulkan/nitro && download https://delta.jan.ai/vulkaninfo -o ./bin && chmod +x ./bin/vulkaninfo", "downloadnitro:darwin": "NITRO_VERSION=$(cat ./bin/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-mac-arm64.tar.gz -e --strip 1 -o ./bin/mac-arm64 && chmod +x ./bin/mac-arm64/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-mac-amd64.tar.gz -e --strip 1 -o ./bin/mac-x64 && chmod +x ./bin/mac-x64/nitro", "downloadnitro:win32": "download.bat", "downloadnitro": "run-script-os", - "build:publish:darwin": "rimraf *.tgz --glob && npm run build && npm run downloadnitro && ../../.github/scripts/auto-sign.sh && cpx \"bin/**\" \"dist/bin\" && npm pack && cpx *.tgz ../../electron/pre-install", - "build:publish:win32": "rimraf *.tgz --glob && npm run build && npm run downloadnitro && cpx \"bin/**\" \"dist/bin\" && npm pack && cpx *.tgz ../../electron/pre-install", - "build:publish:linux": "rimraf *.tgz --glob && npm run build && npm run downloadnitro && cpx \"bin/**\" \"dist/bin\" && npm pack && cpx *.tgz ../../electron/pre-install", + "build:publish:darwin": "rimraf *.tgz --glob && npm run build && npm run downloadnitro && ../../.github/scripts/auto-sign.sh && cpx \"bin/**\" \"dist/bin\" && npm pack && cpx *.tgz ../../pre-install", + "build:publish:win32": "rimraf *.tgz --glob && npm run build && npm run downloadnitro && cpx \"bin/**\" \"dist/bin\" && npm pack && cpx *.tgz ../../pre-install", + "build:publish:linux": "rimraf *.tgz --glob && npm run build && npm run downloadnitro && cpx \"bin/**\" \"dist/bin\" && npm pack && cpx *.tgz ../../pre-install", "build:publish": "run-script-os" }, "exports": { @@ -35,12 +35,12 @@ "rollup-plugin-sourcemaps": "^0.6.3", "rollup-plugin-typescript2": "^0.36.0", "run-script-os": "^1.1.6", - "typescript": "^5.2.2" + "typescript": "^5.2.2", + "@types/os-utils": "^0.0.4", + "@rollup/plugin-replace": "^5.0.5" }, "dependencies": { "@janhq/core": "file:../../core", - "@rollup/plugin-replace": "^5.0.5", - "@types/os-utils": "^0.0.4", "fetch-retry": "^5.0.6", "path-browserify": "^1.0.1", "rxjs": "^7.8.1", diff --git a/extensions/inference-nitro-extension/rollup.config.ts b/extensions/inference-nitro-extension/rollup.config.ts index 374a054cd5..ec8943f9cd 100644 --- a/extensions/inference-nitro-extension/rollup.config.ts +++ b/extensions/inference-nitro-extension/rollup.config.ts @@ -1,31 +1,34 @@ -import resolve from "@rollup/plugin-node-resolve"; -import commonjs from "@rollup/plugin-commonjs"; -import sourceMaps from "rollup-plugin-sourcemaps"; -import typescript from "rollup-plugin-typescript2"; -import json from "@rollup/plugin-json"; -import replace from "@rollup/plugin-replace"; -const packageJson = require("./package.json"); +import resolve from '@rollup/plugin-node-resolve' +import commonjs from '@rollup/plugin-commonjs' +import sourceMaps from 'rollup-plugin-sourcemaps' +import typescript from 'rollup-plugin-typescript2' +import json from '@rollup/plugin-json' +import replace from '@rollup/plugin-replace' +const packageJson = require('./package.json') -const pkg = require("./package.json"); +const pkg = require('./package.json') export default [ { input: `src/index.ts`, - output: [{ file: pkg.main, format: "es", sourcemap: true }], + output: [{ file: pkg.main, format: 'es', sourcemap: true }], // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') external: [], watch: { - include: "src/**", + include: 'src/**', }, plugins: [ replace({ NODE: JSON.stringify(`${packageJson.name}/${packageJson.node}`), INFERENCE_URL: JSON.stringify( process.env.INFERENCE_URL || - "http://127.0.0.1:3928/inferences/llamacpp/chat_completion" + 'http://127.0.0.1:3928/inferences/llamacpp/chat_completion' ), TROUBLESHOOTING_URL: JSON.stringify( - "https://jan.ai/guides/troubleshooting" + 'https://jan.ai/guides/troubleshooting' + ), + JAN_SERVER_INFERENCE_URL: JSON.stringify( + 'http://localhost:1337/v1/chat/completions' ), }), // Allow json resolution @@ -39,7 +42,7 @@ export default [ // which external modules to include in the bundle // https://github.com/rollup/rollup-plugin-node-resolve#usage resolve({ - extensions: [".js", ".ts", ".svelte"], + extensions: ['.js', '.ts', '.svelte'], }), // Resolve source maps to the original source @@ -49,12 +52,12 @@ export default [ { input: `src/node/index.ts`, output: [ - { file: "dist/node/index.cjs.js", format: "cjs", sourcemap: true }, + { file: 'dist/node/index.cjs.js', format: 'cjs', sourcemap: true }, ], // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') - external: ["@janhq/core/node"], + external: ['@janhq/core/node'], watch: { - include: "src/node/**", + include: 'src/node/**', }, plugins: [ // Allow json resolution @@ -67,11 +70,11 @@ export default [ // which external modules to include in the bundle // https://github.com/rollup/rollup-plugin-node-resolve#usage resolve({ - extensions: [".ts", ".js", ".json"], + extensions: ['.ts', '.js', '.json'], }), // Resolve source maps to the original source sourceMaps(), ], }, -]; +] diff --git a/extensions/inference-nitro-extension/src/@types/global.d.ts b/extensions/inference-nitro-extension/src/@types/global.d.ts index bc126337f6..3a3d2aa325 100644 --- a/extensions/inference-nitro-extension/src/@types/global.d.ts +++ b/extensions/inference-nitro-extension/src/@types/global.d.ts @@ -1,12 +1,13 @@ -declare const NODE: string; -declare const INFERENCE_URL: string; -declare const TROUBLESHOOTING_URL: string; +declare const NODE: string +declare const INFERENCE_URL: string +declare const TROUBLESHOOTING_URL: string +declare const JAN_SERVER_INFERENCE_URL: string /** * The response from the initModel function. * @property error - An error message if the model fails to load. */ interface ModelOperationResponse { - error?: any; - modelFile?: string; + error?: any + modelFile?: string } diff --git a/extensions/inference-nitro-extension/src/helpers/sse.ts b/extensions/inference-nitro-extension/src/helpers/sse.ts index c6352383d4..06176c9b9d 100644 --- a/extensions/inference-nitro-extension/src/helpers/sse.ts +++ b/extensions/inference-nitro-extension/src/helpers/sse.ts @@ -1,11 +1,12 @@ -import { Model } from "@janhq/core"; -import { Observable } from "rxjs"; +import { Model } from '@janhq/core' +import { Observable } from 'rxjs' /** * Sends a request to the inference server to generate a response based on the recent messages. * @param recentMessages - An array of recent messages to use as context for the inference. * @returns An Observable that emits the generated response as a string. */ export function requestInference( + inferenceUrl: string, recentMessages: any[], model: Model, controller?: AbortController @@ -16,50 +17,50 @@ export function requestInference( model: model.id, stream: true, ...model.parameters, - }); - fetch(INFERENCE_URL, { - method: "POST", + }) + fetch(inferenceUrl, { + method: 'POST', headers: { - "Content-Type": "application/json", - "Access-Control-Allow-Origin": "*", - Accept: model.parameters.stream - ? "text/event-stream" - : "application/json", + 'Content-Type': 'application/json', + 'Access-Control-Allow-Origin': '*', + 'Accept': model.parameters.stream + ? 'text/event-stream' + : 'application/json', }, body: requestBody, signal: controller?.signal, }) .then(async (response) => { if (model.parameters.stream === false) { - const data = await response.json(); - subscriber.next(data.choices[0]?.message?.content ?? ""); + const data = await response.json() + subscriber.next(data.choices[0]?.message?.content ?? '') } else { - const stream = response.body; - const decoder = new TextDecoder("utf-8"); - const reader = stream?.getReader(); - let content = ""; + const stream = response.body + const decoder = new TextDecoder('utf-8') + const reader = stream?.getReader() + let content = '' while (true && reader) { - const { done, value } = await reader.read(); + const { done, value } = await reader.read() if (done) { - break; + break } - const text = decoder.decode(value); - const lines = text.trim().split("\n"); + const text = decoder.decode(value) + const lines = text.trim().split('\n') for (const line of lines) { - if (line.startsWith("data: ") && !line.includes("data: [DONE]")) { - const data = JSON.parse(line.replace("data: ", "")); - content += data.choices[0]?.delta?.content ?? ""; - if (content.startsWith("assistant: ")) { - content = content.replace("assistant: ", ""); + if (line.startsWith('data: ') && !line.includes('data: [DONE]')) { + const data = JSON.parse(line.replace('data: ', '')) + content += data.choices[0]?.delta?.content ?? '' + if (content.startsWith('assistant: ')) { + content = content.replace('assistant: ', '') } - subscriber.next(content); + subscriber.next(content) } } } } - subscriber.complete(); + subscriber.complete() }) - .catch((err) => subscriber.error(err)); - }); + .catch((err) => subscriber.error(err)) + }) } diff --git a/extensions/inference-nitro-extension/src/index.ts b/extensions/inference-nitro-extension/src/index.ts index 2b0021ba0b..979b4cfac9 100644 --- a/extensions/inference-nitro-extension/src/index.ts +++ b/extensions/inference-nitro-extension/src/index.ts @@ -10,6 +10,7 @@ import { ChatCompletionRole, ContentType, MessageRequest, + MessageRequestType, MessageStatus, ThreadContent, ThreadMessage, @@ -25,9 +26,10 @@ import { ModelEvent, InferenceEvent, ModelSettingParams, -} from "@janhq/core"; -import { requestInference } from "./helpers/sse"; -import { ulid } from "ulid"; + getJanDataFolderPath, +} from '@janhq/core' +import { requestInference } from './helpers/sse' +import { ulid } from 'ulid' /** * A class that implements the InferenceExtension interface from the @janhq/core package. @@ -35,16 +37,16 @@ import { ulid } from "ulid"; * It also subscribes to events emitted by the @janhq/core package and handles new message requests. */ export default class JanInferenceNitroExtension extends InferenceExtension { - private static readonly _homeDir = "file://engines"; - private static readonly _settingsDir = "file://settings"; - private static readonly _engineMetadataFileName = "nitro.json"; + private static readonly _homeDir = 'file://engines' + private static readonly _settingsDir = 'file://settings' + private static readonly _engineMetadataFileName = 'nitro.json' /** * Checking the health for Nitro's process each 5 secs. */ - private static readonly _intervalHealthCheck = 5 * 1000; + private static readonly _intervalHealthCheck = 5 * 1000 - private _currentModel: Model | undefined; + private _currentModel: Model | undefined private _engineSettings: ModelSettingParams = { ctx_len: 2048, @@ -52,55 +54,63 @@ export default class JanInferenceNitroExtension extends InferenceExtension { cpu_threads: 1, cont_batching: false, embedding: true, - }; + } - controller = new AbortController(); - isCancelled = false; + controller = new AbortController() + isCancelled = false /** * The interval id for the health check. Used to stop the health check. */ - private getNitroProcesHealthIntervalId: NodeJS.Timeout | undefined = - undefined; + private getNitroProcesHealthIntervalId: NodeJS.Timeout | undefined = undefined /** * Tracking the current state of nitro process. */ - private nitroProcessInfo: any = undefined; + private nitroProcessInfo: any = undefined + + private inferenceUrl = '' /** * Subscribes to events emitted by the @janhq/core package. */ async onLoad() { if (!(await fs.existsSync(JanInferenceNitroExtension._homeDir))) { - await fs - .mkdirSync(JanInferenceNitroExtension._homeDir) - .catch((err: Error) => console.debug(err)); + try { + await fs.mkdirSync(JanInferenceNitroExtension._homeDir) + } catch (e) { + console.debug(e) + } + } + + // init inference url + // @ts-ignore + const electronApi = window?.electronAPI + this.inferenceUrl = INFERENCE_URL + if (!electronApi) { + this.inferenceUrl = `${window.core?.api?.baseApiUrl}/v1/chat/completions` } + console.debug('Inference url: ', this.inferenceUrl) if (!(await fs.existsSync(JanInferenceNitroExtension._settingsDir))) - await fs.mkdirSync(JanInferenceNitroExtension._settingsDir); - this.writeDefaultEngineSettings(); + await fs.mkdirSync(JanInferenceNitroExtension._settingsDir) + this.writeDefaultEngineSettings() // Events subscription events.on(MessageEvent.OnMessageSent, (data: MessageRequest) => - this.onMessageRequest(data), - ); + this.onMessageRequest(data) + ) - events.on(ModelEvent.OnModelInit, (model: Model) => - this.onModelInit(model), - ); + events.on(ModelEvent.OnModelInit, (model: Model) => this.onModelInit(model)) - events.on(ModelEvent.OnModelStop, (model: Model) => - this.onModelStop(model), - ); + events.on(ModelEvent.OnModelStop, (model: Model) => this.onModelStop(model)) events.on(InferenceEvent.OnInferenceStopped, () => - this.onInferenceStopped(), - ); + this.onInferenceStopped() + ) // Attempt to fetch nvidia info - await executeOnMain(NODE, "updateNvidiaInfo", {}); + await executeOnMain(NODE, 'updateNvidiaInfo', {}) } /** @@ -113,56 +123,62 @@ export default class JanInferenceNitroExtension extends InferenceExtension { const engineFile = await joinPath([ JanInferenceNitroExtension._homeDir, JanInferenceNitroExtension._engineMetadataFileName, - ]); + ]) if (await fs.existsSync(engineFile)) { - const engine = await fs.readFileSync(engineFile, "utf-8"); + const engine = await fs.readFileSync(engineFile, 'utf-8') this._engineSettings = - typeof engine === "object" ? engine : JSON.parse(engine); + typeof engine === 'object' ? engine : JSON.parse(engine) } else { await fs.writeFileSync( engineFile, - JSON.stringify(this._engineSettings, null, 2), - ); + JSON.stringify(this._engineSettings, null, 2) + ) } } catch (err) { - console.error(err); + console.error(err) } } private async onModelInit(model: Model) { - if (model.engine !== InferenceEngine.nitro) return; - - const modelFullPath = await joinPath(["models", model.id]); - - this._currentModel = model; - const nitroInitResult = await executeOnMain(NODE, "runModel", { - modelFullPath, + if (model.engine !== InferenceEngine.nitro) return + + const modelFolder = await joinPath([ + await getJanDataFolderPath(), + 'models', + model.id, + ]) + this._currentModel = model + const nitroInitResult = await executeOnMain(NODE, 'runModel', { + modelFolder, model, - }); + }) if (nitroInitResult?.error) { - events.emit(ModelEvent.OnModelFail, model); - return; + events.emit(ModelEvent.OnModelFail, { + ...model, + error: nitroInitResult.error, + }) + return } - events.emit(ModelEvent.OnModelReady, model); + events.emit(ModelEvent.OnModelReady, model) this.getNitroProcesHealthIntervalId = setInterval( () => this.periodicallyGetNitroHealth(), - JanInferenceNitroExtension._intervalHealthCheck, - ); + JanInferenceNitroExtension._intervalHealthCheck + ) } private async onModelStop(model: Model) { - if (model.engine !== "nitro") return; + if (model.engine !== 'nitro') return - await executeOnMain(NODE, "stopModel"); - events.emit(ModelEvent.OnModelStopped, {}); + await executeOnMain(NODE, 'stopModel') + events.emit(ModelEvent.OnModelStopped, {}) // stop the periocally health check if (this.getNitroProcesHealthIntervalId) { - clearInterval(this.getNitroProcesHealthIntervalId); - this.getNitroProcesHealthIntervalId = undefined; + clearInterval(this.getNitroProcesHealthIntervalId) + this.getNitroProcesHealthIntervalId = undefined } } @@ -170,19 +186,19 @@ export default class JanInferenceNitroExtension extends InferenceExtension { * Periodically check for nitro process's health. */ private async periodicallyGetNitroHealth(): Promise { - const health = await executeOnMain(NODE, "getCurrentNitroProcessInfo"); + const health = await executeOnMain(NODE, 'getCurrentNitroProcessInfo') - const isRunning = this.nitroProcessInfo?.isRunning ?? false; + const isRunning = this.nitroProcessInfo?.isRunning ?? false if (isRunning && health.isRunning === false) { - console.debug("Nitro process is stopped"); - events.emit(ModelEvent.OnModelStopped, {}); + console.debug('Nitro process is stopped') + events.emit(ModelEvent.OnModelStopped, {}) } - this.nitroProcessInfo = health; + this.nitroProcessInfo = health } private async onInferenceStopped() { - this.isCancelled = true; - this.controller?.abort(); + this.isCancelled = true + this.controller?.abort() } /** @@ -191,31 +207,35 @@ export default class JanInferenceNitroExtension extends InferenceExtension { * @returns {Promise} A promise that resolves with the inference response. */ async inference(data: MessageRequest): Promise { - const timestamp = Date.now(); + const timestamp = Date.now() const message: ThreadMessage = { thread_id: data.threadId, created: timestamp, updated: timestamp, status: MessageStatus.Ready, - id: "", + id: '', role: ChatCompletionRole.Assistant, - object: "thread.message", + object: 'thread.message', content: [], - }; + } return new Promise(async (resolve, reject) => { - if (!this._currentModel) return Promise.reject("No model loaded"); + if (!this._currentModel) return Promise.reject('No model loaded') - requestInference(data.messages ?? [], this._currentModel).subscribe({ + requestInference( + this.inferenceUrl, + data.messages ?? [], + this._currentModel + ).subscribe({ next: (_content: any) => {}, complete: async () => { - resolve(message); + resolve(message) }, error: async (err: any) => { - reject(err); + reject(err) }, - }); - }); + }) + }) } /** @@ -226,32 +246,41 @@ export default class JanInferenceNitroExtension extends InferenceExtension { */ private async onMessageRequest(data: MessageRequest) { if (data.model?.engine !== InferenceEngine.nitro || !this._currentModel) { - return; + return } - const timestamp = Date.now(); + const timestamp = Date.now() const message: ThreadMessage = { id: ulid(), thread_id: data.threadId, + type: data.type, assistant_id: data.assistantId, role: ChatCompletionRole.Assistant, content: [], status: MessageStatus.Pending, created: timestamp, updated: timestamp, - object: "thread.message", - }; - events.emit(MessageEvent.OnMessageResponse, message); + object: 'thread.message', + } + + if (data.type !== MessageRequestType.Summary) { + events.emit(MessageEvent.OnMessageResponse, message) + } - this.isCancelled = false; - this.controller = new AbortController(); + this.isCancelled = false + this.controller = new AbortController() // @ts-ignore const model: Model = { ...(this._currentModel || {}), ...(data.model || {}), - }; - requestInference(data.messages ?? [], model, this.controller).subscribe({ + } + requestInference( + this.inferenceUrl, + data.messages ?? [], + model, + this.controller + ).subscribe({ next: (content: any) => { const messageContent: ThreadContent = { type: ContentType.Text, @@ -259,26 +288,26 @@ export default class JanInferenceNitroExtension extends InferenceExtension { value: content.trim(), annotations: [], }, - }; - message.content = [messageContent]; - events.emit(MessageEvent.OnMessageUpdate, message); + } + message.content = [messageContent] + events.emit(MessageEvent.OnMessageUpdate, message) }, complete: async () => { message.status = message.content.length ? MessageStatus.Ready - : MessageStatus.Error; - events.emit(MessageEvent.OnMessageUpdate, message); + : MessageStatus.Error + events.emit(MessageEvent.OnMessageUpdate, message) }, error: async (err: any) => { if (this.isCancelled || message.content.length) { - message.status = MessageStatus.Stopped; - events.emit(MessageEvent.OnMessageUpdate, message); - return; + message.status = MessageStatus.Stopped + events.emit(MessageEvent.OnMessageUpdate, message) + return } - message.status = MessageStatus.Error; - events.emit(MessageEvent.OnMessageUpdate, message); - log(`[APP]::Error: ${err.message}`); + message.status = MessageStatus.Error + events.emit(MessageEvent.OnMessageUpdate, message) + log(`[APP]::Error: ${err.message}`) }, - }); + }) } } diff --git a/extensions/inference-nitro-extension/src/node/accelerator.ts b/extensions/inference-nitro-extension/src/node/accelerator.ts new file mode 100644 index 0000000000..972f88681b --- /dev/null +++ b/extensions/inference-nitro-extension/src/node/accelerator.ts @@ -0,0 +1,240 @@ +import { writeFileSync, existsSync, readFileSync } from 'fs' +import { exec, spawn } from 'child_process' +import path from 'path' +import { getJanDataFolderPath, log } from '@janhq/core/node' + +/** + * Default GPU settings + * TODO: This needs to be refactored to support multiple accelerators + **/ +const DEFALT_SETTINGS = { + notify: true, + run_mode: 'cpu', + nvidia_driver: { + exist: false, + version: '', + }, + cuda: { + exist: false, + version: '', + }, + gpus: [], + gpu_highest_vram: '', + gpus_in_use: [], + is_initial: true, + // TODO: This needs to be set based on user toggle in settings + vulkan: { + enabled: true, + gpu_in_use: '1', + }, +} + +/** + * Path to the settings file + **/ +export const GPU_INFO_FILE = path.join( + getJanDataFolderPath(), + 'settings', + 'settings.json' +) + +/** + * Current nitro process + */ +let nitroProcessInfo: NitroProcessInfo | undefined = undefined + +/** + * Nitro process info + */ +export interface NitroProcessInfo { + isRunning: boolean +} + +/** + * This will retrive GPU informations and persist settings.json + * Will be called when the extension is loaded to turn on GPU acceleration if supported + */ +export async function updateNvidiaInfo() { + if (process.platform !== 'darwin') { + let data + try { + data = JSON.parse(readFileSync(GPU_INFO_FILE, 'utf-8')) + } catch (error) { + data = DEFALT_SETTINGS + writeFileSync(GPU_INFO_FILE, JSON.stringify(data, null, 2)) + } + updateNvidiaDriverInfo() + updateGpuInfo() + } +} + +/** + * Retrieve current nitro process + */ +export const getNitroProcessInfo = (subprocess: any): NitroProcessInfo => { + nitroProcessInfo = { + isRunning: subprocess != null, + } + return nitroProcessInfo +} + +/** + * Validate nvidia and cuda for linux and windows + */ +export async function updateNvidiaDriverInfo(): Promise { + exec( + 'nvidia-smi --query-gpu=driver_version --format=csv,noheader', + (error, stdout) => { + let data = JSON.parse(readFileSync(GPU_INFO_FILE, 'utf-8')) + + if (!error) { + const firstLine = stdout.split('\n')[0].trim() + data['nvidia_driver'].exist = true + data['nvidia_driver'].version = firstLine + } else { + data['nvidia_driver'].exist = false + } + + writeFileSync(GPU_INFO_FILE, JSON.stringify(data, null, 2)) + Promise.resolve() + } + ) +} + +/** + * Check if file exists in paths + */ +export function checkFileExistenceInPaths( + file: string, + paths: string[] +): boolean { + return paths.some((p) => existsSync(path.join(p, file))) +} + +/** + * Validate cuda for linux and windows + */ +export function updateCudaExistence( + data: Record = DEFALT_SETTINGS +): Record { + let filesCuda12: string[] + let filesCuda11: string[] + let paths: string[] + let cudaVersion: string = '' + + if (process.platform === 'win32') { + filesCuda12 = ['cublas64_12.dll', 'cudart64_12.dll', 'cublasLt64_12.dll'] + filesCuda11 = ['cublas64_11.dll', 'cudart64_11.dll', 'cublasLt64_11.dll'] + paths = process.env.PATH ? process.env.PATH.split(path.delimiter) : [] + } else { + filesCuda12 = ['libcudart.so.12', 'libcublas.so.12', 'libcublasLt.so.12'] + filesCuda11 = ['libcudart.so.11.0', 'libcublas.so.11', 'libcublasLt.so.11'] + paths = process.env.LD_LIBRARY_PATH + ? process.env.LD_LIBRARY_PATH.split(path.delimiter) + : [] + paths.push('/usr/lib/x86_64-linux-gnu/') + } + + let cudaExists = filesCuda12.every( + (file) => existsSync(file) || checkFileExistenceInPaths(file, paths) + ) + + if (!cudaExists) { + cudaExists = filesCuda11.every( + (file) => existsSync(file) || checkFileExistenceInPaths(file, paths) + ) + if (cudaExists) { + cudaVersion = '11' + } + } else { + cudaVersion = '12' + } + + data['cuda'].exist = cudaExists + data['cuda'].version = cudaVersion + console.log(data['is_initial'], data['gpus_in_use']) + if (cudaExists && data['is_initial'] && data['gpus_in_use'].length > 0) { + data.run_mode = 'gpu' + } + data.is_initial = false + return data +} + +/** + * Get GPU information + */ +export async function updateGpuInfo(): Promise { + let data = JSON.parse(readFileSync(GPU_INFO_FILE, 'utf-8')) + + // Cuda + if (data['vulkan'] === true) { + // Vulkan + exec( + process.platform === 'win32' + ? `${__dirname}\\..\\bin\\vulkaninfoSDK.exe --summary` + : `${__dirname}/../bin/vulkaninfo --summary`, + (error, stdout) => { + if (!error) { + const output = stdout.toString() + log(output) + const gpuRegex = /GPU(\d+):(?:[\s\S]*?)deviceName\s*=\s*(.*)/g + + let gpus = [] + let match + while ((match = gpuRegex.exec(output)) !== null) { + const id = match[1] + const name = match[2] + gpus.push({ id, vram: 0, name }) + } + data.gpus = gpus + + if (!data['gpus_in_use'] || data['gpus_in_use'].length === 0) { + data.gpus_in_use = [data.gpus.length > 1 ? '1' : '0'] + } + + data = updateCudaExistence(data) + writeFileSync(GPU_INFO_FILE, JSON.stringify(data, null, 2)) + } + Promise.resolve() + } + ) + } else { + exec( + 'nvidia-smi --query-gpu=index,memory.total,name --format=csv,noheader,nounits', + (error, stdout) => { + if (!error) { + log(stdout) + // Get GPU info and gpu has higher memory first + let highestVram = 0 + let highestVramId = '0' + let gpus = stdout + .trim() + .split('\n') + .map((line) => { + let [id, vram, name] = line.split(', ') + vram = vram.replace(/\r/g, '') + if (parseFloat(vram) > highestVram) { + highestVram = parseFloat(vram) + highestVramId = id + } + return { id, vram, name } + }) + + data.gpus = gpus + data.gpu_highest_vram = highestVramId + } else { + data.gpus = [] + data.gpu_highest_vram = '' + } + + if (!data['gpus_in_use'] || data['gpus_in_use'].length === 0) { + data.gpus_in_use = [data['gpu_highest_vram']] + } + + data = updateCudaExistence(data) + writeFileSync(GPU_INFO_FILE, JSON.stringify(data, null, 2)) + Promise.resolve() + } + ) + } +} diff --git a/extensions/inference-nitro-extension/src/node/execute.ts b/extensions/inference-nitro-extension/src/node/execute.ts index ca266639c6..08baba0d53 100644 --- a/extensions/inference-nitro-extension/src/node/execute.ts +++ b/extensions/inference-nitro-extension/src/node/execute.ts @@ -1,65 +1,79 @@ -import { readFileSync } from "fs"; -import * as path from "path"; -import { NVIDIA_INFO_FILE } from "./nvidia"; +import { readFileSync } from 'fs' +import * as path from 'path' +import { GPU_INFO_FILE } from './accelerator' export interface NitroExecutableOptions { - executablePath: string; - cudaVisibleDevices: string; + executablePath: string + cudaVisibleDevices: string + vkVisibleDevices: string } /** * Find which executable file to run based on the current platform. * @returns The name of the executable file to run. */ export const executableNitroFile = (): NitroExecutableOptions => { - let binaryFolder = path.join(__dirname, "..", "bin"); // Current directory by default - let cudaVisibleDevices = ""; - let binaryName = "nitro"; + let binaryFolder = path.join(__dirname, '..', 'bin') // Current directory by default + let cudaVisibleDevices = '' + let vkVisibleDevices = '' + let binaryName = 'nitro' /** * The binary folder is different for each platform. */ - if (process.platform === "win32") { + if (process.platform === 'win32') { /** - * For Windows: win-cpu, win-cuda-11-7, win-cuda-12-0 + * For Windows: win-cpu, win-vulkan, win-cuda-11-7, win-cuda-12-0 */ - let nvidiaInfo = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8")); - if (nvidiaInfo["run_mode"] === "cpu") { - binaryFolder = path.join(binaryFolder, "win-cpu"); + let gpuInfo = JSON.parse(readFileSync(GPU_INFO_FILE, 'utf-8')) + if (gpuInfo['run_mode'] === 'cpu') { + binaryFolder = path.join(binaryFolder, 'win-cpu') } else { - if (nvidiaInfo["cuda"].version === "12") { - binaryFolder = path.join(binaryFolder, "win-cuda-12-0"); + if (gpuInfo['cuda']?.version === '11') { + binaryFolder = path.join(binaryFolder, 'win-cuda-11-7') } else { - binaryFolder = path.join(binaryFolder, "win-cuda-11-7"); + binaryFolder = path.join(binaryFolder, 'win-cuda-12-0') } - cudaVisibleDevices = nvidiaInfo["gpu_highest_vram"]; + cudaVisibleDevices = gpuInfo['gpus_in_use'].join(',') } - binaryName = "nitro.exe"; - } else if (process.platform === "darwin") { + if (gpuInfo['vulkan'] === true) { + binaryFolder = path.join(__dirname, '..', 'bin') + binaryFolder = path.join(binaryFolder, 'win-vulkan') + vkVisibleDevices = gpuInfo['gpus_in_use'].toString() + } + binaryName = 'nitro.exe' + } else if (process.platform === 'darwin') { /** * For MacOS: mac-arm64 (Silicon), mac-x64 (InteL) */ - if (process.arch === "arm64") { - binaryFolder = path.join(binaryFolder, "mac-arm64"); + if (process.arch === 'arm64') { + binaryFolder = path.join(binaryFolder, 'mac-arm64') } else { - binaryFolder = path.join(binaryFolder, "mac-x64"); + binaryFolder = path.join(binaryFolder, 'mac-x64') } } else { /** - * For Linux: linux-cpu, linux-cuda-11-7, linux-cuda-12-0 + * For Linux: linux-cpu, linux-vulkan, linux-cuda-11-7, linux-cuda-12-0 */ - let nvidiaInfo = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8")); - if (nvidiaInfo["run_mode"] === "cpu") { - binaryFolder = path.join(binaryFolder, "linux-cpu"); + let gpuInfo = JSON.parse(readFileSync(GPU_INFO_FILE, 'utf-8')) + if (gpuInfo['run_mode'] === 'cpu') { + binaryFolder = path.join(binaryFolder, 'linux-cpu') } else { - if (nvidiaInfo["cuda"].version === "12") { - binaryFolder = path.join(binaryFolder, "linux-cuda-12-0"); + if (gpuInfo['cuda']?.version === '11') { + binaryFolder = path.join(binaryFolder, 'linux-cuda-11-7') } else { - binaryFolder = path.join(binaryFolder, "linux-cuda-11-7"); + binaryFolder = path.join(binaryFolder, 'linux-cuda-12-0') } - cudaVisibleDevices = nvidiaInfo["gpu_highest_vram"]; + cudaVisibleDevices = gpuInfo['gpus_in_use'].join(',') + } + + if (gpuInfo['vulkan'] === true) { + binaryFolder = path.join(__dirname, '..', 'bin') + binaryFolder = path.join(binaryFolder, 'win-vulkan') + vkVisibleDevices = gpuInfo['gpus_in_use'].toString() } } return { executablePath: path.join(binaryFolder, binaryName), cudaVisibleDevices, - }; -}; + vkVisibleDevices, + } +} diff --git a/extensions/inference-nitro-extension/src/node/index.ts b/extensions/inference-nitro-extension/src/node/index.ts index 7ba90b556b..9b2684a6c5 100644 --- a/extensions/inference-nitro-extension/src/node/index.ts +++ b/extensions/inference-nitro-extension/src/node/index.ts @@ -1,55 +1,50 @@ -import fs from "fs"; -import path from "path"; -import { ChildProcessWithoutNullStreams, spawn } from "child_process"; -import tcpPortUsed from "tcp-port-used"; -import fetchRT from "fetch-retry"; -import { - log, - getJanDataFolderPath, - getSystemResourceInfo, -} from "@janhq/core/node"; -import { getNitroProcessInfo, updateNvidiaInfo } from "./nvidia"; +import fs from 'fs' +import path from 'path' +import { ChildProcessWithoutNullStreams, spawn } from 'child_process' +import tcpPortUsed from 'tcp-port-used' +import fetchRT from 'fetch-retry' +import { log, getSystemResourceInfo } from '@janhq/core/node' +import { getNitroProcessInfo, updateNvidiaInfo } from './accelerator' import { Model, InferenceEngine, ModelSettingParams, PromptTemplate, -} from "@janhq/core"; -import { executableNitroFile } from "./execute"; +} from '@janhq/core' +import { executableNitroFile } from './execute' // Polyfill fetch with retry -const fetchRetry = fetchRT(fetch); +const fetchRetry = fetchRT(fetch) /** * The response object for model init operation. */ interface ModelInitOptions { - modelFullPath: string; - model: Model; + modelFolder: string + model: Model } // The PORT to use for the Nitro subprocess -const PORT = 3928; +const PORT = 3928 // The HOST address to use for the Nitro subprocess -const LOCAL_HOST = "127.0.0.1"; +const LOCAL_HOST = '127.0.0.1' // The URL for the Nitro subprocess -const NITRO_HTTP_SERVER_URL = `http://${LOCAL_HOST}:${PORT}`; +const NITRO_HTTP_SERVER_URL = `http://${LOCAL_HOST}:${PORT}` // The URL for the Nitro subprocess to load a model -const NITRO_HTTP_LOAD_MODEL_URL = `${NITRO_HTTP_SERVER_URL}/inferences/llamacpp/loadmodel`; +const NITRO_HTTP_LOAD_MODEL_URL = `${NITRO_HTTP_SERVER_URL}/inferences/llamacpp/loadmodel` // The URL for the Nitro subprocess to validate a model -const NITRO_HTTP_VALIDATE_MODEL_URL = `${NITRO_HTTP_SERVER_URL}/inferences/llamacpp/modelstatus`; +const NITRO_HTTP_VALIDATE_MODEL_URL = `${NITRO_HTTP_SERVER_URL}/inferences/llamacpp/modelstatus` // The URL for the Nitro subprocess to kill itself -const NITRO_HTTP_KILL_URL = `${NITRO_HTTP_SERVER_URL}/processmanager/destroy`; +const NITRO_HTTP_KILL_URL = `${NITRO_HTTP_SERVER_URL}/processmanager/destroy` // The supported model format // TODO: Should be an array to support more models -const SUPPORTED_MODEL_FORMAT = ".gguf"; +const SUPPORTED_MODEL_FORMAT = '.gguf' // The subprocess instance for Nitro -let subprocess: ChildProcessWithoutNullStreams | undefined = undefined; -// The current model file url -let currentModelFile: string = ""; +let subprocess: ChildProcessWithoutNullStreams | undefined = undefined + // The current model settings -let currentSettings: ModelSettingParams | undefined = undefined; +let currentSettings: ModelSettingParams | undefined = undefined /** * Stops a Nitro subprocess. @@ -57,7 +52,7 @@ let currentSettings: ModelSettingParams | undefined = undefined; * @returns A Promise that resolves when the subprocess is terminated successfully, or rejects with an error message if the subprocess fails to terminate. */ function stopModel(): Promise { - return killSubprocess(); + return killSubprocess() } /** @@ -67,62 +62,79 @@ function stopModel(): Promise { * TODO: Should pass absolute of the model file instead of just the name - So we can modurize the module.ts to npm package */ async function runModel( - wrapper: ModelInitOptions, + wrapper: ModelInitOptions ): Promise { if (wrapper.model.engine !== InferenceEngine.nitro) { // Not a nitro model - return Promise.resolve(); + return Promise.resolve() } - currentModelFile = wrapper.modelFullPath; - const janRoot = await getJanDataFolderPath(); - if (!currentModelFile.includes(janRoot)) { - currentModelFile = path.join(janRoot, currentModelFile); - } - const files: string[] = fs.readdirSync(currentModelFile); - - // Look for GGUF model file - const ggufBinFile = files.find( - (file) => - file === path.basename(currentModelFile) || - file.toLowerCase().includes(SUPPORTED_MODEL_FORMAT), - ); - - if (!ggufBinFile) return Promise.reject("No GGUF model file found"); - - currentModelFile = path.join(currentModelFile, ggufBinFile); - if (wrapper.model.engine !== InferenceEngine.nitro) { - return Promise.reject("Not a nitro model"); + return Promise.reject('Not a nitro model') } else { - const nitroResourceProbe = await getSystemResourceInfo(); + const nitroResourceProbe = await getSystemResourceInfo() // Convert settings.prompt_template to system_prompt, user_prompt, ai_prompt if (wrapper.model.settings.prompt_template) { - const promptTemplate = wrapper.model.settings.prompt_template; - const prompt = promptTemplateConverter(promptTemplate); + const promptTemplate = wrapper.model.settings.prompt_template + const prompt = promptTemplateConverter(promptTemplate) if (prompt?.error) { - return Promise.reject(prompt.error); + return Promise.reject(prompt.error) } - wrapper.model.settings.system_prompt = prompt.system_prompt; - wrapper.model.settings.user_prompt = prompt.user_prompt; - wrapper.model.settings.ai_prompt = prompt.ai_prompt; + wrapper.model.settings.system_prompt = prompt.system_prompt + wrapper.model.settings.user_prompt = prompt.user_prompt + wrapper.model.settings.ai_prompt = prompt.ai_prompt + } + + // modelFolder is the absolute path to the running model folder + // e.g. ~/jan/models/llama-2 + let modelFolder = wrapper.modelFolder + + let llama_model_path = wrapper.model.settings.llama_model_path + + // Absolute model path support + if ( + wrapper.model?.sources.length && + wrapper.model.sources.every((e) => fs.existsSync(e.url)) + ) { + llama_model_path = + wrapper.model.sources.length === 1 + ? wrapper.model.sources[0].url + : wrapper.model.sources.find((e) => + e.url.includes(llama_model_path ?? wrapper.model.id) + )?.url } - const modelFolderPath = path.join(janRoot, "models", wrapper.model.id); - const modelPath = wrapper.model.settings.llama_model_path - ? path.join(modelFolderPath, wrapper.model.settings.llama_model_path) - : currentModelFile; + if (!llama_model_path || !path.isAbsolute(llama_model_path)) { + // Look for GGUF model file + const modelFiles: string[] = fs.readdirSync(modelFolder) + const ggufBinFile = modelFiles.find( + (file) => + // 1. Prioritize llama_model_path (predefined) + (llama_model_path && file === llama_model_path) || + // 2. Prioritize GGUF File (manual import) + file.toLowerCase().includes(SUPPORTED_MODEL_FORMAT) || + // 3. Fallback Model ID (for backward compatibility) + file === wrapper.model.id + ) + if (ggufBinFile) llama_model_path = path.join(modelFolder, ggufBinFile) + } + + // Look for absolute source path for single model + + if (!llama_model_path) return Promise.reject('No GGUF model file found') currentSettings = { ...wrapper.model.settings, - llama_model_path: modelPath, + llama_model_path, // This is critical and requires real CPU physical core count (or performance core) cpu_threads: Math.max(1, nitroResourceProbe.numCpuPhysicalCore), ...(wrapper.model.settings.mmproj && { - mmproj: path.join(modelFolderPath, wrapper.model.settings.mmproj), + mmproj: path.isAbsolute(wrapper.model.settings.mmproj) + ? wrapper.model.settings.mmproj + : path.join(modelFolder, wrapper.model.settings.mmproj), }), - }; - return runNitroAndLoadModel(); + } + return runNitroAndLoadModel() } } @@ -142,10 +154,10 @@ async function runNitroAndLoadModel() { * Should wait for awhile to make sure the port is free and subprocess is killed * The tested threshold is 500ms **/ - if (process.platform === "win32") { - return new Promise((resolve) => setTimeout(resolve, 500)); + if (process.platform === 'win32') { + return new Promise((resolve) => setTimeout(resolve, 500)) } else { - return Promise.resolve(); + return Promise.resolve() } }) .then(spawnNitroProcess) @@ -153,9 +165,9 @@ async function runNitroAndLoadModel() { .then(validateModelStatus) .catch((err) => { // TODO: Broadcast error so app could display proper error message - log(`[NITRO]::Error: ${err}`); - return { error: err }; - }); + log(`[NITRO]::Error: ${err}`) + return { error: err } + }) } /** @@ -165,43 +177,43 @@ async function runNitroAndLoadModel() { */ function promptTemplateConverter(promptTemplate: string): PromptTemplate { // Split the string using the markers - const systemMarker = "{system_message}"; - const promptMarker = "{prompt}"; + const systemMarker = '{system_message}' + const promptMarker = '{prompt}' if ( promptTemplate.includes(systemMarker) && promptTemplate.includes(promptMarker) ) { // Find the indices of the markers - const systemIndex = promptTemplate.indexOf(systemMarker); - const promptIndex = promptTemplate.indexOf(promptMarker); + const systemIndex = promptTemplate.indexOf(systemMarker) + const promptIndex = promptTemplate.indexOf(promptMarker) // Extract the parts of the string - const system_prompt = promptTemplate.substring(0, systemIndex); + const system_prompt = promptTemplate.substring(0, systemIndex) const user_prompt = promptTemplate.substring( systemIndex + systemMarker.length, - promptIndex, - ); + promptIndex + ) const ai_prompt = promptTemplate.substring( - promptIndex + promptMarker.length, - ); + promptIndex + promptMarker.length + ) // Return the split parts - return { system_prompt, user_prompt, ai_prompt }; + return { system_prompt, user_prompt, ai_prompt } } else if (promptTemplate.includes(promptMarker)) { // Extract the parts of the string for the case where only promptMarker is present - const promptIndex = promptTemplate.indexOf(promptMarker); - const user_prompt = promptTemplate.substring(0, promptIndex); + const promptIndex = promptTemplate.indexOf(promptMarker) + const user_prompt = promptTemplate.substring(0, promptIndex) const ai_prompt = promptTemplate.substring( - promptIndex + promptMarker.length, - ); + promptIndex + promptMarker.length + ) // Return the split parts - return { user_prompt, ai_prompt }; + return { user_prompt, ai_prompt } } // Return an error if none of the conditions are met - return { error: "Cannot split prompt template" }; + return { error: 'Cannot split prompt template' } } /** @@ -210,13 +222,13 @@ function promptTemplateConverter(promptTemplate: string): PromptTemplate { */ function loadLLMModel(settings: any): Promise { if (!settings?.ngl) { - settings.ngl = 100; + settings.ngl = 100 } - log(`[NITRO]::Debug: Loading model with params ${JSON.stringify(settings)}`); + log(`[NITRO]::Debug: Loading model with params ${JSON.stringify(settings)}`) return fetchRetry(NITRO_HTTP_LOAD_MODEL_URL, { - method: "POST", + method: 'POST', headers: { - "Content-Type": "application/json", + 'Content-Type': 'application/json', }, body: JSON.stringify(settings), retries: 3, @@ -225,15 +237,15 @@ function loadLLMModel(settings: any): Promise { .then((res) => { log( `[NITRO]::Debug: Load model success with response ${JSON.stringify( - res, - )}`, - ); - return Promise.resolve(res); + res + )}` + ) + return Promise.resolve(res) }) .catch((err) => { - log(`[NITRO]::Error: Load model failed with error ${err}`); - return Promise.reject(err); - }); + log(`[NITRO]::Error: Load model failed with error ${err}`) + return Promise.reject(err) + }) } /** @@ -246,9 +258,9 @@ async function validateModelStatus(): Promise { // Send a GET request to the validation URL. // Retry the request up to 3 times if it fails, with a delay of 500 milliseconds between retries. return fetchRetry(NITRO_HTTP_VALIDATE_MODEL_URL, { - method: "GET", + method: 'GET', headers: { - "Content-Type": "application/json", + 'Content-Type': 'application/json', }, retries: 5, retryDelay: 500, @@ -257,10 +269,10 @@ async function validateModelStatus(): Promise { `[NITRO]::Debug: Validate model state with response ${JSON.stringify( res.status )}` - ); + ) // If the response is OK, check model_loaded status. if (res.ok) { - const body = await res.json(); + const body = await res.json() // If the model is loaded, return an empty object. // Otherwise, return an object with an error message. if (body.model_loaded) { @@ -268,17 +280,17 @@ async function validateModelStatus(): Promise { `[NITRO]::Debug: Validate model state success with response ${JSON.stringify( body )}` - ); - return Promise.resolve(); + ) + return Promise.resolve() } } log( `[NITRO]::Debug: Validate model state failed with response ${JSON.stringify( res.statusText )}` - ); - return Promise.reject("Validate model status failed"); - }); + ) + return Promise.reject('Validate model status failed') + }) } /** @@ -286,21 +298,27 @@ async function validateModelStatus(): Promise { * @returns A Promise that resolves when the subprocess is terminated successfully, or rejects with an error message if the subprocess fails to terminate. */ async function killSubprocess(): Promise { - const controller = new AbortController(); - setTimeout(() => controller.abort(), 5000); - log(`[NITRO]::Debug: Request to kill Nitro`); + const controller = new AbortController() + setTimeout(() => controller.abort(), 5000) + log(`[NITRO]::Debug: Request to kill Nitro`) return fetch(NITRO_HTTP_KILL_URL, { - method: "DELETE", + method: 'DELETE', signal: controller.signal, }) .then(() => { - subprocess?.kill(); - subprocess = undefined; + subprocess?.kill() + subprocess = undefined }) - .catch(() => {}) + .catch(() => {}) // Do nothing with this attempt .then(() => tcpPortUsed.waitUntilFree(PORT, 300, 5000)) - .then(() => log(`[NITRO]::Debug: Nitro process is terminated`)); + .then(() => log(`[NITRO]::Debug: Nitro process is terminated`)) + .catch((err) => { + log( + `[NITRO]::Debug: Could not kill running process on port ${PORT}. Might be another process running on the same port? ${err}` + ) + throw 'PORT_NOT_AVAILABLE' + }) } /** @@ -308,49 +326,53 @@ async function killSubprocess(): Promise { * @returns A promise that resolves when the Nitro subprocess is started. */ function spawnNitroProcess(): Promise { - log(`[NITRO]::Debug: Spawning Nitro subprocess...`); + log(`[NITRO]::Debug: Spawning Nitro subprocess...`) return new Promise(async (resolve, reject) => { - let binaryFolder = path.join(__dirname, "..", "bin"); // Current directory by default - let executableOptions = executableNitroFile(); + let binaryFolder = path.join(__dirname, '..', 'bin') // Current directory by default + let executableOptions = executableNitroFile() - const args: string[] = ["1", LOCAL_HOST, PORT.toString()]; + const args: string[] = ['1', LOCAL_HOST, PORT.toString()] // Execute the binary log( - `[NITRO]::Debug: Spawn nitro at path: ${executableOptions.executablePath}, and args: ${args}`, - ); + `[NITRO]::Debug: Spawn nitro at path: ${executableOptions.executablePath}, and args: ${args}` + ) subprocess = spawn( executableOptions.executablePath, - ["1", LOCAL_HOST, PORT.toString()], + ['1', LOCAL_HOST, PORT.toString()], { cwd: binaryFolder, env: { ...process.env, CUDA_VISIBLE_DEVICES: executableOptions.cudaVisibleDevices, + // Vulkan - Support 1 device at a time for now + ...(executableOptions.vkVisibleDevices?.length > 0 && { + GGML_VULKAN_DEVICE: executableOptions.vkVisibleDevices[0], + }), }, - }, - ); + } + ) // Handle subprocess output - subprocess.stdout.on("data", (data: any) => { - log(`[NITRO]::Debug: ${data}`); - }); + subprocess.stdout.on('data', (data: any) => { + log(`[NITRO]::Debug: ${data}`) + }) - subprocess.stderr.on("data", (data: any) => { - log(`[NITRO]::Error: ${data}`); - }); + subprocess.stderr.on('data', (data: any) => { + log(`[NITRO]::Error: ${data}`) + }) - subprocess.on("close", (code: any) => { - log(`[NITRO]::Debug: Nitro exited with code: ${code}`); - subprocess = undefined; - reject(`child process exited with code ${code}`); - }); + subprocess.on('close', (code: any) => { + log(`[NITRO]::Debug: Nitro exited with code: ${code}`) + subprocess = undefined + reject(`child process exited with code ${code}`) + }) tcpPortUsed.waitUntilUsed(PORT, 300, 30000).then(() => { - log(`[NITRO]::Debug: Nitro is ready`); - resolve(); - }); - }); + log(`[NITRO]::Debug: Nitro is ready`) + resolve() + }) + }) } /** @@ -360,7 +382,7 @@ function spawnNitroProcess(): Promise { */ function dispose() { // clean other registered resources here - killSubprocess(); + killSubprocess() } export default { @@ -370,4 +392,4 @@ export default { dispose, updateNvidiaInfo, getCurrentNitroProcessInfo: () => getNitroProcessInfo(subprocess), -}; +} diff --git a/extensions/inference-nitro-extension/src/node/nvidia.ts b/extensions/inference-nitro-extension/src/node/nvidia.ts deleted file mode 100644 index 13e43290b6..0000000000 --- a/extensions/inference-nitro-extension/src/node/nvidia.ts +++ /dev/null @@ -1,200 +0,0 @@ -import { writeFileSync, existsSync, readFileSync } from "fs"; -import { exec } from "child_process"; -import path from "path"; -import { getJanDataFolderPath } from "@janhq/core/node"; - -/** - * Default GPU settings - **/ -const DEFALT_SETTINGS = { - notify: true, - run_mode: "cpu", - nvidia_driver: { - exist: false, - version: "", - }, - cuda: { - exist: false, - version: "", - }, - gpus: [], - gpu_highest_vram: "", -}; - -/** - * Path to the settings file - **/ -export const NVIDIA_INFO_FILE = path.join( - getJanDataFolderPath(), - "settings", - "settings.json" -); - -/** - * Current nitro process - */ -let nitroProcessInfo: NitroProcessInfo | undefined = undefined; - -/** - * Nitro process info - */ -export interface NitroProcessInfo { - isRunning: boolean; -} - -/** - * This will retrive GPU informations and persist settings.json - * Will be called when the extension is loaded to turn on GPU acceleration if supported - */ -export async function updateNvidiaInfo() { - if (process.platform !== "darwin") { - await Promise.all([ - updateNvidiaDriverInfo(), - updateCudaExistence(), - updateGpuInfo(), - ]); - } -} - -/** - * Retrieve current nitro process - */ -export const getNitroProcessInfo = (subprocess: any): NitroProcessInfo => { - nitroProcessInfo = { - isRunning: subprocess != null, - }; - return nitroProcessInfo; -}; - -/** - * Validate nvidia and cuda for linux and windows - */ -export async function updateNvidiaDriverInfo(): Promise { - exec( - "nvidia-smi --query-gpu=driver_version --format=csv,noheader", - (error, stdout) => { - let data; - try { - data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8")); - } catch (error) { - data = DEFALT_SETTINGS; - } - - if (!error) { - const firstLine = stdout.split("\n")[0].trim(); - data["nvidia_driver"].exist = true; - data["nvidia_driver"].version = firstLine; - } else { - data["nvidia_driver"].exist = false; - } - - writeFileSync(NVIDIA_INFO_FILE, JSON.stringify(data, null, 2)); - Promise.resolve(); - } - ); -} - -/** - * Check if file exists in paths - */ -export function checkFileExistenceInPaths( - file: string, - paths: string[] -): boolean { - return paths.some((p) => existsSync(path.join(p, file))); -} - -/** - * Validate cuda for linux and windows - */ -export function updateCudaExistence() { - let filesCuda12: string[]; - let filesCuda11: string[]; - let paths: string[]; - let cudaVersion: string = ""; - - if (process.platform === "win32") { - filesCuda12 = ["cublas64_12.dll", "cudart64_12.dll", "cublasLt64_12.dll"]; - filesCuda11 = ["cublas64_11.dll", "cudart64_11.dll", "cublasLt64_11.dll"]; - paths = process.env.PATH ? process.env.PATH.split(path.delimiter) : []; - } else { - filesCuda12 = ["libcudart.so.12", "libcublas.so.12", "libcublasLt.so.12"]; - filesCuda11 = ["libcudart.so.11.0", "libcublas.so.11", "libcublasLt.so.11"]; - paths = process.env.LD_LIBRARY_PATH - ? process.env.LD_LIBRARY_PATH.split(path.delimiter) - : []; - paths.push("/usr/lib/x86_64-linux-gnu/"); - } - - let cudaExists = filesCuda12.every( - (file) => existsSync(file) || checkFileExistenceInPaths(file, paths) - ); - - if (!cudaExists) { - cudaExists = filesCuda11.every( - (file) => existsSync(file) || checkFileExistenceInPaths(file, paths) - ); - if (cudaExists) { - cudaVersion = "11"; - } - } else { - cudaVersion = "12"; - } - - let data; - try { - data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8")); - } catch (error) { - data = DEFALT_SETTINGS; - } - - data["cuda"].exist = cudaExists; - data["cuda"].version = cudaVersion; - if (cudaExists) { - data.run_mode = "gpu"; - } - writeFileSync(NVIDIA_INFO_FILE, JSON.stringify(data, null, 2)); -} - -/** - * Get GPU information - */ -export async function updateGpuInfo(): Promise { - exec( - "nvidia-smi --query-gpu=index,memory.total --format=csv,noheader,nounits", - (error, stdout) => { - let data; - try { - data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8")); - } catch (error) { - data = DEFALT_SETTINGS; - } - - if (!error) { - // Get GPU info and gpu has higher memory first - let highestVram = 0; - let highestVramId = "0"; - let gpus = stdout - .trim() - .split("\n") - .map((line) => { - let [id, vram] = line.split(", "); - vram = vram.replace(/\r/g, ""); - if (parseFloat(vram) > highestVram) { - highestVram = parseFloat(vram); - highestVramId = id; - } - return { id, vram }; - }); - - data["gpus"] = gpus; - data["gpu_highest_vram"] = highestVramId; - } else { - data["gpus"] = []; - } - - writeFileSync(NVIDIA_INFO_FILE, JSON.stringify(data, null, 2)); - Promise.resolve(); - } - ); -} diff --git a/extensions/inference-openai-extension/package.json b/extensions/inference-openai-extension/package.json index 5fa0ce974f..5efdbf874f 100644 --- a/extensions/inference-openai-extension/package.json +++ b/extensions/inference-openai-extension/package.json @@ -8,7 +8,7 @@ "license": "AGPL-3.0", "scripts": { "build": "tsc -b . && webpack --config webpack.config.js", - "build:publish": "rimraf *.tgz --glob && npm run build && npm pack && cpx *.tgz ../../electron/pre-install" + "build:publish": "rimraf *.tgz --glob && npm run build && npm pack && cpx *.tgz ../../pre-install" }, "exports": { ".": "./dist/index.js", @@ -18,13 +18,13 @@ "cpx": "^1.5.0", "rimraf": "^3.0.2", "webpack": "^5.88.2", - "webpack-cli": "^5.1.4" + "webpack-cli": "^5.1.4", + "ts-loader": "^9.5.0" }, "dependencies": { "@janhq/core": "file:../../core", "fetch-retry": "^5.0.6", "path-browserify": "^1.0.1", - "ts-loader": "^9.5.0", "ulid": "^2.3.0" }, "engines": { diff --git a/extensions/inference-openai-extension/src/@types/global.d.ts b/extensions/inference-openai-extension/src/@types/global.d.ts index 84f86c1458..a49bb5a2f2 100644 --- a/extensions/inference-openai-extension/src/@types/global.d.ts +++ b/extensions/inference-openai-extension/src/@types/global.d.ts @@ -1,26 +1,26 @@ -declare const MODULE: string; -declare const OPENAI_DOMAIN: string; +declare const MODULE: string +declare const OPENAI_DOMAIN: string declare interface EngineSettings { - full_url?: string; - api_key?: string; + full_url?: string + api_key?: string } enum OpenAIChatCompletionModelName { - "gpt-3.5-turbo-instruct" = "gpt-3.5-turbo-instruct", - "gpt-3.5-turbo-instruct-0914" = "gpt-3.5-turbo-instruct-0914", - "gpt-4-1106-preview" = "gpt-4-1106-preview", - "gpt-3.5-turbo-0613" = "gpt-3.5-turbo-0613", - "gpt-3.5-turbo-0301" = "gpt-3.5-turbo-0301", - "gpt-3.5-turbo" = "gpt-3.5-turbo", - "gpt-3.5-turbo-16k-0613" = "gpt-3.5-turbo-16k-0613", - "gpt-3.5-turbo-1106" = "gpt-3.5-turbo-1106", - "gpt-4-vision-preview" = "gpt-4-vision-preview", - "gpt-4" = "gpt-4", - "gpt-4-0314" = "gpt-4-0314", - "gpt-4-0613" = "gpt-4-0613", + 'gpt-3.5-turbo-instruct' = 'gpt-3.5-turbo-instruct', + 'gpt-3.5-turbo-instruct-0914' = 'gpt-3.5-turbo-instruct-0914', + 'gpt-4-1106-preview' = 'gpt-4-1106-preview', + 'gpt-3.5-turbo-0613' = 'gpt-3.5-turbo-0613', + 'gpt-3.5-turbo-0301' = 'gpt-3.5-turbo-0301', + 'gpt-3.5-turbo' = 'gpt-3.5-turbo', + 'gpt-3.5-turbo-16k-0613' = 'gpt-3.5-turbo-16k-0613', + 'gpt-3.5-turbo-1106' = 'gpt-3.5-turbo-1106', + 'gpt-4-vision-preview' = 'gpt-4-vision-preview', + 'gpt-4' = 'gpt-4', + 'gpt-4-0314' = 'gpt-4-0314', + 'gpt-4-0613' = 'gpt-4-0613', } -declare type OpenAIModel = Omit & { - id: OpenAIChatCompletionModelName; -}; +declare type OpenAIModel = Omit & { + id: OpenAIChatCompletionModelName +} diff --git a/extensions/inference-openai-extension/src/helpers/sse.ts b/extensions/inference-openai-extension/src/helpers/sse.ts index fb75816e7e..11db382827 100644 --- a/extensions/inference-openai-extension/src/helpers/sse.ts +++ b/extensions/inference-openai-extension/src/helpers/sse.ts @@ -1,4 +1,4 @@ -import { Observable } from "rxjs"; +import { Observable } from 'rxjs' /** * Sends a request to the inference server to generate a response based on the recent messages. @@ -14,26 +14,26 @@ export function requestInference( controller?: AbortController ): Observable { return new Observable((subscriber) => { - let model_id: string = model.id; + let model_id: string = model.id if (engine.full_url.includes(OPENAI_DOMAIN)) { - model_id = engine.full_url.split("/")[5]; + model_id = engine.full_url.split('/')[5] } const requestBody = JSON.stringify({ messages: recentMessages, stream: true, model: model_id, ...model.parameters, - }); + }) fetch(`${engine.full_url}`, { - method: "POST", + method: 'POST', headers: { - "Content-Type": "application/json", - Accept: model.parameters.stream - ? "text/event-stream" - : "application/json", - "Access-Control-Allow-Origin": "*", - Authorization: `Bearer ${engine.api_key}`, - "api-key": `${engine.api_key}`, + 'Content-Type': 'application/json', + 'Accept': model.parameters.stream + ? 'text/event-stream' + : 'application/json', + 'Access-Control-Allow-Origin': '*', + 'Authorization': `Bearer ${engine.api_key}`, + 'api-key': `${engine.api_key}`, }, body: requestBody, signal: controller?.signal, @@ -41,41 +41,41 @@ export function requestInference( .then(async (response) => { if (!response.ok) { subscriber.next( - (await response.json()).error?.message ?? "Error occured" - ); - subscriber.complete(); - return; + (await response.json()).error?.message ?? 'Error occurred.' + ) + subscriber.complete() + return } if (model.parameters.stream === false) { - const data = await response.json(); - subscriber.next(data.choices[0]?.message?.content ?? ""); + const data = await response.json() + subscriber.next(data.choices[0]?.message?.content ?? '') } else { - const stream = response.body; - const decoder = new TextDecoder("utf-8"); - const reader = stream?.getReader(); - let content = ""; + const stream = response.body + const decoder = new TextDecoder('utf-8') + const reader = stream?.getReader() + let content = '' while (true && reader) { - const { done, value } = await reader.read(); + const { done, value } = await reader.read() if (done) { - break; + break } - const text = decoder.decode(value); - const lines = text.trim().split("\n"); + const text = decoder.decode(value) + const lines = text.trim().split('\n') for (const line of lines) { - if (line.startsWith("data: ") && !line.includes("data: [DONE]")) { - const data = JSON.parse(line.replace("data: ", "")); - content += data.choices[0]?.delta?.content ?? ""; - if (content.startsWith("assistant: ")) { - content = content.replace("assistant: ", ""); + if (line.startsWith('data: ') && !line.includes('data: [DONE]')) { + const data = JSON.parse(line.replace('data: ', '')) + content += data.choices[0]?.delta?.content ?? '' + if (content.startsWith('assistant: ')) { + content = content.replace('assistant: ', '') } - subscriber.next(content); + subscriber.next(content) } } } } - subscriber.complete(); + subscriber.complete() }) - .catch((err) => subscriber.error(err)); - }); + .catch((err) => subscriber.error(err)) + }) } diff --git a/extensions/inference-openai-extension/src/index.ts b/extensions/inference-openai-extension/src/index.ts index fd1230bc7e..4811717421 100644 --- a/extensions/inference-openai-extension/src/index.ts +++ b/extensions/inference-openai-extension/src/index.ts @@ -18,14 +18,15 @@ import { InferenceEngine, BaseExtension, MessageEvent, + MessageRequestType, ModelEvent, InferenceEvent, AppConfigurationEventName, joinPath, -} from "@janhq/core"; -import { requestInference } from "./helpers/sse"; -import { ulid } from "ulid"; -import { join } from "path"; +} from '@janhq/core' +import { requestInference } from './helpers/sse' +import { ulid } from 'ulid' +import { join } from 'path' /** * A class that implements the InferenceExtension interface from the @janhq/core package. @@ -33,18 +34,18 @@ import { join } from "path"; * It also subscribes to events emitted by the @janhq/core package and handles new message requests. */ export default class JanInferenceOpenAIExtension extends BaseExtension { - private static readonly _engineDir = "file://engines"; - private static readonly _engineMetadataFileName = "openai.json"; + private static readonly _engineDir = 'file://engines' + private static readonly _engineMetadataFileName = 'openai.json' - private static _currentModel: OpenAIModel; + private static _currentModel: OpenAIModel private static _engineSettings: EngineSettings = { - full_url: "https://api.openai.com/v1/chat/completions", - api_key: "sk-", - }; + full_url: 'https://api.openai.com/v1/chat/completions', + api_key: 'sk-', + } - controller = new AbortController(); - isCancelled = false; + controller = new AbortController() + isCancelled = false /** * Subscribes to events emitted by the @janhq/core package. @@ -53,40 +54,40 @@ export default class JanInferenceOpenAIExtension extends BaseExtension { if (!(await fs.existsSync(JanInferenceOpenAIExtension._engineDir))) { await fs .mkdirSync(JanInferenceOpenAIExtension._engineDir) - .catch((err) => console.debug(err)); + .catch((err) => console.debug(err)) } - JanInferenceOpenAIExtension.writeDefaultEngineSettings(); + JanInferenceOpenAIExtension.writeDefaultEngineSettings() // Events subscription events.on(MessageEvent.OnMessageSent, (data) => - JanInferenceOpenAIExtension.handleMessageRequest(data, this), - ); + JanInferenceOpenAIExtension.handleMessageRequest(data, this) + ) events.on(ModelEvent.OnModelInit, (model: OpenAIModel) => { - JanInferenceOpenAIExtension.handleModelInit(model); - }); + JanInferenceOpenAIExtension.handleModelInit(model) + }) events.on(ModelEvent.OnModelStop, (model: OpenAIModel) => { - JanInferenceOpenAIExtension.handleModelStop(model); - }); + JanInferenceOpenAIExtension.handleModelStop(model) + }) events.on(InferenceEvent.OnInferenceStopped, () => { - JanInferenceOpenAIExtension.handleInferenceStopped(this); - }); + JanInferenceOpenAIExtension.handleInferenceStopped(this) + }) const settingsFilePath = await joinPath([ JanInferenceOpenAIExtension._engineDir, JanInferenceOpenAIExtension._engineMetadataFileName, - ]); + ]) events.on( AppConfigurationEventName.OnConfigurationUpdate, (settingsKey: string) => { // Update settings on changes if (settingsKey === settingsFilePath) - JanInferenceOpenAIExtension.writeDefaultEngineSettings(); - }, - ); + JanInferenceOpenAIExtension.writeDefaultEngineSettings() + } + ) } /** @@ -98,45 +99,45 @@ export default class JanInferenceOpenAIExtension extends BaseExtension { try { const engineFile = join( JanInferenceOpenAIExtension._engineDir, - JanInferenceOpenAIExtension._engineMetadataFileName, - ); + JanInferenceOpenAIExtension._engineMetadataFileName + ) if (await fs.existsSync(engineFile)) { - const engine = await fs.readFileSync(engineFile, "utf-8"); + const engine = await fs.readFileSync(engineFile, 'utf-8') JanInferenceOpenAIExtension._engineSettings = - typeof engine === "object" ? engine : JSON.parse(engine); + typeof engine === 'object' ? engine : JSON.parse(engine) } else { await fs.writeFileSync( engineFile, - JSON.stringify(JanInferenceOpenAIExtension._engineSettings, null, 2), - ); + JSON.stringify(JanInferenceOpenAIExtension._engineSettings, null, 2) + ) } } catch (err) { - console.error(err); + console.error(err) } } private static async handleModelInit(model: OpenAIModel) { if (model.engine !== InferenceEngine.openai) { - return; + return } else { - JanInferenceOpenAIExtension._currentModel = model; - JanInferenceOpenAIExtension.writeDefaultEngineSettings(); + JanInferenceOpenAIExtension._currentModel = model + JanInferenceOpenAIExtension.writeDefaultEngineSettings() // Todo: Check model list with API key - events.emit(ModelEvent.OnModelReady, model); + events.emit(ModelEvent.OnModelReady, model) } } private static async handleModelStop(model: OpenAIModel) { - if (model.engine !== "openai") { - return; + if (model.engine !== 'openai') { + return } - events.emit(ModelEvent.OnModelStopped, model); + events.emit(ModelEvent.OnModelStopped, model) } private static async handleInferenceStopped( - instance: JanInferenceOpenAIExtension, + instance: JanInferenceOpenAIExtension ) { - instance.isCancelled = true; - instance.controller?.abort(); + instance.isCancelled = true + instance.controller?.abort() } /** @@ -147,28 +148,32 @@ export default class JanInferenceOpenAIExtension extends BaseExtension { */ private static async handleMessageRequest( data: MessageRequest, - instance: JanInferenceOpenAIExtension, + instance: JanInferenceOpenAIExtension ) { - if (data.model.engine !== "openai") { - return; + if (data.model.engine !== 'openai') { + return } - const timestamp = Date.now(); + const timestamp = Date.now() const message: ThreadMessage = { id: ulid(), thread_id: data.threadId, + type: data.type, assistant_id: data.assistantId, role: ChatCompletionRole.Assistant, content: [], status: MessageStatus.Pending, created: timestamp, updated: timestamp, - object: "thread.message", - }; - events.emit(MessageEvent.OnMessageResponse, message); + object: 'thread.message', + } - instance.isCancelled = false; - instance.controller = new AbortController(); + if (data.type !== MessageRequestType.Summary) { + events.emit(MessageEvent.OnMessageResponse, message) + } + + instance.isCancelled = false + instance.controller = new AbortController() requestInference( data?.messages ?? [], @@ -177,7 +182,7 @@ export default class JanInferenceOpenAIExtension extends BaseExtension { ...JanInferenceOpenAIExtension._currentModel, parameters: data.model.parameters, }, - instance.controller, + instance.controller ).subscribe({ next: (content) => { const messageContent: ThreadContent = { @@ -186,33 +191,33 @@ export default class JanInferenceOpenAIExtension extends BaseExtension { value: content.trim(), annotations: [], }, - }; - message.content = [messageContent]; - events.emit(MessageEvent.OnMessageUpdate, message); + } + message.content = [messageContent] + events.emit(MessageEvent.OnMessageUpdate, message) }, complete: async () => { message.status = message.content.length ? MessageStatus.Ready - : MessageStatus.Error; - events.emit(MessageEvent.OnMessageUpdate, message); + : MessageStatus.Error + events.emit(MessageEvent.OnMessageUpdate, message) }, error: async (err) => { if (instance.isCancelled || message.content.length > 0) { - message.status = MessageStatus.Stopped; - events.emit(MessageEvent.OnMessageUpdate, message); - return; + message.status = MessageStatus.Stopped + events.emit(MessageEvent.OnMessageUpdate, message) + return } const messageContent: ThreadContent = { type: ContentType.Text, text: { - value: "Error occurred: " + err.message, + value: 'Error occurred: ' + err.message, annotations: [], }, - }; - message.content = [messageContent]; - message.status = MessageStatus.Error; - events.emit(MessageEvent.OnMessageUpdate, message); + } + message.content = [messageContent] + message.status = MessageStatus.Error + events.emit(MessageEvent.OnMessageUpdate, message) }, - }); + }) } } diff --git a/extensions/inference-openai-extension/tsconfig.json b/extensions/inference-openai-extension/tsconfig.json index 7bfdd90096..2477d58ce5 100644 --- a/extensions/inference-openai-extension/tsconfig.json +++ b/extensions/inference-openai-extension/tsconfig.json @@ -8,7 +8,7 @@ "forceConsistentCasingInFileNames": true, "strict": false, "skipLibCheck": true, - "rootDir": "./src", + "rootDir": "./src" }, - "include": ["./src"], + "include": ["./src"] } diff --git a/extensions/inference-openai-extension/webpack.config.js b/extensions/inference-openai-extension/webpack.config.js index 72b7d90c10..ee2e3b6243 100644 --- a/extensions/inference-openai-extension/webpack.config.js +++ b/extensions/inference-openai-extension/webpack.config.js @@ -1,16 +1,16 @@ -const path = require("path"); -const webpack = require("webpack"); -const packageJson = require("./package.json"); +const path = require('path') +const webpack = require('webpack') +const packageJson = require('./package.json') module.exports = { experiments: { outputModule: true }, - entry: "./src/index.ts", // Adjust the entry point to match your project's main file - mode: "production", + entry: './src/index.ts', // Adjust the entry point to match your project's main file + mode: 'production', module: { rules: [ { test: /\.tsx?$/, - use: "ts-loader", + use: 'ts-loader', exclude: /node_modules/, }, ], @@ -18,22 +18,22 @@ module.exports = { plugins: [ new webpack.DefinePlugin({ MODULE: JSON.stringify(`${packageJson.name}/${packageJson.module}`), - OPENAI_DOMAIN: JSON.stringify("openai.azure.com"), + OPENAI_DOMAIN: JSON.stringify('openai.azure.com'), }), ], output: { - filename: "index.js", // Adjust the output file name as needed - path: path.resolve(__dirname, "dist"), - library: { type: "module" }, // Specify ESM output format + filename: 'index.js', // Adjust the output file name as needed + path: path.resolve(__dirname, 'dist'), + library: { type: 'module' }, // Specify ESM output format }, resolve: { - extensions: [".ts", ".js"], + extensions: ['.ts', '.js'], fallback: { - path: require.resolve("path-browserify"), + path: require.resolve('path-browserify'), }, }, optimization: { minimize: false, }, // Add loaders and other configuration as needed for your project -}; +} diff --git a/extensions/inference-triton-trtllm-extension/package.json b/extensions/inference-triton-trtllm-extension/package.json index 1d27f9f188..455f8030e2 100644 --- a/extensions/inference-triton-trtllm-extension/package.json +++ b/extensions/inference-triton-trtllm-extension/package.json @@ -8,7 +8,7 @@ "license": "AGPL-3.0", "scripts": { "build": "tsc -b . && webpack --config webpack.config.js", - "build:publish": "rimraf *.tgz --glob && npm run build && npm pack && cpx *.tgz ../../electron/pre-install" + "build:publish": "rimraf *.tgz --glob && npm run build && npm pack && cpx *.tgz ../../pre-install" }, "exports": { ".": "./dist/index.js", @@ -18,13 +18,13 @@ "cpx": "^1.5.0", "rimraf": "^3.0.2", "webpack": "^5.88.2", - "webpack-cli": "^5.1.4" + "webpack-cli": "^5.1.4", + "ts-loader": "^9.5.0" }, "dependencies": { "@janhq/core": "file:../../core", "fetch-retry": "^5.0.6", "path-browserify": "^1.0.1", - "ts-loader": "^9.5.0", "ulid": "^2.3.0", "rxjs": "^7.8.1" }, diff --git a/extensions/inference-triton-trtllm-extension/src/@types/global.d.ts b/extensions/inference-triton-trtllm-extension/src/@types/global.d.ts index 6224b8e68c..c834feba01 100644 --- a/extensions/inference-triton-trtllm-extension/src/@types/global.d.ts +++ b/extensions/inference-triton-trtllm-extension/src/@types/global.d.ts @@ -1,5 +1,5 @@ -import { Model } from "@janhq/core"; +import { Model } from '@janhq/core' declare interface EngineSettings { - base_url?: string; + base_url?: string } diff --git a/extensions/inference-triton-trtllm-extension/src/helpers/sse.ts b/extensions/inference-triton-trtllm-extension/src/helpers/sse.ts index da20fa32d9..9aff612654 100644 --- a/extensions/inference-triton-trtllm-extension/src/helpers/sse.ts +++ b/extensions/inference-triton-trtllm-extension/src/helpers/sse.ts @@ -1,6 +1,6 @@ -import { Observable } from "rxjs"; -import { EngineSettings } from "../@types/global"; -import { Model } from "@janhq/core"; +import { Observable } from 'rxjs' +import { EngineSettings } from '../@types/global' +import { Model } from '@janhq/core' /** * Sends a request to the inference server to generate a response based on the recent messages. @@ -16,48 +16,48 @@ export function requestInference( controller?: AbortController ): Observable { return new Observable((subscriber) => { - const text_input = recentMessages.map((message) => message.text).join("\n"); + const text_input = recentMessages.map((message) => message.text).join('\n') const requestBody = JSON.stringify({ text_input: text_input, max_tokens: 4096, temperature: 0, - bad_words: "", - stop_words: "[DONE]", - stream: true - }); + bad_words: '', + stop_words: '[DONE]', + stream: true, + }) fetch(`${engine.base_url}/v2/models/ensemble/generate_stream`, { - method: "POST", + method: 'POST', headers: { - "Content-Type": "application/json", - Accept: "text/event-stream", - "Access-Control-Allow-Origin": "*", + 'Content-Type': 'application/json', + 'Accept': 'text/event-stream', + 'Access-Control-Allow-Origin': '*', }, body: requestBody, signal: controller?.signal, }) .then(async (response) => { - const stream = response.body; - const decoder = new TextDecoder("utf-8"); - const reader = stream?.getReader(); - let content = ""; + const stream = response.body + const decoder = new TextDecoder('utf-8') + const reader = stream?.getReader() + let content = '' while (true && reader) { - const { done, value } = await reader.read(); + const { done, value } = await reader.read() if (done) { - break; + break } - const text = decoder.decode(value); - const lines = text.trim().split("\n"); + const text = decoder.decode(value) + const lines = text.trim().split('\n') for (const line of lines) { - if (line.startsWith("data: ") && !line.includes("data: [DONE]")) { - const data = JSON.parse(line.replace("data: ", "")); - content += data.choices[0]?.delta?.content ?? ""; - subscriber.next(content); + if (line.startsWith('data: ') && !line.includes('data: [DONE]')) { + const data = JSON.parse(line.replace('data: ', '')) + content += data.choices[0]?.delta?.content ?? '' + subscriber.next(content) } } } - subscriber.complete(); + subscriber.complete() }) - .catch((err) => subscriber.error(err)); - }); + .catch((err) => subscriber.error(err)) + }) } diff --git a/extensions/inference-triton-trtllm-extension/src/index.ts b/extensions/inference-triton-trtllm-extension/src/index.ts index 11ddf78933..f009a81e03 100644 --- a/extensions/inference-triton-trtllm-extension/src/index.ts +++ b/extensions/inference-triton-trtllm-extension/src/index.ts @@ -20,51 +20,49 @@ import { BaseExtension, MessageEvent, ModelEvent, -} from "@janhq/core"; -import { requestInference } from "./helpers/sse"; -import { ulid } from "ulid"; -import { join } from "path"; -import { EngineSettings } from "./@types/global"; +} from '@janhq/core' +import { requestInference } from './helpers/sse' +import { ulid } from 'ulid' +import { join } from 'path' +import { EngineSettings } from './@types/global' /** * A class that implements the InferenceExtension interface from the @janhq/core package. * The class provides methods for initializing and stopping a model, and for making inference requests. * It also subscribes to events emitted by the @janhq/core package and handles new message requests. */ -export default class JanInferenceTritonTrtLLMExtension - extends BaseExtension -{ - private static readonly _homeDir = "file://engines"; - private static readonly _engineMetadataFileName = "triton_trtllm.json"; +export default class JanInferenceTritonTrtLLMExtension extends BaseExtension { + private static readonly _homeDir = 'file://engines' + private static readonly _engineMetadataFileName = 'triton_trtllm.json' - static _currentModel: Model; + static _currentModel: Model static _engineSettings: EngineSettings = { - base_url: "", - }; + base_url: '', + } - controller = new AbortController(); - isCancelled = false; + controller = new AbortController() + isCancelled = false /** * Subscribes to events emitted by the @janhq/core package. */ async onLoad() { if (!(await fs.existsSync(JanInferenceTritonTrtLLMExtension._homeDir))) - JanInferenceTritonTrtLLMExtension.writeDefaultEngineSettings(); + JanInferenceTritonTrtLLMExtension.writeDefaultEngineSettings() // Events subscription events.on(MessageEvent.OnMessageSent, (data) => JanInferenceTritonTrtLLMExtension.handleMessageRequest(data, this) - ); + ) events.on(ModelEvent.OnModelInit, (model: Model) => { - JanInferenceTritonTrtLLMExtension.handleModelInit(model); - }); + JanInferenceTritonTrtLLMExtension.handleModelInit(model) + }) events.on(ModelEvent.OnModelStop, (model: Model) => { - JanInferenceTritonTrtLLMExtension.handleModelStop(model); - }); + JanInferenceTritonTrtLLMExtension.handleModelStop(model) + }) } /** @@ -81,7 +79,7 @@ export default class JanInferenceTritonTrtLLMExtension modelId: string, settings?: ModelSettingParams ): Promise { - return; + return } static async writeDefaultEngineSettings() { @@ -89,11 +87,11 @@ export default class JanInferenceTritonTrtLLMExtension const engine_json = join( JanInferenceTritonTrtLLMExtension._homeDir, JanInferenceTritonTrtLLMExtension._engineMetadataFileName - ); + ) if (await fs.existsSync(engine_json)) { - const engine = await fs.readFileSync(engine_json, "utf-8"); + const engine = await fs.readFileSync(engine_json, 'utf-8') JanInferenceTritonTrtLLMExtension._engineSettings = - typeof engine === "object" ? engine : JSON.parse(engine); + typeof engine === 'object' ? engine : JSON.parse(engine) } else { await fs.writeFileSync( engine_json, @@ -102,10 +100,10 @@ export default class JanInferenceTritonTrtLLMExtension null, 2 ) - ); + ) } } catch (err) { - console.error(err); + console.error(err) } } /** @@ -119,26 +117,26 @@ export default class JanInferenceTritonTrtLLMExtension * @returns {Promise} A promise that resolves when the streaming is stopped. */ async stopInference(): Promise { - this.isCancelled = true; - this.controller?.abort(); + this.isCancelled = true + this.controller?.abort() } private static async handleModelInit(model: Model) { - if (model.engine !== "triton_trtllm") { - return; + if (model.engine !== 'triton_trtllm') { + return } else { - JanInferenceTritonTrtLLMExtension._currentModel = model; - JanInferenceTritonTrtLLMExtension.writeDefaultEngineSettings(); + JanInferenceTritonTrtLLMExtension._currentModel = model + JanInferenceTritonTrtLLMExtension.writeDefaultEngineSettings() // Todo: Check model list with API key - events.emit(ModelEvent.OnModelReady, model); + events.emit(ModelEvent.OnModelReady, model) } } private static async handleModelStop(model: Model) { - if (model.engine !== "triton_trtllm") { - return; + if (model.engine !== 'triton_trtllm') { + return } - events.emit(ModelEvent.OnModelStopped, model); + events.emit(ModelEvent.OnModelStopped, model) } /** @@ -151,11 +149,11 @@ export default class JanInferenceTritonTrtLLMExtension data: MessageRequest, instance: JanInferenceTritonTrtLLMExtension ) { - if (data.model.engine !== "triton_trtllm") { - return; + if (data.model.engine !== 'triton_trtllm') { + return } - const timestamp = Date.now(); + const timestamp = Date.now() const message: ThreadMessage = { id: ulid(), thread_id: data.threadId, @@ -165,12 +163,12 @@ export default class JanInferenceTritonTrtLLMExtension status: MessageStatus.Pending, created: timestamp, updated: timestamp, - object: "thread.message", - }; - events.emit(MessageEvent.OnMessageResponse, message); + object: 'thread.message', + } + events.emit(MessageEvent.OnMessageResponse, message) - instance.isCancelled = false; - instance.controller = new AbortController(); + instance.isCancelled = false + instance.controller = new AbortController() requestInference( data?.messages ?? [], @@ -188,33 +186,33 @@ export default class JanInferenceTritonTrtLLMExtension value: content.trim(), annotations: [], }, - }; - message.content = [messageContent]; - events.emit(MessageEvent.OnMessageUpdate, message); + } + message.content = [messageContent] + events.emit(MessageEvent.OnMessageUpdate, message) }, complete: async () => { message.status = message.content.length ? MessageStatus.Ready - : MessageStatus.Error; - events.emit(MessageEvent.OnMessageUpdate, message); + : MessageStatus.Error + events.emit(MessageEvent.OnMessageUpdate, message) }, error: async (err) => { if (instance.isCancelled || message.content.length) { - message.status = MessageStatus.Error; - events.emit(MessageEvent.OnMessageUpdate, message); - return; + message.status = MessageStatus.Error + events.emit(MessageEvent.OnMessageUpdate, message) + return } const messageContent: ThreadContent = { type: ContentType.Text, text: { - value: "Error occurred: " + err.message, + value: 'Error occurred: ' + err.message, annotations: [], }, - }; - message.content = [messageContent]; - message.status = MessageStatus.Ready; - events.emit(MessageEvent.OnMessageUpdate, message); + } + message.content = [messageContent] + message.status = MessageStatus.Ready + events.emit(MessageEvent.OnMessageUpdate, message) }, - }); + }) } } diff --git a/extensions/inference-triton-trtllm-extension/tsconfig.json b/extensions/inference-triton-trtllm-extension/tsconfig.json index 7bfdd90096..2477d58ce5 100644 --- a/extensions/inference-triton-trtllm-extension/tsconfig.json +++ b/extensions/inference-triton-trtllm-extension/tsconfig.json @@ -8,7 +8,7 @@ "forceConsistentCasingInFileNames": true, "strict": false, "skipLibCheck": true, - "rootDir": "./src", + "rootDir": "./src" }, - "include": ["./src"], + "include": ["./src"] } diff --git a/extensions/inference-triton-trtllm-extension/webpack.config.js b/extensions/inference-triton-trtllm-extension/webpack.config.js index 57a0adb0a2..e83370a1ac 100644 --- a/extensions/inference-triton-trtllm-extension/webpack.config.js +++ b/extensions/inference-triton-trtllm-extension/webpack.config.js @@ -1,16 +1,16 @@ -const path = require("path"); -const webpack = require("webpack"); -const packageJson = require("./package.json"); +const path = require('path') +const webpack = require('webpack') +const packageJson = require('./package.json') module.exports = { experiments: { outputModule: true }, - entry: "./src/index.ts", // Adjust the entry point to match your project's main file - mode: "production", + entry: './src/index.ts', // Adjust the entry point to match your project's main file + mode: 'production', module: { rules: [ { test: /\.tsx?$/, - use: "ts-loader", + use: 'ts-loader', exclude: /node_modules/, }, ], @@ -21,18 +21,18 @@ module.exports = { }), ], output: { - filename: "index.js", // Adjust the output file name as needed - path: path.resolve(__dirname, "dist"), - library: { type: "module" }, // Specify ESM output format + filename: 'index.js', // Adjust the output file name as needed + path: path.resolve(__dirname, 'dist'), + library: { type: 'module' }, // Specify ESM output format }, resolve: { - extensions: [".ts", ".js"], + extensions: ['.ts', '.js'], fallback: { - path: require.resolve("path-browserify"), + path: require.resolve('path-browserify'), }, }, optimization: { minimize: false, }, // Add loaders and other configuration as needed for your project -}; +} diff --git a/extensions/model-extension/.prettierrc b/extensions/model-extension/.prettierrc deleted file mode 100644 index 46f1abcb02..0000000000 --- a/extensions/model-extension/.prettierrc +++ /dev/null @@ -1,8 +0,0 @@ -{ - "semi": false, - "singleQuote": true, - "quoteProps": "consistent", - "trailingComma": "es5", - "endOfLine": "auto", - "plugins": ["prettier-plugin-tailwindcss"] -} diff --git a/extensions/model-extension/package.json b/extensions/model-extension/package.json index 86f177d149..e99122bcf6 100644 --- a/extensions/model-extension/package.json +++ b/extensions/model-extension/package.json @@ -1,6 +1,6 @@ { "name": "@janhq/model-extension", - "version": "1.0.23", + "version": "1.0.25", "description": "Model Management Extension provides model exploration and seamless downloads", "main": "dist/index.js", "module": "dist/module.js", @@ -8,13 +8,14 @@ "license": "AGPL-3.0", "scripts": { "build": "tsc -b . && webpack --config webpack.config.js", - "build:publish": "rimraf *.tgz --glob && npm run build && npm pack && cpx *.tgz ../../electron/pre-install" + "build:publish": "rimraf *.tgz --glob && npm run build && npm pack && cpx *.tgz ../../pre-install" }, "devDependencies": { "cpx": "^1.5.0", "rimraf": "^3.0.2", "webpack": "^5.88.2", - "webpack-cli": "^5.1.4" + "webpack-cli": "^5.1.4", + "ts-loader": "^9.5.0" }, "files": [ "dist/*", @@ -23,7 +24,6 @@ ], "dependencies": { "@janhq/core": "file:../../core", - "path-browserify": "^1.0.1", - "ts-loader": "^9.5.0" + "path-browserify": "^1.0.1" } } diff --git a/extensions/model-extension/src/@types/global.d.ts b/extensions/model-extension/src/@types/global.d.ts index e998455f2e..7a9202a627 100644 --- a/extensions/model-extension/src/@types/global.d.ts +++ b/extensions/model-extension/src/@types/global.d.ts @@ -1,3 +1,15 @@ -declare const EXTENSION_NAME: string -declare const MODULE_PATH: string -declare const VERSION: stringå +export {} +declare global { + declare const EXTENSION_NAME: string + declare const MODULE_PATH: string + declare const VERSION: string + + interface Core { + api: APIFunctions + events: EventEmitter + } + interface Window { + core?: Core | undefined + electronAPI?: any | undefined + } +} diff --git a/extensions/model-extension/src/helpers/path.ts b/extensions/model-extension/src/helpers/path.ts new file mode 100644 index 0000000000..cbb151aa6c --- /dev/null +++ b/extensions/model-extension/src/helpers/path.ts @@ -0,0 +1,11 @@ +/** + * try to retrieve the download file name from the source url + */ + +export function extractFileName(url: string, fileExtension: string): string { + const extractedFileName = url.split('/').pop() + const fileName = extractedFileName.toLowerCase().endsWith(fileExtension) + ? extractedFileName + : extractedFileName + fileExtension + return fileName +} diff --git a/extensions/model-extension/src/index.ts b/extensions/model-extension/src/index.ts index b9fa7731e2..926e65ee50 100644 --- a/extensions/model-extension/src/index.ts +++ b/extensions/model-extension/src/index.ts @@ -8,8 +8,15 @@ import { ModelExtension, Model, getJanDataFolderPath, + events, + DownloadEvent, + DownloadRoute, + ModelEvent, + DownloadState, } from '@janhq/core' +import { extractFileName } from './helpers/path' + /** * A extension for models */ @@ -29,6 +36,8 @@ export default class JanModelExtension extends ModelExtension { */ async onLoad() { this.copyModelsToHomeDir() + // Handle Desktop Events + this.handleDesktopEvents() } /** @@ -61,6 +70,8 @@ export default class JanModelExtension extends ModelExtension { // Finished migration localStorage.setItem(`${EXTENSION_NAME}-version`, VERSION) + + events.emit(ModelEvent.OnModelsUpdate, {}) } catch (err) { console.error(err) } @@ -83,31 +94,66 @@ export default class JanModelExtension extends ModelExtension { if (model.sources.length > 1) { // path to model binaries for (const source of model.sources) { - let path = this.extractFileName(source.url) + let path = extractFileName( + source.url, + JanModelExtension._supportedModelFormat + ) if (source.filename) { path = await joinPath([modelDirPath, source.filename]) } downloadFile(source.url, path, network) } + // TODO: handle multiple binaries for web later } else { - const fileName = this.extractFileName(model.sources[0]?.url) + const fileName = extractFileName( + model.sources[0]?.url, + JanModelExtension._supportedModelFormat + ) const path = await joinPath([modelDirPath, fileName]) downloadFile(model.sources[0]?.url, path, network) + + if (window && window.core?.api && window.core.api.baseApiUrl) { + this.startPollingDownloadProgress(model.id) + } } } /** - * try to retrieve the download file name from the source url + * Specifically for Jan server. */ - private extractFileName(url: string): string { - const extractedFileName = url.split('/').pop() - const fileName = extractedFileName - .toLowerCase() - .endsWith(JanModelExtension._supportedModelFormat) - ? extractedFileName - : extractedFileName + JanModelExtension._supportedModelFormat - return fileName + private async startPollingDownloadProgress(modelId: string): Promise { + // wait for some seconds before polling + await new Promise((resolve) => setTimeout(resolve, 3000)) + + return new Promise((resolve) => { + const interval = setInterval(async () => { + fetch( + `${window.core.api.baseApiUrl}/v1/download/${DownloadRoute.getDownloadProgress}/${modelId}`, + { + method: 'GET', + headers: { contentType: 'application/json' }, + } + ).then(async (res) => { + const state: DownloadState = await res.json() + if (state.downloadState === 'end') { + events.emit(DownloadEvent.onFileDownloadSuccess, state) + clearInterval(interval) + resolve() + return + } + + if (state.downloadState === 'error') { + events.emit(DownloadEvent.onFileDownloadError, state) + clearInterval(interval) + resolve() + return + } + + events.emit(DownloadEvent.onFileDownloadUpdate, state) + }) + }, 1000) + }) } /** @@ -174,15 +220,20 @@ export default class JanModelExtension extends ModelExtension { async getDownloadedModels(): Promise { return await this.getModelsMetadata( async (modelDir: string, model: Model) => { - if (model.engine !== JanModelExtension._offlineInferenceEngine) { + if (model.engine !== JanModelExtension._offlineInferenceEngine) return true - } + + // model binaries (sources) are absolute path & exist + const existFiles = await Promise.all( + model.sources.map((source) => fs.existsSync(source.url)) + ) + if (existFiles.every((exist) => exist)) return true + return await fs .readdirSync(await joinPath([JanModelExtension._homeDir, modelDir])) .then((files: string[]) => { - // or model binary exists in the directory - // model binary name can match model ID or be a .gguf file and not be an incompleted model file - // TODO: Check diff between urls, filenames + // Model binary exists in the directory + // Model binary name can match model ID or be a .gguf file and not be an incompleted model file return ( files.includes(modelDir) || files.filter( @@ -228,8 +279,19 @@ export default class JanModelExtension extends ModelExtension { if (await fs.existsSync(jsonPath)) { // if we have the model.json file, read it let model = await this.readModelMetadata(jsonPath) + model = typeof model === 'object' ? model : JSON.parse(model) + // This to ensure backward compatibility with `model.json` with `source_url` + if (model['source_url'] != null) { + model['sources'] = [ + { + filename: model.id, + url: model['source_url'], + }, + ] + } + if (selector && !(await selector?.(dirName, model))) { return } @@ -243,31 +305,18 @@ export default class JanModelExtension extends ModelExtension { }) const results = await Promise.allSettled(readJsonPromises) const modelData = results.map((result) => { - if (result.status === 'fulfilled') { + if (result.status === 'fulfilled' && result.value) { try { - // This to ensure backward compatibility with `model.json` with `source_url` - const tmpModel = + const model = typeof result.value === 'object' ? result.value : JSON.parse(result.value) - if (tmpModel['source_url'] != null) { - tmpModel['source'] = [ - { - filename: tmpModel.id, - url: tmpModel['source_url'], - }, - ] - } - - return tmpModel as Model + return model as Model } catch { console.debug(`Unable to parse model metadata: ${result.value}`) - return undefined } - } else { - console.error(result.reason) - return undefined } + return undefined }) return modelData.filter((e) => !!e) @@ -318,7 +367,7 @@ export default class JanModelExtension extends ModelExtension { return } - const defaultModel = await this.getDefaultModel() as Model + const defaultModel = (await this.getDefaultModel()) as Model if (!defaultModel) { console.error('Unable to find default model') return @@ -382,4 +431,28 @@ export default class JanModelExtension extends ModelExtension { async getConfiguredModels(): Promise { return this.getModelsMetadata() } + + handleDesktopEvents() { + if (window && window.electronAPI) { + window.electronAPI.onFileDownloadUpdate( + async (_event: string, state: DownloadState | undefined) => { + if (!state) return + state.downloadState = 'downloading' + events.emit(DownloadEvent.onFileDownloadUpdate, state) + } + ) + window.electronAPI.onFileDownloadError( + async (_event: string, state: DownloadState) => { + state.downloadState = 'error' + events.emit(DownloadEvent.onFileDownloadError, state) + } + ) + window.electronAPI.onFileDownloadSuccess( + async (_event: string, state: DownloadState) => { + state.downloadState = 'end' + events.emit(DownloadEvent.onFileDownloadSuccess, state) + } + ) + } + } } diff --git a/extensions/monitoring-extension/package.json b/extensions/monitoring-extension/package.json index 9935e536ee..582f7cd7b8 100644 --- a/extensions/monitoring-extension/package.json +++ b/extensions/monitoring-extension/package.json @@ -1,6 +1,6 @@ { "name": "@janhq/monitoring-extension", - "version": "1.0.9", + "version": "1.0.10", "description": "This extension provides system health and OS level data", "main": "dist/index.js", "module": "dist/module.js", @@ -8,17 +8,17 @@ "license": "AGPL-3.0", "scripts": { "build": "tsc -b . && webpack --config webpack.config.js", - "build:publish": "rimraf *.tgz --glob && npm run build && npm pack && cpx *.tgz ../../electron/pre-install" + "build:publish": "rimraf *.tgz --glob && npm run build && npm pack && cpx *.tgz ../../pre-install" }, "devDependencies": { "rimraf": "^3.0.2", "webpack": "^5.88.2", - "webpack-cli": "^5.1.4" + "webpack-cli": "^5.1.4", + "ts-loader": "^9.5.0" }, "dependencies": { "@janhq/core": "file:../../core", - "node-os-utils": "^1.3.7", - "ts-loader": "^9.5.0" + "node-os-utils": "^1.3.7" }, "files": [ "dist/*", @@ -26,6 +26,7 @@ "README.md" ], "bundleDependencies": [ - "node-os-utils" + "node-os-utils", + "@janhq/core" ] } diff --git a/extensions/monitoring-extension/src/@types/global.d.ts b/extensions/monitoring-extension/src/@types/global.d.ts index 3b45ccc5ad..8106353cfc 100644 --- a/extensions/monitoring-extension/src/@types/global.d.ts +++ b/extensions/monitoring-extension/src/@types/global.d.ts @@ -1 +1 @@ -declare const MODULE: string; +declare const MODULE: string diff --git a/extensions/monitoring-extension/src/index.ts b/extensions/monitoring-extension/src/index.ts index 9297a770f5..ce9b2fc148 100644 --- a/extensions/monitoring-extension/src/index.ts +++ b/extensions/monitoring-extension/src/index.ts @@ -1,4 +1,4 @@ -import { MonitoringExtension, executeOnMain } from "@janhq/core"; +import { MonitoringExtension, executeOnMain } from '@janhq/core' /** * JanMonitoringExtension is a extension that provides system monitoring functionality. @@ -20,7 +20,7 @@ export default class JanMonitoringExtension extends MonitoringExtension { * @returns A Promise that resolves to an object containing information about the system resources. */ getResourcesInfo(): Promise { - return executeOnMain(MODULE, "getResourcesInfo"); + return executeOnMain(MODULE, 'getResourcesInfo') } /** @@ -28,6 +28,6 @@ export default class JanMonitoringExtension extends MonitoringExtension { * @returns A Promise that resolves to an object containing information about the current system load. */ getCurrentLoad(): Promise { - return executeOnMain(MODULE, "getCurrentLoad"); + return executeOnMain(MODULE, 'getCurrentLoad') } } diff --git a/extensions/monitoring-extension/src/module.ts b/extensions/monitoring-extension/src/module.ts index 86b553d526..27781a5d6f 100644 --- a/extensions/monitoring-extension/src/module.ts +++ b/extensions/monitoring-extension/src/module.ts @@ -1,33 +1,92 @@ -const nodeOsUtils = require("node-os-utils"); +const nodeOsUtils = require('node-os-utils') +const getJanDataFolderPath = require('@janhq/core/node').getJanDataFolderPath +const path = require('path') +const { readFileSync } = require('fs') +const exec = require('child_process').exec + +const NVIDIA_INFO_FILE = path.join( + getJanDataFolderPath(), + 'settings', + 'settings.json' +) const getResourcesInfo = () => new Promise((resolve) => { nodeOsUtils.mem.used().then((ramUsedInfo) => { - const totalMemory = ramUsedInfo.totalMemMb * 1024 * 1024; - const usedMemory = ramUsedInfo.usedMemMb * 1024 * 1024; + const totalMemory = ramUsedInfo.totalMemMb * 1024 * 1024 + const usedMemory = ramUsedInfo.usedMemMb * 1024 * 1024 const response = { mem: { totalMemory, usedMemory, }, - }; - resolve(response); - }); - }); + } + resolve(response) + }) + }) const getCurrentLoad = () => - new Promise((resolve) => { + new Promise((resolve, reject) => { nodeOsUtils.cpu.usage().then((cpuPercentage) => { - const response = { - cpu: { - usage: cpuPercentage, - }, - }; - resolve(response); - }); - }); + let data = { + run_mode: 'cpu', + gpus_in_use: [], + } + if (process.platform !== 'darwin') { + data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, 'utf-8')) + } + if (data.run_mode === 'gpu' && data.gpus_in_use.length > 0) { + const gpuIds = data['gpus_in_use'].join(',') + if (gpuIds !== '' && data['vulkan'] !== true) { + exec( + `nvidia-smi --query-gpu=index,name,temperature.gpu,utilization.gpu,memory.total,memory.free,utilization.memory --format=csv,noheader,nounits --id=${gpuIds}`, + (error, stdout, _) => { + if (error) { + console.error(`exec error: ${error}`) + reject(error) + return + } + const gpuInfo = stdout + .trim() + .split('\n') + .map((line) => { + const [ + id, + name, + temperature, + utilization, + memoryTotal, + memoryFree, + memoryUtilization, + ] = line.split(', ').map((item) => item.replace(/\r/g, '')) + return { + id, + name, + temperature, + utilization, + memoryTotal, + memoryFree, + memoryUtilization, + } + }) + resolve({ + cpu: { usage: cpuPercentage }, + gpu: gpuInfo, + }) + } + ) + } else { + // Handle the case where gpuIds is empty + resolve({ cpu: { usage: cpuPercentage }, gpu: [] }) + } + } else { + // Handle the case where run_mode is not 'gpu' or no GPUs are in use + resolve({ cpu: { usage: cpuPercentage }, gpu: [] }) + } + }) + }) module.exports = { getResourcesInfo, getCurrentLoad, -}; +} diff --git a/extensions/monitoring-extension/webpack.config.js b/extensions/monitoring-extension/webpack.config.js index f54059222f..c8c3a34f79 100644 --- a/extensions/monitoring-extension/webpack.config.js +++ b/extensions/monitoring-extension/webpack.config.js @@ -1,24 +1,24 @@ -const path = require("path"); -const webpack = require("webpack"); -const packageJson = require("./package.json"); +const path = require('path') +const webpack = require('webpack') +const packageJson = require('./package.json') module.exports = { experiments: { outputModule: true }, - entry: "./src/index.ts", // Adjust the entry point to match your project's main file - mode: "production", + entry: './src/index.ts', // Adjust the entry point to match your project's main file + mode: 'production', module: { rules: [ { test: /\.tsx?$/, - use: "ts-loader", + use: 'ts-loader', exclude: /node_modules/, }, ], }, output: { - filename: "index.js", // Adjust the output file name as needed - path: path.resolve(__dirname, "dist"), - library: { type: "module" }, // Specify ESM output format + filename: 'index.js', // Adjust the output file name as needed + path: path.resolve(__dirname, 'dist'), + library: { type: 'module' }, // Specify ESM output format }, plugins: [ new webpack.DefinePlugin({ @@ -26,10 +26,10 @@ module.exports = { }), ], resolve: { - extensions: [".ts", ".js"], + extensions: ['.ts', '.js'], }, optimization: { minimize: false, }, // Add loaders and other configuration as needed for your project -}; +} diff --git a/models/dolphin-phi-2/model.json b/models/dolphin-phi-2/model.json new file mode 100644 index 0000000000..c25ff8f694 --- /dev/null +++ b/models/dolphin-phi-2/model.json @@ -0,0 +1,32 @@ +{ + "sources": [ + { + "url": "https://huggingface.co/TheBloke/dolphin-2_6-phi-2-GGUF/resolve/main/dolphin-2_6-phi-2.Q8_0.gguf", + "filename": "dolphin-2_6-phi-2.Q8_0.gguf" + } + ], + "id": "dolphin-phi-2", + "object": "model", + "name": "Dolphin Phi-2 2.7B Q8", + "version": "1.0", + "description": "Dolphin Phi-2 is a 2.7B model, fine-tuned for chat, excelling in common sense and logical reasoning benchmarks.", + "format": "gguf", + "settings": { + "ctx_len": 4096, + "prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant", + "llama_model_path": "dolphin-2_6-phi-2.Q8_0.gguf" + }, + "parameters": { + "max_tokens": 4096, + "stop": ["<|im_end|>"] + }, + "metadata": { + "author": "Cognitive Computations, Microsoft", + "tags": [ + "3B", + "Finetuned" + ], + "size": 2960000000 + }, + "engine": "nitro" + } diff --git a/models/llamacorn-1.1b/model.json b/models/llamacorn-1.1b/model.json new file mode 100644 index 0000000000..056fb90504 --- /dev/null +++ b/models/llamacorn-1.1b/model.json @@ -0,0 +1,37 @@ +{ + "sources": [ + { + "url":"https://huggingface.co/janhq/llamacorn-1.1b-chat-GGUF/resolve/main/llamacorn-1.1b-chat.Q8_0.gguf", + "filename": "llamacorn-1.1b-chat.Q8_0.gguf" + } + ], + "id": "llamacorn-1.1b", + "object": "model", + "name": "LlamaCorn 1.1B Q8", + "version": "1.0", + "description": "LlamaCorn is designed to improve chat functionality from TinyLlama.", + "format": "gguf", + "settings": { + "ctx_len": 2048, + "prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant", + "llama_model_path": "llamacorn-1.1b-chat.Q8_0.gguf" + }, + "parameters": { + "temperature": 0.7, + "top_p": 0.95, + "stream": true, + "max_tokens": 2048, + "stop": [], + "frequency_penalty": 0, + "presence_penalty": 0 + }, + "metadata": { + "author": "Jan", + "tags": [ + "Tiny", + "Finetuned" + ], + "size": 1170000000 + }, + "engine": "nitro" + } \ No newline at end of file diff --git a/models/mistral-ins-7b-q4/cover.png b/models/mistral-ins-7b-q4/cover.png index 000445ecba..73b82e5996 100644 Binary files a/models/mistral-ins-7b-q4/cover.png and b/models/mistral-ins-7b-q4/cover.png differ diff --git a/models/mistral-ins-7b-q4/model.json b/models/mistral-ins-7b-q4/model.json index bfdaffa907..75e0cbf9f0 100644 --- a/models/mistral-ins-7b-q4/model.json +++ b/models/mistral-ins-7b-q4/model.json @@ -29,7 +29,7 @@ "author": "MistralAI, The Bloke", "tags": ["Featured", "7B", "Foundational Model"], "size": 4370000000, - "cover": "https://raw.githubusercontent.com/janhq/jan/main/models/mistral-ins-7b-q4/cover.png" + "cover": "https://raw.githubusercontent.com/janhq/jan/dev/models/mistral-ins-7b-q4/cover.png" }, "engine": "nitro" } diff --git a/models/openchat-3.5-7b/model.json b/models/openchat-3.5-7b/model.json index 294f7d2694..18db33f8e6 100644 --- a/models/openchat-3.5-7b/model.json +++ b/models/openchat-3.5-7b/model.json @@ -1,8 +1,8 @@ { "sources": [ { - "filename": "openchat-3.5-1210.Q4_K_M.gguf", - "url": "https://huggingface.co/TheBloke/openchat-3.5-1210-GGUF/resolve/main/openchat-3.5-1210.Q4_K_M.gguf" + "filename": "openchat-3.5-0106.Q4_K_M.gguf", + "url": "https://huggingface.co/TheBloke/openchat-3.5-0106-GGUF/resolve/main/openchat-3.5-0106.Q4_K_M.gguf" } ], "id": "openchat-3.5-7b", @@ -14,7 +14,7 @@ "settings": { "ctx_len": 4096, "prompt_template": "GPT4 Correct User: {prompt}<|end_of_turn|>GPT4 Correct Assistant:", - "llama_model_path": "openchat-3.5-1210.Q4_K_M.gguf" + "llama_model_path": "openchat-3.5-0106.Q4_K_M.gguf" }, "parameters": { "temperature": 0.7, diff --git a/models/openhermes-neural-7b/cover.png b/models/openhermes-neural-7b/cover.png index 5b9da0aefe..8976d84490 100644 Binary files a/models/openhermes-neural-7b/cover.png and b/models/openhermes-neural-7b/cover.png differ diff --git a/models/openhermes-neural-7b/model.json b/models/openhermes-neural-7b/model.json index 87e1df143a..a13a0f2b85 100644 --- a/models/openhermes-neural-7b/model.json +++ b/models/openhermes-neural-7b/model.json @@ -28,7 +28,7 @@ "author": "Intel, Jan", "tags": ["7B", "Merged", "Featured"], "size": 4370000000, - "cover": "https://raw.githubusercontent.com/janhq/jan/main/models/openhermes-neural-7b/cover.png" + "cover": "https://raw.githubusercontent.com/janhq/jan/dev/models/openhermes-neural-7b/cover.png" }, "engine": "nitro" } diff --git a/models/stable-zephyr-3b/model.json b/models/stable-zephyr-3b/model.json new file mode 100644 index 0000000000..724299ea5e --- /dev/null +++ b/models/stable-zephyr-3b/model.json @@ -0,0 +1,34 @@ +{ + "sources": [ + { + "url": "https://huggingface.co/TheBloke/stablelm-zephyr-3b-GGUF/resolve/main/stablelm-zephyr-3b.Q8_0.gguf", + "filename": "stablelm-zephyr-3b.Q8_0.gguf" + } + ], + "id": "stable-zephyr-3b", + "object": "model", + "name": "Stable Zephyr 3B Q8", + "version": "1.0", + "description": "StableLM Zephyr 3B is trained for safe and reliable chatting.", + "format": "gguf", + "settings": { + "ctx_len": 4096, + "prompt_template": "<|user|>\n{prompt}<|endoftext|>\n<|assistant|>", + "llama_model_path": "stablelm-zephyr-3b.Q8_0.gguf" + }, + "parameters": { + "temperature": 0.7, + "top_p": 0.95, + "stream": true, + "max_tokens": 4096, + "stop": ["<|endoftext|>"], + "frequency_penalty": 0, + "presence_penalty": 0 + }, + "metadata": { + "author": "StabilityAI", + "tags": ["3B", "Finetuned"], + "size": 2970000000 + }, + "engine": "nitro" + } \ No newline at end of file diff --git a/models/trinity-v1.2-7b/cover.png b/models/trinity-v1.2-7b/cover.png index a548e3c173..fbef0bb560 100644 Binary files a/models/trinity-v1.2-7b/cover.png and b/models/trinity-v1.2-7b/cover.png differ diff --git a/models/trinity-v1.2-7b/model.json b/models/trinity-v1.2-7b/model.json index 2dda120e65..9476296421 100644 --- a/models/trinity-v1.2-7b/model.json +++ b/models/trinity-v1.2-7b/model.json @@ -28,7 +28,7 @@ "author": "Jan", "tags": ["7B", "Merged", "Featured"], "size": 4370000000, - "cover": "https://raw.githubusercontent.com/janhq/jan/main/models/trinity-v1.2-7b/cover.png" + "cover": "https://raw.githubusercontent.com/janhq/jan/dev/models/trinity-v1.2-7b/cover.png" }, "engine": "nitro" } diff --git a/models/yarn-mistral-7b/model.json b/models/yarn-mistral-7b/model.json deleted file mode 100644 index ee6de13198..0000000000 --- a/models/yarn-mistral-7b/model.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "sources": [ - { - "url": "https://huggingface.co/TheBloke/Yarn-Mistral-7B-128k-GGUF/resolve/main/yarn-mistral-7b-128k.Q4_K_M.gguf" - } - ], - "id": "yarn-mistral-7b", - "object": "model", - "name": "Yarn Mistral 7B Q4", - "version": "1.0", - "description": "Yarn Mistral 7B is a language model for long context and supports a 128k token context window.", - "format": "gguf", - "settings": { - "ctx_len": 4096, - "prompt_template": "{prompt}" - }, - "parameters": { - "temperature": 0.7, - "top_p": 0.95, - "stream": true, - "max_tokens": 4096, - "frequency_penalty": 0, - "presence_penalty": 0 - }, - "metadata": { - "author": "NousResearch, The Bloke", - "tags": ["7B", "Finetuned"], - "size": 4370000000 - }, - "engine": "nitro" -} diff --git a/package.json b/package.json index 4b8bc4af08..957934fdaa 100644 --- a/package.json +++ b/package.json @@ -21,22 +21,23 @@ "lint": "yarn workspace jan lint && yarn workspace jan-web lint", "test:unit": "yarn workspace @janhq/core test", "test": "yarn workspace jan test:e2e", - "copy:assets": "cpx \"models/**\" \"electron/models/\" && cpx \"docs/openapi/**\" \"electron/docs/openapi\"", + "copy:assets": "cpx \"models/**\" \"electron/models/\" && cpx \"pre-install/*.tgz\" \"electron/pre-install/\" && cpx \"docs/openapi/**\" \"electron/docs/openapi\"", "dev:electron": "yarn copy:assets && yarn workspace jan dev", "dev:web": "yarn workspace jan-web dev", - "dev:server": "yarn workspace @janhq/server dev", + "dev:server": "yarn copy:assets && yarn workspace @janhq/server dev", "dev": "concurrently --kill-others \"yarn dev:web\" \"wait-on http://localhost:3000 && yarn dev:electron\"", "test-local": "yarn lint && yarn build:test && yarn test", "dev:uikit": "yarn workspace @janhq/uikit install && yarn workspace @janhq/uikit dev", "build:uikit": "yarn workspace @janhq/uikit install && yarn workspace @janhq/uikit build", - "build:server": "cd server && yarn install && yarn run build", + "build:server": "yarn copy:assets && cd server && yarn install && yarn run build", "build:core": "cd core && yarn install && yarn run build", "build:web": "yarn workspace jan-web build && cpx \"web/out/**\" \"electron/renderer/\"", "build:electron": "yarn copy:assets && yarn workspace jan build", "build:electron:test": "yarn workspace jan build:test", - "build:extensions:windows": "rimraf ./electron/pre-install/*.tgz && powershell -command \"$jobs = Get-ChildItem -Path './extensions' -Directory | ForEach-Object { Start-Job -Name ($_.Name) -ScriptBlock { param($_dir); try { Set-Location $_dir; npm install; npm run build:publish; Write-Output 'Build successful in ' + $_dir } catch { Write-Error 'Error in ' + $_dir; throw } } -ArgumentList $_.FullName }; $jobs | Wait-Job; $jobs | ForEach-Object { Receive-Job -Job $_ -Keep } | ForEach-Object { Write-Host $_ }; $failed = $jobs | Where-Object { $_.State -ne 'Completed' -or $_.ChildJobs[0].JobStateInfo.State -ne 'Completed' }; if ($failed) { Exit 1 }\"", - "build:extensions:linux": "rimraf ./electron/pre-install/*.tgz && find ./extensions -mindepth 1 -maxdepth 1 -type d -print0 | xargs -0 -n 1 -P 4 -I {} sh -c 'cd {} && npm install && npm run build:publish'", - "build:extensions:darwin": "rimraf ./electron/pre-install/*.tgz && find ./extensions -mindepth 1 -maxdepth 1 -type d -print0 | xargs -0 -n 1 -P 4 -I {} sh -c 'cd {} && npm install && npm run build:publish'", + "build:extensions:windows": "rimraf ./pre-install/*.tgz && powershell -command \"$jobs = Get-ChildItem -Path './extensions' -Directory | ForEach-Object { Start-Job -Name ($_.Name) -ScriptBlock { param($_dir); try { Set-Location $_dir; npm install; npm run build:publish; Write-Output 'Build successful in ' + $_dir } catch { Write-Error 'Error in ' + $_dir; throw } } -ArgumentList $_.FullName }; $jobs | Wait-Job; $jobs | ForEach-Object { Receive-Job -Job $_ -Keep } | ForEach-Object { Write-Host $_ }; $failed = $jobs | Where-Object { $_.State -ne 'Completed' -or $_.ChildJobs[0].JobStateInfo.State -ne 'Completed' }; if ($failed) { Exit 1 }\"", + "build:extensions:linux": "rimraf ./pre-install/*.tgz && find ./extensions -mindepth 1 -maxdepth 1 -type d -print0 | xargs -0 -n 1 -P 4 -I {} sh -c 'cd {} && npm install && npm run build:publish'", + "build:extensions:darwin": "rimraf ./pre-install/*.tgz && find ./extensions -mindepth 1 -maxdepth 1 -type d -print0 | xargs -0 -n 1 -P 4 -I {} sh -c 'cd {} && npm install && npm run build:publish'", + "build:extensions:server": "yarn workspace build:extensions ", "build:extensions": "run-script-os", "build:test": "yarn copy:assets && yarn build:web && yarn workspace jan build:test", "build": "yarn build:web && yarn build:electron", diff --git a/pre-install/.gitkeep b/pre-install/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/server/helpers/setup.ts b/server/helpers/setup.ts new file mode 100644 index 0000000000..7d8f8914a1 --- /dev/null +++ b/server/helpers/setup.ts @@ -0,0 +1,73 @@ +import { join, extname } from 'path' +import { existsSync, readdirSync, writeFileSync, mkdirSync } from 'fs' +import { init, installExtensions } from '@janhq/core/node' + +export async function setup() { + /** + * Setup Jan Data Directory + */ + const appDir = process.env.JAN_DATA_DIRECTORY ?? join(__dirname, '..', 'jan') + + console.debug(`Create app data directory at ${appDir}...`) + if (!existsSync(appDir)) mkdirSync(appDir) + //@ts-ignore + global.core = { + // Define appPath function for app to retrieve app path globaly + appPath: () => appDir, + } + init({ + extensionsPath: join(appDir, 'extensions'), + }) + + /** + * Write app configurations. See #1619 + */ + console.debug('Writing config file...') + writeFileSync( + join(appDir, 'settings.json'), + JSON.stringify({ + data_folder: appDir, + }), + 'utf-8' + ) + + if (!existsSync(join(appDir, 'settings'))) { + console.debug('Writing nvidia config file...') + mkdirSync(join(appDir, 'settings')) + writeFileSync( + join(appDir, 'settings', 'settings.json'), + JSON.stringify( + { + notify: true, + run_mode: 'cpu', + nvidia_driver: { + exist: false, + version: '', + }, + cuda: { + exist: false, + version: '', + }, + gpus: [], + gpu_highest_vram: '', + gpus_in_use: [], + is_initial: true, + }), + 'utf-8' + ) + } + + /** + * Install extensions + */ + + console.debug('Installing extensions...') + + const baseExtensionPath = join(__dirname, '../../..', 'pre-install') + const extensions = readdirSync(baseExtensionPath) + .filter((file) => extname(file) === '.tgz') + .map((file) => join(baseExtensionPath, file)) + + await installExtensions(extensions) + console.debug('Extensions installed') +} diff --git a/server/index.ts b/server/index.ts index 05bfdca961..98cc8385d1 100644 --- a/server/index.ts +++ b/server/index.ts @@ -1,26 +1,27 @@ -import fastify from "fastify"; -import dotenv from "dotenv"; +import fastify from 'fastify' +import dotenv from 'dotenv' import { getServerLogPath, v1Router, logServer, getJanExtensionsPath, -} from "@janhq/core/node"; -import { join } from "path"; +} from '@janhq/core/node' +import { join } from 'path' +import tcpPortUsed from 'tcp-port-used' // Load environment variables -dotenv.config(); +dotenv.config() // Define default settings -const JAN_API_HOST = process.env.JAN_API_HOST || "127.0.0.1"; -const JAN_API_PORT = Number.parseInt(process.env.JAN_API_PORT || "1337"); +const JAN_API_HOST = process.env.JAN_API_HOST || '127.0.0.1' +const JAN_API_PORT = Number.parseInt(process.env.JAN_API_PORT || '1337') // Initialize server settings -let server: any | undefined = undefined; -let hostSetting: string = JAN_API_HOST; -let portSetting: number = JAN_API_PORT; -let corsEnabled: boolean = true; -let isVerbose: boolean = true; +let server: any | undefined = undefined +let hostSetting: string = JAN_API_HOST +let portSetting: number = JAN_API_PORT +let corsEnabled: boolean = true +let isVerbose: boolean = true /** * Server configurations @@ -32,80 +33,93 @@ let isVerbose: boolean = true; * @param baseDir - Base directory for the OpenAPI schema file */ export interface ServerConfig { - host?: string; - port?: number; - isCorsEnabled?: boolean; - isVerboseEnabled?: boolean; - schemaPath?: string; - baseDir?: string; + host?: string + port?: number + isCorsEnabled?: boolean + isVerboseEnabled?: boolean + schemaPath?: string + baseDir?: string + storageAdataper?: any } /** * Function to start the server * @param configs - Server configurations */ -export const startServer = async (configs?: ServerConfig) => { +export const startServer = async (configs?: ServerConfig): Promise => { + if (configs?.port && configs?.host) { + const inUse = await tcpPortUsed.check(Number(configs.port), configs.host) + if (inUse) { + const errorMessage = `Port ${configs.port} is already in use.` + logServer(errorMessage) + throw new Error(errorMessage) + } + } + // Update server settings - isVerbose = configs?.isVerboseEnabled ?? true; - hostSetting = configs?.host ?? JAN_API_HOST; - portSetting = configs?.port ?? JAN_API_PORT; - corsEnabled = configs?.isCorsEnabled ?? true; - const serverLogPath = getServerLogPath(); + isVerbose = configs?.isVerboseEnabled ?? true + hostSetting = configs?.host ?? JAN_API_HOST + portSetting = configs?.port ?? JAN_API_PORT + corsEnabled = configs?.isCorsEnabled ?? true + const serverLogPath = getServerLogPath() // Start the server try { // Log server start - if (isVerbose) logServer(`Debug: Starting JAN API server...`); + if (isVerbose) logServer(`Debug: Starting JAN API server...`) // Initialize Fastify server with logging server = fastify({ logger: { - level: "info", + level: 'info', file: serverLogPath, }, - }); + }) // Register CORS if enabled - if (corsEnabled) await server.register(require("@fastify/cors"), {}); + if (corsEnabled) await server.register(require('@fastify/cors'), {}) // Register Swagger for API documentation - await server.register(require("@fastify/swagger"), { - mode: "static", + await server.register(require('@fastify/swagger'), { + mode: 'static', specification: { - path: configs?.schemaPath ?? "./../docs/openapi/jan.yaml", - baseDir: configs?.baseDir ?? "./../docs/openapi", + path: configs?.schemaPath ?? './../docs/openapi/jan.yaml', + baseDir: configs?.baseDir ?? './../docs/openapi', }, - }); + }) // Register Swagger UI - await server.register(require("@fastify/swagger-ui"), { - routePrefix: "/", - baseDir: configs?.baseDir ?? join(__dirname, "../..", "./docs/openapi"), + await server.register(require('@fastify/swagger-ui'), { + routePrefix: '/', + baseDir: configs?.baseDir ?? join(__dirname, '../..', './docs/openapi'), uiConfig: { - docExpansion: "full", + docExpansion: 'full', deepLinking: false, }, staticCSP: false, transformSpecificationClone: true, - }); + }) // Register static file serving for extensions // TODO: Watch extension files changes and reload await server.register( (childContext: any, _: any, done: any) => { - childContext.register(require("@fastify/static"), { + childContext.register(require('@fastify/static'), { root: getJanExtensionsPath(), wildcard: false, - }); + }) - done(); + done() }, - { prefix: "extensions" } - ); + { prefix: 'extensions' } + ) - // Register API routes - await server.register(v1Router, { prefix: "/v1" }); + // Register proxy middleware + if (configs?.storageAdataper) + server.addHook('preHandler', configs.storageAdataper) + // Register API routes + await server.register(v1Router, { prefix: '/v1' }) // Start listening for requests await server .listen({ @@ -117,13 +131,15 @@ export const startServer = async (configs?: ServerConfig) => { if (isVerbose) logServer( `Debug: JAN API listening at: http://${hostSetting}:${portSetting}` - ); - }); + ) + }) + return true } catch (e) { // Log any errors - if (isVerbose) logServer(`Error: ${e}`); + if (isVerbose) logServer(`Error: ${e}`) } -}; + return false +} /** * Function to stop the server @@ -131,11 +147,11 @@ export const startServer = async (configs?: ServerConfig) => { export const stopServer = async () => { try { // Log server stop - if (isVerbose) logServer(`Debug: Server stopped`); + if (isVerbose) logServer(`Debug: Server stopped`) // Stop the server - await server.close(); + await server.close() } catch (e) { // Log any errors - if (isVerbose) logServer(`Error: ${e}`); + if (isVerbose) logServer(`Error: ${e}`) } -}; +} diff --git a/server/main.ts b/server/main.ts index c3eb691356..71fb111062 100644 --- a/server/main.ts +++ b/server/main.ts @@ -1,3 +1,7 @@ -import { startServer } from "./index"; - -startServer(); +import { s3 } from './middleware/s3' +import { setup } from './helpers/setup' +import { startServer as start } from './index' +/** + * Setup extensions and start the server + */ +setup().then(() => start({ storageAdataper: s3 })) diff --git a/server/middleware/s3.ts b/server/middleware/s3.ts new file mode 100644 index 0000000000..28971a42b4 --- /dev/null +++ b/server/middleware/s3.ts @@ -0,0 +1,70 @@ +import { join } from 'path' + +// Middleware to intercept requests and proxy if certain conditions are met +const config = { + endpoint: process.env.AWS_ENDPOINT, + region: process.env.AWS_REGION, + credentials: { + accessKeyId: process.env.AWS_ACCESS_KEY_ID, + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, + }, +} + +const S3_BUCKET_NAME = process.env.S3_BUCKET_NAME + +const fs = require('@cyclic.sh/s3fs')(S3_BUCKET_NAME, config) +const PROXY_PREFIX = '/v1/fs' +const PROXY_ROUTES = ['/threads', '/messages'] + +export const s3 = (req: any, reply: any, done: any) => { + // Proxy FS requests to S3 using S3FS + if (req.url.startsWith(PROXY_PREFIX)) { + const route = req.url.split('/').pop() + const args = parseRequestArgs(req) + + // Proxy matched requests to the s3fs module + if (args.length && PROXY_ROUTES.some((route) => args[0].includes(route))) { + try { + // Handle customized route + // S3FS does not handle appendFileSync + if (route === 'appendFileSync') { + let result = handAppendFileSync(args) + + reply.status(200).send(result) + return + } + // Reroute the other requests to the s3fs module + const result = fs[route](...args) + reply.status(200).send(result) + return + } catch (ex) { + console.log(ex) + } + } + } + // Let other requests go through + done() +} + +const parseRequestArgs = (req: Request) => { + const { + getJanDataFolderPath, + normalizeFilePath, + } = require('@janhq/core/node') + + return JSON.parse(req.body as any).map((arg: any) => + typeof arg === 'string' && + (arg.startsWith(`file:/`) || arg.startsWith(`file:\\`)) + ? join(getJanDataFolderPath(), normalizeFilePath(arg)) + : arg + ) +} + +const handAppendFileSync = (args: any[]) => { + if (fs.existsSync(args[0])) { + const data = fs.readFileSync(args[0], 'utf-8') + return fs.writeFileSync(args[0], data + args[1]) + } else { + return fs.writeFileSync(args[0], args[1]) + } +} diff --git a/server/nodemon.json b/server/nodemon.json deleted file mode 100644 index 0ea41ca96b..0000000000 --- a/server/nodemon.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "watch": ["main.ts", "v1"], - "ext": "ts, json", - "exec": "tsc && node ./build/main.js" -} \ No newline at end of file diff --git a/server/package.json b/server/package.json index f61730da4a..d9a2bbc9ab 100644 --- a/server/package.json +++ b/server/package.json @@ -18,26 +18,29 @@ }, "dependencies": { "@alumna/reflect": "^1.1.3", + "@cyclic.sh/s3fs": "^1.2.9", "@fastify/cors": "^8.4.2", "@fastify/static": "^6.12.0", "@fastify/swagger": "^8.13.0", "@fastify/swagger-ui": "2.0.1", "@janhq/core": "link:./core", + "@npmcli/arborist": "^7.3.1", "dotenv": "^16.3.1", "fastify": "^4.24.3", - "request": "^2.88.2", "fetch-retry": "^5.0.6", - "tcp-port-used": "^1.0.2", - "request-progress": "^3.0.0" + "node-fetch": "2", + "request": "^2.88.2", + "request-progress": "^3.0.0", + "tcp-port-used": "^1.0.2" }, "devDependencies": { "@types/body-parser": "^1.19.5", "@types/npmcli__arborist": "^5.6.4", + "@types/tcp-port-used": "^1.0.4", "@typescript-eslint/eslint-plugin": "^6.7.3", "@typescript-eslint/parser": "^6.7.3", "eslint-plugin-react": "^7.33.2", "run-script-os": "^1.1.6", - "@types/tcp-port-used": "^1.0.4", "typescript": "^5.2.2" } } diff --git a/server/tsconfig.json b/server/tsconfig.json index 2c4fc4a64e..dd27b89323 100644 --- a/server/tsconfig.json +++ b/server/tsconfig.json @@ -20,5 +20,5 @@ // "sourceMap": true, "include": ["./**/*.ts"], - "exclude": ["core", "build", "dist", "tests", "node_modules"] + "exclude": ["core", "build", "dist", "tests", "node_modules", "extensions"] } diff --git a/uikit/src/input/styles.scss b/uikit/src/input/styles.scss index 9990da8b4c..e649f494da 100644 --- a/uikit/src/input/styles.scss +++ b/uikit/src/input/styles.scss @@ -1,6 +1,6 @@ .input { @apply border-border placeholder:text-muted-foreground flex h-9 w-full rounded-lg border bg-transparent px-3 py-1 transition-colors; - @apply disabled:cursor-not-allowed disabled:bg-zinc-100 disabled:dark:bg-zinc-800 disabled:dark:text-zinc-600; + @apply disabled:text-muted-foreground disabled:cursor-not-allowed disabled:bg-zinc-100 disabled:dark:bg-zinc-800 disabled:dark:text-zinc-600; @apply focus-within:outline-none focus-visible:outline-0 focus-visible:ring-2 focus-visible:ring-blue-500 focus-visible:ring-offset-1; @apply file:border-0 file:bg-transparent file:font-medium; } diff --git a/uikit/src/select/styles.scss b/uikit/src/select/styles.scss index bc5b6c0cc2..90485723ab 100644 --- a/uikit/src/select/styles.scss +++ b/uikit/src/select/styles.scss @@ -1,6 +1,6 @@ .select { @apply placeholder:text-muted-foreground border-border flex h-9 w-full items-center justify-between whitespace-nowrap rounded-md border bg-transparent px-3 py-2 text-sm shadow-sm disabled:cursor-not-allowed [&>span]:line-clamp-1; - @apply disabled:cursor-not-allowed disabled:bg-zinc-100 disabled:dark:bg-zinc-800 disabled:dark:text-zinc-600; + @apply disabled:text-muted-foreground disabled:cursor-not-allowed disabled:bg-zinc-100 disabled:dark:bg-zinc-800 disabled:dark:text-zinc-600; @apply focus-within:outline-none focus-visible:outline-0 focus-visible:ring-2 focus-visible:ring-blue-500 focus-visible:ring-offset-1; &-caret { @@ -21,6 +21,7 @@ &-item { @apply hover:bg-secondary relative my-1 block w-full cursor-pointer select-none items-center rounded-sm px-4 py-2 text-sm data-[disabled]:pointer-events-none data-[disabled]:opacity-50; + @apply focus:outline-none focus-visible:outline-0; } &-trigger-viewport { diff --git a/uikit/types/declaration.d.ts b/uikit/types/declaration.d.ts index 85b1a7136f..f8e975fa54 100644 --- a/uikit/types/declaration.d.ts +++ b/uikit/types/declaration.d.ts @@ -1,4 +1,4 @@ declare module '*.scss' { - const content: Record; - export default content; -} \ No newline at end of file + const content: Record + export default content +} diff --git a/web/.prettierignore b/web/.prettierignore deleted file mode 100644 index 02d9145c14..0000000000 --- a/web/.prettierignore +++ /dev/null @@ -1,5 +0,0 @@ -.next/ -node_modules/ -dist/ -*.hbs -*.mdx \ No newline at end of file diff --git a/web/.prettierrc b/web/.prettierrc deleted file mode 100644 index 46f1abcb02..0000000000 --- a/web/.prettierrc +++ /dev/null @@ -1,8 +0,0 @@ -{ - "semi": false, - "singleQuote": true, - "quoteProps": "consistent", - "trailingComma": "es5", - "endOfLine": "auto", - "plugins": ["prettier-plugin-tailwindcss"] -} diff --git a/web/app/error.tsx b/web/app/error.tsx new file mode 100644 index 0000000000..25b24b9ef5 --- /dev/null +++ b/web/app/error.tsx @@ -0,0 +1,89 @@ +'use client' // Error components must be Client Components + +import { useEffect, useState } from 'react' + +export default function Error({ + error, +}: { + error: Error & { digest?: string } + reset: () => void +}) { + const [showFull, setShowFull] = useState(false) + useEffect(() => { + // Log the error to an error reporting service + console.error(error) + }, [error]) + + return ( + <> +
+
+
+
+ + + + +
+
+

+ Oops! Unexpected error occurred. +

+

+ Something went wrong. Try to{' '} + {' '} + or
feel free to{' '} + + contact us + {' '} + if the problem presists. +

+
+ Error: + {error.message} +
+ 
+                {showFull ? error.stack : error.stack?.slice(0, 200)}
+
+ +
+
+
+
+ + ) +} diff --git a/web/containers/CardSidebar/index.tsx b/web/containers/CardSidebar/index.tsx index 38a8678d9e..89ff60e664 100644 --- a/web/containers/CardSidebar/index.tsx +++ b/web/containers/CardSidebar/index.tsx @@ -22,6 +22,7 @@ interface Props { rightAction?: ReactNode title: string asChild?: boolean + isShow?: boolean hideMoreVerticalAction?: boolean } export default function CardSidebar({ @@ -30,8 +31,9 @@ export default function CardSidebar({ asChild, rightAction, hideMoreVerticalAction, + isShow, }: Props) { - const [show, setShow] = useState(true) + const [show, setShow] = useState(isShow ?? false) const [more, setMore] = useState(false) const [menu, setMenu] = useState(null) const [toggle, setToggle] = useState(null) @@ -67,8 +69,8 @@ export default function CardSidebar({ show && 'rotate-180' )} /> + {title} - {title}
{rightAction && rightAction} @@ -156,7 +158,10 @@ export default function CardSidebar({ ) : ( <> - Opens {title}.json. + Opens{' '} + + {title === 'Tools' ? 'assistant' : title}.json. +  Changes affect all new threads. )} diff --git a/web/containers/DropdownListSidebar/index.tsx b/web/containers/DropdownListSidebar/index.tsx index 140a1aba15..191c7bcbe8 100644 --- a/web/containers/DropdownListSidebar/index.tsx +++ b/web/containers/DropdownListSidebar/index.tsx @@ -14,7 +14,14 @@ import { import { atom, useAtom, useAtomValue, useSetAtom } from 'jotai' -import { MonitorIcon } from 'lucide-react' +import { + MonitorIcon, + LayoutGridIcon, + FoldersIcon, + GlobeIcon, + CheckIcon, + CopyIcon, +} from 'lucide-react' import { twMerge } from 'tailwind-merge' @@ -22,6 +29,7 @@ import { MainViewState } from '@/constants/screens' import { useActiveModel } from '@/hooks/useActiveModel' +import { useClipboard } from '@/hooks/useClipboard' import { useMainViewState } from '@/hooks/useMainViewState' import useRecommendedModel from '@/hooks/useRecommendedModel' @@ -42,6 +50,8 @@ import { export const selectedModelAtom = atom(undefined) +const engineOptions = ['Local', 'Remote'] + // TODO: Move all of the unscoped logics outside of the component const DropdownListSidebar = ({ strictedThread = true, @@ -51,13 +61,24 @@ const DropdownListSidebar = ({ const activeThread = useAtomValue(activeThreadAtom) const [selectedModel, setSelectedModel] = useAtom(selectedModelAtom) const setThreadModelParams = useSetAtom(setThreadModelParamsAtom) - + const [isTabActive, setIsTabActive] = useState(0) const { stateModel } = useActiveModel() const [serverEnabled, setServerEnabled] = useAtom(serverEnabledAtom) const { setMainViewState } = useMainViewState() const [loader, setLoader] = useState(0) const { recommendedModel, downloadedModels } = useRecommendedModel() const { updateModelParameter } = useUpdateModelParameters() + const clipboard = useClipboard({ timeout: 1000 }) + const [copyId, setCopyId] = useState('') + + const localModel = downloadedModels.filter( + (model) => model.engine === InferenceEngine.nitro + ) + const remoteModel = downloadedModels.filter( + (model) => model.engine === InferenceEngine.openai + ) + + const modelOptions = isTabActive === 0 ? localModel : remoteModel useEffect(() => { if (!activeThread) return @@ -73,7 +94,7 @@ const DropdownListSidebar = ({ // This is fake loader please fix this when we have realtime percentage when load model useEffect(() => { - if (stateModel.loading) { + if (stateModel.model === selectedModel?.id && stateModel.loading) { if (loader === 24) { setTimeout(() => { setLoader(loader + 1) @@ -94,7 +115,7 @@ const DropdownListSidebar = ({ } else { setLoader(0) } - }, [stateModel.loading, loader]) + }, [stateModel.loading, loader, selectedModel, stateModel.model]) const onValueSelected = useCallback( async (modelId: string) => { @@ -138,12 +159,16 @@ const DropdownListSidebar = ({ return null } + const selectedModelLoading = + stateModel.model === selectedModel?.id && stateModel.loading + return ( <>