From 254a79ccbe7c23e8f452c82aa187b5ebdb2d7bfb Mon Sep 17 00:00:00 2001 From: Louis Date: Fri, 22 Mar 2024 17:53:20 +0700 Subject: [PATCH] fix: turborepo extensions (#2392) * fix: turborepo extensions Update package.json Update Makefile Update Makefile Update Makefile Update Makefile Update Makefile Update package.json * chore: turbo cache * fix: install extensions in parallel * fix: timeout issue * Turbo cache using s3 * Remove cache task --------- Co-authored-by: Hien To Co-authored-by: Service Account --- .github/workflows/codeQL-analysis.yml | 104 ------------------ .../jan-electron-linter-and-test.yml | 8 +- Makefile | 5 +- extensions/package.json | 11 ++ extensions/turbo.json | 15 +++ package.json | 25 ++--- turbo.json | 41 +++++-- web/package.json | 2 +- 8 files changed, 77 insertions(+), 134 deletions(-) delete mode 100644 .github/workflows/codeQL-analysis.yml create mode 100644 extensions/package.json create mode 100644 extensions/turbo.json diff --git a/.github/workflows/codeQL-analysis.yml b/.github/workflows/codeQL-analysis.yml deleted file mode 100644 index 00301b8405..0000000000 --- a/.github/workflows/codeQL-analysis.yml +++ /dev/null @@ -1,104 +0,0 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# -name: "CodeQL" - -on: - push: - branches: - - main - - dev - paths: - - "electron/**" - - .github/workflows/jan-electron-linter-and-test.yml - - "web/**" - - "uikit/**" - - "package.json" - - "node_modules/**" - - "yarn.lock" - - "core/**" - - "extensions/**" - - "!README.md" - - "Makefile" - - pull_request: - branches: - - main - - dev - paths: - - "electron/**" - - .github/workflows/jan-electron-linter-and-test.yml - - "web/**" - - "uikit/**" - - "package.json" - - "node_modules/**" - - "yarn.lock" - - "Makefile" - -jobs: - analyze: - name: Analyze - # Runner size impacts CodeQL analysis time. To learn more, please see: - # - https://gh.io/recommended-hardware-resources-for-running-codeql - # - https://gh.io/supported-runners-and-hardware-resources - # - https://gh.io/using-larger-runners - # Consider using larger runners for possible analysis time improvements. - runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} - timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }} - permissions: - actions: read - contents: read - security-events: write - - strategy: - fail-fast: false - matrix: - language: ["javascript-typescript"] - # CodeQL supports [ 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' ] - # Use only 'java-kotlin' to analyze code written in Java, Kotlin or both - # Use only 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both - # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v3 - with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - - # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs - # queries: security-extended,security-and-quality - - # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@v3 - - # ℹī¸ Command-line programs to run using the OS shell. - # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun - - # If the Autobuild fails above, remove it and uncomment the following three lines. - # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. - - # - run: | - # echo "Run, Build Application using script" - # ./location_of_script_within_repo/buildscript.sh - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 - with: - category: "/language:${{matrix.language}}" diff --git a/.github/workflows/jan-electron-linter-and-test.yml b/.github/workflows/jan-electron-linter-and-test.yml index 05d320bbd5..3c0e22de48 100644 --- a/.github/workflows/jan-electron-linter-and-test.yml +++ b/.github/workflows/jan-electron-linter-and-test.yml @@ -51,7 +51,7 @@ jobs: uses: actions/checkout@v3 - name: Installing node - uses: actions/setup-node@v1 + uses: actions/setup-node@v3 with: node-version: 20 @@ -87,7 +87,7 @@ jobs: uses: actions/checkout@v3 - name: Installing node - uses: actions/setup-node@v1 + uses: actions/setup-node@v3 with: node-version: 20 @@ -149,7 +149,7 @@ jobs: uses: actions/checkout@v3 - name: Installing node - uses: actions/setup-node@v1 + uses: actions/setup-node@v3 with: node-version: 20 @@ -157,7 +157,7 @@ jobs: continue-on-error: true run: | make clean - + - name: Linter and test run: | export DISPLAY=$(w -h | awk 'NR==1 {print $2}') diff --git a/Makefile b/Makefile index 0a53a18edc..77d7e90597 100644 --- a/Makefile +++ b/Makefile @@ -17,6 +17,7 @@ install-and-build: build-uikit ifeq ($(OS),Windows_NT) yarn config set network-timeout 300000 endif + yarn global add turbo yarn build:core yarn build:server yarn install @@ -24,9 +25,9 @@ endif check-file-counts: install-and-build ifeq ($(OS),Windows_NT) - powershell -Command "if ((Get-ChildItem -Path pre-install -Filter *.tgz | Measure-Object | Select-Object -ExpandProperty Count) -ne (Get-ChildItem -Path extensions -Directory | Measure-Object | Select-Object -ExpandProperty Count)) { Write-Host 'Number of .tgz files in pre-install does not match the number of subdirectories in extension'; exit 1 } else { Write-Host 'Extension build successful' }" + powershell -Command "if ((Get-ChildItem -Path pre-install -Filter *.tgz | Measure-Object | Select-Object -ExpandProperty Count) -ne (Get-ChildItem -Path extensions -Directory | Where-Object Name -like *-extension* | Measure-Object | Select-Object -ExpandProperty Count)) { Write-Host 'Number of .tgz files in pre-install does not match the number of subdirectories in extensions with package.json'; exit 1 } else { Write-Host 'Extension build successful' }" else - @tgz_count=$$(find pre-install -type f -name "*.tgz" | wc -l); dir_count=$$(find extensions -mindepth 1 -maxdepth 1 -type d | wc -l); if [ $$tgz_count -ne $$dir_count ]; then echo "Number of .tgz files in pre-install ($$tgz_count) does not match the number of subdirectories in extension ($$dir_count)"; exit 1; else echo "Extension build successful"; fi + @tgz_count=$$(find pre-install -type f -name "*.tgz" | wc -l); dir_count=$$(find extensions -mindepth 1 -maxdepth 1 -type d -exec test -e '{}/package.json' \; -print | wc -l); if [ $$tgz_count -ne $$dir_count ]; then echo "Number of .tgz files in pre-install ($$tgz_count) does not match the number of subdirectories in extension ($$dir_count)"; exit 1; else echo "Extension build successful"; fi endif dev: check-file-counts diff --git a/extensions/package.json b/extensions/package.json new file mode 100644 index 0000000000..b5b34bf79c --- /dev/null +++ b/extensions/package.json @@ -0,0 +1,11 @@ +{ + "private": true, + "workspaces": { + "packages": [ + "**" + ], + "nohoist": [ + "**" + ] + } +} diff --git a/extensions/turbo.json b/extensions/turbo.json new file mode 100644 index 0000000000..ec87739bd2 --- /dev/null +++ b/extensions/turbo.json @@ -0,0 +1,15 @@ +{ + "$schema": "https://turbo.build/schema.json", + "pipeline": { + "build": { + "outputs": [".dist/**", "*.tgz"] + }, + "build:publish": { + "dependsOn": ["^build"] + }, + "dev": { + "cache": false + }, + "type-check": {} + } +} diff --git a/package.json b/package.json index 79ed81e7a9..8f85b7eeb9 100644 --- a/package.json +++ b/package.json @@ -18,31 +18,30 @@ ] }, "scripts": { - "lint": "yarn workspace jan lint && yarn workspace jan-web lint", + "lint": "yarn workspace jan lint && yarn workspace @janhq/web lint", "test:unit": "yarn workspace @janhq/core test", "test": "yarn workspace jan test:e2e", + "test-local": "yarn lint && yarn build:test && yarn test", + "pre-install:darwin": "find extensions -type f -path \"**/*.tgz\" -exec cp {} pre-install \\;", + "pre-install:linux": "find extensions -type f -path \"**/*.tgz\" -exec cp {} pre-install \\;", + "pre-install:win32": "powershell -Command \"Get-ChildItem -Path \"extensions\" -Recurse -File -Filter \"*.tgz\" | ForEach-Object { Copy-Item -Path $_.FullName -Destination \"pre-install\" }\"", + "pre-install": "run-script-os", "copy:assets": "cpx \"models/**\" \"electron/models/\" && cpx \"pre-install/*.tgz\" \"electron/pre-install/\" && cpx \"docs/openapi/**\" \"electron/docs/openapi\"", "dev:electron": "yarn copy:assets && yarn workspace jan dev", - "dev:web": "yarn workspace jan-web dev", + "dev:web": "yarn workspace @janhq/web dev", "dev:server": "yarn copy:assets && yarn workspace @janhq/server dev", - "dev": "concurrently --kill-others \"yarn dev:web\" \"wait-on http://localhost:3000 && yarn dev:electron\"", - "test-local": "yarn lint && yarn build:test && yarn test", + "dev": "turbo run dev --parallel --filter=!@janhq/server", "dev:uikit": "yarn workspace @janhq/uikit install && yarn workspace @janhq/uikit dev", "build:uikit": "yarn workspace @janhq/uikit install && yarn workspace @janhq/uikit build", "build:server": "yarn copy:assets && cd server && yarn install && yarn run build", "build:core": "cd core && yarn install && yarn run build", - "build:web": "yarn workspace jan-web build && cpx \"web/out/**\" \"electron/renderer/\"", + "build:web": "yarn workspace @janhq/web build && cpx \"web/out/**\" \"electron/renderer/\"", "build:electron": "yarn copy:assets && yarn workspace jan build", "build:electron:test": "yarn workspace jan build:test", - "build:extensions:windows": "rimraf ./pre-install/*.tgz && powershell -command \"$jobs = Get-ChildItem -Path './extensions' -Directory | ForEach-Object { Start-Job -Name ($_.Name) -ScriptBlock { param($_dir); try { Set-Location $_dir; yarn; yarn build:publish; Write-Output 'Build successful in ' + $_dir } catch { Write-Error 'Error in ' + $_dir; throw } } -ArgumentList $_.FullName }; $jobs | Wait-Job; $jobs | ForEach-Object { Receive-Job -Job $_ -Keep } | ForEach-Object { Write-Host $_ }; $failed = $jobs | Where-Object { $_.State -ne 'Completed' -or $_.ChildJobs[0].JobStateInfo.State -ne 'Completed' }; if ($failed) { Exit 1 }\"", - "build:extensions:linux": "rimraf ./pre-install/*.tgz && find ./extensions -mindepth 1 -maxdepth 1 -type d -print0 | xargs -0 -n 1 -P 4 -I {} sh -c 'cd {}; yarn; yarn build:publish'", - "build:extensions:darwin": "rimraf ./pre-install/*.tgz && find ./extensions -mindepth 1 -maxdepth 1 -type d -print0 | xargs -0 -n 1 -P 4 -I {} sh -c 'cd {}; yarn; yarn build:publish'", - "build:extensions:server": "yarn workspace build:extensions ", - "build:extensions": "run-script-os", - "build:test": "yarn copy:assets && yarn build:web && yarn workspace jan build:test", + "build:extensions": "rimraf ./pre-install/*.tgz && turbo run @janhq/core#build && cd extensions && yarn install && turbo run build:publish && cd .. && yarn pre-install", + "build:test": "yarn copy:assets && turbo run @janhq/web#build && cpx \"web/out/**\" \"electron/renderer/\" && turbo run build:test", "build": "yarn build:web && yarn build:electron", - "build:publish": "yarn copy:assets && yarn build:web && yarn workspace jan build:publish", - "turbo:electron": "turbo run dev --parallel --filter=!@janhq/server" + "build:publish": "yarn copy:assets && yarn build:web && yarn workspace jan build:publish" }, "devDependencies": { "concurrently": "^8.2.1", diff --git a/turbo.json b/turbo.json index 0e53ece2d2..c6176830c7 100644 --- a/turbo.json +++ b/turbo.json @@ -2,27 +2,48 @@ "$schema": "https://turbo.build/schema.json", "pipeline": { "build": { - "outputs": [".next/**", "!.next/cache/**"] + "outputs": [".next/**", "!.next/cache/**", "dist/**"] }, "dev": { "cache": false }, - "web#build": { - "dependsOn": ["@janhq/core#build"] - }, - "web:dev": { + "@janhq/web#dev": { "cache": false, "persistent": true, "dependsOn": ["@janhq/core#build", "@janhq/uikit#build"] }, - "electron:dev": { + "@janhq/server#build": { + "outputs": ["dist/**"], + "dependsOn": ["@janhq/core#build"] + }, + "jan#dev": { "cache": false, "persistent": true, - "dependsOn": ["@janhq/core#build", "@janhq/server#build", "jan#compile"] + "dependsOn": ["@janhq/core#build", "@janhq/server#build"] }, - "electron#build": { - "dependsOn": ["web#build", "server#build", "core#build"], - "cache": false + "@janhq/core#build": { + "outputs": ["dist/**"] + }, + "@janhq/web#build": { + "outputs": ["out/**"], + "dependsOn": ["@janhq/core#build", "@janhq/uikit#build"] + }, + "jan#build": { + "outputs": ["dist/**"], + "dependsOn": [ + "@janhq/core#build", + "@janhq/server#build", + "@janhq/web#build" + ] + }, + "jan#build:test": { + "outputs": ["dist/**"], + "cache": false, + "dependsOn": [ + "@janhq/core#build", + "@janhq/server#build", + "@janhq/web#build" + ] }, "type-check": {} } diff --git a/web/package.json b/web/package.json index 84d489c007..94675151e9 100644 --- a/web/package.json +++ b/web/package.json @@ -1,5 +1,5 @@ { - "name": "jan-web", + "name": "@janhq/web", "version": "0.1.0", "private": true, "homepage": "./",