fix: turborepo extensions (janhq#2392)

* fix: turborepo extensions

Update package.json

Update Makefile

Update Makefile

Update Makefile

Update Makefile

Update Makefile

Update package.json

* chore: turbo cache

* fix: install extensions in parallel

* fix: timeout issue

* Turbo cache using s3

* Remove cache task

---------

Co-authored-by: Hien To <[email protected]>
Co-authored-by: Service Account <[email protected]>

.github/workflows/jan-electron-linter-and-test.yml

@@ -51,7 +51,7 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Installing node
-        uses: actions/setup-node@v1
+        uses: actions/setup-node@v3
         with:
           node-version: 20
 
@@ -87,7 +87,7 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Installing node
-        uses: actions/setup-node@v1
+        uses: actions/setup-node@v3
         with:
           node-version: 20
 
@@ -149,15 +149,15 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Installing node
-        uses: actions/setup-node@v1
+        uses: actions/setup-node@v3
         with:
           node-version: 20
 
       - name: "Cleanup cache"
         continue-on-error: true
         run: |
           make clean
-
+            
       - name: Linter and test
         run: |
           export DISPLAY=$(w -h | awk 'NR==1 {print $2}')

Makefile

@@ -17,16 +17,17 @@ install-and-build: build-uikit
 ifeq ($(OS),Windows_NT)
 	yarn config set network-timeout 300000
 endif
+	yarn global add turbo
 	yarn build:core
 	yarn build:server
 	yarn install
 	yarn build:extensions
 
 check-file-counts: install-and-build
 ifeq ($(OS),Windows_NT)
-	powershell -Command "if ((Get-ChildItem -Path pre-install -Filter *.tgz | Measure-Object | Select-Object -ExpandProperty Count) -ne (Get-ChildItem -Path extensions -Directory | Measure-Object | Select-Object -ExpandProperty Count)) { Write-Host 'Number of .tgz files in pre-install does not match the number of subdirectories in extension'; exit 1 } else { Write-Host 'Extension build successful' }"
+	powershell -Command "if ((Get-ChildItem -Path pre-install -Filter *.tgz | Measure-Object | Select-Object -ExpandProperty Count) -ne (Get-ChildItem -Path extensions -Directory | Where-Object Name -like *-extension* | Measure-Object | Select-Object -ExpandProperty Count)) { Write-Host 'Number of .tgz files in pre-install does not match the number of subdirectories in extensions with package.json'; exit 1 } else { Write-Host 'Extension build successful' }"
 else
-	@tgz_count=$$(find pre-install -type f -name "*.tgz" | wc -l); dir_count=$$(find extensions -mindepth 1 -maxdepth 1 -type d | wc -l); if [ $$tgz_count -ne $$dir_count ]; then echo "Number of .tgz files in pre-install ($$tgz_count) does not match the number of subdirectories in extension ($$dir_count)"; exit 1; else echo "Extension build successful"; fi
+	@tgz_count=$$(find pre-install -type f -name "*.tgz" | wc -l); dir_count=$$(find extensions -mindepth 1 -maxdepth 1 -type d -exec test -e '{}/package.json' \; -print | wc -l); if [ $$tgz_count -ne $$dir_count ]; then echo "Number of .tgz files in pre-install ($$tgz_count) does not match the number of subdirectories in extension ($$dir_count)"; exit 1; else echo "Extension build successful"; fi
 endif
 
 dev: check-file-counts

extensions/package.json

@@ -0,0 +1,11 @@
+{
+  "private": true,
+  "workspaces": {
+    "packages": [
+      "**"
+    ],
+    "nohoist": [
+      "**"
+    ]
+  }
+}

extensions/turbo.json

@@ -0,0 +1,15 @@
+{
+  "$schema": "https://turbo.build/schema.json",
+  "pipeline": {
+    "build": {
+      "outputs": [".dist/**", "*.tgz"]
+    },
+    "build:publish": {
+      "dependsOn": ["^build"]
+    },
+    "dev": {
+      "cache": false
+    },
+    "type-check": {}
+  }
+}

package.json

@@ -18,31 +18,30 @@
     ]
   },
   "scripts": {
-    "lint": "yarn workspace jan lint && yarn workspace jan-web lint",
+    "lint": "yarn workspace jan lint && yarn workspace @janhq/web lint",
     "test:unit": "yarn workspace @janhq/core test",
     "test": "yarn workspace jan test:e2e",
+    "test-local": "yarn lint && yarn build:test && yarn test",
+    "pre-install:darwin": "find extensions -type f -path \"**/*.tgz\" -exec cp {} pre-install \\;",
+    "pre-install:linux": "find extensions -type f -path \"**/*.tgz\" -exec cp {} pre-install \\;",
+    "pre-install:win32": "powershell -Command \"Get-ChildItem -Path \"extensions\" -Recurse -File -Filter \"*.tgz\" | ForEach-Object { Copy-Item -Path $_.FullName -Destination \"pre-install\" }\"",
+    "pre-install": "run-script-os",
     "copy:assets": "cpx \"models/**\" \"electron/models/\" && cpx \"pre-install/*.tgz\" \"electron/pre-install/\" && cpx \"docs/openapi/**\" \"electron/docs/openapi\"",
     "dev:electron": "yarn copy:assets && yarn workspace jan dev",
-    "dev:web": "yarn workspace jan-web dev",
+    "dev:web": "yarn workspace @janhq/web dev",
     "dev:server": "yarn copy:assets && yarn workspace @janhq/server dev",
-    "dev": "concurrently --kill-others \"yarn dev:web\" \"wait-on http://localhost:3000 && yarn dev:electron\"",
-    "test-local": "yarn lint && yarn build:test && yarn test",
+    "dev": "turbo run dev --parallel --filter=!@janhq/server",
     "dev:uikit": "yarn workspace @janhq/uikit install && yarn workspace @janhq/uikit dev",
     "build:uikit": "yarn workspace @janhq/uikit install && yarn workspace @janhq/uikit build",
     "build:server": "yarn copy:assets && cd server && yarn install && yarn run build",
     "build:core": "cd core && yarn install && yarn run build",
-    "build:web": "yarn workspace jan-web build && cpx \"web/out/**\" \"electron/renderer/\"",
+    "build:web": "yarn workspace @janhq/web build && cpx \"web/out/**\" \"electron/renderer/\"",
     "build:electron": "yarn copy:assets && yarn workspace jan build",
     "build:electron:test": "yarn workspace jan build:test",
-    "build:extensions:windows": "rimraf ./pre-install/*.tgz && powershell -command \"$jobs = Get-ChildItem -Path './extensions' -Directory | ForEach-Object { Start-Job -Name ($_.Name) -ScriptBlock { param($_dir); try { Set-Location $_dir; yarn; yarn build:publish; Write-Output 'Build successful in ' + $_dir } catch { Write-Error 'Error in ' + $_dir; throw } } -ArgumentList $_.FullName }; $jobs | Wait-Job; $jobs | ForEach-Object { Receive-Job -Job $_ -Keep } | ForEach-Object { Write-Host $_ }; $failed = $jobs | Where-Object { $_.State -ne 'Completed' -or $_.ChildJobs[0].JobStateInfo.State -ne 'Completed' }; if ($failed) { Exit 1 }\"",
-    "build:extensions:linux": "rimraf ./pre-install/*.tgz && find ./extensions -mindepth 1 -maxdepth 1 -type d -print0 | xargs -0 -n 1 -P 4 -I {} sh -c 'cd {}; yarn; yarn build:publish'",
-    "build:extensions:darwin": "rimraf ./pre-install/*.tgz && find ./extensions -mindepth 1 -maxdepth 1 -type d -print0 | xargs -0 -n 1 -P 4 -I {} sh -c 'cd {}; yarn; yarn build:publish'",
-    "build:extensions:server": "yarn workspace build:extensions ",
-    "build:extensions": "run-script-os",
-    "build:test": "yarn copy:assets && yarn build:web && yarn workspace jan build:test",
+    "build:extensions": "rimraf ./pre-install/*.tgz && turbo run @janhq/core#build && cd extensions && yarn install && turbo run build:publish && cd .. && yarn pre-install",
+    "build:test": "yarn copy:assets && turbo run @janhq/web#build && cpx \"web/out/**\" \"electron/renderer/\" && turbo run build:test",
     "build": "yarn build:web && yarn build:electron",
-    "build:publish": "yarn copy:assets && yarn build:web && yarn workspace jan build:publish",
-    "turbo:electron": "turbo run dev --parallel --filter=!@janhq/server"
+    "build:publish": "yarn copy:assets && yarn build:web && yarn workspace jan build:publish"
   },
   "devDependencies": {
     "concurrently": "^8.2.1",

turbo.json

@@ -2,27 +2,48 @@
   "$schema": "https://turbo.build/schema.json",
   "pipeline": {
     "build": {
-      "outputs": [".next/**", "!.next/cache/**"]
+      "outputs": [".next/**", "!.next/cache/**", "dist/**"]
     },
     "dev": {
       "cache": false
     },
-    "web#build": {
-      "dependsOn": ["@janhq/core#build"]
-    },
-    "web:dev": {
+    "@janhq/web#dev": {
       "cache": false,
       "persistent": true,
       "dependsOn": ["@janhq/core#build", "@janhq/uikit#build"]
     },
-    "electron:dev": {
+    "@janhq/server#build": {
+      "outputs": ["dist/**"],
+      "dependsOn": ["@janhq/core#build"]
+    },
+    "jan#dev": {
       "cache": false,
       "persistent": true,
-      "dependsOn": ["@janhq/core#build", "@janhq/server#build", "jan#compile"]
+      "dependsOn": ["@janhq/core#build", "@janhq/server#build"]
     },
-    "electron#build": {
-      "dependsOn": ["web#build", "server#build", "core#build"],
-      "cache": false
+    "@janhq/core#build": {
+      "outputs": ["dist/**"]
+    },
+    "@janhq/web#build": {
+      "outputs": ["out/**"],
+      "dependsOn": ["@janhq/core#build", "@janhq/uikit#build"]
+    },
+    "jan#build": {
+      "outputs": ["dist/**"],
+      "dependsOn": [
+        "@janhq/core#build",
+        "@janhq/server#build",
+        "@janhq/web#build"
+      ]
+    },
+    "jan#build:test": {
+      "outputs": ["dist/**"],
+      "cache": false,
+      "dependsOn": [
+        "@janhq/core#build",
+        "@janhq/server#build",
+        "@janhq/web#build"
+      ]
     },
     "type-check": {}
   }

web/package.json

@@ -1,5 +1,5 @@
 {
-  "name": "jan-web",
+  "name": "@janhq/web",
   "version": "0.1.0",
   "private": true,
   "homepage": "./",