pnpm setup (bluesky-social#1532)

* use pnpm

* fix dependency issues, replace yarn and lerna scripts

* remove the main/dist scripts

* update Dockerfiles

* use pnpm

* fix dependency issues, replace yarn and lerna scripts

* remove the main/dist scripts

* update Dockerfiles

* update bin script

* remove unused zod dep

* fix type errors in pds

* add types prop to packages

* remove unused, bump lock

* fix test running

* build before test

* fix pino types

* format

* pds depends on dev-env in test

* refer to src instead of built packages

* pds relies on bsky in test too

* remove yarn.lock

* add -r flag to root test

* test push to aws

* remove docker test

* add publishConfig to new package

* move services to top level dir

(cherry picked from commit f5012bec33435a4473e9960066807623334f3aff)

* update workflow paths

(cherry picked from commit 5c70f0176d381ca35d6be10cfa173e22373a5b5d)

* add main-to-dist script

* use script in all packages, remove old Dockerfiles

* remove old bsky service

* remove newline

* test container builds

* Revert "test container builds"

This reverts commit c228611.

* remove unused config

* test build containers

* pnpm in syntax

* bump dd-trace

* shamefully hoist

* even more shame

* hoist, externalize deps

* clean install for prod and smaller containers

* dont build branches

---------

Co-authored-by: dholms <[email protected]>

.github/workflows/build-and-push-bsky-aws.yaml

@@ -47,7 +47,7 @@ jobs:
         with:
           context: .
           push: ${{ github.event_name != 'pull_request' }}
-          file: ./packages/bsky/Dockerfile
+          file: ./services/bsky/Dockerfile
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=gha

.github/workflows/build-and-push-bsky-ghcr.yaml

@@ -49,7 +49,7 @@ jobs:
         with:
           context: .
           push: ${{ github.event_name != 'pull_request' }}
-          file: ./packages/bsky/Dockerfile
+          file: ./services/bsky/Dockerfile
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=gha

.github/workflows/build-and-push-pds-aws.yaml

@@ -47,7 +47,7 @@ jobs:
         with:
           context: .
           push: ${{ github.event_name != 'pull_request' }}
-          file: ./packages/pds/Dockerfile
+          file: ./services/pds/Dockerfile
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=gha

.github/workflows/build-and-push-pds-ghcr.yaml

@@ -49,7 +49,7 @@ jobs:
         with:
           context: .
           push: ${{ github.event_name != 'pull_request' }}
-          file: ./packages/pds/Dockerfile
+          file: ./services/pds/Dockerfile
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=gha

.github/workflows/repo.yaml

@@ -12,32 +12,41 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+      - uses: pnpm/action-setup@v2
+        with:
+          version: 8
       - uses: actions/setup-node@v3
         with:
           node-version: 18
-          cache: "yarn"
-      - run: yarn install --frozen-lockfile
-      - run: yarn build
+          cache: "pnpm"
+      - run: pnpm i --frozen-lockfile
+      - run: pnpm build
   test:
     strategy:
       matrix:
         shard: [1/8, 2/8, 3/8, 4/8, 5/8, 6/8, 7/8, 8/8]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+      - uses: pnpm/action-setup@v2
+        with:
+          version: 8
       - uses: actions/setup-node@v3
         with:
           node-version: 18
-          cache: "yarn"
-      - run: yarn install --frozen-lockfile
-      - run: yarn test:withFlags --maxWorkers=1 --shard=${{ matrix.shard }} --passWithNoTests
+          cache: "pnpm"
+      - run: pnpm i --frozen-lockfile
+      - run: pnpm test:withFlags --maxWorkers=1 --shard=${{ matrix.shard }} --passWithNoTests
   verify:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+      - uses: pnpm/action-setup@v2
+        with:
+          version: 8
       - uses: actions/setup-node@v3
         with:
           node-version: 18
-          cache: "yarn"
-      - run: yarn install --frozen-lockfile
-      - run: yarn verify
+          cache: "pnpm"
+      - run: pnpm install --frozen-lockfile
+      - run: pnpm verify

.npmrc

@@ -0,0 +1 @@
+enable-pre-post-scripts = true

Makefile

@@ -12,46 +12,46 @@ help: ## Print info about all commands
 
 .PHONY: build
 build: ## Compile all modules
-	yarn build
+	pnpm build
 
 .PHONY: test
 test: ## Run all tests
-	yarn test
+	pnpm test
 
 .PHONY: run-dev-env
 run-dev-env: ## Run a "development environment" shell
-	cd packages/dev-env; yarn run start
+	cd packages/dev-env; pnpm run start
 
 .PHONY: run-dev-pds
 run-dev-pds: ## Run PDS locally
 	if [ ! -f "packages/pds/.dev.env" ]; then cp packages/pds/example.dev.env packages/pds/.dev.env; fi
-	cd packages/pds; ENV=dev yarn run start | yarn exec pino-pretty
+	cd packages/pds; ENV=dev pnpm run start | pnpm exec pino-pretty
 
 .PHONY: run-dev-bsky
 run-dev-bsky: ## Run appview ('bsky') locally
 	if [ ! -f "packages/bsky/.dev.env" ]; then cp packages/bsky/example.dev.env packages/bsky/.dev.env; fi
-	cd packages/bsky; ENV=dev yarn run start | yarn exec pino-pretty
+	cd packages/bsky; ENV=dev pnpm run start | pnpm exec pino-pretty
 
 .PHONY: codegen
 codegen: ## Re-generate packages from lexicon/ files
-	cd packages/api; yarn run codegen
-	cd packages/pds; yarn run codegen
-	cd packages/bsky; yarn run codegen
+	cd packages/api; pnpm run codegen
+	cd packages/pds; pnpm run codegen
+	cd packages/bsky; pnpm run codegen
 
 .PHONY: lint
 lint: ## Run style checks and verify syntax
-	yarn verify
+	pnpm verify
 
 .PHONY: fmt
 fmt: ## Run syntax re-formatting
-	yarn prettier
+	pnpm prettier
 
 .PHONY: deps
-deps: ## Installs dependent libs using 'yarn install'
-	yarn install --frozen-lockfile
+deps: ## Installs dependent libs using 'pnpm install'
+	pnpm install --frozen-lockfile
 
 .PHONY: nvm-setup
-nvm-setup: ## Use NVM to install and activate node+yarn
+nvm-setup: ## Use NVM to install and activate node+pnpm
 	nvm install 18
 	nvm use 18
-	npm install --global yarn
+	npm install --global pnpm

package.json

@@ -9,12 +9,12 @@
     "node": ">=18"
   },
   "scripts": {
-    "prepublish": "yarn build",
-    "verify": "lerna run verify --stream",
-    "prettier": "lerna run prettier",
-    "build": "lerna run build",
-    "test": "LOG_ENABLED=false NODE_ENV=development ./packages/dev-infra/with-test-redis-and-db.sh lerna run test --stream",
-    "test:withFlags": "LOG_ENABLED=false NODE_ENV=development ./packages/dev-infra/with-test-redis-and-db.sh lerna run test --stream --"
+    "verify": "pnpm -r --stream lint && pnpm -r --stream prettier",
+    "format": "pnpm -r --stream prettier:fix",
+    "build": "pnpm -r --stream build",
+    "update-main-to-dist": "pnpm -r --stream update-main-to-dist",
+    "test": "LOG_ENABLED=false NODE_ENV=development ./packages/dev-infra/with-test-redis-and-db.sh pnpm --stream -r test",
+    "test:withFlags": "LOG_ENABLED=false NODE_ENV=development ./packages/dev-infra/with-test-redis-and-db.sh pnpm --stream -r test --"
   },
   "devDependencies": {
     "@babel/core": "^7.18.6",
@@ -35,7 +35,6 @@
     "eslint-config-prettier": "^8.5.0",
     "eslint-plugin-prettier": "^4.2.1",
     "jest": "^28.1.2",
-    "lerna": "^4.0.0",
     "node-gyp": "^9.3.1",
     "npm-run-all": "^4.1.5",
     "pino-pretty": "^9.1.0",

packages/README.md

@@ -21,11 +21,11 @@
 
 Only applicable to packages which contain benchmarks(`jest.bench.config.js`).
 
-You can run benchmarks with `yarn bench`.
+You can run benchmarks with `pnpm bench`.
 
 ### Attaching a profiler
 
-Running `yarn bench:profile` will launch `bench` with `--inspect-brk` flag.
+Running `pnpm bench:profile` will launch `bench` with `--inspect-brk` flag.
 Execution will be paused until a debugger is attached, you can read more 
 about node debuggers [here](https://nodejs.org/en/docs/guides/debugging-getting-started#inspector-clients)
 

packages/api/build.js

@@ -1,16 +1,8 @@
-const pkgJson = require('@npmcli/package-json')
 const { nodeExternalsPlugin } = require('esbuild-node-externals')
 
 const buildShallow =
   process.argv.includes('--shallow') || process.env.ATP_BUILD_SHALLOW === 'true'
 
-if (process.argv.includes('--update-main-to-dist')) {
-  return pkgJson
-    .load(__dirname)
-    .then((pkg) => pkg.update({ main: 'dist/index.js' }))
-    .then((pkg) => pkg.save())
-}
-
 require('esbuild').build({
   logLevel: 'info',
   entryPoints: ['src/index.ts'],

packages/api/package.json

@@ -2,15 +2,16 @@
   "name": "@atproto/api",
   "version": "0.6.10",
   "main": "src/index.ts",
+  "publishConfig": {
+    "main": "dist/index.js",
+    "types": "dist/index.d.ts"
+  },
   "scripts": {
-    "codegen": "yarn docgen && node ./scripts/generate-code.mjs && lex gen-api ./src/client ../../lexicons/com/atproto/*/* ../../lexicons/app/bsky/*/*",
+    "codegen": "pnpm docgen && node ./scripts/generate-code.mjs && lex gen-api ./src/client ../../lexicons/com/atproto/*/* ../../lexicons/app/bsky/*/*",
     "docgen": "node ./scripts/generate-docs.mjs",
     "build": "node ./build.js",
     "postbuild": "tsc --build tsconfig.build.json",
-    "update-main-to-dist": "node ./update-pkg.js --update-main-to-dist",
-    "update-main-to-src": "node ./update-pkg.js --update-main-to-src",
-    "prepublish": "npm run update-main-to-dist",
-    "postpublish": "npm run update-main-to-src",
+    "update-main-to-dist": "node ../../update-main-to-dist.js packages/api",
     "test": "jest",
     "bench": "jest --config jest.bench.config.js",
     "bench:profile": "node --inspect-brk ../../node_modules/.bin/jest --config jest.bench.config.js"
@@ -22,15 +23,17 @@
     "directory": "packages/api"
   },
   "dependencies": {
-    "@atproto/common-web": "*",
-    "@atproto/syntax": "*",
-    "@atproto/xrpc": "*",
+    "@atproto/common-web": "workspace:^",
+    "@atproto/lexicon": "workspace:^",
+    "@atproto/syntax": "workspace:^",
+    "@atproto/xrpc": "workspace:^",
+    "multiformats": "^9.9.0",
     "tlds": "^1.234.0",
     "typed-emitter": "^2.1.0"
   },
   "devDependencies": {
-    "@atproto/lex-cli": "*",
-    "@atproto/pds": "*",
+    "@atproto/lex-cli": "workspace:^",
+    "@atproto/pds": "workspace:^",
     "common-tags": "^1.8.2"
   }
 }

packages/aws/build.js

@@ -1,16 +1,8 @@
-const pkgJson = require('@npmcli/package-json')
 const { nodeExternalsPlugin } = require('esbuild-node-externals')
 
 const buildShallow =
   process.argv.includes('--shallow') || process.env.ATP_BUILD_SHALLOW === 'true'
 
-if (process.argv.includes('--update-main-to-dist')) {
-  return pkgJson
-    .load(__dirname)
-    .then((pkg) => pkg.update({ main: 'dist/index.js' }))
-    .then((pkg) => pkg.save())
-}
-
 require('esbuild').build({
   logLevel: 'info',
   entryPoints: ['src/index.ts'],

packages/aws/package.json

@@ -2,6 +2,10 @@
   "name": "@atproto/aws",
   "version": "0.1.0",
   "main": "src/index.ts",
+  "publishConfig": {
+    "main": "dist/index.js",
+    "types": "dist/src/index.d.ts"
+  },
   "license": "MIT",
   "repository": {
     "type": "git",
@@ -12,21 +16,23 @@
     "prettier": "prettier --check src/",
     "prettier:fix": "prettier --write src/",
     "lint": "eslint . --ext .ts,.tsx",
-    "lint:fix": "yarn lint --fix",
+    "lint:fix": "pnpm lint --fix",
     "verify": "run-p prettier lint",
-    "verify:fix": "yarn prettier:fix && yarn lint:fix",
+    "verify:fix": "pnpm prettier:fix && pnpm lint:fix",
     "build": "node ./build.js",
-    "postbuild": "tsc --build tsconfig.build.json"
+    "postbuild": "tsc --build tsconfig.build.json",
+    "update-main-to-dist": "node ../../update-main-to-dist.js packages/aws"
   },
   "dependencies": {
-    "@atproto/crypto": "*",
+    "@atproto/crypto": "workspace:^",
+    "@atproto/repo": "workspace:^",
     "@aws-sdk/client-cloudfront": "^3.261.0",
     "@aws-sdk/client-kms": "^3.196.0",
     "@aws-sdk/client-s3": "^3.224.0",
     "@aws-sdk/lib-storage": "^3.226.0",
     "@noble/curves": "^1.1.0",
     "key-encoder": "^2.0.3",
-    "multiformats": "^9.6.4",
+    "multiformats": "^9.9.0",
     "uint8arrays": "3.0.0"
   }
 }

packages/bsky/build.js

@@ -1,16 +1,8 @@
-const pkgJson = require('@npmcli/package-json')
 const { nodeExternalsPlugin } = require('esbuild-node-externals')
 
 const buildShallow =
   process.argv.includes('--shallow') || process.env.ATP_BUILD_SHALLOW === 'true'
 
-if (process.argv.includes('--update-main-to-dist')) {
-  return pkgJson
-    .load(__dirname)
-    .then((pkg) => pkg.update({ main: 'dist/index.js' }))
-    .then((pkg) => pkg.save())
-}
-
 require('esbuild').build({
   logLevel: 'info',
   entryPoints: ['src/index.ts', 'src/db/index.ts'],