Bsky app view (bluesky-social#716)

* Init pulling bsky app view from pds package into its own package, remove sqlite db dialect

* Cull bsky config, services, auth, etc.

* Sweep app view xrpc methods, tidy deps, add storage back for img server

* Run repo subscription on bsky app view

* Collapse db migrations down for bsky app view

* Tidy app view bin

* Remove mute functionality from app view, delegate to pds

* Initial tidy/culling of bsky app view tests

* Passing bsky app view db, server, and repo subscription tests

* Passing bsky app view duplicate-records tests

* Bsky app view test tidy/cull

* In bsky app view replace repo_root, ipld_block, did_handle with actor and record tables. Remove assertions/confirmations.

* Update bsky impl for simpler actor and record tables, removed asserion/confirmations. Skip indexing unknown collections.

* Setup actor handles by did in bsky app view

* Passing indexing tests on bsky app view

* Passing image tests on bsky app view

* Fix bsky actor reindexing, support custom lock id for testing repo subs

* Sweep bsky view tests, misc tests, passing

* Tidy bsky deps

* Include did in resized image uris

* Update bsky image process server to use getBlob

* Update image server tests, misc fixes

* Implement bsky blob resolver

* Wire local image processing server to local blob resolver, test blob resolver

* Tidy

* Tidy

* Tidy

* Tidy app view init

* Fix handle resolution, tidy

* Add utils for partitioning indexing by did

* Update repo sub to parallelize work per repo

* Dep tidy

* Tidy bsky tests for updated repo sub destroy()

* Update thead indexing to handle out-of-order posts

* Sketch out strategy in bsky for handling too-big commits

* Set content-type on sync.getBlob

* Add logging for failed transmissions in bsky blob resolver

* Tidy

* Tidy bsky repo indexing and supporting repo interfaces

* Sort in app view based on combo of creation and indexing times

* Fix types

* Add retry utils to bsky

* Add retries to http requests made by bsky

* Test repo indexing

* Update bsky db/model for lex refactor

* Update bsky lexicons for lex refactor

* Update bsky actor service for lex refactor

* Update bsky feed service for lex refactor

* Update bsky indexing service for lex refactor

* Update bsky repo subscription for lex refactor

* Tidy bsky repo sub

* Add unspecced endpoints to bsky app view, update entrypoint

* Update bsky xrpc utils for lex refactor

* Update bsky xrpc methods for lex refactor

* Update bsky test seeds for lex refactor, tidy api entrypoint

* Update bsky non-view tests for lex refactor

* Update bsky likes view test for lex refactor, minor fix

* Update bsky author feed tests for lex refactor, minor test util fix

* Update bsky follow, profile, repost, search view tests for lex refactor

* Update bsky timeline view tests for lex refactor

* Replace bsky out-of-order thread indexing logic

* Update bsky thread view tests for lex refactor, general test tidying

* Handle rebases and too-big commits in repo subscription, tracking commit data cid

* Tidy

* Ensure did resolver reports "not found" only when positively not found

* Handle tombstones and handle updates in bsky

* Test indexing handle updates and did tombstones

* Support cors on bsky

* Allow app view to serve most routes unauthed

* Tests for bsky unauthed views

* Tidy bsky service entrypoint and dockerfile

* Remove unused storage interfaces from bsky

* Bsky entrypoint and dockerfile fixes, tidy

* Add workflow for bsky build to aws

* Use more standard db env variables, make migration creds optional

* Make bsky repo subscription optional

* Fix lex->json serialization in bsky

* Split bsky actor sync state into its own table

* Skip invalid records on indexing full repo, tidy

* Tidy

* Leader test timing

* Tidy/lint

* Fix bsky config overrides

.github/workflows/build-and-push-bsky-aws.yaml

@@ -0,0 +1,54 @@
+name: build-and-push-bsky-aws
+on:
+  push:
+    branches:
+      - main
+      - bsky-app-view
+env:
+  REGISTRY: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_REGISTRY }}
+  USERNAME: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_USERNAME }}
+  PASSWORD: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_PASSWORD }}
+  IMAGE_NAME: bsky-app-view
+
+jobs:
+  bsky-container-aws:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+      id-token: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Setup Docker buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Log into registry ${{ env.REGISTRY }}
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ env.USERNAME}}
+          password: ${{ env.PASSWORD }}
+
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=sha,enable=true,priority=100,prefix=,suffix=,format=long
+
+      - name: Build and push Docker image
+        id: build-and-push
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          push: ${{ github.event_name != 'pull_request' }}
+          file: ./packages/bsky/Dockerfile
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

packages/bsky/.env

@@ -0,0 +1 @@
+DEBUG_MODE="1"

packages/bsky/.eslintignore

@@ -0,0 +1 @@
+lexicon

packages/bsky/.prettierignore

@@ -0,0 +1 @@
+src/lexicon/**/*

packages/bsky/Dockerfile

@@ -0,0 +1,50 @@
+FROM node:18-alpine as build
+
+# Move files into the image and install
+WORKDIR /app
+COPY ./*.* ./
+# NOTE bsky's transitive dependencies go here: if that changes, this needs to be updated.
+COPY ./packages/bsky ./packages/bsky
+COPY ./packages/api ./packages/api
+COPY ./packages/common ./packages/common
+COPY ./packages/common-web ./packages/common-web
+COPY ./packages/crypto ./packages/crypto
+COPY ./packages/did-resolver ./packages/did-resolver
+COPY ./packages/identifier ./packages/identifier
+COPY ./packages/lexicon ./packages/lexicon
+COPY ./packages/nsid ./packages/nsid
+COPY ./packages/repo ./packages/repo
+COPY ./packages/uri ./packages/uri
+COPY ./packages/xrpc ./packages/xrpc
+COPY ./packages/xrpc-server ./packages/xrpc-server
+RUN ATP_BUILD_SHALLOW=true yarn install --frozen-lockfile > /dev/null
+RUN yarn workspaces run build --update-main-to-dist > /dev/null
+# Remove non-prod deps
+RUN yarn install --production --ignore-scripts --prefer-offline > /dev/null
+
+WORKDIR packages/bsky/service
+RUN yarn install --frozen-lockfile > /dev/null
+
+# Uses assets from build stage to reduce build size
+FROM node:18-alpine
+
+# RUN npm install -g yarn
+RUN apk add --update dumb-init
+
+# Avoid zombie processes, handle signal forwarding
+ENTRYPOINT ["dumb-init", "--"]
+
+WORKDIR /app/packages/bsky/service
+COPY --from=build /app /app
+
+EXPOSE 3000
+ENV PORT=3000
+ENV NODE_ENV=production
+
+# https://github.com/nodejs/docker-node/blob/master/docs/BestPractices.md#non-root-user
+USER node
+CMD ["node", "--enable-source-maps", "index.js"]
+
+LABEL org.opencontainers.image.source=https://github.com/bluesky-social/atproto
+LABEL org.opencontainers.image.description="Bsky App View"
+LABEL org.opencontainers.image.licenses=MIT

packages/bsky/README.md

@@ -0,0 +1,3 @@
+# Bsky App View
+
+The Bsky App View. This contains the indexers and XRPC methods for reading data from the Bsky application.

packages/bsky/babel.config.js

@@ -0,0 +1,3 @@
+module.exports = {
+  presets: [['@babel/preset-env']],
+}

packages/bsky/bin/migration-create.ts

@@ -0,0 +1,38 @@
+#!/usr/bin/env ts-node
+
+import * as fs from 'fs/promises'
+import * as path from 'path'
+
+export async function main() {
+  const now = new Date()
+  const prefix = now.toISOString().replace(/[^a-z0-9]/gi, '') // Order of migrations matches alphabetical order of their names
+  const name = process.argv[2]
+  if (!name || !name.match(/^[a-z0-9-]+$/)) {
+    process.exitCode = 1
+    return console.error(
+      'Must pass a migration name consisting of lowercase digits, numbers, and dashes.',
+    )
+  }
+  const filename = `${prefix}-${name}`
+  const dir = path.join(__dirname, '..', 'src', 'db', 'migrations')
+
+  await fs.writeFile(path.join(dir, `${filename}.ts`), template, { flag: 'wx' })
+  await fs.writeFile(
+    path.join(dir, 'index.ts'),
+    `export * as _${prefix} from './${filename}'\n`,
+    { flag: 'a' },
+  )
+}
+
+const template = `import { Kysely } from 'kysely'
+
+export async function up(db: Kysely<unknown>): Promise<void> {
+  // Migration code
+}
+
+export async function down(db: Kysely<unknown>): Promise<void> {
+  // Migration code
+}
+`
+
+main()

packages/bsky/build.js

@@ -0,0 +1,27 @@
+const pkgJson = require('@npmcli/package-json')
+const { nodeExternalsPlugin } = require('esbuild-node-externals')
+
+const buildShallow =
+  process.argv.includes('--shallow') || process.env.ATP_BUILD_SHALLOW === 'true'
+
+if (process.argv.includes('--update-main-to-dist')) {
+  return pkgJson
+    .load(__dirname)
+    .then((pkg) => pkg.update({ main: 'dist/index.js' }))
+    .then((pkg) => pkg.save())
+}
+
+require('esbuild').build({
+  logLevel: 'info',
+  entryPoints: ['src/index.ts', 'src/bin.ts', 'src/db/index.ts'],
+  bundle: true,
+  sourcemap: true,
+  outdir: 'dist',
+  platform: 'node',
+  external: [
+    // Referenced in pg driver, but optional and we don't use it
+    'pg-native',
+    'sharp',
+  ],
+  plugins: buildShallow ? [nodeExternalsPlugin()] : [],
+})

packages/bsky/example.dev.env

@@ -0,0 +1,5 @@
+DB_POSTGRES_URL="postgres://bsky:yksb@localhost/pds_dev"
+DEBUG_MODE=1
+LOG_ENABLED="true"
+LOG_LEVEL=debug
+LOG_DESTINATION=1

packages/bsky/jest.config.js

@@ -0,0 +1,6 @@
+const base = require('../../jest.config.base.js')
+
+module.exports = {
+  ...base,
+  displayName: 'Bsky App View',
+}

packages/bsky/package.json

@@ -0,0 +1,65 @@
+{
+  "name": "@atproto/bsky",
+  "version": "0.0.0",
+  "license": "MIT",
+  "main": "src/index.ts",
+  "bin": "dist/bin.ts",
+  "scripts": {
+    "codegen": "lex gen-server ./src/lexicon ../../lexicons/com/atproto/*/* ../../lexicons/app/bsky/*/*",
+    "build": "node ./build.js",
+    "postbuild": "tsc --build tsconfig.build.json",
+    "start": "node --enable-source-maps dist/bin.js",
+    "test": "../pg/with-test-db.sh jest",
+    "test:log": "tail -50 test.log | pino-pretty",
+    "test:updateSnapshot": "jest --updateSnapshot",
+    "prettier": "prettier --check src/",
+    "prettier:fix": "prettier --write src/",
+    "lint": "eslint . --ext .ts,.tsx",
+    "lint:fix": "yarn lint --fix",
+    "verify": "run-p prettier lint",
+    "verify:fix": "yarn prettier:fix && yarn lint:fix",
+    "migration:create": "ts-node ./bin/migration-create.ts",
+    "update-main-to-dist": "node ./update-pkg.js --update-main-to-dist",
+    "update-main-to-src": "node ./update-pkg.js --update-main-to-src",
+    "prepublish": "npm run update-main-to-dist",
+    "postpublish": "npm run update-main-to-src"
+  },
+  "dependencies": {
+    "@atproto/api": "*",
+    "@atproto/common": "*",
+    "@atproto/crypto": "*",
+    "@atproto/did-resolver": "*",
+    "@atproto/identifier": "*",
+    "@atproto/lexicon": "*",
+    "@atproto/repo": "*",
+    "@atproto/uri": "*",
+    "@atproto/xrpc-server": "*",
+    "@did-plc/lib": "^0.0.1",
+    "cors": "^2.8.5",
+    "dotenv": "^16.0.0",
+    "express": "^4.17.2",
+    "express-async-errors": "^3.1.1",
+    "http-errors": "^2.0.0",
+    "http-terminator": "^3.2.0",
+    "kysely": "^0.22.0",
+    "multiformats": "^9.6.4",
+    "p-queue": "^6.6.2",
+    "pg": "^8.8.0",
+    "pino": "^8.6.1",
+    "pino-http": "^8.2.1",
+    "sharp": "^0.31.2",
+    "uint8arrays": "3.0.0"
+  },
+  "devDependencies": {
+    "@atproto/api": "*",
+    "@atproto/lex-cli": "*",
+    "@atproto/pds": "*",
+    "@atproto/xrpc": "*",
+    "@did-plc/server": "^0.0.1",
+    "@types/cors": "^2.8.12",
+    "@types/express": "^4.17.13",
+    "@types/pg": "^8.6.5",
+    "@types/sharp": "^0.31.0",
+    "axios": "^0.27.2"
+  }
+}

packages/bsky/service/index.js

@@ -0,0 +1,82 @@
+'use strict' /* eslint-disable */
+
+require('dd-trace/init') // Only works with commonjs
+  .tracer.use('express', {
+    hooks: {
+      request: (span, req) => {
+        maintainXrpcResource(span, req)
+      },
+    },
+  })
+
+// Tracer code above must come before anything else
+const path = require('path')
+const { Database, ServerConfig, BskyAppView } = require('@atproto/bsky')
+
+const main = async () => {
+  const env = getEnv()
+  // Migrate using credentialed user
+  const migrateDb = Database.postgres({
+    url: env.dbMigratePostgresUrl,
+    schema: env.dbPostgresSchema,
+  })
+  await migrateDb.migrateToLatestOrThrow()
+  await migrateDb.close()
+  // Use lower-credentialed user to run the app
+  const db = Database.postgres({
+    url: env.dbPostgresUrl,
+    schema: env.dbSchema,
+  })
+  const cfg = ServerConfig.readEnv({
+    port: env.port,
+    version: env.version,
+    repoProvider: env.repoProvider,
+    dbPostgresUrl: env.dbPostgresUrl,
+    dbPostgresSchema: env.dbPostgresSchema,
+    publicUrl: env.publicUrl,
+    didPlcUrl: env.didPlcUrl,
+    imgUriSalt: env.imgUriSalt,
+    imgUriKey: env.imgUriKey,
+    imgUriEndpoint: env.imgUriEndpoint,
+    blobCacheLocation: env.blobCacheLocation,
+  })
+  const bsky = BskyAppView.create({ db, config: cfg })
+  await bsky.start()
+  // Graceful shutdown (see also https://aws.amazon.com/blogs/containers/graceful-shutdowns-with-ecs/)
+  process.on('SIGTERM', async () => {
+    await bsky.destroy()
+  })
+}
+
+const getEnv = () => ({
+  port: parseInt(process.env.PORT),
+  version: process.env.BSKY_VERSION,
+  repoProvider: process.env.REPO_PROVIDER,
+  dbPostgresUrl: process.env.DB_POSTGRES_URL,
+  dbMigratePostgresUrl:
+    process.env.DB_MIGRATE_POSTGRES_URL || process.env.DB_POSTGRES_URL,
+  dbPostgresSchema: process.env.DB_POSTGRES_SCHEMA,
+  publicUrl: process.env.PUBLIC_URL,
+  didPlcUrl: process.env.DID_PLC_URL,
+  imgUriSalt: process.env.IMG_URI_SALT,
+  imgUriKey: process.env.IMG_URI_KEY,
+  imgUriEndpoint: process.env.IMG_URI_ENDPOINT,
+  blobCacheLocation: process.env.BLOB_CACHE_LOC,
+})
+
+const maintainXrpcResource = (span, req) => {
+  // Show actual xrpc method as resource rather than the route pattern
+  if (span && req.originalUrl?.startsWith('/xrpc/')) {
+    span.setTag(
+      'resource.name',
+      [
+        req.method,
+        path.posix.join(req.baseUrl || '', req.path || '', '/').slice(0, -1), // Ensures no trailing slash
+      ]
+        .filter(Boolean)
+        .join(' '),
+    )
+  }
+}
+
+main()

packages/bsky/service/package.json

@@ -0,0 +1,7 @@
+{
+  "name": "bsky-app-view-service",
+  "private": true,
+  "dependencies": {
+    "dd-trace": "^3.8.0"
+  }
+}