Merge remote-tracking branch 'origin' into eric/app-796-likes-tab

* origin:
  Runtime flags in PDS, appview-proxy flags (bluesky-social#1491)

packages/pds/package.json

@@ -56,6 +56,7 @@
     "http-terminator": "^3.2.0",
     "jsonwebtoken": "^8.5.1",
     "kysely": "^0.22.0",
+    "lru-cache": "^10.0.1",
     "multiformats": "^9.6.4",
     "nodemailer": "^6.8.0",
     "nodemailer-html-to-text": "^3.2.0",

packages/pds/src/api/com/atproto/identity/resolveHandle.ts

@@ -34,7 +34,7 @@ export default function (server: Server, ctx: AppContext) {
 
     // this is not someone on our server, but we help with resolving anyway
 
-    if (!did && ctx.canProxyRead(req)) {
+    if (!did && (await ctx.canProxyRead(req))) {
       did = await tryResolveFromAppview(ctx.appviewAgent, handle)
     }
 

packages/pds/src/api/com/atproto/repo/getRecord.ts

@@ -26,7 +26,7 @@ export default function (server: Server, ctx: AppContext) {
       }
     }
 
-    if (ctx.canProxyRead(req)) {
+    if (await ctx.canProxyRead(req)) {
       const res = await ctx.appviewAgent.api.com.atproto.repo.getRecord(params)
       return {
         encoding: 'application/json',

packages/pds/src/app-view/api/app/bsky/actor/getProfile.ts

@@ -13,7 +13,7 @@ export default function (server: Server, ctx: AppContext) {
     handler: async ({ req, auth, params }) => {
       const requester =
         auth.credentials.type === 'access' ? auth.credentials.did : null
-      if (ctx.canProxyRead(req)) {
+      if (await ctx.canProxyRead(req, requester)) {
         const res = await ctx.appviewAgent.api.app.bsky.actor.getProfile(
           params,
           requester

packages/pds/src/app-view/api/app/bsky/actor/getProfiles.ts

@@ -9,7 +9,7 @@ export default function (server: Server, ctx: AppContext) {
     auth: ctx.accessVerifier,
     handler: async ({ req, auth, params }) => {
       const requester = auth.credentials.did
-      if (ctx.canProxyRead(req)) {
+      if (await ctx.canProxyRead(req, requester)) {
         const res = await ctx.appviewAgent.api.app.bsky.actor.getProfiles(
           params,
           await ctx.serviceAuthHeaders(requester),

packages/pds/src/app-view/api/app/bsky/actor/getSuggestions.ts

@@ -7,7 +7,7 @@ export default function (server: Server, ctx: AppContext) {
     auth: ctx.accessVerifier,
     handler: async ({ req, params, auth }) => {
       const requester = auth.credentials.did
-      if (ctx.canProxyRead(req)) {
+      if (await ctx.canProxyRead(req, requester)) {
         const res = await ctx.appviewAgent.api.app.bsky.actor.getSuggestions(
           params,
           await ctx.serviceAuthHeaders(requester),

packages/pds/src/app-view/api/app/bsky/actor/searchActors.ts

@@ -16,7 +16,7 @@ export default function (server: Server, ctx: AppContext) {
     auth: ctx.accessVerifier,
     handler: async ({ req, auth, params }) => {
       const requester = auth.credentials.did
-      if (ctx.canProxyRead(req)) {
+      if (await ctx.canProxyRead(req, requester)) {
         const res = await ctx.appviewAgent.api.app.bsky.actor.searchActors(
           params,
           await ctx.serviceAuthHeaders(requester),

packages/pds/src/app-view/api/app/bsky/actor/searchActorsTypeahead.ts

@@ -14,7 +14,7 @@ export default function (server: Server, ctx: AppContext) {
     auth: ctx.accessVerifier,
     handler: async ({ req, params, auth }) => {
       const requester = auth.credentials.did
-      if (ctx.canProxyRead(req)) {
+      if (await ctx.canProxyRead(req, requester)) {
         const res =
           await ctx.appviewAgent.api.app.bsky.actor.searchActorsTypeahead(
             params,

packages/pds/src/app-view/api/app/bsky/feed/getActorFeeds.ts

@@ -8,7 +8,7 @@ export default function (server: Server, ctx: AppContext) {
     auth: ctx.accessVerifier,
     handler: async ({ req, auth, params }) => {
       const requester = auth.credentials.did
-      if (ctx.canProxyRead(req)) {
+      if (await ctx.canProxyRead(req, requester)) {
         const res = await ctx.appviewAgent.api.app.bsky.feed.getActorFeeds(
           params,
           await ctx.serviceAuthHeaders(requester),

packages/pds/src/app-view/api/app/bsky/feed/getAuthorFeed.ts

@@ -15,7 +15,7 @@ export default function (server: Server, ctx: AppContext) {
     handler: async ({ req, params, auth }) => {
       const requester =
         auth.credentials.type === 'access' ? auth.credentials.did : null
-      if (ctx.canProxyRead(req)) {
+      if (await ctx.canProxyRead(req, requester)) {
         const res = await ctx.appviewAgent.api.app.bsky.feed.getAuthorFeed(
           params,
           requester

packages/pds/src/app-view/api/app/bsky/feed/getFeed.ts

@@ -30,7 +30,7 @@ export default function (server: Server, ctx: AppContext) {
     handler: async ({ req, params, auth }) => {
       const requester = auth.credentials.did
 
-      if (ctx.canProxyRead(req)) {
+      if (await ctx.canProxyRead(req, requester)) {
         const { data: feed } =
           await ctx.appviewAgent.api.app.bsky.feed.getFeedGenerator(
             { feed: params.feed },

packages/pds/src/app-view/api/app/bsky/feed/getFeedGenerator.ts

@@ -12,7 +12,7 @@ export default function (server: Server, ctx: AppContext) {
     auth: ctx.accessVerifier,
     handler: async ({ req, params, auth }) => {
       const requester = auth.credentials.did
-      if (ctx.canProxyRead(req)) {
+      if (await ctx.canProxyRead(req, requester)) {
         const res = await ctx.appviewAgent.api.app.bsky.feed.getFeedGenerator(
           params,
           await ctx.serviceAuthHeaders(requester),

packages/pds/src/app-view/api/app/bsky/feed/getFeedGenerators.ts

@@ -6,7 +6,7 @@ export default function (server: Server, ctx: AppContext) {
     auth: ctx.accessVerifier,
     handler: async ({ req, params, auth }) => {
       const requester = auth.credentials.did
-      if (ctx.canProxyRead(req)) {
+      if (await ctx.canProxyRead(req, requester)) {
         const res = await ctx.appviewAgent.api.app.bsky.feed.getFeedGenerators(
           params,
           await ctx.serviceAuthHeaders(requester),

packages/pds/src/app-view/api/app/bsky/feed/getLikes.ts

@@ -9,7 +9,7 @@ export default function (server: Server, ctx: AppContext) {
     auth: ctx.accessVerifier,
     handler: async ({ req, params, auth }) => {
       const requester = auth.credentials.did
-      if (ctx.canProxyRead(req)) {
+      if (await ctx.canProxyRead(req, requester)) {
         const res = await ctx.appviewAgent.api.app.bsky.feed.getLikes(
           params,
           await ctx.serviceAuthHeaders(requester),

packages/pds/src/app-view/api/app/bsky/feed/getPostThread.ts

@@ -47,7 +47,7 @@ export default function (server: Server, ctx: AppContext) {
     auth: ctx.accessVerifier,
     handler: async ({ req, params, auth }) => {
       const requester = auth.credentials.did
-      if (ctx.canProxyRead(req)) {
+      if (await ctx.canProxyRead(req, requester)) {
         try {
           const res = await ctx.appviewAgent.api.app.bsky.feed.getPostThread(
             params,

packages/pds/src/app-view/api/app/bsky/feed/getPosts.ts

@@ -8,7 +8,7 @@ export default function (server: Server, ctx: AppContext) {
     auth: ctx.accessVerifier,
     handler: async ({ req, params, auth }) => {
       const requester = auth.credentials.did
-      if (ctx.canProxyRead(req)) {
+      if (await ctx.canProxyRead(req, requester)) {
         const res = await ctx.appviewAgent.api.app.bsky.feed.getPosts(
           params,
           await ctx.serviceAuthHeaders(requester),

packages/pds/src/app-view/api/app/bsky/feed/getRepostedBy.ts

@@ -8,7 +8,7 @@ export default function (server: Server, ctx: AppContext) {
     auth: ctx.accessVerifier,
     handler: async ({ req, params, auth }) => {
       const requester = auth.credentials.did
-      if (ctx.canProxyRead(req)) {
+      if (await ctx.canProxyRead(req, requester)) {
         const res = await ctx.appviewAgent.api.app.bsky.feed.getRepostedBy(
           params,
           await ctx.serviceAuthHeaders(requester),

packages/pds/src/app-view/api/app/bsky/feed/getTimeline.ts

@@ -19,7 +19,7 @@ export default function (server: Server, ctx: AppContext) {
         throw new InvalidRequestError(`Unsupported algorithm: ${algorithm}`)
       }
 
-      if (ctx.canProxyRead(req)) {
+      if (await ctx.canProxyRead(req, requester)) {
         const res = await ctx.appviewAgent.api.app.bsky.feed.getTimeline(
           params,
           await ctx.serviceAuthHeaders(requester),

packages/pds/src/app-view/api/app/bsky/graph/getBlocks.ts

@@ -8,7 +8,7 @@ export default function (server: Server, ctx: AppContext) {
     auth: ctx.accessVerifier,
     handler: async ({ req, params, auth }) => {
       const requester = auth.credentials.did
-      if (ctx.canProxyRead(req)) {
+      if (await ctx.canProxyRead(req, requester)) {
         const res = await ctx.appviewAgent.api.app.bsky.graph.getBlocks(
           params,
           await ctx.serviceAuthHeaders(requester),

packages/pds/src/app-view/api/app/bsky/graph/getFollowers.ts

@@ -11,7 +11,7 @@ export default function (server: Server, ctx: AppContext) {
     handler: async ({ req, params, auth }) => {
       const requester =
         auth.credentials.type === 'access' ? auth.credentials.did : null
-      if (ctx.canProxyRead(req)) {
+      if (await ctx.canProxyRead(req, requester)) {
         const res = await ctx.appviewAgent.api.app.bsky.graph.getFollowers(
           params,
           requester

packages/pds/src/app-view/api/app/bsky/graph/getFollows.ts

@@ -11,7 +11,7 @@ export default function (server: Server, ctx: AppContext) {
     handler: async ({ req, params, auth }) => {
       const requester =
         auth.credentials.type === 'access' ? auth.credentials.did : null
-      if (ctx.canProxyRead(req)) {
+      if (await ctx.canProxyRead(req, requester)) {
         const res = await ctx.appviewAgent.api.app.bsky.graph.getFollows(
           params,
           requester

packages/pds/src/app-view/api/app/bsky/graph/getList.ts

@@ -8,7 +8,7 @@ export default function (server: Server, ctx: AppContext) {
     auth: ctx.accessVerifier,
     handler: async ({ req, params, auth }) => {
       const requester = auth.credentials.did
-      if (ctx.canProxyRead(req)) {
+      if (await ctx.canProxyRead(req, requester)) {
         const res = await ctx.appviewAgent.api.app.bsky.graph.getList(
           params,
           await ctx.serviceAuthHeaders(requester),

packages/pds/src/app-view/api/app/bsky/graph/getListMutes.ts

@@ -7,7 +7,7 @@ export default function (server: Server, ctx: AppContext) {
     auth: ctx.accessVerifier,
     handler: async ({ req, params, auth }) => {
       const requester = auth.credentials.did
-      if (ctx.canProxyRead(req)) {
+      if (await ctx.canProxyRead(req, requester)) {
         const res = await ctx.appviewAgent.api.app.bsky.graph.getListMutes(
           params,
           await ctx.serviceAuthHeaders(requester),

packages/pds/src/app-view/api/app/bsky/graph/getLists.ts

@@ -8,7 +8,7 @@ export default function (server: Server, ctx: AppContext) {
     auth: ctx.accessVerifier,
     handler: async ({ req, params, auth }) => {
       const requester = auth.credentials.did
-      if (ctx.canProxyRead(req)) {
+      if (await ctx.canProxyRead(req, requester)) {
         const res = await ctx.appviewAgent.api.app.bsky.graph.getLists(
           params,
           await ctx.serviceAuthHeaders(requester),

packages/pds/src/app-view/api/app/bsky/graph/getMutes.ts

@@ -8,7 +8,7 @@ export default function (server: Server, ctx: AppContext) {
     auth: ctx.accessVerifier,
     handler: async ({ req, auth, params }) => {
       const requester = auth.credentials.did
-      if (ctx.canProxyRead(req)) {
+      if (await ctx.canProxyRead(req, requester)) {
         const res = await ctx.appviewAgent.api.app.bsky.graph.getMutes(
           params,
           await ctx.serviceAuthHeaders(requester),

packages/pds/src/app-view/api/app/bsky/notification/getUnreadCount.ts

@@ -8,7 +8,7 @@ export default function (server: Server, ctx: AppContext) {
     auth: ctx.accessVerifier,
     handler: async ({ req, auth, params }) => {
       const requester = auth.credentials.did
-      if (ctx.canProxyRead(req)) {
+      if (await ctx.canProxyRead(req, requester)) {
         const res =
           await ctx.appviewAgent.api.app.bsky.notification.getUnreadCount(
             params,

packages/pds/src/app-view/api/app/bsky/notification/listNotifications.ts

@@ -12,7 +12,7 @@ export default function (server: Server, ctx: AppContext) {
     auth: ctx.accessVerifier,
     handler: async ({ req, params, auth }) => {
       const requester = auth.credentials.did
-      if (ctx.canProxyRead(req)) {
+      if (await ctx.canProxyRead(req, requester)) {
         const res =
           await ctx.appviewAgent.api.app.bsky.notification.listNotifications(
             params,

packages/pds/src/app-view/api/app/bsky/unspecced.ts

@@ -27,7 +27,7 @@ export default function (server: Server, ctx: AppContext) {
     auth: ctx.accessVerifier,
     handler: async ({ req, params, auth }) => {
       const requester = auth.credentials.did
-      if (ctx.canProxyRead(req)) {
+      if (await ctx.canProxyRead(req, requester)) {
         const hotClassicUri = Object.keys(ctx.algos).find((uri) =>
           uri.endsWith('/hot-classic'),
         )

packages/pds/src/context.ts

@@ -21,6 +21,7 @@ import { MountedAlgos } from './feed-gen/types'
 import { Crawlers } from './crawlers'
 import { LabelCache } from './label-cache'
 import { ContentReporter } from './content-reporter'
+import { RuntimeFlags } from './runtime-flags'
 
 export class AppContext {
   constructor(
@@ -42,6 +43,7 @@ export class AppContext {
       sequencerLeader: SequencerLeader | null
       labeler: Labeler
       labelCache: LabelCache
+      runtimeFlags: RuntimeFlags
       contentReporter?: ContentReporter
       backgroundQueue: BackgroundQueue
       appviewAgent?: AtpAgent
@@ -138,6 +140,10 @@ export class AppContext {
     return this.opts.labelCache
   }
 
+  get runtimeFlags(): RuntimeFlags {
+    return this.opts.runtimeFlags
+  }
+
   get contentReporter(): ContentReporter | undefined {
     return this.opts.contentReporter
   }
@@ -185,12 +191,24 @@ export class AppContext {
     return this.opts.appviewAgent
   }
 
-  canProxyRead(req: express.Request): boolean {
-    return (
-      this.cfg.bskyAppViewProxy &&
-      this.cfg.bskyAppViewEndpoint !== undefined &&
-      req.get('x-appview-proxy') !== undefined
-    )
+  async canProxyRead(
+    req: express.Request,
+    did?: string | null,
+  ): Promise<boolean> {
+    if (!this.cfg.bskyAppViewProxy || !this.cfg.bskyAppViewEndpoint) {
+      return false
+    }
+    if (req.get('x-appview-proxy') !== undefined) {
+      return true
+    }
+    // e.g. /xrpc/a.b.c.d/ -> a.b.c.d/ -> a.b.c.d
+    const endpoint = req.path.replace('/xrpc/', '').replaceAll('/', '')
+    if (!did) {
+      // when no did assigned, only proxy reads if threshold is at max of 10
+      const threshold = this.runtimeFlags.appviewProxy.getThreshold(endpoint)
+      return threshold === 10
+    }
+    return await this.runtimeFlags.appviewProxy.shouldProxy(endpoint, did)
   }
 
   canProxyFeedConstruction(req: express.Request): boolean {

packages/pds/src/db/database-schema.ts

@@ -23,9 +23,11 @@ import * as listMute from './tables/list-mute'
 import * as label from './tables/label'
 import * as repoSeq from './tables/repo-seq'
 import * as appMigration from './tables/app-migration'
+import * as runtimeFlag from './tables/runtime-flag'
 import * as appView from '../app-view/db'
 
 export type DatabaseSchemaType = appView.DatabaseSchemaType &
+  runtimeFlag.PartialDB &
   appMigration.PartialDB &
   userAccount.PartialDB &
   userState.PartialDB &

packages/pds/src/db/migrations/20230818T134357818Z-runtime-flags.ts

@@ -0,0 +1,13 @@
+import { Kysely } from 'kysely'
+
+export async function up(db: Kysely<unknown>): Promise<void> {
+  await db.schema
+    .createTable('runtime_flag')
+    .addColumn('name', 'varchar', (col) => col.primaryKey())
+    .addColumn('value', 'varchar', (col) => col.notNull())
+    .execute()
+}
+
+export async function down(db: Kysely<unknown>): Promise<void> {
+  await db.schema.dropTable('runtime_flag').execute()
+}

packages/pds/src/db/migrations/index.ts

@@ -61,3 +61,4 @@ export * as _20230801T195109532Z from './20230801T195109532Z-remove-moderation-f
 export * as _20230807T035309811Z from './20230807T035309811Z-feed-item-delete-invite-for-user-idx'
 export * as _20230808T172813122Z from './20230808T172813122Z-repo-rev'
 export * as _20230810T203412859Z from './20230810T203412859Z-action-duration'
+export * as _20230818T134357818Z from './20230818T134357818Z-runtime-flags'

packages/pds/src/db/tables/runtime-flag.ts

@@ -0,0 +1,8 @@
+export interface RuntimeFlag {
+  name: string
+  value: string
+}
+
+export const tableName = 'runtime_flag'
+
+export type PartialDB = { [tableName]: RuntimeFlag }

packages/pds/src/index.ts

@@ -46,6 +46,7 @@ import { Crawlers } from './crawlers'
 import { LabelCache } from './label-cache'
 import { ContentReporter } from './content-reporter'
 import { ModerationService } from './services/moderation'
+import { RuntimeFlags } from './runtime-flags'
 
 export type { MountedAlgos } from './feed-gen/types'
 export type { ServerConfigValues } from './config'
@@ -227,6 +228,8 @@ export class PDS {
       crawlers,
     })
 
+    const runtimeFlags = new RuntimeFlags(db)
+
     const ctx = new AppContext({
       db,
       blobstore,
@@ -241,6 +244,7 @@ export class PDS {
       sequencerLeader,
       labeler,
       labelCache,
+      runtimeFlags,
       contentReporter,
       services,
       mailer,
@@ -309,6 +313,7 @@ export class PDS {
     await this.ctx.sequencer.start()
     await this.ctx.db.startListeningToChannels()
     this.ctx.labelCache.start()
+    await this.ctx.runtimeFlags.start()
     const server = this.app.listen(this.ctx.cfg.port)
     this.server = server
     this.server.keepAliveTimeout = 90000
@@ -318,6 +323,7 @@ export class PDS {
   }
 
   async destroy(): Promise<void> {
+    this.ctx.runtimeFlags.destroy()
     this.ctx.labelCache.stop()
     await this.ctx.sequencerLeader?.destroy()
     await this.terminator?.terminate()