Detached QPs and hidden replies (bluesky-social#2675)

* Add new postgate lex, hiddeReplies to threadgate, codegen * Add protobufs * Add to mock dataplane * Add matching postgate method to feed hydration methods * Add to getRecord * Add to HydrationState * Fix typo * Add to mergeStates, fetch embeds in threads * Integrate into embed views * Add test for QPs in threads * Add feed test * Fix naming convention in protos * Add #viewRemoved record view, rename postgate.json * Integrate new view * Filter hidden replies from feeds * Filter out replies at the handler level, do not filter for author feeds * Fix lint * Move hidden reply check to view layer * Reduce, reuse, recycle * Rename to lowercase * Rename layer vars * Add quote gate props to postgate (bluesky-social#2693) * Add quote gate props to postgate * Consistent naming * Fix record structure * Codegen * Show hidden replies in author feed * Allow reposts of hidden replies * Lex and codegen * Add violates_quote_gate to proto * Consistent naming, codegen * Integrate violatesQuotegate and canQuotepost * Remove rules, codegen * Hydrate all postgates for all requested posts * Match other impl * Add test, need to split these out * Format * Hydrate first nested embeds too * Add postgate test suite * Add violatesQuoteGate to dataplane * Ingest and set violatesQuoteGate, return on meta * Return removed embed for quotes that violate gate * Add test * Dedupe URIs before fetching postgates * Update snaps * Snap * Format * Updating naming conventions for postgate-related attributes * Correct naming * Consistency * Proto too * Rename to viewDetached * Codegen * Rename everything * Codegen * Quotes that violate a quote gate can still be quoted themselves * Couple more renames * Snaps * Ensure reply ref is tombstoned for hidden replies * Split out hidden replies tests and create fresh fixture * Hydrate threadgates for reply notifications, filter hidden replies * Remove snap * Add flaky test * Rename violatesEmbeddingRules * Fix flaky test * Only write to db if violatesEmbeddingRules is true * DRY up post uri -> gate uri logic * isThreadgateListRule * Don't share users object between tests * No pascal * Remove default params * Find -> some * canQuotepost -> canEmbed, remove optional * Fix quoteee typo * await follows * Throw in post uri -> gate utils * Ensure fetch threadgates for reply roots * Don't hydrate threadgates twice * DRY up uri -> did parsing * Clean up parsePostgate logic * Format * Revert change * Revert change * Replace a couple more uri->did conversions * Only filter replies from feeds if viewer hid them * Revert, filter out replies that are hidden from feeds * Remove old test * Replace uri->did util * Revert change to unused file * Only validatePostEmbed and check postgates for post records * Ensure notifications aren't generated down a hidden reply chain * Changeset * Cleanup * Fix notification filtering logic * Simplify * Don't notify for invalid embeds * Use new APIs * Add hasPostGate and hasThreadGate flags from dataplane * Only fetch postgates if post has one * Only fetch threadgates if post has one or was deleted * Remove notification filtering * Don't hydrate threadgates for notifications * Move hidden replies in feeds to match block handling * Do no filtering of hidden replies in feeds * Revert "Don't hydrate threadgates for notifications" This reverts commit 1dcec0b. * Revert "Remove notification filtering" This reverts commit 1e7069d. * Filter notifications for OP only * Add additional check to hidden replies test * Move noty filter logic into method handler * Update .changeset/perfect-parrots-appear.md Co-authored-by: devin ivy <[email protected]> * Update packages/bsky/tests/seed/postgates.ts Co-authored-by: devin ivy <[email protected]> * Another structuredClone * Update packages/bsky/src/hydration/hydrator.ts Co-authored-by: devin ivy <[email protected]> * Better comment * Update packages/bsky/src/data-plane/server/indexing/plugins/post.ts Co-authored-by: devin ivy <[email protected]> * Regen protos to match dataplane * Update quotes snap to include embeddingDisabled * Clarify usage of post uri -> gate utils --------- Co-authored-by: devin ivy <[email protected]>
hboon · Aug 21, 2024 · aba664f · aba664f
1 parent 2a0c088
commit aba664f
Show file tree

Hide file tree

Showing 75 changed files with 2,297 additions and 113 deletions.
diff --git a/.changeset/perfect-parrots-appear.md b/.changeset/perfect-parrots-appear.md
@@ -0,0 +1,7 @@
+---
+"@atproto/bsky": patch
+"@atproto/api": patch
+"@atproto/pds": patch
+---
+
+Adds `postgate` records to power quote gating and detached quote posts, plus `hiddenReplies` to the `threadgate` record.
diff --git a/lexicons/app/bsky/embed/record.json b/lexicons/app/bsky/embed/record.json
@@ -20,6 +20,7 @@
             "#viewRecord",
             "#viewNotFound",
             "#viewBlocked",
+            "#viewDetached",
             "app.bsky.feed.defs#generatorView",
             "app.bsky.graph.defs#listView",
             "app.bsky.labeler.defs#labelerView",
@@ -80,6 +81,14 @@
         "blocked": { "type": "boolean", "const": true },
         "author": { "type": "ref", "ref": "app.bsky.feed.defs#blockedAuthor" }
       }
+    },
+    "viewDetached": {
+      "type": "object",
+      "required": ["uri", "detached"],
+      "properties": {
+        "uri": { "type": "string", "format": "at-uri" },
+        "detached": { "type": "boolean", "const": true }
+      }
     }
   }
 }
diff --git a/lexicons/app/bsky/feed/defs.json b/lexicons/app/bsky/feed/defs.json
@@ -42,7 +42,8 @@
         "repost": { "type": "string", "format": "at-uri" },
         "like": { "type": "string", "format": "at-uri" },
         "threadMuted": { "type": "boolean" },
-        "replyDisabled": { "type": "boolean" }
+        "replyDisabled": { "type": "boolean" },
+        "embeddingDisabled": { "type": "boolean" }
       }
     },
     "feedViewPost": {

diff --git a/lexicons/app/bsky/feed/postgate.json b/lexicons/app/bsky/feed/postgate.json
@@ -0,0 +1,45 @@
+{
+  "lexicon": 1,
+  "id": "app.bsky.feed.postgate",
+  "defs": {
+    "main": {
+      "type": "record",
+      "key": "tid",
+      "description": "Record defining interaction rules for a post. The record key (rkey) of the postgate record must match the record key of the post, and that record must be in the same repository.",
+      "record": {
+        "type": "object",
+        "required": ["post", "createdAt"],
+        "properties": {
+          "createdAt": { "type": "string", "format": "datetime" },
+          "post": {
+            "type": "string",
+            "format": "at-uri",
+            "description": "Reference (AT-URI) to the post record."
+          },
+          "detachedEmbeddingUris": {
+            "type": "array",
+            "maxLength": 50,
+            "items": {
+              "type": "string",
+              "format": "at-uri"
+            },
+            "description": "List of AT-URIs embedding this post that the author has detached from."
+          },
+          "embeddingRules": {
+            "type": "array",
+            "maxLength": 5,
+            "items": {
+              "type": "union",
+              "refs": ["#disableRule"]
+            }
+          }
+        }
+      }
+    },
+    "disableRule": {
+      "type": "object",
+      "description": "Disables embedding of this post.",
+      "properties": {}
+    }
+  }
+}
diff --git a/lexicons/app/bsky/feed/threadgate.json b/lexicons/app/bsky/feed/threadgate.json
@@ -23,7 +23,16 @@
               "refs": ["#mentionRule", "#followingRule", "#listRule"]
             }
           },
-          "createdAt": { "type": "string", "format": "datetime" }
+          "createdAt": { "type": "string", "format": "datetime" },
+          "hiddenReplies": {
+            "type": "array",
+            "maxLength": 50,
+            "items": {
+              "type": "string",
+              "format": "at-uri"
+            },
+            "description": "List of hidden reply URIs."
+          }
         }
       }
     },

diff --git a/packages/api/src/client/index.ts b/packages/api/src/client/index.ts
@@ -116,6 +116,7 @@ import * as AppBskyFeedGetSuggestedFeeds from './types/app/bsky/feed/getSuggeste
 import * as AppBskyFeedGetTimeline from './types/app/bsky/feed/getTimeline'
 import * as AppBskyFeedLike from './types/app/bsky/feed/like'
 import * as AppBskyFeedPost from './types/app/bsky/feed/post'
+import * as AppBskyFeedPostgate from './types/app/bsky/feed/postgate'
 import * as AppBskyFeedRepost from './types/app/bsky/feed/repost'
 import * as AppBskyFeedSearchPosts from './types/app/bsky/feed/searchPosts'
 import * as AppBskyFeedSendInteractions from './types/app/bsky/feed/sendInteractions'
@@ -314,6 +315,7 @@ export * as AppBskyFeedGetSuggestedFeeds from './types/app/bsky/feed/getSuggeste
 export * as AppBskyFeedGetTimeline from './types/app/bsky/feed/getTimeline'
 export * as AppBskyFeedLike from './types/app/bsky/feed/like'
 export * as AppBskyFeedPost from './types/app/bsky/feed/post'
+export * as AppBskyFeedPostgate from './types/app/bsky/feed/postgate'
 export * as AppBskyFeedRepost from './types/app/bsky/feed/repost'
 export * as AppBskyFeedSearchPosts from './types/app/bsky/feed/searchPosts'
 export * as AppBskyFeedSendInteractions from './types/app/bsky/feed/sendInteractions'
@@ -1601,6 +1603,7 @@ export class AppBskyFeedNS {
   generator: GeneratorRecord
   like: LikeRecord
   post: PostRecord
+  postgate: PostgateRecord
   repost: RepostRecord
   threadgate: ThreadgateRecord
 
@@ -1609,6 +1612,7 @@ export class AppBskyFeedNS {
     this.generator = new GeneratorRecord(client)
     this.like = new LikeRecord(client)
     this.post = new PostRecord(client)
+    this.postgate = new PostgateRecord(client)
     this.repost = new RepostRecord(client)
     this.threadgate = new ThreadgateRecord(client)
   }
@@ -1995,6 +1999,67 @@ export class PostRecord {
   }
 }
 
+export class PostgateRecord {
+  _client: XrpcClient
+
+  constructor(client: XrpcClient) {
+    this._client = client
+  }
+
+  async list(
+    params: Omit<ComAtprotoRepoListRecords.QueryParams, 'collection'>,
+  ): Promise<{
+    cursor?: string
+    records: { uri: string; value: AppBskyFeedPostgate.Record }[]
+  }> {
+    const res = await this._client.call('com.atproto.repo.listRecords', {
+      collection: 'app.bsky.feed.postgate',
+      ...params,
+    })
+    return res.data
+  }
+
+  async get(
+    params: Omit<ComAtprotoRepoGetRecord.QueryParams, 'collection'>,
+  ): Promise<{ uri: string; cid: string; value: AppBskyFeedPostgate.Record }> {
+    const res = await this._client.call('com.atproto.repo.getRecord', {
+      collection: 'app.bsky.feed.postgate',
+      ...params,
+    })
+    return res.data
+  }
+
+  async create(
+    params: Omit<
+      ComAtprotoRepoCreateRecord.InputSchema,
+      'collection' | 'record'
+    >,
+    record: AppBskyFeedPostgate.Record,
+    headers?: Record<string, string>,
+  ): Promise<{ uri: string; cid: string }> {
+    record.$type = 'app.bsky.feed.postgate'
+    const res = await this._client.call(
+      'com.atproto.repo.createRecord',
+      undefined,
+      { collection: 'app.bsky.feed.postgate', ...params, record },
+      { encoding: 'application/json', headers },
+    )
+    return res.data
+  }
+
+  async delete(
+    params: Omit<ComAtprotoRepoDeleteRecord.InputSchema, 'collection'>,
+    headers?: Record<string, string>,
+  ): Promise<void> {
+    await this._client.call(
+      'com.atproto.repo.deleteRecord',
+      undefined,
+      { collection: 'app.bsky.feed.postgate', ...params },
+      { headers },
+    )
+  }
+}
+
 export class RepostRecord {
   _client: XrpcClient
 

diff --git a/packages/api/src/client/lexicons.ts b/packages/api/src/client/lexicons.ts
@@ -4989,6 +4989,7 @@ export const schemaDict = {
               'lex:app.bsky.embed.record#viewRecord',
               'lex:app.bsky.embed.record#viewNotFound',
               'lex:app.bsky.embed.record#viewBlocked',
+              'lex:app.bsky.embed.record#viewDetached',
               'lex:app.bsky.feed.defs#generatorView',
               'lex:app.bsky.graph.defs#listView',
               'lex:app.bsky.labeler.defs#labelerView',
@@ -5083,6 +5084,20 @@ export const schemaDict = {
           },
         },
       },
+      viewDetached: {
+        type: 'object',
+        required: ['uri', 'detached'],
+        properties: {
+          uri: {
+            type: 'string',
+            format: 'at-uri',
+          },
+          detached: {
+            type: 'boolean',
+            const: true,
+          },
+        },
+      },
     },
   },
   AppBskyEmbedRecordWithMedia: {
@@ -5208,6 +5223,9 @@ export const schemaDict = {
           replyDisabled: {
             type: 'boolean',
           },
+          embeddingDisabled: {
+            type: 'boolean',
+          },
         },
       },
       feedViewPost: {
@@ -6662,6 +6680,56 @@ export const schemaDict = {
       },
     },
   },
+  AppBskyFeedPostgate: {
+    lexicon: 1,
+    id: 'app.bsky.feed.postgate',
+    defs: {
+      main: {
+        type: 'record',
+        key: 'tid',
+        description:
+          'Record defining interaction rules for a post. The record key (rkey) of the postgate record must match the record key of the post, and that record must be in the same repository.',
+        record: {
+          type: 'object',
+          required: ['post', 'createdAt'],
+          properties: {
+            createdAt: {
+              type: 'string',
+              format: 'datetime',
+            },
+            post: {
+              type: 'string',
+              format: 'at-uri',
+              description: 'Reference (AT-URI) to the post record.',
+            },
+            detachedEmbeddingUris: {
+              type: 'array',
+              maxLength: 50,
+              items: {
+                type: 'string',
+                format: 'at-uri',
+              },
+              description:
+                'List of AT-URIs embedding this post that the author has detached from.',
+            },
+            embeddingRules: {
+              type: 'array',
+              maxLength: 5,
+              items: {
+                type: 'union',
+                refs: ['lex:app.bsky.feed.postgate#disableRule'],
+              },
+            },
+          },
+        },
+      },
+      disableRule: {
+        type: 'object',
+        description: 'Disables embedding of this post.',
+        properties: {},
+      },
+    },
+  },
   AppBskyFeedRepost: {
     lexicon: 1,
     id: 'app.bsky.feed.repost',
@@ -6873,6 +6941,15 @@ export const schemaDict = {
               type: 'string',
               format: 'datetime',
             },
+            hiddenReplies: {
+              type: 'array',
+              maxLength: 50,
+              items: {
+                type: 'string',
+                format: 'at-uri',
+              },
+              description: 'List of hidden reply URIs.',
+            },
           },
         },
       },
@@ -11864,6 +11941,7 @@ export const ids = {
   AppBskyFeedGetTimeline: 'app.bsky.feed.getTimeline',
   AppBskyFeedLike: 'app.bsky.feed.like',
   AppBskyFeedPost: 'app.bsky.feed.post',
+  AppBskyFeedPostgate: 'app.bsky.feed.postgate',
   AppBskyFeedRepost: 'app.bsky.feed.repost',
   AppBskyFeedSearchPosts: 'app.bsky.feed.searchPosts',
   AppBskyFeedSendInteractions: 'app.bsky.feed.sendInteractions',

diff --git a/packages/api/src/client/types/app/bsky/embed/record.ts b/packages/api/src/client/types/app/bsky/embed/record.ts
@@ -38,6 +38,7 @@ export interface View {
     | ViewRecord
     | ViewNotFound
     | ViewBlocked
+    | ViewDetached
     | AppBskyFeedDefs.GeneratorView
     | AppBskyGraphDefs.ListView
     | AppBskyLabelerDefs.LabelerView
@@ -125,3 +126,21 @@ export function isViewBlocked(v: unknown): v is ViewBlocked {
 export function validateViewBlocked(v: unknown): ValidationResult {
   return lexicons.validate('app.bsky.embed.record#viewBlocked', v)
 }
+
+export interface ViewDetached {
+  uri: string
+  detached: true
+  [k: string]: unknown
+}
+
+export function isViewDetached(v: unknown): v is ViewDetached {
+  return (
+    isObj(v) &&
+    hasProp(v, '$type') &&
+    v.$type === 'app.bsky.embed.record#viewDetached'
+  )
+}
+
+export function validateViewDetached(v: unknown): ValidationResult {
+  return lexicons.validate('app.bsky.embed.record#viewDetached', v)
+}
diff --git a/packages/api/src/client/types/app/bsky/feed/defs.ts b/packages/api/src/client/types/app/bsky/feed/defs.ts
@@ -52,6 +52,7 @@ export interface ViewerState {
   like?: string
   threadMuted?: boolean
   replyDisabled?: boolean
+  embeddingDisabled?: boolean
   [k: string]: unknown
 }