Get blobs (bluesky-social#606)

* refactor repo routes * basic blob route * getBlob route * tidy * move getBlob to sync * allow mimetype on getBlob * creator on blob table * migration * migration * handle deletes & check db on getBlob * fix content type bug * back to octet-stream * Update packages/pds/src/api/com/atproto/sync/getBlob.ts Co-authored-by: devin ivy <[email protected]> * fix up migrations * pr feedback * fixing up merge & migration * patched up migration --------- Co-authored-by: devin ivy <[email protected]>
kemerald25 · Mar 14, 2023 · afa28c7 · afa28c7
1 parent 5b20053
commit afa28c7
Show file tree

Hide file tree

Showing 30 changed files with 785 additions and 254 deletions.
diff --git a/lexicons/com/atproto/sync/getBlob.json b/lexicons/com/atproto/sync/getBlob.json
@@ -0,0 +1,21 @@
+{
+  "lexicon": 1,
+  "id": "com.atproto.sync.getBlob",
+  "defs": {
+    "main": {
+      "type": "query",
+      "description": "Get a blob associated with a given repo.",
+      "parameters": {
+        "type": "params",
+        "required": ["did", "cid"],
+        "properties": {
+          "did": {"type": "string", "description": "The DID of the repo."},
+          "cid": {"type": "string", "description": "The CID of the blob to fetch"}
+        }
+      },
+      "output": {
+        "encoding": "*/*"
+      }
+    }
+  }
+}
diff --git a/packages/api/src/client/index.ts b/packages/api/src/client/index.ts
@@ -49,6 +49,7 @@ import * as ComAtprotoSessionCreate from './types/com/atproto/session/create'
 import * as ComAtprotoSessionDelete from './types/com/atproto/session/delete'
 import * as ComAtprotoSessionGet from './types/com/atproto/session/get'
 import * as ComAtprotoSessionRefresh from './types/com/atproto/session/refresh'
+import * as ComAtprotoSyncGetBlob from './types/com/atproto/sync/getBlob'
 import * as ComAtprotoSyncGetBlocks from './types/com/atproto/sync/getBlocks'
 import * as ComAtprotoSyncGetCheckout from './types/com/atproto/sync/getCheckout'
 import * as ComAtprotoSyncGetCommitPath from './types/com/atproto/sync/getCommitPath'
@@ -139,6 +140,7 @@ export * as ComAtprotoSessionCreate from './types/com/atproto/session/create'
 export * as ComAtprotoSessionDelete from './types/com/atproto/session/delete'
 export * as ComAtprotoSessionGet from './types/com/atproto/session/get'
 export * as ComAtprotoSessionRefresh from './types/com/atproto/session/refresh'
+export * as ComAtprotoSyncGetBlob from './types/com/atproto/sync/getBlob'
 export * as ComAtprotoSyncGetBlocks from './types/com/atproto/sync/getBlocks'
 export * as ComAtprotoSyncGetCheckout from './types/com/atproto/sync/getCheckout'
 export * as ComAtprotoSyncGetCommitPath from './types/com/atproto/sync/getCommitPath'
@@ -703,6 +705,17 @@ export class SyncNS {
     this._service = service
   }
 
+  getBlob(
+    params?: ComAtprotoSyncGetBlob.QueryParams,
+    opts?: ComAtprotoSyncGetBlob.CallOptions,
+  ): Promise<ComAtprotoSyncGetBlob.Response> {
+    return this._service.xrpc
+      .call('com.atproto.sync.getBlob', params, undefined, opts)
+      .catch((e) => {
+        throw ComAtprotoSyncGetBlob.toKnownErr(e)
+      })
+  }
+
   getBlocks(
     params?: ComAtprotoSyncGetBlocks.QueryParams,
     opts?: ComAtprotoSyncGetBlocks.CallOptions,

diff --git a/packages/api/src/client/lexicons.ts b/packages/api/src/client/lexicons.ts
@@ -2025,6 +2025,33 @@ export const schemaDict = {
       },
     },
   },
+  ComAtprotoSyncGetBlob: {
+    lexicon: 1,
+    id: 'com.atproto.sync.getBlob',
+    defs: {
+      main: {
+        type: 'query',
+        description: 'Get a blob associated with a given repo.',
+        parameters: {
+          type: 'params',
+          required: ['did', 'cid'],
+          properties: {
+            did: {
+              type: 'string',
+              description: 'The DID of the repo.',
+            },
+            cid: {
+              type: 'string',
+              description: 'The CID of the blob to fetch',
+            },
+          },
+        },
+        output: {
+          encoding: '*/*',
+        },
+      },
+    },
+  },
   ComAtprotoSyncGetBlocks: {
     lexicon: 1,
     id: 'com.atproto.sync.getBlocks',
@@ -4171,6 +4198,7 @@ export const ids = {
   ComAtprotoSessionDelete: 'com.atproto.session.delete',
   ComAtprotoSessionGet: 'com.atproto.session.get',
   ComAtprotoSessionRefresh: 'com.atproto.session.refresh',
+  ComAtprotoSyncGetBlob: 'com.atproto.sync.getBlob',
   ComAtprotoSyncGetBlocks: 'com.atproto.sync.getBlocks',
   ComAtprotoSyncGetCheckout: 'com.atproto.sync.getCheckout',
   ComAtprotoSyncGetCommitPath: 'com.atproto.sync.getCommitPath',

diff --git a/packages/api/src/client/types/com/atproto/sync/getBlob.ts b/packages/api/src/client/types/com/atproto/sync/getBlob.ts
@@ -0,0 +1,32 @@
+/**
+ * GENERATED CODE - DO NOT MODIFY
+ */
+import { Headers, XRPCError } from '@atproto/xrpc'
+import { ValidationResult } from '@atproto/lexicon'
+import { isObj, hasProp } from '../../../../util'
+import { lexicons } from '../../../../lexicons'
+
+export interface QueryParams {
+  /** The DID of the repo. */
+  did: string
+  /** The CID of the blob to fetch */
+  cid: string
+}
+
+export type InputSchema = undefined
+
+export interface CallOptions {
+  headers?: Headers
+}
+
+export interface Response {
+  success: boolean
+  headers: Headers
+  data: Uint8Array
+}
+
+export function toKnownErr(e: any) {
+  if (e instanceof XRPCError) {
+  }
+  return e
+}
diff --git a/packages/aws/src/s3.ts b/packages/aws/src/s3.ts
@@ -1,6 +1,6 @@
 import * as aws from '@aws-sdk/client-s3'
 import { Upload } from '@aws-sdk/lib-storage'
-import { BlobStore } from '@atproto/repo'
+import { BlobStore, BlobNotFoundError } from '@atproto/repo'
 import { randomStr } from '@atproto/crypto'
 import { CID } from 'multiformats/cid'
 import stream from 'stream'
@@ -56,10 +56,16 @@ export class S3BlobStore implements BlobStore {
   }
 
   async makePermanent(key: string, cid: CID): Promise<void> {
-    await this.move({
-      from: this.getTmpPath(key),
-      to: this.getStoredPath(cid),
-    })
+    const alreadyHas = await this.hasStored(cid)
+    if (!alreadyHas) {
+      await this.move({
+        from: this.getTmpPath(key),
+        to: this.getStoredPath(cid),
+      })
+    } else {
+      // already saved, so we no-op & just delete the temp
+      await this.deleteKey(this.getTmpPath(key))
+    }
   }
 
   async putPermanent(
@@ -98,7 +104,7 @@ export class S3BlobStore implements BlobStore {
     if (res.Body) {
       return res.Body
     } else {
-      throw new Error(`Could not get blob: ${cid.toString()}`)
+      throw new BlobNotFoundError()
     }
   }
 
@@ -113,9 +119,25 @@ export class S3BlobStore implements BlobStore {
   }
 
   async delete(cid: CID): Promise<void> {
+    await this.deleteKey(this.getStoredPath(cid))
+  }
+
+  async hasStored(cid: CID): Promise<boolean> {
+    try {
+      const res = await this.client.headObject({
+        Bucket: this.bucket,
+        Key: this.getStoredPath(cid),
+      })
+      return res.$metadata.httpStatusCode === 200
+    } catch (err) {
+      return false
+    }
+  }
+
+  private async deleteKey(key: string) {
     await this.client.deleteObject({
       Bucket: this.bucket,
-      Key: this.getStoredPath(cid),
+      Key: key,
     })
   }
 

diff --git a/packages/pds/src/api/com/atproto/blob.ts b/packages/pds/src/api/com/atproto/blob.ts
@@ -4,10 +4,11 @@ import AppContext from '../../../context'
 export default function (server: Server, ctx: AppContext) {
   server.com.atproto.blob.upload({
     auth: ctx.accessVerifierCheckTakedown,
-    handler: async ({ input }) => {
+    handler: async ({ auth, input }) => {
+      const requester = auth.credentials.did
       const cid = await ctx.services
         .repo(ctx.db)
-        .blobs.addUntetheredBlob(input.encoding, input.body)
+        .blobs.addUntetheredBlob(requester, input.encoding, input.body)
 
       return {
         encoding: 'application/json',