SQL Blockstore (bluesky-social#276)

* setting up sql-blockstore

* drop table migration

* unblock sqlite tx

* correct binary types

* lol woops

* block encoding utilities

* no longer store raw record in record table

* fix dev-env

* pr feedback

* tx check

packages/common/src/blocks.ts

@@ -0,0 +1,35 @@
+import { CID } from 'multiformats/cid'
+import * as Block from 'multiformats/block'
+import { sha256 as blockHasher } from 'multiformats/hashes/sha2'
+import * as blockCodec from '@ipld/dag-cbor'
+
+export const valueToIpldBlock = async (
+  data: unknown,
+): Promise<Block.Block<unknown>> => {
+  return Block.encode({
+    value: data,
+    codec: blockCodec,
+    hasher: blockHasher,
+  })
+}
+
+export const cidForData = async (data: unknown): Promise<CID> => {
+  const block = await valueToIpldBlock(data)
+  return block.cid
+}
+
+export const valueToIpldBytes = (value: unknown): Uint8Array => {
+  return blockCodec.encode(value)
+}
+
+export const ipldBytesToValue = (bytes: Uint8Array) => {
+  return blockCodec.decode(bytes)
+}
+
+export const ipldBytesToRecord = (bytes: Uint8Array): object => {
+  const val = ipldBytesToValue(bytes)
+  if (typeof val !== 'object' || val === null) {
+    throw new Error(`Expected object, got: ${val}`)
+  }
+  return val
+}

packages/common/src/index.ts

@@ -3,6 +3,6 @@ export * as util from './util'
 
 export * from './util'
 export * from './tid'
-export * from './cid'
+export * from './blocks'
 export * from './logger'
 export * from './types'

packages/dev-env/src/index.ts

@@ -58,11 +58,10 @@ export class DevEnvServer {
 
         const db = await PDSDatabase.memory()
         await db.migrateToLatestOrThrow()
-        const serverBlockstore = new MemoryBlockstore()
         const keypair = await crypto.EcdsaKeypair.create()
 
         this.inst = await onServerReady(
-          PDSServer(serverBlockstore, db, keypair, {
+          PDSServer(db, keypair, {
             debugMode: true,
             scheme: 'http',
             hostname: 'localhost',

packages/pds/src/api/app/bsky/getAuthorFeed.ts

@@ -55,7 +55,7 @@ export default function (server: Server) {
       let postsAndRepostsQb = db.db
         .selectFrom(postsQb.union(repostsQb).as('posts_and_reposts'))
         .innerJoin('app_bsky_post as post', 'post.uri', 'postUri')
-        .innerJoin('record', 'record.uri', 'postUri')
+        .innerJoin('ipld_block', 'ipld_block.cid', 'post.cid')
         .innerJoin('user as author', 'author.did', 'post.creator')
         .leftJoin(
           'app_bsky_profile as author_profile',
@@ -73,8 +73,8 @@ export default function (server: Server) {
           'postUri',
           'postCid',
           'cursor',
-          'record.raw as recordRaw',
-          'record.indexedAt as indexedAt',
+          'ipld_block.content as recordBytes',
+          'ipld_block.indexedAt as indexedAt',
           'author.did as authorDid',
           'author.username as authorName',
           'author_profile.displayName as authorDisplayName',

packages/pds/src/api/app/bsky/getHomeFeed.ts

@@ -71,7 +71,7 @@ export default function (server: Server) {
       let postsAndRepostsQb = db.db
         .selectFrom(postsQb.union(repostsQb).as('posts_and_reposts'))
         .innerJoin('app_bsky_post as post', 'post.uri', 'postUri')
-        .innerJoin('record', 'record.uri', 'postUri')
+        .innerJoin('ipld_block', 'ipld_block.cid', 'post.cid')
         .innerJoin('user as author', 'author.did', 'post.creator')
         .leftJoin(
           'app_bsky_profile as author_profile',
@@ -89,8 +89,8 @@ export default function (server: Server) {
           'postUri',
           'postCid',
           'cursor',
-          'record.raw as recordRaw',
-          'record.indexedAt as indexedAt',
+          'ipld_block.content as recordBytes',
+          'ipld_block.indexedAt as indexedAt',
           'author.did as authorDid',
           'author.username as authorName',
           'author_profile.displayName as authorDisplayName',

packages/pds/src/api/app/bsky/getLikedBy.ts

@@ -13,15 +13,14 @@ export default function (server: Server) {
       let builder = db.db
         .selectFrom('app_bsky_like as like')
         .where('like.subject', '=', uri)
-        .innerJoin('record', 'like.uri', 'record.uri')
         .innerJoin('user', 'like.creator', 'user.did')
         .leftJoin('app_bsky_profile as profile', 'profile.creator', 'user.did')
         .select([
           'user.did as did',
           'user.username as name',
           'profile.displayName as displayName',
           'like.createdAt as createdAt',
-          'record.indexedAt as indexedAt',
+          'like.indexedAt as indexedAt',
         ])
 
       if (cid) {

packages/pds/src/api/app/bsky/getNotifications.ts

@@ -1,5 +1,6 @@
-import { Server } from '../../../lexicon'
 import { AuthRequiredError, InvalidRequestError } from '@atproto/xrpc-server'
+import * as common from '@atproto/common'
+import { Server } from '../../../lexicon'
 import * as GetNotifications from '../../../lexicon/types/app/bsky/getNotifications'
 import * as locals from '../../../locals'
 import { paginate } from '../../../db/util'
@@ -19,7 +20,7 @@ export default function (server: Server) {
       let notifBuilder = db.db
         .selectFrom('user_notification as notif')
         .where('notif.userDid', '=', requester)
-        .innerJoin('record', 'record.uri', 'notif.recordUri')
+        .innerJoin('ipld_block', 'ipld_block.cid', 'notif.recordCid')
         .innerJoin('user as author', 'author.did', 'notif.author')
         .leftJoin(
           'app_bsky_profile as author_profile',
@@ -35,8 +36,8 @@ export default function (server: Server) {
           'notif.reason as reason',
           'notif.reasonSubject as reasonSubject',
           'notif.indexedAt as createdAt',
-          'record.raw as record',
-          'record.indexedAt as indexedAt',
+          'ipld_block.content as recordBytes',
+          'ipld_block.indexedAt as indexedAt',
           'notif.recordUri as uri',
         ])
 
@@ -69,7 +70,7 @@ export default function (server: Server) {
         },
         reason: notif.reason,
         reasonSubject: notif.reasonSubject || undefined,
-        record: JSON.parse(notif.record),
+        record: common.ipldBytesToRecord(notif.recordBytes),
         isRead: notif.createdAt <= user.lastSeenNotifs,
         indexedAt: notif.indexedAt,
       }))

packages/pds/src/api/app/bsky/getPostThread.ts

@@ -1,5 +1,6 @@
 import { Kysely } from 'kysely'
 import { AuthRequiredError, InvalidRequestError } from '@atproto/xrpc-server'
+import * as common from '@atproto/common'
 import { Server } from '../../../lexicon'
 import * as GetPostThread from '../../../lexicon/types/app/bsky/getPostThread'
 import * as locals from '../../../locals'
@@ -72,7 +73,7 @@ const postInfoBuilder = (db: Kysely<DatabaseSchema>, requester: string) => {
   const { ref } = db.dynamic
   return db
     .selectFrom('app_bsky_post as post')
-    .innerJoin('record', 'record.uri', 'post.uri')
+    .innerJoin('ipld_block', 'ipld_block.cid', 'post.cid')
     .innerJoin('user as author', 'author.did', 'post.creator')
     .leftJoin(
       'app_bsky_profile as author_profile',
@@ -86,8 +87,8 @@ const postInfoBuilder = (db: Kysely<DatabaseSchema>, requester: string) => {
       'author.did as authorDid',
       'author.username as authorName',
       'author_profile.displayName as authorDisplayName',
-      'record.raw as rawRecord',
-      'record.indexedAt as indexedAt',
+      'ipld_block.content as recordBytes',
+      'ipld_block.indexedAt as indexedAt',
       db
         .selectFrom('app_bsky_like')
         .select(countAll.as('count'))
@@ -132,7 +133,7 @@ const rowToPost = (
       name: row.authorName,
       displayName: row.authorDisplayName || undefined,
     },
-    record: JSON.parse(row.rawRecord),
+    record: common.ipldBytesToRecord(row.recordBytes),
     parent: parent ? { ...parent } : undefined,
     replyCount: row.replyCount || 0,
     likeCount: row.likeCount || 0,

packages/pds/src/api/app/bsky/getRepostedBy.ts

@@ -13,15 +13,14 @@ export default function (server: Server) {
       let builder = db.db
         .selectFrom('app_bsky_repost as repost')
         .where('repost.subject', '=', uri)
-        .innerJoin('record', 'repost.uri', 'record.uri')
         .innerJoin('user', 'repost.creator', 'user.did')
         .leftJoin('app_bsky_profile as profile', 'profile.creator', 'user.did')
         .select([
           'user.did as did',
           'user.username as name',
           'profile.displayName as displayName',
           'repost.createdAt as createdAt',
-          'record.indexedAt as indexedAt',
+          'repost.indexedAt as indexedAt',
         ])
 
       if (cid) {

packages/pds/src/api/app/bsky/getUserFollowers.ts

@@ -19,19 +19,18 @@ export default function (server: Server) {
       let followersReq = db.db
         .selectFrom('app_bsky_follow as follow')
         .where('follow.subject', '=', subject.did)
-        .innerJoin('record', 'record.uri', 'follow.uri')
-        .innerJoin('user as creator', 'creator.did', 'record.did')
+        .innerJoin('user as creator', 'creator.did', 'follow.creator')
         .leftJoin(
           'app_bsky_profile as profile',
           'profile.creator',
-          'record.did',
+          'follow.creator',
         )
         .select([
           'creator.did as did',
           'creator.username as name',
           'profile.displayName as displayName',
           'follow.createdAt as createdAt',
-          'record.indexedAt as indexedAt',
+          'follow.indexedAt as indexedAt',
         ])
 
       followersReq = paginate(followersReq, {

packages/pds/src/api/app/bsky/getUserFollows.ts

@@ -19,7 +19,6 @@ export default function (server: Server) {
       let followsReq = db.db
         .selectFrom('app_bsky_follow as follow')
         .where('follow.creator', '=', creator.did)
-        .innerJoin('record', 'record.uri', 'follow.uri')
         .innerJoin('user as subject', 'subject.did', 'follow.subject')
         .leftJoin(
           'app_bsky_profile as profile',
@@ -31,7 +30,7 @@ export default function (server: Server) {
           'subject.username as name',
           'profile.displayName as displayName',
           'follow.createdAt as createdAt',
-          'record.indexedAt as indexedAt',
+          'follow.indexedAt as indexedAt',
         ])
 
       followsReq = paginate(followsReq, {

packages/pds/src/api/app/bsky/updateProfile.ts

@@ -4,14 +4,15 @@ import * as locals from '../../../locals'
 import * as schema from '../../../lexicon/schemas'
 import { AtUri } from '@atproto/uri'
 import { RepoStructure } from '@atproto/repo'
+import SqlBlockstore from '../../../sql-blockstore'
 import { CID } from 'multiformats/cid'
 import * as Profile from '../../../lexicon/types/app/bsky/profile'
 
 const profileNsid = schema.ids.AppBskyProfile
 
 export default function (server: Server) {
   server.app.bsky.updateProfile(async (_params, input, req, res) => {
-    const { auth, db, blockstore, logger } = locals.get(res)
+    const { auth, db, logger } = locals.get(res)
 
     const requester = auth.getUserDid(req)
     if (!requester) {
@@ -21,13 +22,15 @@ export default function (server: Server) {
     const uri = new AtUri(`${requester}/${profileNsid}/self`)
 
     const { profileCid, updated } = await db.transaction(
-      async (txnDb): Promise<{ profileCid: CID; updated: Profile.Record }> => {
-        const currRoot = await txnDb.getRepoRoot(requester, true)
+      async (dbTxn): Promise<{ profileCid: CID; updated: Profile.Record }> => {
+        const currRoot = await dbTxn.getRepoRoot(requester, true)
         if (!currRoot) {
           throw new InvalidRequestError(
             `${requester} is not a registered repo on this server`,
           )
         }
+        const now = new Date().toISOString()
+        const blockstore = new SqlBlockstore(dbTxn, requester, now)
         const repo = await RepoStructure.load(blockstore, currRoot)
         const current = await repo.getRecord(profileNsid, 'self')
         if (!db.records.profile.matchesSchema(current)) {
@@ -47,7 +50,7 @@ export default function (server: Server) {
           )
         }
 
-        const currBadges = await txnDb.db
+        const currBadges = await dbTxn.db
           .selectFrom('app_bsky_profile_badge')
           .selectAll()
           .where('profileUri', '=', uri.toString())
@@ -72,31 +75,27 @@ export default function (server: Server) {
         const profileCid = await repo.blockstore.put(updated)
 
         // Update profile record
-        await txnDb.db
+        await dbTxn.db
           .updateTable('record')
-          .set({
-            raw: JSON.stringify(updated),
-            cid: profileCid.toString(),
-            indexedAt: new Date().toISOString(),
-          })
+          .set({ cid: profileCid.toString() })
           .where('uri', '=', uri.toString())
           .execute()
 
         // Update profile app index
-        await txnDb.db
+        await dbTxn.db
           .updateTable('app_bsky_profile')
           .set({
             cid: profileCid.toString(),
             displayName: updated.displayName,
             description: updated.description,
-            indexedAt: new Date().toISOString(),
+            indexedAt: now,
           })
           .where('uri', '=', uri.toString())
           .execute()
 
         // Remove old badges
         if (toDelete.length > 0) {
-          await txnDb.db
+          await dbTxn.db
             .deleteFrom('app_bsky_profile_badge')
             .where('profileUri', '=', uri.toString())
             .where('badgeUri', 'in', toDelete)
@@ -105,7 +104,7 @@ export default function (server: Server) {
 
         // Add new badges
         if (toAdd.length > 0) {
-          await txnDb.db
+          await dbTxn.db
             .insertInto('app_bsky_profile_badge')
             .values(toAdd)
             .execute()
@@ -119,7 +118,7 @@ export default function (server: Server) {
             cid: profileCid,
           })
           .createCommit(authStore, async (prev, curr) => {
-            const success = await txnDb.updateRepoRoot(requester, curr, prev)
+            const success = await dbTxn.updateRepoRoot(requester, curr, prev)
             if (!success) {
               logger.error({ did: requester, curr, prev }, 'repo update failed')
               throw new Error('Could not update repo root')

packages/pds/src/api/app/bsky/util/feed.ts

@@ -1,3 +1,4 @@
+import * as common from '@atproto/common'
 import * as GetAuthorFeed from '../../../../lexicon/types/app/bsky/getAuthorFeed'
 import * as GetHomeFeed from '../../../../lexicon/types/app/bsky/getHomeFeed'
 
@@ -19,7 +20,7 @@ export const rowToFeedItem = (row: FeedRow): FeedItem => ({
           displayName: row.originatorDisplayName ?? undefined,
         }
       : undefined,
-  record: JSON.parse(row.recordRaw),
+  record: common.ipldBytesToRecord(row.recordBytes),
   replyCount: row.replyCount,
   repostCount: row.repostCount,
   likeCount: row.likeCount,
@@ -42,7 +43,7 @@ type FeedRow = {
   postUri: string
   postCid: string
   cursor: string
-  recordRaw: string
+  recordBytes: Uint8Array
   indexedAt: string
   authorDid: string
   authorName: string