Verify deltas (bluesky-social#57)

* starting verification code

* missed some "programs"

* finished pass at verify algo

* emitting proper updates for chagnes

* verify ucans on update

* reorg delta file

* cleanup etc

* TESTS!

* optional emit

common/src/blockstore/ipld-store.ts

@@ -6,8 +6,11 @@ import { BlockWriter } from '@ipld/car/writer'
 
 import MemoryBlockstore from './memory-blockstore.js'
 import * as check from '../common/check.js'
+import * as util from '../common/util.js'
 import { BlockstoreI } from './types.js'
 import { PersistentBlockstore } from './persistent-blockstore.js'
+import { BlockReader } from '@ipld/car/api'
+import CidSet from '../repo/cid-set.js'
 
 type AllowedIpldRecordVal = string | number | CID | CID[] | Uint8Array | null
 
@@ -60,6 +63,22 @@ export class IpldStore {
     }
   }
 
+  async has(cid: CID): Promise<boolean> {
+    return this.rawBlockstore.has(cid)
+  }
+
+  async isMissing(cid: CID): Promise<boolean> {
+    const has = await this.has(cid)
+    return !has
+  }
+
+  async checkMissing(cids: CidSet): Promise<CidSet> {
+    const missing = await util.asyncFilter(cids.toList(), (c) => {
+      return this.isMissing(c)
+    })
+    return new CidSet(missing)
+  }
+
   async getBytes(cid: CID): Promise<Uint8Array> {
     return this.rawBlockstore.get(cid)
   }
@@ -75,6 +94,12 @@ export class IpldStore {
   async addToCar(car: BlockWriter, cid: CID) {
     car.put({ cid, bytes: await this.getBytes(cid) })
   }
+
+  async loadCar(car: BlockReader): Promise<void> {
+    for await (const block of car.blocks()) {
+      await this.putBytes(block.cid, block.bytes)
+    }
+  }
 }
 
 export default IpldStore

common/src/blockstore/memory-blockstore.ts

@@ -18,6 +18,10 @@ export class MemoryBlockstore implements BlockstoreI {
     this.map.set(k.toString(), v)
   }
 
+  async has(k: CID): Promise<boolean> {
+    return this.map.has(k.toString())
+  }
+
   async destroy(): Promise<void> {
     this.map.clear()
   }

common/src/blockstore/persistent-blockstore.ts

@@ -21,6 +21,15 @@ export class PersistentBlockstore implements BlockstoreI {
     await this.store.put(cid.toString(), bytes)
   }
 
+  async has(cid: CID): Promise<boolean> {
+    try {
+      await this.get(cid)
+      return true
+    } catch (_) {
+      return false
+    }
+  }
+
   async destroy(): Promise<void> {
     await this.store.clear()
     await this.store.close()

common/src/blockstore/types.ts

@@ -3,5 +3,6 @@ import { CID } from 'multiformats/cid'
 export interface BlockstoreI {
   get(cid: CID): Promise<Uint8Array>
   put(cid: CID, bytes: Uint8Array): Promise<void>
+  has(cid: CID): Promise<boolean>
   destroy(): Promise<void>
 }

common/src/common/util.ts

@@ -44,3 +44,11 @@ export const s32decode = (s: string): number => {
   }
   return i
 }
+
+export const asyncFilter = async <T>(
+  arr: T[],
+  fn: (t: T) => Promise<boolean>,
+) => {
+  const results = await Promise.all(arr.map((t) => fn(t)))
+  return arr.filter((_, i) => results[i])
+}

common/src/index.ts

@@ -7,8 +7,8 @@ export * from './microblog/index.js'
 export * from './microblog/types.js'
 export * from './microblog/delegator.js'
 export * as check from './common/check.js'
-export * as service from './network/service.js'
 export * as util from './common/util.js'
+export * as service from './network/service.js'
 export * as ucanCheck from './auth/ucan-checks.js'
 export * as auth from './auth/index.js'
 

common/src/repo/cid-set.ts

@@ -19,9 +19,7 @@ export class CidSet {
   }
 
   subtractSet(toSubtract: CidSet): CidSet {
-    toSubtract.toList().forEach((cid) => {
-      this.set.delete(cid.toString())
-    })
+    toSubtract.toList().map((c) => this.delete(c))
     return this
   }
 

common/src/repo/delta.ts

@@ -0,0 +1,279 @@
+import { CID } from 'multiformats'
+import { BlueskyCapability } from '../auth/bluesky-capability.js'
+import { Collection, TIDEntry, DIDEntry, IdMapping } from '../repo/types.js'
+import CidSet from './cid-set.js'
+import TID from './tid.js'
+import * as auth from '../auth/index.js'
+
+// AUTHORIZATION HELPERS
+// ----------------------
+
+export const capabilityForEvent = (
+  did: string,
+  event: Event,
+): BlueskyCapability => {
+  if (isRelationshipEvent(event)) {
+    return auth.writeCap(did, 'relationships')
+  }
+  if (isObjectEvent(event)) {
+    return auth.writeCap(did, event.namespace, event.collection, event.tid)
+  }
+  if (isNamespaceEvent(event)) {
+    return auth.writeCap(did, event.namespace)
+  }
+  throw new Error(`Could not identity event: ${event}`)
+}
+
+// DELTA TYPES & HELPERS
+// ----------------------
+
+type Delete = {
+  key: string
+}
+
+type Add = {
+  key: string
+  cid: CID
+}
+
+type Update = {
+  key: string
+  old: CID
+  cid: CID
+}
+
+type Diff = {
+  adds: Add[]
+  updates: Update[]
+  deletes: Delete[]
+}
+
+export const idMapDiff = (
+  prevMap: IdMapping,
+  currMap: IdMapping,
+  newCids: CidSet,
+): Diff => {
+  const diff: Diff = {
+    adds: [],
+    updates: [],
+    deletes: [],
+  }
+  // find deletions
+  for (const key of Object.keys(prevMap)) {
+    if (!currMap[key]) {
+      diff.deletes.push({ key })
+    }
+  }
+  // find additions & changes
+  for (const key of Object.keys(currMap)) {
+    const old = prevMap[key]
+    const cid = currMap[key]
+    if (old && old.equals(cid)) continue
+    if (!newCids.has(cid)) {
+      throw new Error(`Cid not included in added cids: ${cid.toString()}`)
+    }
+    if (old) {
+      diff.updates.push({ key, old, cid })
+    } else {
+      diff.adds.push({ key, cid })
+    }
+  }
+  return diff
+}
+
+export const tidEntriesDiff = (
+  prevEntries: TIDEntry[],
+  currEntries: TIDEntry[],
+  newCids: CidSet,
+): Diff => {
+  const idPrevMap = tidEntriesToIdMapping(prevEntries)
+  const idCurrMap = tidEntriesToIdMapping(currEntries)
+  return idMapDiff(idPrevMap, idCurrMap, newCids)
+}
+
+export const tidEntriesToIdMapping = (entries: TIDEntry[]): IdMapping => {
+  return entries.reduce((acc, cur) => {
+    acc[cur.tid.toString()] = cur.cid
+    return acc
+  }, {} as IdMapping)
+}
+
+export const didEntriesDiff = (
+  prevEntries: DIDEntry[],
+  currEntries: DIDEntry[],
+  newCids: CidSet,
+): Diff => {
+  const idPrevMap = didEntriesToIdMapping(prevEntries)
+  const idCurrMap = didEntriesToIdMapping(currEntries)
+  return idMapDiff(idPrevMap, idCurrMap, newCids)
+}
+
+export const didEntriesToIdMapping = (entries: DIDEntry[]): IdMapping => {
+  return entries.reduce((acc, cur) => {
+    acc[cur.did] = cur.cid
+    return acc
+  }, {} as IdMapping)
+}
+
+// EVENT TYPES & HELPERS
+// ----------------------
+
+export enum EventType {
+  AddedRelationship = 'added_relationship',
+  UpdatedRelationship = 'updated_relationship',
+  DeletedRelationship = 'deleted_relationship',
+  AddedObject = 'added_object',
+  UpdatedObject = 'updated_object',
+  DeletedObject = 'deleted_object',
+  DeletedNamespace = 'deleted_namespace',
+}
+
+export type RelationshipEvent =
+  | AddedRelationship
+  | UpdatedRelationship
+  | DeletedRelationship
+
+export type ObjectEvent = AddedObject | UpdatedObject | DeletedObject
+
+export type NamespaceEvent = DeletedNamespace
+
+export type Event = RelationshipEvent | ObjectEvent | NamespaceEvent
+
+export type AddedObject = {
+  event: EventType.AddedObject
+  namespace: string
+  collection: Collection
+  tid: TID
+  cid: CID
+}
+
+export type UpdatedObject = {
+  event: EventType.UpdatedObject
+  namespace: string
+  collection: Collection
+  tid: TID
+  cid: CID
+  prevCid: CID
+}
+
+export type DeletedObject = {
+  event: EventType.DeletedObject
+  namespace: string
+  collection: Collection
+  tid: TID
+}
+
+export type AddedRelationship = {
+  event: EventType.AddedRelationship
+  did: string
+  cid: CID
+}
+
+export type UpdatedRelationship = {
+  event: EventType.UpdatedRelationship
+  did: string
+  cid: CID
+  prevCid: CID
+}
+
+export type DeletedRelationship = {
+  event: EventType.DeletedRelationship
+  did: string
+}
+
+export type DeletedNamespace = {
+  event: EventType.DeletedNamespace
+  namespace: string
+}
+
+export const addedObject = (
+  namespace: string,
+  collection: Collection,
+  tid: TID,
+  cid: CID,
+): AddedObject => ({
+  event: EventType.AddedObject,
+  namespace,
+  collection,
+  tid,
+  cid,
+})
+
+export const updatedObject = (
+  namespace: string,
+  collection: Collection,
+  tid: TID,
+  cid: CID,
+  prevCid: CID,
+): UpdatedObject => ({
+  event: EventType.UpdatedObject,
+  namespace,
+  collection,
+  tid,
+  cid,
+  prevCid,
+})
+
+export const deletedObject = (
+  namespace: string,
+  collection: Collection,
+  tid: TID,
+): DeletedObject => ({
+  event: EventType.DeletedObject,
+  namespace,
+  collection,
+  tid,
+})
+
+export const addedRelationship = (
+  did: string,
+  cid: CID,
+): AddedRelationship => ({
+  event: EventType.AddedRelationship,
+  did,
+  cid,
+})
+
+export const updatedRelationship = (
+  did: string,
+  cid: CID,
+  prevCid: CID,
+): UpdatedRelationship => ({
+  event: EventType.UpdatedRelationship,
+  did,
+  cid,
+  prevCid,
+})
+
+export const deletedRelationship = (did: string): DeletedRelationship => ({
+  event: EventType.DeletedRelationship,
+  did,
+})
+
+export const deletedNamespace = (namespace: string): DeletedNamespace => ({
+  event: EventType.DeletedNamespace,
+  namespace,
+})
+
+export const isRelationshipEvent = (
+  event: Event,
+): event is RelationshipEvent => {
+  return (
+    event.event === EventType.AddedRelationship ||
+    event.event === EventType.UpdatedRelationship ||
+    event.event === EventType.DeletedRelationship
+  )
+}
+
+export const isObjectEvent = (event: Event): event is ObjectEvent => {
+  return (
+    event.event === EventType.AddedObject ||
+    event.event === EventType.UpdatedObject ||
+    event.event === EventType.DeletedObject ||
+    event.event === EventType.DeletedNamespace
+  )
+}
+
+export const isNamespaceEvent = (event: Event): event is NamespaceEvent => {
+  return event.event === EventType.DeletedNamespace
+}