More comprehensive testing for PDS views, using snapshots (bluesky-so…

…cial#223)

* Create basic seed for server based on view tests

* Utilize basic seed in view tests

* Split-up pds view tests

* Trying out snapshot testing for author feed

* Tidy

* Generalize normalization for snapshot testing

* Snapshot test home feed, fix inclusion of own reposts

* Fix typo

* Add follow snapshot tests, fix ordering and db date handling

* Add like snapshot tests, fix ordering and db date handling

* Add notification snapshot tests

* Add profile snapshot tests

* Add reposts snapshot tests, fix ordering and db date handling

* Add thread snapshot tests

* Fix typo

* Remove unneeded space

packages/server/src/api/todo/social/getHomeFeed.ts

@@ -12,10 +12,7 @@ import {
   queryPostsWithReposts,
   queryResultToFeedItem,
 } from './util'
-import {
-  isNotRepostClause,
-  postOrRepostIndexedAtClause,
-} from '../../../db/util'
+import { postOrRepostIndexedAtClause } from '../../../db/util'
 
 export default function (server: Server) {
   server.todo.social.getHomeFeed(
@@ -43,7 +40,8 @@ export default function (server: Server) {
       if (feedAlgorithm === FeedAlgorithm.Firehose) {
         // All posts, except requester's reposts
         queryPostsWithReposts(builder).where(
-          `post.creator != :requester or ${isNotRepostClause}`,
+          // The null check handles ANSI nulls
+          '(repost.creator != :requester or repost.creator is null)',
           { requester },
         )
       } else if (feedAlgorithm === FeedAlgorithm.ReverseChronological) {
@@ -57,7 +55,7 @@ export default function (server: Server) {
             .getQuery()
         }
         queryPostsWithReposts(builder)
-          .where(`(post.creator != :requester or ${isNotRepostClause})`, {
+          .where(`(repost.creator != :requester or repost.creator is null)`, {
             requester,
           })
           .andWhere(

packages/server/src/api/todo/social/getLikedBy.ts

@@ -5,6 +5,7 @@ import { LikeIndex } from '../../../db/records/like'
 import { ProfileIndex } from '../../../db/records/profile'
 import { User } from '../../../db/user'
 import * as locals from '../../../locals'
+import { dateFromDb, dateToDb } from '../../../db/util'
 
 export default function (server: Server) {
   server.todo.social.getLikedBy(
@@ -26,10 +27,12 @@ export default function (server: Server) {
         .leftJoin(User, 'user', 'like.creator = user.did')
         .leftJoin(ProfileIndex, 'profile', 'profile.creator = user.did')
         .where('like.subject = :uri', { uri })
-        .orderBy('like.createdAt')
+        .orderBy('like.createdAt', 'DESC')
 
       if (before !== undefined) {
-        builder.andWhere('like.createdAt < :before', { before })
+        builder.andWhere('like.createdAt < :before', {
+          before: dateToDb(before),
+        })
       }
       if (limit !== undefined) {
         builder.limit(limit)
@@ -40,7 +43,7 @@ export default function (server: Server) {
         did: row.did,
         name: row.name,
         displayName: row.displayName || undefined,
-        createdAt: row.createdAt,
+        createdAt: dateFromDb(row.createdAt),
         indexedAt: row.indexedAt,
       }))
 

packages/server/src/api/todo/social/getNotifications.ts

@@ -30,7 +30,6 @@ export default function (server: Server) {
           'notif.createdAt AS createdAt',
           'record.raw AS record',
           'record.indexedAt AS indexedAt',
-
           'notif.recordUri AS uri',
         ])
         .from(UserNotification, 'notif')
@@ -72,6 +71,7 @@ export default function (server: Server) {
         record: JSON.parse(notif.record),
         isRead: notif.createdAt <= user.lastSeenNotifs,
         indexedAt: notif.indexedAt,
+        // @TODO do we need createdAt so that it can be used as a cursor?
       }))
       return {
         encoding: 'application/json',

packages/server/src/api/todo/social/getRepostedBy.ts

@@ -5,6 +5,7 @@ import { ProfileIndex } from '../../../db/records/profile'
 import { User } from '../../../db/user'
 import { RepostIndex } from '../../../db/records/repost'
 import * as locals from '../../../locals'
+import { dateFromDb, dateToDb } from '../../../db/util'
 
 export default function (server: Server) {
   server.todo.social.getRepostedBy(
@@ -26,10 +27,12 @@ export default function (server: Server) {
         .leftJoin(User, 'user', 'repost.creator = user.did')
         .leftJoin(ProfileIndex, 'profile', 'profile.creator = user.did')
         .where('repost.subject = :uri', { uri })
-        .orderBy('repost.createdAt')
+        .orderBy('repost.createdAt', 'DESC')
 
       if (before !== undefined) {
-        builder.andWhere('repost.createdAt < :before', { before })
+        builder.andWhere('repost.createdAt < :before', {
+          before: dateToDb(before),
+        })
       }
       if (limit !== undefined) {
         builder.limit(limit)
@@ -40,7 +43,7 @@ export default function (server: Server) {
         did: row.did,
         name: row.name,
         displayName: row.displayName || undefined,
-        createdAt: row.createdAt,
+        createdAt: dateFromDb(row.createdAt),
         indexedAt: row.indexedAt,
       }))
 

packages/server/src/api/todo/social/getUserFollowers.ts

@@ -7,6 +7,7 @@ import { ProfileIndex } from '../../../db/records/profile'
 import { User } from '../../../db/user'
 import * as util from './util'
 import * as locals from '../../../locals'
+import { dateFromDb, dateToDb } from '../../../db/util'
 
 export default function (server: Server) {
   server.todo.social.getUserFollowers(
@@ -32,10 +33,12 @@ export default function (server: Server) {
         .innerJoin(User, 'creator', 'creator.did = record.did')
         .leftJoin(ProfileIndex, 'profile', 'profile.creator = record.did')
         .where('follow.subject = :subject', { subject: subject.did })
-        .orderBy('follow.createdAt')
+        .orderBy('follow.createdAt', 'DESC')
 
       if (before !== undefined) {
-        followersReq.andWhere('follow.createdAt < :before', { before })
+        followersReq.andWhere('follow.createdAt < :before', {
+          before: dateToDb(before),
+        })
       }
       if (limit !== undefined) {
         followersReq.limit(limit)
@@ -46,7 +49,7 @@ export default function (server: Server) {
         did: row.did,
         name: row.name,
         displayName: row.displayName || undefined,
-        createdAt: row.createdAt,
+        createdAt: dateFromDb(row.createdAt),
         indexedAt: row.indexedAt,
       }))
 

packages/server/src/api/todo/social/getUserFollows.ts

@@ -7,6 +7,7 @@ import { ProfileIndex } from '../../../db/records/profile'
 import { User } from '../../../db/user'
 import * as util from './util'
 import * as locals from '../../../locals'
+import { dateFromDb, dateToDb } from '../../../db/util'
 
 export default function (server: Server) {
   server.todo.social.getUserFollows(
@@ -32,10 +33,12 @@ export default function (server: Server) {
         .innerJoin(User, 'subject', 'follow.subject = subject.did')
         .leftJoin(ProfileIndex, 'profile', 'profile.creator = follow.subject')
         .where('follow.creator = :creator', { creator: creator.did })
-        .orderBy('follow.createdAt')
+        .orderBy('follow.createdAt', 'DESC')
 
       if (before !== undefined) {
-        followsReq.andWhere('follow.createdAt < :before', { before })
+        followsReq.andWhere('follow.createdAt < :before', {
+          before: dateToDb(before),
+        })
       }
       if (limit !== undefined) {
         followsReq.limit(limit)
@@ -46,7 +49,7 @@ export default function (server: Server) {
         did: row.did,
         name: row.name,
         displayName: row.displayName || undefined,
-        createdAt: row.createdAt,
+        createdAt: dateFromDb(row.createdAt),
         indexedAt: row.indexedAt,
       }))
 

packages/server/src/db/util.ts

@@ -19,6 +19,25 @@ export const postOrRepostIndexedAtClause = `iif(${isNotRepostClause}, post.index
 
 type Subquery = (qb: SelectQueryBuilder<any>) => SelectQueryBuilder<any>
 
+// datetimes go to/from the database in the format 'YYYY-MM-DD HH:MM:SS'
+// whereas ISO datetimes take the format 'YYYY-MM-DDTHH:MM:SSZ', so we convert.
+
+// E.g. 2022-10-08 04:05:22.079 -> 2022-10-08T04:05:22.079Z
+export const dateFromDb = (date: string) => {
+  if (date.endsWith('Z') && date.includes('T')) {
+    return date
+  }
+  return new Date(date + 'Z').toISOString()
+}
+
+// E.g. 2022-10-08T04:05:22.079Z -> 2022-10-08 04:05:22.079
+export const dateToDb = (date: string) => {
+  if (!date.endsWith('Z') && date.includes(' ')) {
+    return date
+  }
+  return date.replace('T', ' ').replace(/Z$/, '')
+}
+
 export const countSubquery = (
   table: EntityTarget<any>,
   subject: string,

packages/server/tests/_util.ts

@@ -1,10 +1,11 @@
 import { MemoryBlockstore } from '@adxp/repo'
 import * as crypto from '@adxp/crypto'
 import * as plc from '@adxp/plc'
+import { AdxUri } from '@adxp/uri'
 import getPort from 'get-port'
 import * as uint8arrays from 'uint8arrays'
-
 import server, { ServerConfig, Database, App } from '../src/index'
+import { TodoSocialGetAuthorFeed, TodoSocialGetHomeFeed } from '@adxp/api'
 
 const USE_TEST_SERVER = true
 
@@ -91,3 +92,90 @@ export const adminAuth = () => {
     )
   )
 }
+
+// Swap out identifiers and dates with stable
+// values for the purpose of snapshot testing
+export const forSnapshot = (obj: unknown) => {
+  const records = { [kTake]: 'record' }
+  const collections = { [kTake]: 'collection' }
+  const users = { [kTake]: 'user' }
+  const unknown = { [kTake]: 'unknown' }
+  return mapLeafValues(obj, (item) => {
+    if (typeof item !== 'string') {
+      return item
+    }
+    const str = item.startsWith('did:plc:') ? `adx://${item}` : item
+    if (str.startsWith('adx://')) {
+      const uri = new AdxUri(str)
+      if (uri.recordKey) {
+        return take(records, str)
+      }
+      if (uri.collection) {
+        return take(collections, str)
+      }
+      if (uri.hostname) {
+        return take(users, str)
+      }
+      return take(unknown, str)
+    }
+    if (str.match(/^\d{4}-\d{2}-\d{2}T/)) {
+      return constantDate
+    }
+    return item
+  })
+}
+
+// Feed testing utils
+
+type FeedItem = TodoSocialGetAuthorFeed.FeedItem &
+  TodoSocialGetHomeFeed.FeedItem
+
+export const getCursors = (feed: FeedItem[]) => feed.map((item) => item.cursor)
+
+export const getSortedCursors = (feed: FeedItem[]) =>
+  getCursors(feed).sort((a, b) => tstamp(b) - tstamp(a))
+
+export const getOriginator = (item: FeedItem) =>
+  item.repostedBy ? item.repostedBy.did : item.author.did
+
+const tstamp = (x: string) => new Date(x).getTime()
+
+// Useful for remapping ids in snapshot testing, to make snapshots deterministic.
+// E.g. you may use this to map this:
+//   [{ uri: 'did://rad'}, { uri: 'did://bad' }, { uri: 'did://rad'}]
+// to this:
+//   [{ uri: '0'}, { uri: '1' }, { uri: '0'}]
+const kTake = Symbol('take')
+export function take(obj, value: string): string
+export function take(obj, value: string | undefined): string | undefined
+export function take(
+  obj: { [s: string]: number; [kTake]?: string },
+  value: string | undefined,
+): string | undefined {
+  if (value === undefined) {
+    return
+  }
+  if (!(value in obj)) {
+    obj[value] = Object.keys(obj).length
+  }
+  const kind = obj[kTake]
+  return typeof kind === 'string'
+    ? `${kind}(${obj[value]})`
+    : String(obj[value])
+}
+
+export const constantDate = new Date(0).toISOString()
+
+const mapLeafValues = (obj: unknown, fn: (val: unknown) => unknown) => {
+  if (Array.isArray(obj)) {
+    return obj.map((item) => mapLeafValues(item, fn))
+  }
+  if (obj && typeof obj === 'object') {
+    return Object.entries(obj).reduce(
+      (collect, [name, value]) =>
+        Object.assign(collect, { [name]: mapLeafValues(value, fn) }),
+      {},
+    )
+  }
+  return fn(obj)
+}

packages/server/tests/account.test.ts

@@ -251,7 +251,7 @@ describe('account', () => {
       return expect(token).toBeDefined()
     }
 
-    // Reset back from updatedPassword to password
+    // Reset back from passwordAlt to password
     await client.todo.adx.resetAccountPassword({}, { token, password })
 
     // Reuse of token fails

packages/server/tests/seeds/follows.ts

@@ -0,0 +1,49 @@
+import { SeedClient } from './client'
+
+export default async (sc: SeedClient) => {
+  await sc.createAccount('alice', users.alice)
+  await sc.createAccount('bob', users.bob)
+  await sc.createAccount('carol', users.carol)
+  await sc.createAccount('dan', users.dan)
+  await sc.createAccount('eve', users.eve)
+  await sc.follow(sc.dids.alice, sc.dids.bob)
+  await sc.follow(sc.dids.alice, sc.dids.carol)
+  await sc.follow(sc.dids.alice, sc.dids.dan)
+  await sc.follow(sc.dids.alice, sc.dids.eve)
+  await sc.follow(sc.dids.carol, sc.dids.alice)
+  await sc.follow(sc.dids.bob, sc.dids.alice)
+  await sc.follow(sc.dids.bob, sc.dids.carol)
+  await sc.follow(sc.dids.dan, sc.dids.alice)
+  await sc.follow(sc.dids.dan, sc.dids.bob)
+  await sc.follow(sc.dids.dan, sc.dids.eve)
+  await sc.follow(sc.dids.eve, sc.dids.alice)
+  await sc.follow(sc.dids.eve, sc.dids.carol)
+}
+
+const users = {
+  alice: {
+    email: '[email protected]',
+    username: 'alice.test',
+    password: 'alice-pass',
+  },
+  bob: {
+    email: '[email protected]',
+    username: 'bob.test',
+    password: 'bob-pass',
+  },
+  carol: {
+    email: '[email protected]',
+    username: 'carol.test',
+    password: 'carol-pass',
+  },
+  dan: {
+    email: '[email protected]',
+    username: 'dan.test',
+    password: 'dan-pass',
+  },
+  eve: {
+    email: '[email protected]',
+    username: 'eve.test',
+    password: 'eve-pass',
+  },
+}

packages/server/tests/seeds/likes.ts

@@ -0,0 +1,15 @@
+import basicSeed from './basic'
+import { SeedClient } from './client'
+
+export default async (sc: SeedClient) => {
+  await basicSeed(sc)
+  await sc.createAccount('eve', {
+    email: '[email protected]',
+    username: 'eve.test',
+    password: 'eve-pass',
+  })
+  await sc.like(sc.dids.eve, sc.posts[sc.dids.alice][1].uriRaw)
+  await sc.like(sc.dids.carol, sc.replies[sc.dids.bob][0].uriRaw)
+  await sc.like(sc.dids.eve, sc.reposts[sc.dids.dan][0].toString())
+  return sc
+}

packages/server/tests/seeds/reposts.ts

@@ -0,0 +1,18 @@
+import basicSeed from './basic'
+import { SeedClient } from './client'
+
+export default async (sc: SeedClient) => {
+  await basicSeed(sc)
+  await sc.createAccount('eve', {
+    email: '[email protected]',
+    username: 'eve.test',
+    password: 'eve-pass',
+  })
+  await sc.repost(sc.dids.bob, sc.posts[sc.dids.alice][2].uriRaw)
+  await sc.repost(sc.dids.carol, sc.posts[sc.dids.alice][2].uriRaw)
+  await sc.repost(sc.dids.dan, sc.posts[sc.dids.alice][2].uriRaw)
+  await sc.repost(sc.dids.eve, sc.posts[sc.dids.alice][2].uriRaw)
+  await sc.repost(sc.dids.dan, sc.replies[sc.dids.bob][0].uriRaw)
+  await sc.repost(sc.dids.eve, sc.replies[sc.dids.bob][0].uriRaw)
+  return sc
+}