Finalize PDS in-process AppView removal (bluesky-social#1198)

* rm tables * rm event-stream & proxied * Remove appview services, move label service to pds * only proxy appview stuff * delete more tables * Start removing message dispatched from pds * more syncing-up removal of message dispatcher in pds * merged * remove feedgens from pds, remove getPopular * remove unused image helper from pds * fixing compiler errors * clean up sharp * rm label service * first pass on cleaning up tests * fix up a bunch of tests * moderation view tests * last admin tests * got a lil overzealous in deletes * clean up unused cfg * clean up label table * simplify admin repo search query/logic * tidy pds entrypoint * in-progress pds config changes * cfg fiddling * finish cleaning up cfg/ctx * comments * building * pds prefix on env * test env * collapse pds migrations down into a single migration * fix up dev-env * tidy * cleanup * fix pds admin tests * fix handle test * fix pds proxy tests * fix subscribe repos test * fix sqlite config in pds tests * add sqlite clause in sequencer-leader * fix actor search w/ sqlite on pds * fixes * fix dev env build * update pds service entrypoint * simple env example * make takedown ids opaque identifiers in the pds * use pds routes for api tests * update pds dockerfile with volume and correct port env var * add a couple env vars to example * add comments to env example * @atproto/pds 0.2.0-beta.0 * @atproto/aws 0.0.1-beta.0 * appview did * @atproto/aws 0.0.1 * enable logs by default * update env example * bugfixing sandbox issues * consistency in pds env var name for appview url * log on pds start and stop, configure version at runtime * @atproto/pds 0.2.0-beta.1 * fix semver matching for pds beta version * v0.2.0-beta.2 * default invites to being not required * fix flaky test * limit db connections in tests * publish 0.2.0-beta.d3 * fix invite required parsing * @atproto/pds 0.2.0-beta.5 * Proxy getPopularFeedGenerators on simplified pds (bluesky-social#1222) proxy getPopularFeedGenerators on pds Co-authored-by: dholms <[email protected]> * tidy migrations * fix service entry * bump version * change auth order * bump version * bump version * add upgradeRepoVersion & fallback url for cdn * bump version * merging * merge pds * building dev-env * merging tests * merge service entry * test fixing * tidy * fix admin search * tidy * tidy * add snap for getListFeed * add backup nameserver cfg * tidy + pr feedback * tidy * tidy env * bit more * re-add dotenv to root package.json * fix dep * build branch * fix tests * Refactor tests to make better use of dev-env (bluesky-social#1690) * refactor pds tests to use dev env * refactor bsky tests * fix pds test * tidy bsky tests * build pds correctly * fix entry point * default logging to false (for now) * format service entry * Switch takedown ids back to ints on pds distribution (bluesky-social#1694) * switch takedown ids back to ints, consistent with live pds * tidy/fix migration * update migration for sqlite * export moderation action reversal * takedown tests * dont build branch --------- Co-authored-by: Devin Ivy <[email protected]>
kemerald25 · Oct 2, 2023 · d664b51 · d664b51
1 parent 3b65224
commit d664b51
Show file tree

Hide file tree

Showing 304 changed files with 3,548 additions and 7,129 deletions.
diff --git a/Makefile b/Makefile
@@ -22,15 +22,9 @@ test: ## Run all tests
 run-dev-env: ## Run a "development environment" shell
 	cd packages/dev-env; pnpm run start
 
-.PHONY: run-dev-pds
-run-dev-pds: ## Run PDS locally
-	if [ ! -f "packages/pds/.dev.env" ]; then cp packages/pds/example.dev.env packages/pds/.dev.env; fi
-	cd packages/pds; ENV=dev pnpm run start | pnpm exec pino-pretty
-
-.PHONY: run-dev-bsky
-run-dev-bsky: ## Run appview ('bsky') locally
-	if [ ! -f "packages/bsky/.dev.env" ]; then cp packages/bsky/example.dev.env packages/bsky/.dev.env; fi
-	cd packages/bsky; ENV=dev pnpm run start | pnpm exec pino-pretty
+.PHONY: run-dev-env-logged
+run-dev-env: ## Run a "development environment" shell
+	LOG_ENABLED=true cd packages/dev-env; pnpm run start | pnpm exec pino-pretty
 
 .PHONY: codegen
 codegen: ## Re-generate packages from lexicon/ files

diff --git a/packages/api/package.json b/packages/api/package.json
@@ -40,7 +40,7 @@
   },
   "devDependencies": {
     "@atproto/lex-cli": "workspace:^",
-    "@atproto/pds": "workspace:^",
+    "@atproto/dev-env": "workspace:^",
     "common-tags": "^1.8.2"
   }
 }
diff --git a/packages/api/tests/agent.test.ts b/packages/api/tests/agent.test.ts
@@ -1,29 +1,23 @@
 import { defaultFetchHandler } from '@atproto/xrpc'
-import {
-  CloseFn,
-  runTestServer,
-  TestServerInfo,
-} from '@atproto/pds/tests/_util'
 import {
   AtpAgent,
   AtpAgentFetchHandlerResponse,
   AtpSessionEvent,
   AtpSessionData,
 } from '..'
+import { TestNetworkNoAppView } from '@atproto/dev-env'
 
 describe('agent', () => {
-  let server: TestServerInfo
-  let close: CloseFn
+  let network: TestNetworkNoAppView
 
   beforeAll(async () => {
-    server = await runTestServer({
+    network = await TestNetworkNoAppView.create({
       dbPostgresSchema: 'api_agent',
     })
-    close = server.close
   })
 
   afterAll(async () => {
-    await close()
+    await network.close()
   })
 
   it('creates a new session on account creation.', async () => {
@@ -34,7 +28,7 @@ describe('agent', () => {
       sessions.push(sess)
     }
 
-    const agent = new AtpAgent({ service: server.url, persistSession })
+    const agent = new AtpAgent({ service: network.pds.url, persistSession })
 
     const res = await agent.createAccount({
       handle: 'user1.test',
@@ -74,7 +68,7 @@ describe('agent', () => {
       sessions.push(sess)
     }
 
-    const agent1 = new AtpAgent({ service: server.url, persistSession })
+    const agent1 = new AtpAgent({ service: network.pds.url, persistSession })
 
     const email = '[email protected]'
     await agent1.createAccount({
@@ -83,7 +77,7 @@ describe('agent', () => {
       password: 'password',
     })
 
-    const agent2 = new AtpAgent({ service: server.url, persistSession })
+    const agent2 = new AtpAgent({ service: network.pds.url, persistSession })
     const res1 = await agent2.login({
       identifier: 'user2.test',
       password: 'password',
@@ -122,7 +116,7 @@ describe('agent', () => {
       sessions.push(sess)
     }
 
-    const agent1 = new AtpAgent({ service: server.url, persistSession })
+    const agent1 = new AtpAgent({ service: network.pds.url, persistSession })
 
     await agent1.createAccount({
       handle: 'user3.test',
@@ -133,7 +127,7 @@ describe('agent', () => {
       throw new Error('No session created')
     }
 
-    const agent2 = new AtpAgent({ service: server.url, persistSession })
+    const agent2 = new AtpAgent({ service: network.pds.url, persistSession })
     const res1 = await agent2.resumeSession(agent1.session)
 
     expect(agent2.hasSession).toEqual(true)
@@ -165,7 +159,7 @@ describe('agent', () => {
       sessions.push(sess)
     }
 
-    const agent = new AtpAgent({ service: server.url, persistSession })
+    const agent = new AtpAgent({ service: network.pds.url, persistSession })
 
     // create an account and a session with it
     await agent.createAccount({
@@ -230,7 +224,7 @@ describe('agent', () => {
       sessions.push(sess)
     }
 
-    const agent = new AtpAgent({ service: server.url, persistSession })
+    const agent = new AtpAgent({ service: network.pds.url, persistSession })
 
     // create an account and a session with it
     await agent.createAccount({
@@ -309,7 +303,7 @@ describe('agent', () => {
       sessions.push(sess)
     }
 
-    const agent = new AtpAgent({ service: server.url, persistSession })
+    const agent = new AtpAgent({ service: network.pds.url, persistSession })
 
     try {
       await agent.login({
@@ -349,7 +343,7 @@ describe('agent', () => {
       sessions.push(sess)
     }
 
-    const agent = new AtpAgent({ service: server.url, persistSession })
+    const agent = new AtpAgent({ service: network.pds.url, persistSession })
 
     // create an account and a session with it
     await agent.createAccount({
@@ -420,7 +414,7 @@ describe('agent', () => {
         newHandlerCallCount++
       }
 
-      const agent = new AtpAgent({ service: server.url, persistSession })
+      const agent = new AtpAgent({ service: network.pds.url, persistSession })
 
       await agent.createAccount({
         handle: 'user7.test',
@@ -452,7 +446,7 @@ describe('agent', () => {
         sessions.push(sess)
       }
 
-      const agent = new AtpAgent({ service: server.url, persistSession })
+      const agent = new AtpAgent({ service: network.pds.url, persistSession })
 
       await expect(
         agent.createAccount({

diff --git a/packages/api/tests/bsky-agent.test.ts b/packages/api/tests/bsky-agent.test.ts
@@ -1,23 +1,17 @@
-import {
-  CloseFn,
-  runTestServer,
-  TestServerInfo,
-} from '@atproto/pds/tests/_util'
+import { TestNetworkNoAppView } from '@atproto/dev-env'
 import { BskyAgent, ComAtprotoRepoPutRecord, AppBskyActorProfile } from '..'
 
 describe('agent', () => {
-  let server: TestServerInfo
-  let close: CloseFn
+  let network: TestNetworkNoAppView
 
   beforeAll(async () => {
-    server = await runTestServer({
+    network = await TestNetworkNoAppView.create({
       dbPostgresSchema: 'bsky_agent',
     })
-    close = server.close
   })
 
   afterAll(async () => {
-    await close()
+    await network.close()
   })
 
   const getProfileDisplayName = async (
@@ -35,7 +29,7 @@ describe('agent', () => {
   }
 
   it('upsertProfile correctly creates and updates profiles.', async () => {
-    const agent = new BskyAgent({ service: server.url })
+    const agent = new BskyAgent({ service: network.pds.url })
 
     await agent.createAccount({
       handle: 'user1.test',
@@ -67,7 +61,7 @@ describe('agent', () => {
   })
 
   it('upsertProfile correctly handles CAS failures.', async () => {
-    const agent = new BskyAgent({ service: server.url })
+    const agent = new BskyAgent({ service: network.pds.url })
 
     await agent.createAccount({
       handle: 'user2.test',
@@ -106,7 +100,7 @@ describe('agent', () => {
   })
 
   it('upsertProfile wont endlessly retry CAS failures.', async () => {
-    const agent = new BskyAgent({ service: server.url })
+    const agent = new BskyAgent({ service: network.pds.url })
 
     await agent.createAccount({
       handle: 'user3.test',
@@ -135,7 +129,7 @@ describe('agent', () => {
   })
 
   it('upsertProfile validates the record.', async () => {
-    const agent = new BskyAgent({ service: server.url })
+    const agent = new BskyAgent({ service: network.pds.url })
 
     await agent.createAccount({
       handle: 'user4.test',
@@ -153,70 +147,70 @@ describe('agent', () => {
 
   describe('app', () => {
     it('should retrieve the api app', () => {
-      const agent = new BskyAgent({ service: server.url })
+      const agent = new BskyAgent({ service: network.pds.url })
       expect(agent.app).toBe(agent.api.app)
     })
   })
 
   describe('post', () => {
     it('should throw if no session', async () => {
-      const agent = new BskyAgent({ service: server.url })
+      const agent = new BskyAgent({ service: network.pds.url })
       await expect(agent.post({ text: 'foo' })).rejects.toThrow('Not logged in')
     })
   })
 
   describe('deletePost', () => {
     it('should throw if no session', async () => {
-      const agent = new BskyAgent({ service: server.url })
+      const agent = new BskyAgent({ service: network.pds.url })
       await expect(agent.deletePost('foo')).rejects.toThrow('Not logged in')
     })
   })
 
   describe('like', () => {
     it('should throw if no session', async () => {
-      const agent = new BskyAgent({ service: server.url })
+      const agent = new BskyAgent({ service: network.pds.url })
       await expect(agent.like('foo', 'bar')).rejects.toThrow('Not logged in')
     })
   })
 
   describe('deleteLike', () => {
     it('should throw if no session', async () => {
-      const agent = new BskyAgent({ service: server.url })
+      const agent = new BskyAgent({ service: network.pds.url })
       await expect(agent.deleteLike('foo')).rejects.toThrow('Not logged in')
     })
   })
 
   describe('repost', () => {
     it('should throw if no session', async () => {
-      const agent = new BskyAgent({ service: server.url })
+      const agent = new BskyAgent({ service: network.pds.url })
       await expect(agent.repost('foo', 'bar')).rejects.toThrow('Not logged in')
     })
   })
 
   describe('deleteRepost', () => {
     it('should throw if no session', async () => {
-      const agent = new BskyAgent({ service: server.url })
+      const agent = new BskyAgent({ service: network.pds.url })
       await expect(agent.deleteRepost('foo')).rejects.toThrow('Not logged in')
     })
   })
 
   describe('follow', () => {
     it('should throw if no session', async () => {
-      const agent = new BskyAgent({ service: server.url })
+      const agent = new BskyAgent({ service: network.pds.url })
       await expect(agent.follow('foo')).rejects.toThrow('Not logged in')
     })
   })
 
   describe('deleteFollow', () => {
     it('should throw if no session', async () => {
-      const agent = new BskyAgent({ service: server.url })
+      const agent = new BskyAgent({ service: network.pds.url })
       await expect(agent.deleteFollow('foo')).rejects.toThrow('Not logged in')
     })
   })
 
   describe('preferences methods', () => {
     it('gets and sets preferences correctly', async () => {
-      const agent = new BskyAgent({ service: server.url })
+      const agent = new BskyAgent({ service: network.pds.url })
 
       await agent.createAccount({
         handle: 'user5.test',
@@ -714,7 +708,7 @@ describe('agent', () => {
     })
 
     it('resolves duplicates correctly', async () => {
-      const agent = new BskyAgent({ service: server.url })
+      const agent = new BskyAgent({ service: network.pds.url })
 
       await agent.createAccount({
         handle: 'user6.test',

diff --git a/packages/api/tests/errors.test.ts b/packages/api/tests/errors.test.ts
@@ -1,25 +1,19 @@
-import {
-  CloseFn,
-  runTestServer,
-  TestServerInfo,
-} from '@atproto/pds/tests/_util'
 import { AtpAgent, ComAtprotoServerCreateAccount } from '..'
+import { TestNetworkNoAppView } from '@atproto/dev-env'
 
 describe('errors', () => {
-  let server: TestServerInfo
+  let network: TestNetworkNoAppView
   let client: AtpAgent
-  let close: CloseFn
 
   beforeAll(async () => {
-    server = await runTestServer({
+    network = await TestNetworkNoAppView.create({
       dbPostgresSchema: 'known_errors',
     })
-    client = new AtpAgent({ service: server.url })
-    close = server.close
+    client = network.pds.getClient()
   })
 
   afterAll(async () => {
-    await close()
+    await network.close()
   })
 
   it('constructs the correct error instance', async () => {

diff --git a/packages/bsky/example.dev.env b/packages/bsky/example.dev.env
diff --git a/packages/bsky/package.json b/packages/bsky/package.json
@@ -43,7 +43,6 @@
     "@isaacs/ttlcache": "^1.4.1",
     "compression": "^1.7.4",
     "cors": "^2.8.5",
-    "dotenv": "^16.0.0",
     "express": "^4.17.2",
     "express-async-errors": "^3.1.1",
     "form-data": "^4.0.0",