Skip to content

Commit

Permalink
Bsky appview admin and moderation endpoints (bluesky-social#840)
Browse files Browse the repository at this point in the history
* Add moderation and labeling model to bsky appview

* Carry over moderation service from pds

* Pass over bsky moderation service to get it working, missing blob support

* Carry over label service from pds to bsky

* Reorg bsky http auth, implement admin auth

* Carry over relevant admin endpoints pds to bsky

* Retrofit bsky admin routes from pds impls

* Implement resolve handle on appview

* Avoid loop in bsky handle resolution

* Add bsky appview to dev-env, opt-in

* Fix bsky searchRepos with empty term, tidy

* Include blobs on bsky admin views

* Stop resolving taken-down blobs on appview

* Tidy

* Carry over pds moderation tests to bsky

* Support image cache invalidation on bsky appview

* Add missing changes for bsky tests

* Test takedowns in bsky views

* Test takedowns on bsky notifs

* In bsky appview ensure label.neg is modeled as a boolean, not an integer bit. Add select column when checking blob takedown.
  • Loading branch information
devinivy authored Apr 24, 2023
1 parent f30887b commit 05e6ebe
Show file tree
Hide file tree
Showing 65 changed files with 5,217 additions and 105 deletions.
14 changes: 13 additions & 1 deletion packages/bsky/service/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ require('dd-trace/init') // Only works with commonjs

// Tracer code above must come before anything else
const path = require('path')
const { CloudfrontInvalidator } = require('@atproto/aws')
const { Database, ServerConfig, BskyAppView } = require('@atproto/bsky')

const main = async () => {
Expand Down Expand Up @@ -40,7 +41,17 @@ const main = async () => {
imgUriEndpoint: env.imgUriEndpoint,
blobCacheLocation: env.blobCacheLocation,
})
const bsky = BskyAppView.create({ db, config: cfg })
const cfInvalidator = env.cfDistributionId
? new CloudfrontInvalidator({
distributionId: env.cfDistributionId,
pathPrefix: cfg.imgUriEndpoint && new URL(cfg.imgUriEndpoint).pathname,
})
: undefined
const bsky = BskyAppView.create({
db,
config: cfg,
imgInvalidator: cfInvalidator,
})
await bsky.start()
// Graceful shutdown (see also https://aws.amazon.com/blogs/containers/graceful-shutdowns-with-ecs/)
process.on('SIGTERM', async () => {
Expand All @@ -62,6 +73,7 @@ const getEnv = () => ({
imgUriKey: process.env.IMG_URI_KEY,
imgUriEndpoint: process.env.IMG_URI_ENDPOINT,
blobCacheLocation: process.env.BLOB_CACHE_LOC,
cfDistributionId: process.env.CF_DISTRIBUTION_ID,
})

const maintainXrpcResource = (span, req) => {
Expand Down
2 changes: 1 addition & 1 deletion packages/bsky/src/api/app/bsky/actor/getProfile.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { InvalidRequestError } from '@atproto/xrpc-server'
import { Server } from '../../../../lexicon'
import { softDeleted } from '../../../../db/util'
import AppContext from '../../../../context'
import { authOptionalVerifier } from '../util'
import { authOptionalVerifier } from '../../../auth'

export default function (server: Server, ctx: AppContext) {
server.app.bsky.actor.getProfile({
Expand Down
3 changes: 1 addition & 2 deletions packages/bsky/src/api/app/bsky/actor/getProfiles.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import { Server } from '../../../../lexicon'
import AppContext from '../../../../context'
import { authOptionalVerifier } from '../util'

import { authOptionalVerifier } from '../../../auth'
export default function (server: Server, ctx: AppContext) {
server.app.bsky.actor.getProfiles({
auth: authOptionalVerifier,
Expand Down
2 changes: 1 addition & 1 deletion packages/bsky/src/api/app/bsky/actor/getSuggestions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import AppContext from '../../../../context'
import { Cursor, GenericKeyset, paginate } from '../../../../db/pagination'
import { countAll, notSoftDeletedClause } from '../../../../db/util'
import { Server } from '../../../../lexicon'
import { authOptionalVerifier } from '../util'
import { authOptionalVerifier } from '../../../auth'

export default function (server: Server, ctx: AppContext) {
server.app.bsky.actor.getSuggestions({
Expand Down
2 changes: 1 addition & 1 deletion packages/bsky/src/api/app/bsky/actor/searchActors.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import {
getUserSearchQuery,
SearchKeyset,
} from '../../../../services/util/search'
import { authOptionalVerifier } from '../util'
import { authOptionalVerifier } from '../../../auth'

export default function (server: Server, ctx: AppContext) {
server.app.bsky.actor.searchActors({
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import AppContext from '../../../../context'
import { Server } from '../../../../lexicon'
import { cleanTerm, getUserSearchQuery } from '../../../../services/util/search'
import { authOptionalVerifier } from '../util'
import { authOptionalVerifier } from '../../../auth'

export default function (server: Server, ctx: AppContext) {
server.app.bsky.actor.searchActorsTypeahead({
Expand Down
2 changes: 1 addition & 1 deletion packages/bsky/src/api/app/bsky/feed/getAuthorFeed.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { Server } from '../../../../lexicon'
import { FeedKeyset, composeFeed } from '../util/feed'
import { paginate } from '../../../../db/pagination'
import AppContext from '../../../../context'
import { authOptionalVerifier } from '../util'
import { authOptionalVerifier } from '../../../auth'

export default function (server: Server, ctx: AppContext) {
server.app.bsky.feed.getAuthorFeed({
Expand Down
2 changes: 1 addition & 1 deletion packages/bsky/src/api/app/bsky/feed/getLikes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { Server } from '../../../../lexicon'
import { paginate, TimeCidKeyset } from '../../../../db/pagination'
import AppContext from '../../../../context'
import { notSoftDeletedClause } from '../../../../db/util'
import { authOptionalVerifier } from '../util'
import { authOptionalVerifier } from '../../../auth'

export default function (server: Server, ctx: AppContext) {
server.app.bsky.feed.getLikes({
Expand Down
2 changes: 1 addition & 1 deletion packages/bsky/src/api/app/bsky/feed/getPostThread.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import {
PostInfoMap,
} from '../../../../services/types'
import { FeedService } from '../../../../services/feed'
import { authOptionalVerifier } from '../util'
import { authOptionalVerifier } from '../../../auth'

export type PostThread = {
post: FeedRow
Expand Down
2 changes: 1 addition & 1 deletion packages/bsky/src/api/app/bsky/feed/getRepostedBy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { Server } from '../../../../lexicon'
import { paginate, TimeCidKeyset } from '../../../../db/pagination'
import AppContext from '../../../../context'
import { notSoftDeletedClause } from '../../../../db/util'
import { authOptionalVerifier } from '../util'
import { authOptionalVerifier } from '../../../auth'

export default function (server: Server, ctx: AppContext) {
server.app.bsky.feed.getRepostedBy({
Expand Down
2 changes: 1 addition & 1 deletion packages/bsky/src/api/app/bsky/feed/getTimeline.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { Server } from '../../../../lexicon'
import { FeedAlgorithm, FeedKeyset, composeFeed } from '../util/feed'
import { paginate } from '../../../../db/pagination'
import AppContext from '../../../../context'
import { authVerifier } from '../util'
import { authVerifier } from '../../../auth'

// @TODO getTimeline() will be replaced by composeTimeline() in the app-view
export default function (server: Server, ctx: AppContext) {
Expand Down
2 changes: 1 addition & 1 deletion packages/bsky/src/api/app/bsky/graph/getFollowers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { Server } from '../../../../lexicon'
import { paginate, TimeCidKeyset } from '../../../../db/pagination'
import AppContext from '../../../../context'
import { notSoftDeletedClause } from '../../../../db/util'
import { authOptionalVerifier } from '../util'
import { authOptionalVerifier } from '../../../auth'

export default function (server: Server, ctx: AppContext) {
server.app.bsky.graph.getFollowers({
Expand Down
2 changes: 1 addition & 1 deletion packages/bsky/src/api/app/bsky/graph/getFollows.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { Server } from '../../../../lexicon'
import { paginate, TimeCidKeyset } from '../../../../db/pagination'
import AppContext from '../../../../context'
import { notSoftDeletedClause } from '../../../../db/util'
import { authOptionalVerifier } from '../util'
import { authOptionalVerifier } from '../../../auth'

export default function (server: Server, ctx: AppContext) {
server.app.bsky.graph.getFollows({
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { Server } from '../../../../lexicon'
import { countAll, notSoftDeletedClause } from '../../../../db/util'
import AppContext from '../../../../context'
import { authVerifier } from '../util'
import { authVerifier } from '../../../auth'

export default function (server: Server, ctx: AppContext) {
server.app.bsky.notification.getUnreadCount({
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { Server } from '../../../../lexicon'
import { paginate, TimeCidKeyset } from '../../../../db/pagination'
import AppContext from '../../../../context'
import { notSoftDeletedClause } from '../../../../db/util'
import { authVerifier } from '../util'
import { authVerifier } from '../../../auth'

export default function (server: Server, ctx: AppContext) {
server.app.bsky.notification.listNotifications({
Expand Down
2 changes: 1 addition & 1 deletion packages/bsky/src/api/app/bsky/unspecced.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import { FeedKeyset, composeFeed } from './util/feed'
import { paginate } from '../../../db/pagination'
import AppContext from '../../../context'
import { FeedRow, FeedItemType } from '../../../services/types'
import { authOptionalVerifier } from './util'
import { authOptionalVerifier } from '../../auth'
import { countAll } from '../../../db/util'

// THIS IS A TEMPORARY UNSPECCED ROUTE
Expand Down
28 changes: 0 additions & 28 deletions packages/bsky/src/api/app/bsky/util/index.ts

This file was deleted.

65 changes: 65 additions & 0 deletions packages/bsky/src/api/auth.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
import express from 'express'
import * as uint8arrays from 'uint8arrays'
import { AuthRequiredError } from '@atproto/xrpc-server'

const BASIC = 'Basic '
const BEARER = 'Bearer '

// @TODO(bsky) treating did as a bearer, just a placeholder for now.
export const authVerifier = (ctx: {
req: express.Request
res: express.Response
}) => {
const { authorization = '' } = ctx.req.headers
if (!authorization.startsWith(BEARER)) {
throw new AuthRequiredError()
}
const did = authorization.replace(BEARER, '').trim()
if (!did.startsWith('did:')) {
throw new AuthRequiredError()
}
return { credentials: { did } }
}

export const authOptionalVerifier = (ctx: {
req: express.Request
res: express.Response
}) => {
if (!ctx.req.headers.authorization) {
return { credentials: { did: null } }
}
return authVerifier(ctx)
}

export const adminVerifier =
(adminPassword: string) =>
(ctx: { req: express.Request; res: express.Response }) => {
const { authorization = '' } = ctx.req.headers
const parsed = parseBasicAuth(authorization)
if (!parsed) {
throw new AuthRequiredError()
}
const { username, password } = parsed
if (username !== 'admin' || password !== adminPassword) {
throw new AuthRequiredError()
}
return { credentials: { admin: true } }
}

export const parseBasicAuth = (
token: string,
): { username: string; password: string } | null => {
if (!token.startsWith(BASIC)) return null
const b64 = token.slice(BASIC.length)
let parsed: string[]
try {
parsed = uint8arrays
.toString(uint8arrays.fromString(b64, 'base64pad'), 'utf8')
.split(':')
} catch (err) {
return null
}
const [username, password] = parsed
if (!username || !password) return null
return { username, password }
}
21 changes: 20 additions & 1 deletion packages/bsky/src/api/blob-resolver.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import axios, { AxiosError } from 'axios'
import { CID } from 'multiformats/cid'
import { ensureValidDid } from '@atproto/identifier'
import { VerifyCidTransform } from '@atproto/common'
import { TAKEDOWN } from '../lexicon/types/com/atproto/admin/defs'
import { DidNotFoundError } from '@atproto/did-resolver'
import AppContext from '../context'
import { httpLogger as log } from '../logger'
Expand All @@ -30,7 +31,25 @@ export const createRouter = (ctx: AppContext): express.Router => {
return next(createError(400, 'Invalid cid'))
}

const { pds } = await ctx.didResolver.resolveAtprotoData(did) // @TODO cache did info
const [{ pds }, takedown] = await Promise.all([
ctx.didResolver.resolveAtprotoData(did), // @TODO cache did info
ctx.db.db
.selectFrom('moderation_action_subject_blob')
.select('actionId')
.innerJoin(
'moderation_action',
'moderation_action.id',
'moderation_action_subject_blob.actionId',
)
.where('cid', '=', cidStr)
.where('action', '=', TAKEDOWN)
.where('reversedAt', 'is', null)
.executeTakeFirst(),
])
if (takedown) {
return next(createError(404, 'Blob not found'))
}

const blobResult = await retryHttp(() =>
getBlob({ pds, did, cid: cidStr }),
)
Expand Down
19 changes: 19 additions & 0 deletions packages/bsky/src/api/com/atproto/admin/getModerationAction.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
import { Server } from '../../../../lexicon'
import AppContext from '../../../../context'
import { adminVerifier } from '../../../auth'

export default function (server: Server, ctx: AppContext) {
server.com.atproto.admin.getModerationAction({
auth: adminVerifier(ctx.cfg.adminPassword),
handler: async ({ params }) => {
const { db, services } = ctx
const { id } = params
const moderationService = services.moderation(db)
const result = await moderationService.getActionOrThrow(id)
return {
encoding: 'application/json',
body: await moderationService.views.actionDetail(result),
}
},
})
}
26 changes: 26 additions & 0 deletions packages/bsky/src/api/com/atproto/admin/getModerationActions.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
import { Server } from '../../../../lexicon'
import AppContext from '../../../../context'
import { adminVerifier } from '../../../auth'

export default function (server: Server, ctx: AppContext) {
server.com.atproto.admin.getModerationActions({
auth: adminVerifier(ctx.cfg.adminPassword),
handler: async ({ params }) => {
const { db, services } = ctx
const { subject, limit = 50, cursor } = params
const moderationService = services.moderation(db)
const results = await moderationService.getActions({
subject,
limit,
cursor,
})
return {
encoding: 'application/json',
body: {
cursor: results.at(-1)?.id.toString() ?? undefined,
actions: await moderationService.views.action(results),
},
}
},
})
}
19 changes: 19 additions & 0 deletions packages/bsky/src/api/com/atproto/admin/getModerationReport.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
import { Server } from '../../../../lexicon'
import AppContext from '../../../../context'
import { adminVerifier } from '../../../auth'

export default function (server: Server, ctx: AppContext) {
server.com.atproto.admin.getModerationReport({
auth: adminVerifier(ctx.cfg.adminPassword),
handler: async ({ params }) => {
const { db, services } = ctx
const { id } = params
const moderationService = services.moderation(db)
const result = await moderationService.getReportOrThrow(id)
return {
encoding: 'application/json',
body: await moderationService.views.reportDetail(result),
}
},
})
}
27 changes: 27 additions & 0 deletions packages/bsky/src/api/com/atproto/admin/getModerationReports.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
import { Server } from '../../../../lexicon'
import AppContext from '../../../../context'
import { adminVerifier } from '../../../auth'

export default function (server: Server, ctx: AppContext) {
server.com.atproto.admin.getModerationReports({
auth: adminVerifier(ctx.cfg.adminPassword),
handler: async ({ params }) => {
const { db, services } = ctx
const { subject, resolved, limit = 50, cursor } = params
const moderationService = services.moderation(db)
const results = await moderationService.getReports({
subject,
resolved,
limit,
cursor,
})
return {
encoding: 'application/json',
body: {
cursor: results.at(-1)?.id.toString() ?? undefined,
reports: await moderationService.views.report(results),
},
}
},
})
}
Loading

0 comments on commit 05e6ebe

Please sign in to comment.