[WIP] Transaction workers to Redis (thirdweb-dev#554)

* wip * WIP: transaction workers in Redis * webhooks and usage tracking * wip * remove prepare worker * multiple transaction hashes * updates to allow all POST end-point to work with new flow * transaction db updates * purge worker * db update * add queued txs * UserOps Updates * build fixes * Error Handling update * updates for errors * simplify purge logic * wip * add getChain * support get all tx by wallet * fix sent redis * updated error handling * wip * separate helper funcs for userOps * typing... * remove env debug * update ttl seconds * fix build, add nonce helpers * update: added implementation to retrieve the contract-address * updates for deployedContractAddress * fixed createAccount build issue * unused nonces worker * unused nonces * send-transaction userOp update * add pretty error message back * Add openapi.json to gitignore * remove debug line * unused nonce * reuse rpc request * remove simulation in send eoa transaction * update sdk, undo rpc client caching * remove unused env var * cleanup and comments * log released nonces * handle resend rpc errors * explicitly list attempts/backoff * fix mine worker bug * better handle rpc error * prune transaction lists and details * prune tx by count * wip * explicit worker init step * add basic auth to queues management * migrate postgres tx * add migration worker * updates for value override, simulation + send Tx/UserOps using data * tx details debug endpoint * Remove debug line * fix example * fix: bullboard not working in prod * move cron init workers * bugfix: account created from ethers signer need provider * Add test suite * bugfix: remove hardcoded anvil chain * fix: only connect provider to aws signer * fix: removed logs * fix: remove debug test values * fix: pass wallet label in test * Updates: Removed logs + queuedAt indexed * Fk/extend ph worker (thirdweb-dev#595) * Update: Added nonce rebase with blockchain logic * removed logs * updates * AWS & GCP KMS provider fix * updated rebaseNonce * feat: take file as arg * chore: update anvil installation instructions in e2e README * Updates: v2 Nonce (thirdweb-dev#597) * Updates * updates * cleaned logs * updates * updated recycleNonce to check if isNan * comments updated * chore: expose NonceResyncQueue to bullboard * Convert nonce-recycled to set and add nonce-sent set helpers * fix: handle removing from sent nonces in fail handler * chore: Remove commented out code in setup.ts file * Update nonceResyncWorker to use stricter wildcard pattern for redis keys * Add isSentNonce helper function for checking if a nonce is in the sent nonces set * refactor and solve nonce resynchronization logic bugs and nits in nonceResyncWorker * chore: Refactor cancelRecycledNoncesWorker to use SMEMBERS instead of LRANGE for retrieving unused nonces * treat `ReplacementGasFeeTowLow` as `NonceAlreadyUsed` * updated logs & removeSentNonce --------- Co-authored-by: Prithvish Baidya <[email protected]> * yarn lock * queue latency metrics * use v5 for local wallets on EOA * handle rpc errors better * fix: batch ioredis operations * updated isReplacementGasFeeTooLow check * graceful shutdown, use multi instead of lua script * add backoff on SendTransactionQueue * fix: default REDIS_MAXMEMORY = 0 * lua script default to 0 * fix queued tx cancel * default concurrency 200 * updated wallet_address -> walletAddress, erc20 balance-of * reverting changes. will move it to a breaking updates PR * checksum addresses to RPC * fix build * use checksumAddress * Make effective gas price optional * set onchainStatus for userop * allow replaying failed txs * always query latest block tag for tx count * fix: regex address validation * Updates: Claim-to using v5 SDK (thirdweb-dev#618) * Updates: Claim-to using v5 SDK * updates * removed caching for getContractV5 --------- Co-authored-by: farhanW3 <[email protected]> Co-authored-by: Prithvish Baidya <[email protected]> Co-authored-by: Farhan Khwaja <[email protected]>
domecloud · Aug 23, 2024 · 0ba2550 · 0ba2550
1 parent 019a78e
commit 0ba2550
Show file tree

Hide file tree

Showing 164 changed files with 5,339 additions and 3,947 deletions.
diff --git a/.gitignore b/.gitignore
@@ -134,6 +134,10 @@ yarn-error.log
 .yarn/install-state.gz
 .pnp.*
 
+# yalc cache
+.yalc
+yalc.lock
+
 swagger.yml
 worker/distWorker/
 .DS_Store
@@ -144,3 +148,6 @@ credentials
 
 # SSL Local Certs folder
 https
+
+# Auto generated OpenAPI file
+openapi.json
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -30,23 +30,14 @@ services:
     restart: always
     ports:
       - 6379:6379
+    deploy:
+      resources:
+        limits:
+          cpus: "1"
+          memory: 1024M
     volumes:
       - redis_data:/data
 
-  bullboard:
-    container_name: bullboard
-    image: deadly0/bull-board:3.2.6
-    restart: always
-    ports:
-      - 3333:3000
-    environment:
-      REDIS_HOST: redis
-      REDIS_PORT: 6379
-      REDIS_USE_TLS: "false"
-      BULL_PREFIX: bull
-    depends_on:
-      - redis
-
   engine:
     profiles:
       - engine

diff --git a/package.json b/package.json
@@ -27,13 +27,15 @@
   },
   "dependencies": {
     "@aws-sdk/client-kms": "^3.398.0",
+    "@bull-board/fastify": "^5.21.1",
+    "@fastify/basic-auth": "^5.1.1",
     "@fastify/cookie": "^8.3.0",
     "@fastify/express": "^2.3.0",
     "@fastify/swagger": "^8.9.0",
     "@fastify/type-provider-typebox": "^3.2.0",
     "@fastify/websocket": "^8.2.0",
     "@google-cloud/kms": "^4.4.0",
-    "@prisma/client": "^5.14.0",
+    "@prisma/client": "5.17.0",
     "@sinclair/typebox": "^0.31.28",
     "@t3-oss/env-core": "^0.6.0",
     "@thirdweb-dev/auth": "^4.1.87",
@@ -43,7 +45,7 @@
     "@types/base-64": "^1.0.2",
     "base-64": "^1.0.0",
     "body-parser": "^1.20.2",
-    "bullmq": "^5.7.8",
+    "bullmq": "^5.11.0",
     "cookie": "^0.5.0",
     "cookie-parser": "^1.4.6",
     "copyfiles": "^2.4.1",
@@ -68,7 +70,6 @@
     "superjson": "^2.2.1",
     "thirdweb": "^5.45.1",
     "uuid": "^9.0.1",
-    "viem": "^1.14.0",
     "winston": "^3.14.1",
     "zod": "^3.23.8"
   },

diff --git a/src/db/client.ts b/src/db/client.ts
@@ -22,7 +22,7 @@ export const knex = pg({
   acquireConnectionTimeout: 30000,
 } as Knex.Config);
 
-export const isDatabaseHealthy = async (): Promise<boolean> => {
+export const isDatabaseReachable = async () => {
   try {
     await prisma.walletDetails.findFirst();
     return true;

diff --git a/src/db/configuration/getConfiguration.ts b/src/db/configuration/getConfiguration.ts
@@ -1,16 +1,19 @@
 import { Configuration } from "@prisma/client";
+import { Static } from "@sinclair/typebox";
 import { LocalWallet } from "@thirdweb-dev/wallets";
 import { ethers } from "ethers";
-import { Config } from "../../schema/config";
+import { Chain } from "thirdweb";
+import { ParsedConfig } from "../../schema/config";
 import { WalletType } from "../../schema/wallet";
 import { mandatoryAllowedCorsUrls } from "../../server/utils/cors-urls";
+import { networkResponseSchema } from "../../utils/cache/getSdk";
 import { decrypt } from "../../utils/crypto";
 import { env } from "../../utils/env";
 import { logger } from "../../utils/logger";
 import { prisma } from "../client";
 import { updateConfiguration } from "./updateConfiguration";
 
-const withWalletConfig = async (config: Configuration): Promise<Config> => {
+const toParsedConfig = async (config: Configuration): Promise<ParsedConfig> => {
   // We destructure the config to omit wallet related fields to prevent direct access
   const {
     awsAccessKeyId,
@@ -25,6 +28,31 @@ const withWalletConfig = async (config: Configuration): Promise<Config> => {
     ...restConfig
   } = config;
 
+  // Parse "chainOverrides" JSON to an array of Chain[] items.
+  let chainOverridesParsed: Chain[] = [];
+  if (config.chainOverrides) {
+    try {
+      const parsed: Static<typeof networkResponseSchema>[] = JSON.parse(
+        config.chainOverrides,
+      );
+      chainOverridesParsed = parsed.map(
+        (chain): Chain => ({
+          id: chain.chainId,
+          name: chain.name,
+          rpc: chain.rpc[0],
+          nativeCurrency: chain.nativeCurrency,
+          testnet: chain.testnet ? true : undefined,
+        }),
+      );
+    } catch (e) {
+      logger({
+        service: "server",
+        level: "error",
+        message: `Failed parsing chainOverrides: ${e}`,
+      });
+    }
+  }
+
   // TODO: Remove backwards compatibility with next breaking change
   if (awsAccessKeyId && awsSecretAccessKey && awsRegion) {
     // First try to load the aws secret using the encryption password
@@ -61,6 +89,7 @@ const withWalletConfig = async (config: Configuration): Promise<Config> => {
       ...restConfig,
       contractSubscriptionsRequeryDelaySeconds:
         contractSubscriptionsRetryDelaySeconds,
+      chainOverridesParsed,
       walletConfiguration: {
         type: WalletType.awsKms,
         awsRegion,
@@ -109,6 +138,7 @@ const withWalletConfig = async (config: Configuration): Promise<Config> => {
       ...restConfig,
       contractSubscriptionsRequeryDelaySeconds:
         contractSubscriptionsRetryDelaySeconds,
+      chainOverridesParsed,
       walletConfiguration: {
         type: WalletType.gcpKms,
         gcpApplicationProjectId,
@@ -124,6 +154,7 @@ const withWalletConfig = async (config: Configuration): Promise<Config> => {
     ...restConfig,
     contractSubscriptionsRequeryDelaySeconds:
       contractSubscriptionsRetryDelaySeconds,
+    chainOverridesParsed,
     walletConfiguration: {
       type: WalletType.local,
     },
@@ -139,7 +170,7 @@ const createAuthWalletEncryptedJson = async () => {
   });
 };
 
-export const getConfiguration = async (): Promise<Config> => {
+export const getConfiguration = async (): Promise<ParsedConfig> => {
   let config = await prisma.configuration.findUnique({
     where: {
       id: "default",
@@ -193,5 +224,6 @@ export const getConfiguration = async (): Promise<Config> => {
     });
   }
 
-  return withWalletConfig(config);
+  const result = await toParsedConfig(config);
+  return result;
 };
diff --git a/src/db/transactions/cleanTxs.ts b/src/db/transactions/cleanTxs.ts