fix(database): [nan-1063] migrate to database package (#2236)

## Describe your changes
As part of another ticket related to webhooks, I want to make a specific
webhook package. I don't want the webhook package to depend on shared
but shared contained the database as of now. This PR moves the database
connection logic to its own package freeing it from shared and making it
easier to use freely in other packages now. Note that the seeders have
to stay due to the usage of services.

## Issue ticket number and link
Contributes to NAN-1063

## Checklist before requesting a review (skip if just adding/editing
APIs & templates)
- [ ] I added tests, otherwise the reason is: 
- [ ] I added observability, otherwise the reason is:
- [ ] I added analytics, otherwise the reason is:

Dockerfile

@@ -20,6 +20,7 @@ COPY packages/webapp/package.json ./packages/webapp/package.json
 COPY packages/data-ingestion/package.json ./packages/data-ingestion/package.json
 COPY packages/kvstore/package.json ./packages/kvstore/package.json
 COPY packages/logs/package.json ./packages/logs/package.json
+COPY packages/database/package.json ./packages/database/package.json
 COPY packages/records/package.json ./packages/records/package.json
 COPY packages/types/package.json ./packages/types/package.json
 COPY packages/scheduler/package.json ./packages/scheduler/package.json

package.json

@@ -19,11 +19,12 @@
         "packages/webapp",
         "packages/utils",
         "packages/data-ingestion",
-        "packages/types"
+        "packages/types",
+        "packages/database"
     ],
     "scripts": {
-        "create:migration": "cd packages/shared/lib/db && knex migrate:make $1 --esm --knexfile ./knexfile.cjs",
-        "undo:migration": "cd packages/shared/lib/db && knex migrate:rollback --esm --knexfile ./knexfile.cjs",
+        "create:migration": "cd packages/database/lib && knex migrate:make $1 --esm --knexfile ./knexfile.cjs",
+        "undo:migration": "cd packages/database/lib && knex migrate:rollback --esm --knexfile ./knexfile.cjs",
         "prettier-format": "prettier --config .prettierrc \"./**/*.{ts,tsx}\" --write",
         "prettier-watch": "onchange './**/*.{ts,tsx}' -- prettier --write {{changed}}",
         "lint": "eslint . --ext .ts,.tsx",

packages/database/.gitignore

@@ -0,0 +1,3 @@
+tsconfig.tsbuildinfo
+dist/*
+node_modules

packages/shared/lib/db/config.ts → packages/database/lib/config.ts

@@ -1,4 +1,4 @@
-import { getDbConfig } from './database.js';
+import { getDbConfig } from './getConfig.js';
 
 const config = getDbConfig({ timeoutMs: 60000 });
 

packages/shared/lib/db/db.integration.test.ts → packages/database/lib/db.integration.test.ts

@@ -1,5 +1,5 @@
 import { expect, describe, it } from 'vitest';
-import { multipleMigrations } from '../db/database.js';
+import { multipleMigrations } from './index.js';
 
 describe('Migration test', async () => {
     it('Should run migrations successfully', async () => {

packages/database/lib/getConfig.ts

@@ -0,0 +1,27 @@
+import type { Knex } from 'knex';
+
+export const defaultSchema = process.env['NANGO_DB_SCHEMA'] || 'nango';
+const additionalSchemas = process.env['NANGO_DB_ADDITIONAL_SCHEMAS']
+    ? process.env['NANGO_DB_ADDITIONAL_SCHEMAS'].split(',').map((schema: string) => schema.trim())
+    : [];
+
+export function getDbConfig({ timeoutMs }: { timeoutMs: number }): Knex.Config {
+    return {
+        client: process.env['NANGO_DB_CLIENT'] || 'pg',
+        connection: process.env['NANGO_DATABASE_URL'] || {
+            host: process.env['NANGO_DB_HOST'] || (process.env['SERVER_RUN_MODE'] === 'DOCKERIZED' ? 'nango-db' : 'localhost'),
+            port: +(process.env['NANGO_DB_PORT'] || 5432),
+            user: process.env['NANGO_DB_USER'] || 'nango',
+            database: process.env['NANGO_DB_NAME'] || 'nango',
+            password: process.env['NANGO_DB_PASSWORD'] || 'nango',
+            ssl: process.env['NANGO_DB_SSL'] != null && process.env['NANGO_DB_SSL'].toLowerCase() === 'true' ? { rejectUnauthorized: false } : undefined,
+            statement_timeout: timeoutMs
+        },
+        pool: {
+            min: parseInt(process.env['NANGO_DB_POOL_MIN'] || '2'),
+            max: parseInt(process.env['NANGO_DB_POOL_MAX'] || '50')
+        },
+        // SearchPath needs the current db and public because extension can only be installed once per DB
+        searchPath: [defaultSchema, 'public', ...additionalSchemas]
+    };
+}

packages/shared/lib/db/database.ts → packages/database/lib/index.ts

@@ -2,32 +2,7 @@ import knex from 'knex';
 import type { Knex } from 'knex';
 import { metrics, retry } from '@nangohq/utils';
 import type { Pool } from 'tarn';
-
-const defaultSchema = process.env['NANGO_DB_SCHEMA'] || 'nango';
-const additionalSchemas = process.env['NANGO_DB_ADDITIONAL_SCHEMAS']
-    ? process.env['NANGO_DB_ADDITIONAL_SCHEMAS'].split(',').map((schema: string) => schema.trim())
-    : [];
-
-export function getDbConfig({ timeoutMs }: { timeoutMs: number }): Knex.Config {
-    return {
-        client: process.env['NANGO_DB_CLIENT'] || 'pg',
-        connection: process.env['NANGO_DATABASE_URL'] || {
-            host: process.env['NANGO_DB_HOST'] || (process.env['SERVER_RUN_MODE'] === 'DOCKERIZED' ? 'nango-db' : 'localhost'),
-            port: +(process.env['NANGO_DB_PORT'] || 5432),
-            user: process.env['NANGO_DB_USER'] || 'nango',
-            database: process.env['NANGO_DB_NAME'] || 'nango',
-            password: process.env['NANGO_DB_PASSWORD'] || 'nango',
-            ssl: process.env['NANGO_DB_SSL'] != null && process.env['NANGO_DB_SSL'].toLowerCase() === 'true' ? { rejectUnauthorized: false } : undefined,
-            statement_timeout: timeoutMs
-        },
-        pool: {
-            min: parseInt(process.env['NANGO_DB_POOL_MIN'] || '2'),
-            max: parseInt(process.env['NANGO_DB_POOL_MAX'] || '50')
-        },
-        // SearchPath needs the current db and public because extension can only be installed once per DB
-        searchPath: [defaultSchema, 'public', ...additionalSchemas]
-    };
-}
+import { defaultSchema, getDbConfig } from './getConfig.js';
 
 export class KnexDatabase {
     knex: Knex;
@@ -103,6 +78,8 @@ export const schema = (): Knex.QueryBuilder => db.knex.queryBuilder();
 
 export const dbNamespace = '_nango_';
 
+export type { Knex };
+
 export const multipleMigrations = async (): Promise<void> => {
     try {
         await db.knex.raw(`CREATE SCHEMA IF NOT EXISTS ${db.schema()}`);

packages/shared/lib/db/knexfile.cjs → packages/database/lib/knexfile.cjs

@@ -1,6 +1,6 @@
 // Knex CLI for migration needs this Knexfile but doesn't play well with ESM modules.
 // That's why the content of the Knex config is moved to ./config.ts to be imported by the app in ESM-fashion, and the Knexfile is only used for the Knex CLI in CommonJS-fashion.
 // eslint-disable-next-line @typescript-eslint/no-var-requires
-const { config } = require('../../dist/db/config.js');
+const { config } = require('../dist/config.js');
 
 module.exports = config;

packages/shared/lib/db/migrations/20221026075018_create_connection.cjs → packages/database/lib/migrations/20221026075018_create_connection.cjs

@@ -10,6 +10,5 @@ exports.up = function (knex, _) {
 };
 
 exports.down = function (knex, _) {
-    var schema = 'nango';
     return knex.schema.dropTable('_nango_connections');
 };

packages/shared/lib/db/migrations/20230804082618_connection_metadata_consolidation.cjs → packages/database/lib/migrations/20230804082618_connection_metadata_consolidation.cjs

@@ -4,7 +4,6 @@
  *       2) move fieldMappings into metadata under a fieldMapping key
  */
 const DB_TABLE = '_nango_connections';
-const TABLE_PREFIX = '_nango_';
 
 exports.up = async function (knex, _) {
     const existingMetaData = await knex.select('id', 'metadata', 'connection_config').from(DB_TABLE).whereNotNull('metadata').andWhere('metadata', '!=', '{}');

packages/shared/lib/db/migrations/20230804110506_remove_params_from_connection_config.cjs → packages/database/lib/migrations/20230804110506_remove_params_from_connection_config.cjs

@@ -4,7 +4,6 @@
  *       2) If it has a connectionConfig.params[string] key remove the params key and move the value to connectionConfig[string]
  */
 const DB_TABLE = '_nango_connections';
-const TABLE_PREFIX = '_nango_';
 
 exports.up = async function (knex, _) {
     const existingCC = await knex.select('id', 'connection_config').from(DB_TABLE).whereNotNull('connection_config').andWhere('connection_config', '!=', '{}');
@@ -19,6 +18,7 @@ exports.up = async function (knex, _) {
             if (key.includes('connectionConfig.params.')) {
                 const newKey = key.replace('connectionConfig.params.', '');
                 connection_config[newKey] = connection_config[key];
+                // eslint-disable-next-line @typescript-eslint/no-dynamic-delete
                 delete connection_config[key];
 
                 await knex.update({ connection_config }).from(DB_TABLE).where({ id });
@@ -29,6 +29,6 @@ exports.up = async function (knex, _) {
     return Promise.resolve();
 };
 
-exports.down = async function (knex, _) {
+exports.down = async function (_knex, _) {
     return Promise.resolve();
 };

packages/shared/lib/db/migrations/20231213124728_drop_sync_data_records_indexes.cjs → packages/database/lib/migrations/20231213124728_drop_sync_data_records_indexes.cjs

@@ -1,6 +1,6 @@
 exports.config = { transaction: false };
 
-exports.up = function(knex) {
+exports.up = function (_knex) {
     return Promise.resolve();
     /*
      * Production only migration
@@ -17,6 +17,6 @@ exports.up = function(knex) {
     */
 };
 
-exports.down = function(knex) {
+exports.down = function (_knex) {
     return Promise.resolve();
 };

packages/database/lib/migrations/20240527094039_add_notifications_table.cjs

@@ -0,0 +1,22 @@
+const DB_TABLE = '_nango_active_logs';
+
+exports.up = async function (knex, _) {
+    return knex.schema.createTable(DB_TABLE, function (table) {
+        table.increments('id').primary();
+        table.string('type', 'varchar(255)').notNullable();
+        table.string('action', 'varchar(255)').notNullable();
+        table.integer('connection_id').unsigned().notNullable();
+        table.foreign('connection_id').references('id').inTable('_nango_connections').onDelete('CASCADE');
+        table.integer('activity_log_id').unsigned().notNullable();
+        table.foreign('activity_log_id').references('id').inTable('_nango_activity_logs').onDelete('CASCADE');
+        table.string('log_id');
+        table.boolean('active').defaultTo(true);
+        table.uuid('sync_id').defaultTo(null);
+        table.foreign('sync_id').references('id').inTable('_nango_syncs').onDelete('CASCADE');
+        table.timestamps(true, true);
+    });
+};
+
+exports.down = async function (knex, _) {
+    return knex.schema.dropTable(DB_TABLE);
+};

packages/database/lib/migrations/20240529180658_add_notifications_table_index.cjs

@@ -0,0 +1,9 @@
+exports.config = { transaction: false };
+
+exports.up = async function (knex, _) {
+    await knex.schema.raw('CREATE INDEX CONCURRENTLY "idx_sync_id_active_true" ON "_nango_active_logs" USING BTREE ("sync_id") WHERE active = true');
+};
+
+exports.down = async function (knex, _) {
+    await knex.schema.raw('DROP INDEX CONCURRENTLY idx_sync_id_active_true');
+};

packages/database/lib/migrations/20240531122550_add_webhooks_table.cjs

@@ -0,0 +1,20 @@
+const DB_TABLE = '_nango_external_webhooks';
+
+exports.up = async function (knex, _) {
+    return knex.schema.createTable(DB_TABLE, function (table) {
+        table.increments('id').primary();
+        table.integer('environment_id').unsigned().notNullable();
+        table.foreign('environment_id').references('id').inTable('_nango_environments').onDelete('CASCADE');
+        table.string('primary_url').notNullable();
+        table.string('secondary_url').notNullable();
+        table.boolean('on_sync_completion_always').defaultTo(false);
+        table.boolean('on_auth_creation').defaultTo(false);
+        table.boolean('on_auth_refesh_error').defaultTo(false);
+        table.boolean('on_sync_error').defaultTo(false);
+        table.timestamps(true, true);
+    });
+};
+
+exports.down = async function (knex, _) {
+    return knex.schema.dropTable(DB_TABLE);
+};

packages/database/package.json

@@ -0,0 +1,32 @@
+{
+    "name": "@nangohq/database",
+    "version": "1.0.0",
+    "type": "module",
+    "main": "./dist/index.js",
+    "module": "./dist/index.js",
+    "private": true,
+    "bundleDependencies": [
+        "@nangohq/utils"
+    ],
+    "scripts": {
+        "build": "rimraf ./dist && tsc"
+    },
+    "repository": {
+        "type": "git",
+        "url": "git+https://github.com/NangoHQ/nango.git",
+        "directory": "packages/database"
+    },
+    "keywords": [],
+    "dependencies": {
+        "@nangohq/utils": "file:../utils",
+        "knex": "3.1.0",
+        "tarn": "3.0.2"
+    },
+    "devDependencies": {
+        "typescript": "^5.3.3",
+        "vitest": "0.33.0"
+    },
+    "files": [
+        "dist/**/*"
+    ]
+}

packages/database/tsconfig.json

@@ -0,0 +1,13 @@
+{
+    "extends": "../../tsconfig.json",
+    "compilerOptions": {
+        "rootDir": "lib",
+        "outDir": "dist"
+    },
+    "references": [
+        {
+            "path": "../utils"
+        }
+    ],
+    "include": ["lib/**/*", "../utils/lib/vitest.d.ts"]
+}

packages/jobs/Dockerfile

@@ -16,6 +16,7 @@ COPY packages/data-ingestion/ packages/data-ingestion/
 COPY packages/kvstore/ packages/kvstore/
 COPY packages/jobs/ packages/jobs/
 COPY packages/logs/ packages/logs/
+COPY packages/database/ packages/database/
 COPY packages/runner/ packages/runner/
 COPY packages/scheduler/ packages/scheduler/
 COPY packages/orchestrator/ packages/orchestrator/

packages/jobs/lib/app.ts

@@ -7,7 +7,8 @@ import { deleteOldActivityLogs } from './crons/deleteOldActivities.js';
 import { deleteSyncsData } from './crons/deleteSyncsData.js';
 import { reconcileTemporalSchedules } from './crons/reconcileTemporalSchedules.js';
 import { getLogger, stringifyError } from '@nangohq/utils';
-import { db, featureFlags } from '@nangohq/shared';
+import { featureFlags } from '@nangohq/shared';
+import db from '@nangohq/database';
 import { envs } from './env.js';
 
 const logger = getLogger('Jobs');