[Github #627] shared refactor groundwork (#635)

* [gh-#627] add shared package

* [gh-#627] shared models refactor

* [gh-#627] update refactor logic

* [gh-#627] add references to shared

* [gh-#627] copy shared package in the Dockerfile

* [gh-#627] standardize tsconfig

* [gh-#627] update db placement

* [gh-#627] update build commands

* [gh-#627] npm i before building

* [gh-#627] move shard up

* [gh-#627] reorder build

* [gh-#627] fix commas

* [gh-#627] set npm config prefix

* [gh-#627] CI build of true

* [gh-#627] type cleanup

* [gh-#627] workspaces true

* [gh-#627] update packages

* [gh-#627] remove db

* [gh-#627] update package ordering

* [gh-#627] dot root directory

docker-compose.yaml

@@ -18,7 +18,7 @@ services:
         image: nangohq/nango-server
         container_name: nango-server
         volumes:
-            - './packages/server/providers.yaml:/usr/nango-server/src/packages/server/providers.yaml'
+            - './packages/shared/providers.yaml:/usr/nango-server/src/packages/shared/providers.yaml'
         restart: always
         ports:
             - '3003:3003'

docs/docs/contribute-api.md

@@ -18,7 +18,7 @@ This page details how to contribute a new API for OAuth to Nango. Documentation
 
 Support for the OAuth flow of different providers is implemented with templates: A small config that tells Nango how to perform the OAuth flow for the specific provider.
 
-All templates of Nango live in a single file called [providers.yaml](https://nango.dev/oauth-providers) in the `server` package.
+All templates of Nango live in a single file called [providers.yaml](https://nango.dev/oauth-providers) in the `shared` package.
 
 Most templates only need to make use of 3 configuration keys, but in some cases you might need more.
 
@@ -91,7 +91,7 @@ Some OAuth providers require `https` callbacks or don't allow `localhost` callba
 
 ### Step 4: Add your new provider to `providers.yaml`
 
-Edit the `packages/server/providers.yaml` file as explained in the Step 0 to add support for the new provider. The provider's API documentation should contain all the details you need on the OAuth flow to complete this step.
+Edit the `packages/shared/providers.yaml` file as explained in the Step 0 to add support for the new provider. The provider's API documentation should contain all the details you need on the OAuth flow to complete this step.
 
 Propagate your changes by running:
 
@@ -100,7 +100,7 @@ docker compose restart nango-server # Force a restart, which will load in the ya
 ```
 
 ```bash
-docker compose run nango-server cat packages/server/providers.yaml # Print the contents of the providers file from inside the container
+docker compose run nango-server cat packages/shared/providers.yaml # Print the contents of the providers file from inside the container
 ```
 
 ### Step 5: Create a new integration & connection in Nango

package.json

@@ -3,6 +3,7 @@
     "type": "module",
     "workspaces": [
         "packages/cli",
+        "packages/shared",
         "packages/frontend",
         "packages/node-client",
         "packages/server"
@@ -13,18 +14,22 @@
         "lint": "eslint . --ext .ts,.tsx",
         "lint:fix": "eslint . --ext .ts,.tsx --fix",
         "install:all": "npm i && cd packages/webapp && npm i && cd ../..",
-        "ts-build": "tsc -b --clean packages/server packages/cli && tsc -b tsconfig.build.json && tsc -b packages/webapp/tsconfig.json",
+        "ts-build": "tsc -b --clean packages/shared packages/server packages/cli && tsc -b tsconfig.build.json && tsc -b packages/webapp/tsconfig.json",
         "docker-build": "docker build -f packages/server/Dockerfile -t nango-server:latest .",
         "webapp-build:hosted": "cd ./packages/webapp && npm run build:hosted && cd ../..",
         "webapp-build:staging": "cd ./packages/webapp && npm run build:staging && cd ../..",
         "webapp-build:prod": "cd ./packages/webapp && npm run build:prod && cd ../..",
         "webapp-build:watch": "tsc -b -w packages/webapp/tsconfig.json",
+        "shared:watch": "cd ./packages/shared && tsc -w",
         "build:hosted": "npm run install:all && npm run ts-build && npm run webapp-build:hosted ",
         "build:staging": "npm run install:all && npm run ts-build && npm run webapp-build:staging",
         "build:prod": "npm run install:all && npm run ts-build && npm run webapp-build:prod",
+        "server:dev:watch": "cd ./packages/server && npm run dev",
+        "webapp:dev:watch": "cd ./packages/webapp && npm run start:hosted",
         "prepare": "husky install",
         "build:watch": "tsc -b -w tsconfig.build.json",
-        "dev:watch": "concurrently --kill-others \"npm run build:watch\" \"npm run webapp-build:watch\""
+        "dev:watch": "concurrently --kill-others \"npm run build:watch\" \"npm run webapp-build:watch\"",
+        "dev:watch:apps": "concurrently --kill-others \"npm run server:dev:watch\" \"npm run webapp:dev:watch\""
     },
     "devDependencies": {
         "@tsconfig/node18-strictest-esm": "^1.0.1",

packages/server/Dockerfile

@@ -7,6 +7,7 @@ WORKDIR /usr/nango-server/src
 COPY packages/server/ packages/server/
 COPY package*.json ./
 COPY packages/webapp/build/ packages/webapp/build/
+COPY packages/shared/ packages/shared/
 
 RUN npm pkg delete scripts.prepare
 RUN npm install --omit=dev

packages/server/lib/clients/oauth1.client.ts

@@ -2,9 +2,8 @@
  * Copyright (c) 2022 Nango, all rights reserved.
  */
 
-import type { ProviderTemplateOAuth1, ProviderTemplate } from '../models.js';
 import oAuth1 from 'oauth';
-import type { ProviderConfig } from '../models.js';
+import type { Config as ProviderConfig, TemplateOAuth1 as ProviderTemplateOAuth1, Template as ProviderTemplate } from '@nangohq/shared';
 
 type OAuth1RequestTokenResult = {
     request_token: string;

packages/server/lib/clients/oauth2.client.ts

@@ -3,17 +3,17 @@
  */
 
 import {
-    ProviderTemplateOAuth2,
-    ProviderAuthModes,
-    ProviderTemplate as ProviderTemplate,
+    Config as ProviderConfig,
+    TemplateOAuth2 as ProviderTemplateOAuth2,
+    AuthModes as ProviderAuthModes,
+    Template as ProviderTemplate,
     OAuth2Credentials,
     OAuthAuthorizationMethod,
     OAuthBodyFormat,
     Connection
-} from '../models.js';
+} from '@nangohq/shared';
 import { AuthorizationCode, AccessToken } from 'simple-oauth2';
 import connectionsManager from '../services/connection.service.js';
-import type { ProviderConfig } from '../models.js';
 import { interpolateString } from '../utils/utils.js';
 import Boom from '@hapi/boom';
 import { NangoError } from '../utils/error.js';

packages/server/lib/clients/provider.client.ts

@@ -1,13 +1,13 @@
 import braintree from 'braintree';
 import {
-    ProviderConfig,
+    Config as ProviderConfig,
     Connection,
     OAuth2Credentials,
-    ProviderAuthModes,
+    AuthModes as ProviderAuthModes,
     AuthorizationTokenResponse,
     RefreshTokenResponse,
-    ProviderTemplateOAuth2
-} from '../models.js';
+    TemplateOAuth2 as ProviderTemplateOAuth2
+} from '@nangohq/shared';
 import axios from 'axios';
 import { isTokenExpired, parseTokenExpirationDate } from '../utils/utils.js';
 import { NangoError } from '../utils/error.js';

packages/server/lib/controllers/access.middleware.ts

@@ -1,6 +1,6 @@
 import type { Request, Response, NextFunction } from 'express';
 import accountService from '../services/account.service.js';
-import type { Account } from '../models.js';
+import type { Account } from '@nangohq/shared';
 import { isCloud, setAccount, isBasicAuthEnabled } from '../utils/utils.js';
 import errorManager from '../utils/error.manager.js';
 import userService from '../services/user.service.js';

packages/server/lib/controllers/activity.controller.ts

@@ -2,7 +2,7 @@ import type { Request, Response } from 'express';
 import type { NextFunction } from 'express';
 
 import { getUserAndAccountFromSession } from '../utils/utils.js';
-import { getLogsByAccount } from '../services/activity.service.js';
+import { getLogsByAccount } from '@nangohq/shared';
 
 class ActivityController {
     public async retrieve(req: Request, res: Response, next: NextFunction) {

packages/server/lib/controllers/auth.controller.ts

@@ -11,7 +11,7 @@ import Mailgun from 'mailgun.js';
 import type { User } from '../models.js';
 import formData from 'form-data';
 import { NangoError } from '../utils/error.js';
-import configService from '../services/config.service.js';
+import { configService } from '@nangohq/shared';
 
 export interface WebUser {
     id: number;

packages/server/lib/controllers/config.controller.ts

@@ -1,28 +1,35 @@
 import type { NextFunction, Request, Response } from 'express';
-import configService from '../services/config.service.js';
-import type { ProviderConfig } from '../models.js';
+import { configService, Config as ProviderConfig } from '@nangohq/shared';
 import analytics from '../utils/analytics.js';
 import { getAccount, getUserAndAccountFromSession, parseConnectionConfigParamsFromTemplate } from '../utils/utils.js';
 import errorManager from '../utils/error.manager.js';
 import connectionService from '../services/connection.service.js';
 import { NangoError } from '../utils/error.js';
 
+interface Integration {
+    uniqueKey: string;
+    provider: string;
+    connectionCount: number;
+    creationDate: Date | undefined;
+    connectionConfigParams?: string[];
+}
+
 class ConfigController {
     /**
      * Webapp
      */
 
     async listProviderConfigsWeb(req: Request, res: Response, next: NextFunction) {
         try {
-            let account = (await getUserAndAccountFromSession(req)).account;
+            const account = (await getUserAndAccountFromSession(req)).account;
 
-            let configs = await configService.listProviderConfigs(account.id);
+            const configs = await configService.listProviderConfigs(account.id);
 
-            let connections = await connectionService.listConnections(account.id);
+            const connections = await connectionService.listConnections(account.id);
 
-            let integrations = configs.map((config) => {
-                let template = configService.getTemplates()[config.provider];
-                let integration: any = {
+            const integrations = configs.map((config: ProviderConfig) => {
+                const template = configService.getTemplates()[config.provider];
+                const integration: Integration = {
                     uniqueKey: config.unique_key,
                     provider: config.provider,
                     connectionCount: connections.filter((connection) => connection.provider === config.unique_key).length,
@@ -35,7 +42,7 @@ class ConfigController {
             });
 
             res.status(200).send({
-                integrations: integrations.sort(function (a, b) {
+                integrations: integrations.sort(function (a: Integration, b: Integration) {
                     return b.creationDate!.getTime() - a.creationDate!.getTime();
                 })
             });
@@ -106,7 +113,7 @@ class ConfigController {
                 account_id: account.id
             };
 
-            let result = await configService.createProviderConfig(config);
+            const result = await configService.createProviderConfig(config);
 
             if (Array.isArray(result) && result.length === 1 && result[0] != null && 'id' in result[0]) {
                 analytics.track('server:config_created', account.id, { provider: config.provider });

packages/server/lib/controllers/connection.controller.ts

@@ -1,16 +1,23 @@
 import type { Request, Response } from 'express';
 import connectionService from '../services/connection.service.js';
 import type { NextFunction } from 'express';
-import configService from '../services/config.service.js';
-import { ProviderConfig, ProviderTemplate, Connection, ProviderAuthModes, ProviderTemplateOAuth2, HTTP_VERB, LogLevel, LogAction } from '../models.js';
 import analytics from '../utils/analytics.js';
 import {
     createActivityLog,
     createActivityLogMessage,
     createActivityLogMessageAndEnd,
     updateProvider as updateProviderActivityLog,
-    updateSuccess as updateSuccessActivityLog
-} from '../services/activity.service.js';
+    updateSuccess as updateSuccessActivityLog,
+    Config as ProviderConfig,
+    Template as ProviderTemplate,
+    AuthModes as ProviderAuthModes,
+    TemplateOAuth2 as ProviderTemplateOAuth2,
+    Connection,
+    LogLevel,
+    LogAction,
+    HTTP_VERB,
+    configService
+} from '@nangohq/shared';
 import { getAccount, getUserAndAccountFromSession } from '../utils/utils.js';
 import { getConnectionCredentials } from '../utils/connection.js';
 import { WSErrBuilder } from '../utils/web-socket-error.js';
@@ -123,7 +130,7 @@ class ConnectionController {
                 await createActivityLogMessageAndEnd({
                     level: 'info',
                     activity_log_id: activityLogId as number,
-                    auth_mode: template.auth_mode,
+                    auth_mode: template?.auth_mode,
                     content: `Token manual refresh fetch was successful for ${providerConfigKey} and connection ${connectionId} from the web UI`,
                     timestamp: Date.now()
                 });
@@ -165,9 +172,9 @@ class ConnectionController {
             }
 
             let uniqueKeyToProvider: { [key: string]: string } = {};
-            let providerConfigKeys = configs.map((config) => config.unique_key);
+            let providerConfigKeys = configs.map((config: ProviderConfig) => config.unique_key);
 
-            providerConfigKeys.forEach((key, i) => (uniqueKeyToProvider[key] = configs[i]!.provider));
+            providerConfigKeys.forEach((key: string, i: number) => (uniqueKeyToProvider[key] = configs[i]!.provider));
 
             let result = connections.map((connection) => {
                 return {
@@ -312,7 +319,7 @@ class ConnectionController {
     async listProviders(_: Request, res: Response, next: NextFunction) {
         try {
             const providers = Object.entries(configService.getTemplates())
-                .map((providerProperties) => {
+                .map((providerProperties: [string, ProviderTemplate]) => {
                     const [provider, properties] = providerProperties;
                     return {
                         name: provider,

packages/server/lib/controllers/oauth.controller.ts

@@ -4,7 +4,6 @@ import * as uuid from 'uuid';
 import simpleOauth2 from 'simple-oauth2';
 import { getSimpleOAuth2ClientConfig } from '../clients/oauth2.client.js';
 import { OAuth1Client } from '../clients/oauth1.client.js';
-import configService from '../services/config.service.js';
 import connectionService from '../services/connection.service.js';
 import {
     getOauthCallbackUrl,
@@ -23,19 +22,18 @@ import {
     findActivityLogBySession,
     updateProviderConfigAndConnectionId as updateProviderConfigAndConnectionIdActivityLog,
     updateSessionId as updateSessionIdActivityLog,
-    addEndTime as addEndTimeActivityLog
-} from '../services/activity.service.js';
-import {
-    ProviderConfig,
-    ProviderTemplate,
-    ProviderTemplateOAuth2,
-    ProviderAuthModes,
+    addEndTime as addEndTimeActivityLog,
+    LogLevel,
+    LogAction,
+    configService,
+    Config as ProviderConfig,
+    Template as ProviderTemplate,
+    TemplateOAuth2 as ProviderTemplateOAuth2,
+    AuthModes as ProviderAuthModes,
     OAuthSession,
     OAuth1RequestTokenResult,
-    AuthCredentials,
-    LogLevel,
-    LogAction
-} from '../models.js';
+    AuthCredentials
+} from '@nangohq/shared';
 import type { NextFunction } from 'express';
 import errorManager from '../utils/error.manager.js';
 import providerClientManager from '../clients/provider.client.js';