feat(logs): implem v2 logging (#1945)

## Context

Contributes to NAN-582

Since we started talking about that a lot of stuff have fluctuated. So
here is a recap:
- When we want to log something we create an `Operation` that will
contain the state and the main data (i.e: account, env, sync,
connection, user)
- Each log message has the exact same schema with `parentId` filled with
the `Operation`'s id
- logs and operation are stored in the same index


## Changes

- Double write of logs to Elasticsearch storage

The write is currently controlled by the env var `NANGO_LOGS_ENABLED` to
be able to disable everything if something goes wrong. The main calls to
ES have been try/catched temporarily to also avoid any issue.

It should cover all logs across the codebase, but it's missing a lot of
context data (i.e: account, env name, sync, connection) that were not
added to limit the size of this PR.


## How to test?

- Set the env
```sh
# (false will log to console only, true will log to OpenSearch)
NANGO_LOGS_ENABLED=false | true 
```
- Launch the backend
```sh
npm run dw
npm run dwa
```
- Launch a sync / action, create a connection, cancel a sync, deploy
integrations
- Go to  http://localhost:9500/messages/_search?sort=createdAt:desc
Its not very readable but you should find all the logs there.



## Next steps

- Complete the missing data
- New API endpoint
- Implement the new UI
- Switch to new UI
- Remove activityLogId param wherever it's now useless (it's still going
to be needed at some place where it's serialized)
- Remove old library

.env.example

@@ -71,9 +71,9 @@ DEFAULT_GITHUB_CLIENT_ID=
 DEFAULT_GITHUB_CLIENT_SECRET=
 
 # ----- Logs
-NANGO_LOGS_ES_URL="http://localhost:9500"
-NANGO_LOGS_ES_USER=""
-NANGO_LOGS_ES_PWD=""
+NANGO_LOGS_OS_URL="http://localhost:9500"
+NANGO_LOGS_OS_USER=""
+NANGO_LOGS_OS_PWD=""
 
 # Hosted Auth Configuration
 WORKOS_API_KEY=

.github/workflows/cli-verification.yaml

@@ -28,6 +28,10 @@ jobs:
                   registry-url: 'https://npm.pkg.github.com'
                   scope: '@nangohq'
                   always-auth: true
+            - name: Build
+              run: |
+                  npm ci
+                  npm run ts-build
             - name: Publish packages to github registry so they can be bumped
               env:
                   NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/integration-flows.yaml

@@ -30,7 +30,6 @@ jobs:
                   # Build, install CLI and verify it can run
                   npm ci
                   npm run ts-build
-                  npm run -w nango postbuild
                   npm install -g ./packages/cli
                   NANGO_CLI_UPGRADE_MODE=ignore nango version --debug
 

dev/docker-compose.dev.yaml

@@ -50,21 +50,21 @@ services:
         networks:
             - nango
 
-    es:
-        container_name: es
-        image: elasticsearch:8.12.2
+    opensearch:
+        container_name: opensearch
+        image: opensearchproject/opensearch:2.13.0
         ulimits:
             memlock:
                 soft: -1
                 hard: -1
         volumes:
-            - esdata1:/usr/share/elasticsearch/data
+            - opensearch-data1:/usr/share/opensearch/data
         ports:
             - 9500:9200
             - 9600:9300
         environment:
             - discovery.type=single-node
-            - xpack.security.enabled=false
+            - plugins.security.disabled=true
         networks:
             - nango
         deploy:
@@ -76,5 +76,5 @@ networks:
     nango:
 
 volumes:
-    esdata1:
+    opensearch-data1:
         driver: local

docker-compose.yaml

@@ -35,15 +35,15 @@ services:
             - LOG_LEVEL=${LOG_LEVEL:-info}
             - TELEMETRY=${TELEMETRY}
             - NANGO_SERVER_WEBSOCKETS_PATH=${NANGO_SERVER_WEBSOCKETS_PATH}
-            - NANGO_LOGS_ES_URL=${NANGO_LOGS_ES_URL:-http://localhost:9500}
+            - NANGO_LOGS_OS_URL=${NANGO_LOGS_OS_URL:-http://localhost:9500}
         volumes:
             - './packages/shared/providers.yaml:/usr/nango-server/src/packages/shared/providers.yaml'
         restart: always
         ports:
             - '${SERVER_PORT:-3003}:${SERVER_PORT:-3003}'
         depends_on:
             - nango-db
-            - nanog-es
+            - nanog-opensearch
         networks:
             - nango
 
@@ -55,26 +55,26 @@ services:
         networks:
             - nango
 
-    nanog-es:
-        image: elasticsearch:8.12.2
+    nanog-opensearch:
+        image: opensearchproject/opensearch:2.13.0
         ulimits:
             memlock:
                 soft: -1
                 hard: -1
         volumes:
-            - esdata1:/usr/share/elasticsearch/data
+            - opensearch-data1:/usr/share/opensearch/data
         ports:
             - 9500:9500
             - 9600:9300
         environment:
             - discovery.type=single-node
-            - xpack.security.enabled=false
+            - plugins.security.disabled=true
         networks:
             - nango
 
 networks:
     nango:
 
 volumes:
-    esdata1:
+    opensearch-data1:
         driver: local

packages/cli/docker/docker-compose.yaml

@@ -36,7 +36,7 @@ services:
             - LOG_LEVEL=${LOG_LEVEL:-info}
             - TELEMETRY=${TELEMETRY}
             - NANGO_SERVER_WEBSOCKETS_PATH=${NANGO_SERVER_WEBSOCKETS_PATH}
-            - NANGO_LOGS_ES_URL=${NANGO_LOGS_ES_URL:-http://localhost:9500}
+            - NANGO_LOGS_OS_URL=${NANGO_LOGS_OS_URL:-http://localhost:9500}
             - RECORDS_DATABASE_URL=${RECORDS_DATABASE_URL:-postgresql://nango:nango@nango-db:5432/nango}
         restart: always
         ports:
@@ -107,26 +107,26 @@ services:
         networks:
             - nango
 
-    nanog-es:
-        image: elasticsearch:8.12.2
+    nanog-opensearch:
+        image: opensearchproject/opensearch:2.13.0
         ulimits:
             memlock:
                 soft: -1
                 hard: -1
         volumes:
-            - esdata1:/usr/share/elasticsearch/data
+            - opensearch-data1:/usr/share/opensearch/data
         ports:
             - 9500:9500
             - 9600:9300
         environment:
             - discovery.type=single-node
-            - xpack.security.enabled=false
+            - plugins.security.disabled=true
         networks:
             - nango
 
 networks:
     nango:
 
 volumes:
-    esdata1:
+    opensearch-data1:
         driver: local

packages/cli/lib/services/dryrun.service.ts

@@ -164,10 +164,19 @@ class DryRunService {
                 return Promise.resolve([]);
             }
         };
+        const logContextGetter = {
+            create: () => {
+                return Promise.resolve({}) as any;
+            },
+            get: () => {
+                return {} as any;
+            }
+        };
 
         const syncRun = new syncRunService({
             integrationService,
             recordsService,
+            logContextGetter,
             writeToDb: false,
             nangoConnection,
             provider,

packages/jobs/Dockerfile

@@ -14,6 +14,7 @@ COPY packages/utils/ packages/utils/
 COPY packages/records/ packages/records/
 COPY packages/data-ingestion/ packages/data-ingestion/
 COPY packages/jobs/ packages/jobs/
+COPY packages/logs/ packages/logs/
 COPY packages/runner/ packages/runner/
 COPY package*.json ./