Skip to content

Commit

Permalink
[Github #632] activity tab db storage (#633)
Browse files Browse the repository at this point in the history 
* [gh-#632] add migration, service, and model

* [gh-#632] use db for storage and retrieval

* [gh-#632] oauth refactor

* [gh-#632] update models and methods

* [gh-#632] add sync type and remove commented out messages

* [gh-#632] remove log compose file

* [gh-#632] add index on account_id field

* [gh-#632] single query using json_agg

* [gh-#632] add cron deletion and notice about the deletion process
  • Loading branch information
@khaliqgant
khaliqgant authored May 11, 2023
1 parent 9ad2a2b commit 3a254b3
Show file tree
Hide file tree
Showing 17 changed files with 685 additions and 455 deletions.
1 change: 0 additions & 1 deletion docker-compose.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ services:
container_name: nango-server
volumes:
- './packages/server/providers.yaml:/usr/nango-server/src/packages/server/providers.yaml'
- './packages/server/NangoActivity.json:/usr/nango-server/src/packages/server/NangoActivity.json'
restart: always
ports:
- '3003:3003'
Expand Down
50 changes: 50 additions & 0 deletions package-lock.json
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

58 changes: 6 additions & 52 deletions packages/server/lib/controllers/activity.controller.ts
View file
Original file line number Diff line number Diff line change
@@ -1,65 +1,19 @@
import type { Request, Response } from 'express';
import type { NextFunction } from 'express';
import fs from 'fs';

import { FILENAME, LogData } from '../utils/file-logger.js';
import { getUserAndAccountFromSession } from '../utils/utils.js';
import { getLogsByAccount } from '../services/activity.service.js';

class ActivityController {
/**
* Retrieve
* @desc read a file path and send back the contents of the file
* @param {Request} req Express request object
* @param {Response} res Express response object
* @param {NextFuncion} next callback function to pass control to the next middleware function in the pipeline.
*/
public async retrieve(_req: Request, res: Response, next: NextFunction) {
public async retrieve(req: Request, res: Response, next: NextFunction) {
try {
const filePath = `./${FILENAME}`;
if (fs.existsSync(filePath)) {
const fileContents = fs.readFileSync(filePath, 'utf8');

if (fileContents.length === 0) {
res.send([]);
} else {
const mergedLogs = this.mergeSessions(JSON.parse(fileContents) as unknown as LogData[]);
res.send(mergedLogs);
}
}
const account = (await getUserAndAccountFromSession(req)).account;
const logs = await getLogsByAccount(account.id);
res.send(logs);
} catch (error) {
next(error);
}
}

/**
* Merge Sessions
* @desc append any messages of oauth continuation entries that have a merge property of true
* to an existing session id and update the end time while maintaing
* log ordering
*/
private mergeSessions(logs: LogData[]) {
const updatedLogs: LogData[] = [];
const sessions: Record<string, number> = {};

for (let i = 0; i < logs.length; i++) {
const log = logs[i];

if (log?.sessionId && !log.merge) {
sessions[log.sessionId] = i;
}

if (log?.merge && typeof sessions[log.sessionId as string] !== 'undefined') {
const mergeIndex: number = updatedLogs.findIndex((updated) => {
return updated.sessionId === log.sessionId && !updated.merge;
});
updatedLogs[mergeIndex]!.messages = [...updatedLogs[mergeIndex]!.messages, ...log.messages];
updatedLogs[mergeIndex]!.end = log.end as number;
updatedLogs[mergeIndex]!.success = log.success;
} else {
updatedLogs.push(log as LogData);
}
}
return updatedLogs.reverse();
}
}

export default new ActivityController();
80 changes: 52 additions & 28 deletions packages/server/lib/controllers/connection.controller.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,17 @@ import type { Request, Response } from 'express';
import connectionService from '../services/connection.service.js';
import type { NextFunction } from 'express';
import configService from '../services/config.service.js';
import { ProviderConfig, ProviderTemplate, Connection, ProviderAuthModes, ProviderTemplateOAuth2, HTTP_VERB } from '../models.js';
import { ProviderConfig, ProviderTemplate, Connection, ProviderAuthModes, ProviderTemplateOAuth2, HTTP_VERB, LogLevel, LogAction } from '../models.js';
import analytics from '../utils/analytics.js';
import {
createActivityLog,
createActivityLogMessage,
createActivityLogMessageAndEnd,
updateProvider as updateProviderActivityLog,
updateSuccess as updateSuccessActivityLog
} from '../services/activity.service.js';
import { getAccount, getUserAndAccountFromSession } from '../utils/utils.js';
import { getConnectionCredentials } from '../utils/connection.js';
import { updateAppLogsAndWrite, LogData, LogLevel, LogAction } from '../utils/file-logger.js';
import { WSErrBuilder } from '../utils/web-socket-error.js';
import errorManager from '../utils/error.manager.js';

Expand All @@ -30,15 +36,24 @@ class ConnectionController {
start: Date.now(),
end: Date.now(),
timestamp: Date.now(),
connectionId: connectionId as string,
providerConfigKey: providerConfigKey as string,
messages: [] as LogData['messages'],
message: '',
provider: ''
connection_id: connectionId as string,
provider_config_key: providerConfigKey as string,
account_id: account.id
};

const activityLogId = await createActivityLog(log);

await createActivityLogMessage({
level: 'error',
activity_log_id: activityLogId as number,
timestamp: Date.now(),
content: `Token fetch was successful for ${providerConfigKey} and connection ${connectionId} from the web UI`
});

if (connectionId == null) {
updateAppLogsAndWrite(log, 'error', {
await createActivityLogMessageAndEnd({
level: 'error',
activity_log_id: activityLogId as number,
timestamp: Date.now(),
content: WSErrBuilder.MissingConnectionId().message
});
Expand All @@ -48,7 +63,9 @@ class ConnectionController {
}

if (providerConfigKey == null) {
updateAppLogsAndWrite(log, 'error', {
await createActivityLogMessageAndEnd({
level: 'error',
activity_log_id: activityLogId as number,
timestamp: Date.now(),
content: WSErrBuilder.MissingProviderConfigKey().message
});
Expand All @@ -60,7 +77,9 @@ class ConnectionController {
const connection: Connection | null = await connectionService.getConnection(connectionId, providerConfigKey, account.id);

if (connection == null) {
updateAppLogsAndWrite(log, 'error', {
await createActivityLogMessageAndEnd({
level: 'error',
activity_log_id: activityLogId as number,
timestamp: Date.now(),
content: 'Unknown connection'
});
Expand All @@ -72,7 +91,9 @@ class ConnectionController {
const config: ProviderConfig | null = await configService.getProviderConfig(connection.provider_config_key, account.id);

if (config == null) {
updateAppLogsAndWrite(log, 'error', {
await createActivityLogMessageAndEnd({
level: 'error',
activity_log_id: activityLogId as number,
timestamp: Date.now(),
content: 'Unknown provider config'
});
Expand All @@ -81,7 +102,7 @@ class ConnectionController {
return;
}

log.provider = config.provider;
await updateProviderActivityLog(activityLogId as number, config.provider);

const template: ProviderTemplate | undefined = configService.getTemplate(config.provider);

Expand All @@ -90,21 +111,21 @@ class ConnectionController {
connection,
config,
template as ProviderTemplateOAuth2,
log,
activityLogId,
false,
'token'
'token' as LogAction
);
}

log.success = true;
await updateSuccessActivityLog(activityLogId as number, true);

if (instantRefresh) {
updateAppLogsAndWrite(log, 'info', {
authMode: template.auth_mode,
await createActivityLogMessageAndEnd({
level: 'info',
activity_log_id: activityLogId as number,
auth_mode: template.auth_mode,
content: `Token manual refresh fetch was successful for ${providerConfigKey} and connection ${connectionId} from the web UI`,
timestamp: Date.now(),
providerConfigKey,
connectionId
timestamp: Date.now()
});
}

Expand Down Expand Up @@ -205,28 +226,31 @@ class ConnectionController {

async getConnectionCreds(req: Request, res: Response, next: NextFunction) {
try {
const accountId = getAccount(res);
const connectionId = req.params['connectionId'] as string;
const providerConfigKey = req.query['provider_config_key'] as string;
const instantRefresh = req.query['force_refresh'] === 'true';

const action: LogAction = 'token';
const log = {
level: 'debug' as LogLevel,
success: true,
action: 'token' as LogAction,
action,
start: Date.now(),
end: Date.now(),
timestamp: Date.now(),
method: req.method as HTTP_VERB,
connectionId,
providerConfigKey,
messages: [] as LogData['messages'],
message: '',
endpoint: ''
connection_id: connectionId as string,
provider_config_key: providerConfigKey as string,
account_id: accountId
};

const connection = await getConnectionCredentials(res, connectionId, providerConfigKey, log, instantRefresh);
const activityLogId = await createActivityLog(log);
const connection = await getConnectionCredentials(res, connectionId, providerConfigKey, activityLogId as number, action, instantRefresh);

updateAppLogsAndWrite(log, 'info', {
await createActivityLogMessageAndEnd({
level: 'info',
activity_log_id: activityLogId as number,
timestamp: Date.now(),
content: 'Connection credentials found successfully',
params: {
Expand Down
Loading

0 comments on commit 3a254b3

Please sign in to comment.