Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(CLI): show on-events scripts in nango deploy confirmation message #3069

Merged
merged 3 commits into from
Nov 27, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fix: split between DBOnEventScript and OnEventScript
  • Loading branch information
TBonnin committed Nov 27, 2024
commit a1d17d1dc69e50f4b2b83e2431c6522f5940ff09
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
import { isError, isSuccess, runServer, shouldBeProtected } from '../../../utils/tests.js';
import { eventTypeMapper, seeders } from '@nangohq/shared';
import { seeders } from '@nangohq/shared';

const endpoint = '/sync/deploy/confirmation';
let api: Awaited<ReturnType<typeof runServer>>;
Expand Down Expand Up @@ -123,7 +123,7 @@ describe(`POST ${endpoint}`, () => {
{
name: existingOnEvent.name,
providerConfigKey: existingOnEvent.providerConfigKey,
event: eventTypeMapper.fromDb(existingOnEvent.event)
event: existingOnEvent.event
}
]
});
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { requireEmptyQuery, zodErrorToHTTP } from '@nangohq/utils';
import type { PostDeployConfirmation, ScriptDifferences } from '@nangohq/types';
import { asyncWrapper } from '../../../utils/asyncWrapper.js';
import { eventTypeMapper, getAndReconcileDifferences, onEventScriptService } from '@nangohq/shared';
import { getAndReconcileDifferences, onEventScriptService } from '@nangohq/shared';
import { getOrchestrator } from '../../../utils/utils.js';
import { logContextGetter } from '@nangohq/logs';
import { validation } from './validation.js';
Expand Down Expand Up @@ -50,14 +50,14 @@ export const postDeployConfirmation = asyncWrapper<PostDeployConfirmation>(async
return {
providerConfigKey: script.providerConfigKey,
name: script.name,
event: eventTypeMapper.fromDb(script.event)
event: script.event
};
}),
deletedOnEventScripts: diff.deleted.map((script) => {
return {
providerConfigKey: script.providerConfigKey,
name: script.name,
event: eventTypeMapper.fromDb(script.event)
event: script.event
};
})
};
Expand Down
2 changes: 1 addition & 1 deletion packages/shared/lib/seeders/onEventScript.seeder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ export async function createOnEventScript({
account: DBTeam;
environment: DBEnvironment;
providerConfigKey: string;
}): Promise<OnEventScript & { providerConfigKey: string }> {
}): Promise<OnEventScript> {
const scripts: OnEventScriptsByProvider[] = [
{
providerConfigKey,
Expand Down
95 changes: 65 additions & 30 deletions packages/shared/lib/services/on-event-scripts.service.ts
Original file line number Diff line number Diff line change
@@ -1,31 +1,61 @@
import db from '@nangohq/database';
import remoteFileService from './file/remote.service.js';
import { env } from '@nangohq/utils';
import type { OnEventScriptsByProvider, OnEventScript, DBTeam, DBEnvironment, OnEventType } from '@nangohq/types';
import type { OnEventScriptsByProvider, DBOnEventScript, DBTeam, DBEnvironment, OnEventType, OnEventScript } from '@nangohq/types';
import { increment } from './sync/config/config.service.js';
import configService from './config.service.js';

const TABLE = 'on_event_scripts';

const EVENT_TYPE_MAPPINGS: Record<OnEventScript['event'], OnEventType> = {
const EVENT_TYPE_MAPPINGS: Record<DBOnEventScript['event'], OnEventType> = {
POST_CONNECTION_CREATION: 'post-connection-creation',
PRE_CONNECTION_DELETION: 'pre-connection-deletion'
} as const;

export const eventTypeMapper = {
fromDb: (event: OnEventScript['event']): OnEventType => {
const eventTypeMapper = {
fromDb: (event: DBOnEventScript['event']): OnEventType => {
return EVENT_TYPE_MAPPINGS[event];
},
toDb: (eventType: OnEventType): OnEventScript['event'] => {
toDb: (eventType: OnEventType): DBOnEventScript['event'] => {
for (const [key, value] of Object.entries(EVENT_TYPE_MAPPINGS)) {
if (value === eventType) {
return key as OnEventScript['event'];
return key as DBOnEventScript['event'];
}
}
throw new Error(`Unknown event type: ${eventType}`); // This should never happen
}
};

const dbMapper = {
to: (script: OnEventScript): DBOnEventScript => {
return {
id: script.id,
config_id: script.configId,
name: script.name,
file_location: script.fileLocation,
version: script.version,
active: script.active,
event: eventTypeMapper.toDb(script.event),
created_at: script.createdAt,
updated_at: script.updatedAt
};
},
from: (dbScript: DBOnEventScript & { provider_config_key: string }): OnEventScript => {
return {
id: dbScript.id,
configId: dbScript.config_id,
providerConfigKey: dbScript.provider_config_key,
name: dbScript.name,
fileLocation: dbScript.file_location,
version: dbScript.version,
active: dbScript.active,
event: eventTypeMapper.fromDb(dbScript.event),
createdAt: dbScript.created_at,
updatedAt: dbScript.updated_at
};
}
};

export const onEventScriptService = {
async update({
environment,
Expand All @@ -35,14 +65,14 @@ export const onEventScriptService = {
environment: DBEnvironment;
account: DBTeam;
onEventScriptsByProvider: OnEventScriptsByProvider[];
}): Promise<(OnEventScript & { providerConfigKey: string })[]> {
}): Promise<OnEventScript[]> {
return db.knex.transaction(async (trx) => {
const onEventInserts: Omit<OnEventScript, 'id' | 'created_at' | 'updated_at'>[] = [];
const onEventInserts: Omit<DBOnEventScript, 'id' | 'created_at' | 'updated_at'>[] = [];

// Deactivate all previous scripts for the environment
// This is done to ensure that we don't have any orphaned scripts when they are removed from nango.yaml
const previousScriptVersions = await trx
.from<OnEventScript>(TABLE)
.from<DBOnEventScript>(TABLE)
.whereRaw(`config_id IN (SELECT id FROM _nango_configs WHERE environment_id = ?)`, [environment.id])
.where({
active: true
Expand Down Expand Up @@ -94,58 +124,63 @@ export const onEventScriptService = {
}
}
if (onEventInserts.length > 0) {
type R = Awaited<ReturnType<typeof onEventScriptService.update>>;
const res = await trx
.with('inserted', (qb) => {
qb.insert(onEventInserts).into(TABLE).returning('*');
})
.select<R>(['inserted.*', '_nango_configs.unique_key as providerConfigKey'])
.select<(DBOnEventScript & { provider_config_key: string })[]>(['inserted.*', '_nango_configs.unique_key as provider_config_key'])
.from('inserted')
.join('_nango_configs', 'inserted.config_id', '_nango_configs.id');
return res;
return res.map(dbMapper.from);
}
return [];
});
},
getByConfig: async (configId: number, event: OnEventType): Promise<OnEventScript[]> => {
return db.knex.from<OnEventScript>(TABLE).where({ config_id: configId, active: true, event: eventTypeMapper.toDb(event) });

getByEnvironmentId: async (environmentId: number): Promise<OnEventScript[]> => {
const existingScriptsQuery = await db.knex
.select<(DBOnEventScript & { provider_config_key: string })[]>(`${TABLE}.*`, '_nango_configs.unique_key as provider_config_key')
.from(TABLE)
.join('_nango_configs', `${TABLE}.config_id`, '_nango_configs.id')
.where({
'_nango_configs.environment_id': environmentId,
[`${TABLE}.active`]: true
});
return existingScriptsQuery.map(dbMapper.from);
},

getByConfig: async (configId: number, event: OnEventType): Promise<DBOnEventScript[]> => {
return db.knex.from<DBOnEventScript>(TABLE).where({ config_id: configId, active: true, event: eventTypeMapper.toDb(event) });
},

diffChanges: async ({
environmentId,
onEventScriptsByProvider
}: {
environmentId: number;
onEventScriptsByProvider: OnEventScriptsByProvider[];
}): Promise<{
added: (Omit<OnEventScript, 'id' | 'file_location' | 'created_at' | 'updated_at'> & { providerConfigKey: string })[];
deleted: (OnEventScript & { providerConfigKey: string })[];
updated: (OnEventScript & { providerConfigKey: string })[];
added: Omit<OnEventScript, 'id' | 'fileLocation' | 'createdAt' | 'updatedAt'>[];
deleted: OnEventScript[];
updated: OnEventScript[];
}> => {
const res: Awaited<ReturnType<typeof onEventScriptService.diffChanges>> = {
added: [],
deleted: [],
updated: []
};

const existingScripts = await db.knex
.select<(OnEventScript & { providerConfigKey: string })[]>(`${TABLE}.*`, '_nango_configs.unique_key as providerConfigKey')
.from(TABLE)
.join('_nango_configs', `${TABLE}.config_id`, '_nango_configs.id')
.where({
'_nango_configs.environment_id': environmentId,
[`${TABLE}.active`]: true
});
const existingScripts = await onEventScriptService.getByEnvironmentId(environmentId);

// Create a map of existing scripts for easier lookup
const existingMap = new Map(existingScripts.map((script) => [`${script.config_id}:${script.name}:${script.event}`, script]));
const existingMap = new Map(existingScripts.map((script) => [`${script.configId}:${script.name}:${script.event}`, script]));

for (const provider of onEventScriptsByProvider) {
const config = await configService.getProviderConfig(provider.providerConfigKey, environmentId);
if (!config || !config.id) continue;

for (const script of provider.scripts) {
const event = eventTypeMapper.toDb(script.event);
const key = `${config.id}:${script.name}:${event}`;
const key = `${config.id}:${script.name}:${script.event}`;

const maybeScript = existingMap.get(key);
if (maybeScript) {
Expand All @@ -157,11 +192,11 @@ export const onEventScriptService = {
} else {
// Script doesn't exist - it's new
res.added.push({
config_id: config.id,
configId: config.id,
name: script.name,
version: '0.0.1',
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Out of curiosity:
Did you talk with Bastien about adding on-events-scripts in the templates, managing their versioning, and being able to enable/disable them via the UI?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No but that's why I pinged him in a previous PR to mention that there would still be a fair amount of work to support the features you mentioned (pre-built, dasbhoard, etc...) for on-events scripts

active: true,
event,
event: script.event,
providerConfigKey: provider.providerConfigKey
});
}
Expand Down
2 changes: 1 addition & 1 deletion packages/types/lib/deploy/api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import type { JSONSchema7 } from 'json-schema';
import type { Endpoint, ApiError } from '../api.js';
import type { IncomingFlowConfig, OnEventScriptsByProvider } from './incomingFlow.js';
import type { SyncDeploymentResult } from './index.js';
import type { OnEventType } from '../nangoYaml/index.js';
import type { OnEventType } from '../scripts/on-events/api.js';

export type PostDeployConfirmation = Endpoint<{
Method: 'POST';
Expand Down
3 changes: 2 additions & 1 deletion packages/types/lib/deploy/incomingFlow.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import type { Merge } from 'type-fest';
import type { NangoModel, NangoSyncEndpointOld, NangoSyncEndpointV2, OnEventType, ScriptTypeLiteral, SyncTypeLiteral } from '../nangoYaml';
import type { NangoModel, NangoSyncEndpointOld, NangoSyncEndpointV2, ScriptTypeLiteral, SyncTypeLiteral } from '../nangoYaml';
import type { OnEventType } from '../scripts/on-events/api';

export interface IncomingScriptFiles {
js: string;
Expand Down
1 change: 1 addition & 0 deletions packages/types/lib/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ export type * from './proxy/api.js';

export type * from './environment/db.js';
export type * from './scripts/on-events/db.js';
export type * from './scripts/on-events/api.js';
export type * from './scripts/syncs/api.js';
export type * from './slackNotifications/db.js';
export type * from './notification/active-logs/db.js';
Expand Down
4 changes: 2 additions & 2 deletions packages/types/lib/nangoYaml/index.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import type { OnEventType } from '../scripts/on-events/api';

export type HTTP_METHOD = 'GET' | 'POST' | 'PATCH' | 'PUT' | 'DELETE';
export type SyncTypeLiteral = 'incremental' | 'full';
export type ScriptFileType = 'actions' | 'syncs' | 'on-events' | 'post-connection-scripts'; // post-connection-scripts is deprecated
Expand Down Expand Up @@ -80,8 +82,6 @@ export type NangoYamlModelField = boolean | number | string | null | string[] |
export type NangoYaml = NangoYamlV1 | NangoYamlV2;

// -------------- Parsed
export type OnEventType = 'post-connection-creation' | 'pre-connection-deletion';

export interface NangoYamlParsed {
yamlVersion: 'v1' | 'v2';
integrations: NangoYamlParsedIntegration[];
Expand Down
16 changes: 16 additions & 0 deletions packages/types/lib/scripts/on-events/api.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
import type { DBOnEventScript } from './db.js';

export type OnEventType = 'post-connection-creation' | 'pre-connection-deletion';

export interface OnEventScript {
id: DBOnEventScript['id'];
configId: DBOnEventScript['config_id'];
providerConfigKey: string;
name: DBOnEventScript['name'];
fileLocation: DBOnEventScript['file_location'];
version: DBOnEventScript['version'];
active: DBOnEventScript['active'];
event: OnEventType;
createdAt: DBOnEventScript['created_at'];
updatedAt: DBOnEventScript['updated_at'];
}
2 changes: 1 addition & 1 deletion packages/types/lib/scripts/on-events/db.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import type { Timestamps } from '../../db.js';

export interface OnEventScript extends Timestamps {
export interface DBOnEventScript extends Timestamps {
id: number;
config_id: number;
name: string;
Expand Down