Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(CLI): show on-events scripts in nango deploy confirmation message #3069

Merged
merged 3 commits into from
Nov 27, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,62 @@ describe(`POST ${endpoint}`, () => {
deletedSyncs: [],
newActions: [],
newSyncs: [],
deletedModels: []
deletedModels: [],
newOnEventScripts: [],
deletedOnEventScripts: []
});
expect(res.res.status).toBe(200);
});

it('should show correct on-events scripts diff', async () => {
const { account, env: environment } = await seeders.seedAccountEnvAndUser();
const { unique_key: providerConfigKey } = await seeders.createConfigSeed(environment, 'notion-123', 'notion');
const existingOnEvent = await seeders.createOnEventScript({ account, environment, providerConfigKey });

const res = await api.fetch(endpoint, {
method: 'POST',
token: environment.secret_key,
body: {
debug: false,
flowConfigs: [],
onEventScriptsByProvider: [
{
providerConfigKey,
scripts: [
{
name: 'new-script',
event: 'post-connection-creation',
fileBody: { js: '', ts: '' }
}
]
}
],
reconcile: false
}
});

isSuccess(res.json);

expect(res.json).toStrictEqual<typeof res.json>({
deletedActions: [],
deletedSyncs: [],
newActions: [],
newSyncs: [],
deletedModels: [],
newOnEventScripts: [
{
name: 'new-script',
providerConfigKey,
event: 'post-connection-creation'
}
],
deletedOnEventScripts: [
{
name: existingOnEvent.name,
providerConfigKey: existingOnEvent.providerConfigKey,
event: existingOnEvent.event
}
]
});
expect(res.res.status).toBe(200);
});
Expand Down
39 changes: 35 additions & 4 deletions packages/server/lib/controllers/sync/deploy/postConfirmation.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { requireEmptyQuery, zodErrorToHTTP } from '@nangohq/utils';
import type { PostDeployConfirmation } from '@nangohq/types';
import type { PostDeployConfirmation, ScriptDifferences } from '@nangohq/types';
import { asyncWrapper } from '../../../utils/asyncWrapper.js';
import { getAndReconcileDifferences } from '@nangohq/shared';
import { getAndReconcileDifferences, onEventScriptService } from '@nangohq/shared';
import { getOrchestrator } from '../../../utils/utils.js';
import { logContextGetter } from '@nangohq/logs';
import { validation } from './validation.js';
Expand All @@ -24,7 +24,7 @@ export const postDeployConfirmation = asyncWrapper<PostDeployConfirmation>(async
const body: PostDeployConfirmation['Body'] = val.data;
const environmentId = res.locals['environment'].id;

const result = await getAndReconcileDifferences({
const syncAndActionDifferences = await getAndReconcileDifferences({
environmentId,
flows: body.flowConfigs,
performAction: false,
Expand All @@ -33,10 +33,41 @@ export const postDeployConfirmation = asyncWrapper<PostDeployConfirmation>(async
logContextGetter,
orchestrator
});
if (!result) {
if (!syncAndActionDifferences) {
res.status(500).send({ error: { code: 'server_error' } });
return;
}

let result: ScriptDifferences;
if (body.onEventScriptsByProvider) {
const diff = await onEventScriptService.diffChanges({
environmentId,
onEventScriptsByProvider: body.onEventScriptsByProvider
});
result = {
...syncAndActionDifferences,
newOnEventScripts: diff.added.map((script) => {
return {
providerConfigKey: script.providerConfigKey,
name: script.name,
event: script.event
};
}),
deletedOnEventScripts: diff.deleted.map((script) => {
return {
providerConfigKey: script.providerConfigKey,
name: script.name,
event: script.event
};
})
};
} else {
result = {
...syncAndActionDifferences,
newOnEventScripts: [],
deletedOnEventScripts: []
};
}

res.status(200).send(result);
});
1 change: 1 addition & 0 deletions packages/shared/lib/seeders/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,3 +6,4 @@ export * from './global.seeder.js';
export * from './sync-job.seeder.js';
export * from './sync.seeder.js';
export * from './user.seeder.js';
export * from './onEventScript.seeder.js';
34 changes: 34 additions & 0 deletions packages/shared/lib/seeders/onEventScript.seeder.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
import type { DBEnvironment, DBTeam, OnEventScript, OnEventScriptsByProvider } from '@nangohq/types';
import { onEventScriptService } from '../services/on-event-scripts.service.js';

export async function createOnEventScript({
account,
environment,
providerConfigKey
}: {
account: DBTeam;
environment: DBEnvironment;
providerConfigKey: string;
}): Promise<OnEventScript> {
const scripts: OnEventScriptsByProvider[] = [
{
providerConfigKey,
scripts: [
{
name: 'test-script',
event: 'post-connection-creation',
fileBody: { js: '', ts: '' }
}
]
}
];
const [added] = await onEventScriptService.update({
environment,
account,
onEventScriptsByProvider: scripts
});
if (!added) {
throw new Error('failed_to_create_on_event_script');
}
return added;
}
142 changes: 125 additions & 17 deletions packages/shared/lib/services/on-event-scripts.service.ts
Original file line number Diff line number Diff line change
@@ -1,20 +1,60 @@
import db from '@nangohq/database';
import remoteFileService from './file/remote.service.js';
import { env } from '@nangohq/utils';
import type { OnEventScriptsByProvider, OnEventScript, DBTeam, DBEnvironment, OnEventType } from '@nangohq/types';
import type { OnEventScriptsByProvider, DBOnEventScript, DBTeam, DBEnvironment, OnEventType, OnEventScript } from '@nangohq/types';
import { increment } from './sync/config/config.service.js';
import configService from './config.service.js';

const TABLE = 'on_event_scripts';

function toDbEvent(eventType: OnEventType): OnEventScript['event'] {
switch (eventType) {
case 'post-connection-creation':
return 'POST_CONNECTION_CREATION';
case 'pre-connection-deletion':
return 'PRE_CONNECTION_DELETION';
const EVENT_TYPE_MAPPINGS: Record<DBOnEventScript['event'], OnEventType> = {
POST_CONNECTION_CREATION: 'post-connection-creation',
PRE_CONNECTION_DELETION: 'pre-connection-deletion'
} as const;

const eventTypeMapper = {
fromDb: (event: DBOnEventScript['event']): OnEventType => {
return EVENT_TYPE_MAPPINGS[event];
},
toDb: (eventType: OnEventType): DBOnEventScript['event'] => {
for (const [key, value] of Object.entries(EVENT_TYPE_MAPPINGS)) {
if (value === eventType) {
return key as DBOnEventScript['event'];
}
}
throw new Error(`Unknown event type: ${eventType}`); // This should never happen
}
};

const dbMapper = {
to: (script: OnEventScript): DBOnEventScript => {
return {
id: script.id,
config_id: script.configId,
name: script.name,
file_location: script.fileLocation,
version: script.version,
active: script.active,
event: eventTypeMapper.toDb(script.event),
created_at: script.createdAt,
updated_at: script.updatedAt
};
},
from: (dbScript: DBOnEventScript & { provider_config_key: string }): OnEventScript => {
return {
id: dbScript.id,
configId: dbScript.config_id,
providerConfigKey: dbScript.provider_config_key,
name: dbScript.name,
fileLocation: dbScript.file_location,
version: dbScript.version,
active: dbScript.active,
event: eventTypeMapper.fromDb(dbScript.event),
createdAt: dbScript.created_at,
updatedAt: dbScript.updated_at
};
}
}
};

export const onEventScriptService = {
async update({
Expand All @@ -25,14 +65,14 @@ export const onEventScriptService = {
environment: DBEnvironment;
account: DBTeam;
onEventScriptsByProvider: OnEventScriptsByProvider[];
}): Promise<(OnEventScript & { providerConfigKey: string })[]> {
}): Promise<OnEventScript[]> {
return db.knex.transaction(async (trx) => {
const onEventInserts: Omit<OnEventScript, 'id' | 'created_at' | 'updated_at'>[] = [];
const onEventInserts: Omit<DBOnEventScript, 'id' | 'created_at' | 'updated_at'>[] = [];

// Deactivate all previous scripts for the environment
// This is done to ensure that we don't have any orphaned scripts when they are removed from nango.yaml
const previousScriptVersions = await trx
.from<OnEventScript>(TABLE)
.from<DBOnEventScript>(TABLE)
.whereRaw(`config_id IN (SELECT id FROM _nango_configs WHERE environment_id = ?)`, [environment.id])
.where({
active: true
Expand All @@ -52,7 +92,7 @@ export const onEventScriptService = {

for (const script of scripts) {
const { name, fileBody, event: scriptEvent } = script;
const event = toDbEvent(scriptEvent);
const event = eventTypeMapper.toDb(scriptEvent);

const previousScriptVersion = previousScriptVersions.find((p) => p.config_id === config.id && p.name === name && p.event === event);
const version = previousScriptVersion ? increment(previousScriptVersion.version) : '0.0.1';
Expand Down Expand Up @@ -84,20 +124,88 @@ export const onEventScriptService = {
}
}
if (onEventInserts.length > 0) {
type R = Awaited<ReturnType<typeof onEventScriptService.update>>;
const res = await trx
.with('inserted', (qb) => {
qb.insert(onEventInserts).into(TABLE).returning('*');
})
.select<R>(['inserted.*', '_nango_configs.unique_key as providerConfigKey'])
.select<(DBOnEventScript & { provider_config_key: string })[]>(['inserted.*', '_nango_configs.unique_key as provider_config_key'])
.from('inserted')
.join('_nango_configs', 'inserted.config_id', '_nango_configs.id');
return res;
return res.map(dbMapper.from);
}
return [];
});
},
getByConfig: async (configId: number, event: OnEventType): Promise<OnEventScript[]> => {
return db.knex.from<OnEventScript>(TABLE).where({ config_id: configId, active: true, event: toDbEvent(event) });

getByEnvironmentId: async (environmentId: number): Promise<OnEventScript[]> => {
const existingScriptsQuery = await db.knex
.select<(DBOnEventScript & { provider_config_key: string })[]>(`${TABLE}.*`, '_nango_configs.unique_key as provider_config_key')
.from(TABLE)
.join('_nango_configs', `${TABLE}.config_id`, '_nango_configs.id')
.where({
'_nango_configs.environment_id': environmentId,
[`${TABLE}.active`]: true
});
return existingScriptsQuery.map(dbMapper.from);
},

getByConfig: async (configId: number, event: OnEventType): Promise<DBOnEventScript[]> => {
return db.knex.from<DBOnEventScript>(TABLE).where({ config_id: configId, active: true, event: eventTypeMapper.toDb(event) });
},

diffChanges: async ({
environmentId,
onEventScriptsByProvider
}: {
environmentId: number;
onEventScriptsByProvider: OnEventScriptsByProvider[];
}): Promise<{
added: Omit<OnEventScript, 'id' | 'fileLocation' | 'createdAt' | 'updatedAt'>[];
deleted: OnEventScript[];
updated: OnEventScript[];
}> => {
const res: Awaited<ReturnType<typeof onEventScriptService.diffChanges>> = {
added: [],
deleted: [],
updated: []
};

const existingScripts = await onEventScriptService.getByEnvironmentId(environmentId);

// Create a map of existing scripts for easier lookup
const previousMap = new Map(existingScripts.map((script) => [`${script.configId}:${script.name}:${script.event}`, script]));

for (const provider of onEventScriptsByProvider) {
const config = await configService.getProviderConfig(provider.providerConfigKey, environmentId);
if (!config || !config.id) continue;

for (const script of provider.scripts) {
const key = `${config.id}:${script.name}:${script.event}`;

const maybeScript = previousMap.get(key);
if (maybeScript) {
// Script already exists - it's an update
res.updated.push(maybeScript);

// Remove from map to track deletions
previousMap.delete(key);
} else {
// Script doesn't exist - it's new
res.added.push({
configId: config.id,
name: script.name,
version: '0.0.1',
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Out of curiosity:
Did you talk with Bastien about adding on-events-scripts in the templates, managing their versioning, and being able to enable/disable them via the UI?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No but that's why I pinged him in a previous PR to mention that there would still be a fair amount of work to support the features you mentioned (pre-built, dasbhoard, etc...) for on-events scripts

active: true,
event: script.event,
providerConfigKey: provider.providerConfigKey
});
}
}
}

// Any remaining scripts in the map were not found - they are deleted
res.deleted.push(...Array.from(previousMap.values()));

return res;
}
};
14 changes: 13 additions & 1 deletion packages/types/lib/deploy/api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import type { JSONSchema7 } from 'json-schema';
import type { Endpoint, ApiError } from '../api.js';
import type { IncomingFlowConfig, OnEventScriptsByProvider } from './incomingFlow.js';
import type { SyncDeploymentResult } from './index.js';
import type { OnEventType } from '../scripts/on-events/api.js';

export type PostDeployConfirmation = Endpoint<{
Method: 'POST';
Expand All @@ -15,7 +16,7 @@ export type PostDeployConfirmation = Endpoint<{
singleDeployMode?: boolean;
jsonSchema?: JSONSchema7 | undefined;
};
Success: SyncAndActionDifferences;
Success: ScriptDifferences;
}>;

export type PostDeploy = Endpoint<{
Expand Down Expand Up @@ -68,10 +69,21 @@ export interface SlimAction {
name: string;
}

export interface SlimOnEventScript {
providerConfigKey: string;
name: string;
event: OnEventType;
}

export interface SyncAndActionDifferences {
newSyncs: SlimSync[];
deletedSyncs: SlimSync[];
newActions: SlimAction[];
deletedActions: SlimAction[];
deletedModels: string[];
}

export interface ScriptDifferences extends SyncAndActionDifferences {
newOnEventScripts: SlimOnEventScript[];
deletedOnEventScripts: SlimOnEventScript[];
}
Loading
Loading