forked from microsoft/BotFramework-Composer
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
fix: moved long running creation steps to worker threads to prevent m…
…ain process overload (microsoft#6323) * Initial addition of worker thread * commneted out long running code and moved to worker (scaffolding and initial implementation) * Removing unused reference * Adding dialog merge work to merge_worker execution * Moving yeomen template execution to worker thread * Added status updarte for creation from within worker * Fixing yeoman spelling mistake * Handling PR changes - file and funciton name changes * PR requested name changes * fixing unit test * Adding autogenerated locale strings * Fixing unit test to mock worker and remove process.env assertion Co-authored-by: Patrick Volum <[email protected]> Co-authored-by: Srinaath Ravichandran <[email protected]> Co-authored-by: Soroush <[email protected]>
- Loading branch information
4 people
authored
Mar 10, 2021
1 parent
823d4e0
commit 5137457
Showing
8 changed files
with
190 additions
and
81 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -468,6 +468,7 @@ export class BotProjectService { | |
| templateVersion, | ||
| name, | ||
| locationRef, | ||
| jobId, | ||
| user | ||
| ); | ||
|
|
||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
50 changes: 50 additions & 0 deletions
50
Composer/packages/server/src/workers/dialogMerge.worker.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,50 @@ | ||
| // Copyright (c) Microsoft Corporation. | ||
| // Licensed under the MIT License. | ||
|
|
||
| import { Worker, isMainThread, workerData, parentPort } from 'worker_threads'; | ||
|
|
||
| import { SchemaMerger } from '@microsoft/bf-dialog/lib/library/schemaMerger'; | ||
|
|
||
| import { Path } from '../utility/path'; | ||
| import { BotProject } from '../models/bot/botProject'; | ||
|
|
||
| if (!isMainThread) { | ||
| const realMerge = new SchemaMerger( | ||
| [workerData.manifestFile, '!**/imported/**', '!**/generated/**'], | ||
| Path.join(workerData.dataDir, 'schemas/sdk'), | ||
| Path.join(workerData.dataDir, 'dialogs/imported'), | ||
| false, | ||
| false, | ||
| console.log, | ||
| console.warn, | ||
| console.error | ||
| ); | ||
|
|
||
| realMerge | ||
| .merge() | ||
| .then(() => { | ||
| process.exit(0); | ||
| }) | ||
| .catch((err) => { | ||
| parentPort?.postMessage({ error: err }); | ||
| process.exit(1); | ||
| }); | ||
| } | ||
|
|
||
| export function runDialogMerge(manifestFile: string, currentProject: BotProject) { | ||
| return new Promise<void>((resolve, reject) => { | ||
| const w = new Worker(__filename, { | ||
| workerData: { manifestFile, dataDir: currentProject.dataDir }, | ||
| }); | ||
| w.on('exit', (returnCode) => { | ||
| if (returnCode === 0) { | ||
| resolve(); | ||
| } | ||
| }); | ||
| w.on('message', (message) => { | ||
| if (message?.error) { | ||
| reject(message.error); | ||
| } | ||
| }); | ||
| }); | ||
| } |
113 changes: 113 additions & 0 deletions
113
Composer/packages/server/src/workers/templateInstallation.worker.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,113 @@ | ||
| // Copyright (c) Microsoft Corporation. | ||
| // Licensed under the MIT License. | ||
|
|
||
| import { Worker, isMainThread, workerData, parentPort } from 'worker_threads'; | ||
|
|
||
| import Environment from 'yeoman-environment'; | ||
| import yeoman from 'yeoman-environment'; | ||
| import TerminalAdapter from 'yeoman-environment/lib/adapter'; | ||
| import formatMessage from 'format-message'; | ||
|
|
||
| import { templateGeneratorPath } from '../settings/env'; | ||
| import log from '../logger'; | ||
| import { BackgroundProcessManager } from '../services/backgroundProcessManager'; | ||
|
|
||
| const installRemoteTemplate = async ( | ||
| yeomanEnv: Environment, | ||
| generatorName: string, | ||
| npmPackageName: string, | ||
| templateVersion: string | ||
| ): Promise<boolean> => { | ||
| yeomanEnv.cwd = templateGeneratorPath; | ||
| try { | ||
| log('Installing generator', npmPackageName); | ||
| templateVersion = templateVersion ? templateVersion : '*'; | ||
| await yeomanEnv.installLocalGenerators({ [npmPackageName]: templateVersion }); | ||
|
|
||
| log('Looking up local packages'); | ||
| await yeomanEnv.lookupLocalPackages(); | ||
| return true; | ||
| } catch { | ||
| return false; | ||
| } | ||
| }; | ||
|
|
||
| const instantiateRemoteTemplate = async ( | ||
| yeomanEnv: Environment, | ||
| generatorName: string, | ||
| dstDir: string, | ||
| projectName: string | ||
| ) => { | ||
| log('About to instantiate a template!', dstDir, generatorName, projectName); | ||
| yeomanEnv.cwd = dstDir; | ||
|
|
||
| await yeomanEnv.run([generatorName, projectName], {}, () => { | ||
| log('Template successfully instantiated', dstDir, generatorName, projectName); | ||
| }); | ||
| }; | ||
|
|
||
| const yeomanWork = async (npmPackageName: string, templateVersion: string, dstDir: string, projectName: string) => { | ||
| const generatorName = npmPackageName.toLowerCase().replace('generator-', ''); | ||
| // create yeoman environment | ||
| parentPort?.postMessage({ status: formatMessage('Getting Yeoman environment') }); | ||
|
|
||
| const yeomanEnv = yeoman.createEnv( | ||
| '', | ||
| { yeomanRepository: templateGeneratorPath }, | ||
| new TerminalAdapter({ console: console }) | ||
| ); | ||
| await yeomanEnv.lookupLocalPackages(); | ||
|
|
||
| parentPort?.postMessage({ status: formatMessage('Installing Yeoman template') }); | ||
|
|
||
| const remoteTemplateAvailable = await installRemoteTemplate( | ||
| yeomanEnv, | ||
| generatorName, | ||
| npmPackageName, | ||
| templateVersion | ||
| ); | ||
| if (remoteTemplateAvailable) { | ||
| parentPort?.postMessage({ status: formatMessage('Instantiating Yeoman template') }); | ||
|
|
||
| await instantiateRemoteTemplate(yeomanEnv, generatorName, dstDir, projectName); | ||
| } else { | ||
| // handle error | ||
| throw new Error(`error hit when installing remote template`); | ||
| } | ||
| }; | ||
|
|
||
| export function runYeomanTemplatePipeline( | ||
| npmPackageName: string, | ||
| templateVersion: string, | ||
| dstDir: string, | ||
| projectName: string, | ||
| jobId: string | ||
| ) { | ||
| return new Promise<void>((resolve, reject) => { | ||
| const w = new Worker(__filename, { | ||
| workerData: { npmPackageName, templateVersion, dstDir, projectName }, | ||
| }); | ||
| w.on('exit', () => { | ||
| resolve(); | ||
| }); | ||
| w.on('message', (message) => { | ||
| if (message?.error) { | ||
| reject(message.error); | ||
| } | ||
| if (message?.status) { | ||
| BackgroundProcessManager.updateProcess(jobId, 202, message?.status); | ||
| } | ||
| }); | ||
| }); | ||
| } | ||
|
|
||
| if (!isMainThread) { | ||
| yeomanWork(workerData.npmPackageName, workerData.templateVersion, workerData.dstDir, workerData.projectName) | ||
| .then(() => { | ||
| process.exit(0); | ||
| }) | ||
| .catch((err) => { | ||
| parentPort?.postMessage({ error: err }); | ||
| process.exit(1); | ||
| }); | ||
| } |