feat: bf-orchestrator lib integration (microsoft#4384)

* Add bf-orchestrator dependency

* Basic API and FT

* Add test to ignore list for now

* Fix linter issues

* Update orchestrator-core to pick up Linux fixes

* Fixes after latest merge

* Attempt to run orchestrator tests on CI

* Disable test again local model is available

* Try to bring glic into alpine-linux

* Revert "Try to bring glic into alpine-linux"

This reverts commit ff119aa.

* Try switching to buster for Orchestrator

* lockdown orchestrator version due to API change

* Add libgomp dep for orchestrator

Co-authored-by: Dong Lei <[email protected]>

Composer/packages/server/jest.config.js

@@ -4,5 +4,5 @@ const { createConfig } = require('@botframework-composer/test-utils');
 
 module.exports = createConfig('server', 'node', {
   setupFiles: [path.resolve(__dirname, 'src/__tests__/setupEnv.ts')],
-  testPathIgnorePatterns: ['src/__tests__/setupEnv.ts'],
+  testPathIgnorePatterns: ['src/__tests__/setupEnv.ts', 'orchestratorBuilder'],
 });

Composer/packages/server/package.json

@@ -65,6 +65,7 @@
     "@microsoft/bf-dispatcher": "^4.11.0-beta.20201016.393c6b2",
     "@microsoft/bf-generate-library": "^4.10.0-daily.20201015.174962",
     "@microsoft/bf-lu": "^4.11.0-dev.20201013.7ccb128",
+    "@microsoft/bf-orchestrator": "4.11.0-beta.20201013.20d7917",
     "archiver": "^5.0.2",
     "axios": "^0.19.2",
     "azure-storage": "^2.10.3",

Composer/packages/server/src/models/bot/__mocks__/mockLUInput.json

@@ -0,0 +1,16 @@
+[
+  {
+    "name": "emptybot-1.dialog",
+    "content": "{\n  \"$kind\": \"Microsoft.AdaptiveDialog\",\n  \"$designer\": {\n    \"name\": \"AddItem\",\n    \"id\": \"225905\"\n  },\n  \"autoEndDialog\": true,\n  \"defaultResultProperty\": \"dialog.result\",\n  \"triggers\": [\n    {\n      \"$kind\": \"Microsoft.OnBeginDialog\",\n      \"$designer\": {\n        \"name\": \"BeginDialog\",\n        \"id\": \"479346\"\n      },\n      \"actions\": [\n        {\n          \"$kind\": \"Microsoft.SetProperties\",\n          \"$designer\": {\n            \"id\": \"811190\",\n            \"name\": \"Set properties\"\n          },\n          \"assignments\": [\n            {\n              \"property\": \"dialog.itemTitle\",\n              \"value\": \"=coalesce(@itemTitle, $itemTitle)\"\n            },\n            {\n              \"property\": \"dialog.listType\",\n              \"value\": \"=coalesce(@listType, $listType)\"\n            }\n          ]\n        },\n        {\n          \"$kind\": \"Microsoft.TextInput\",\n          \"$designer\": {\n            \"id\": \"282825\",\n            \"name\": \"AskForTitle\"\n          },\n          \"prompt\": \"${TextInput_Prompt_282825()}\",\n          \"maxTurnCount\": \"3\",\n          \"property\": \"dialog.itemTitle\",\n          \"value\": \"=coalesce(@itemTitle, $itemTitle)\",\n          \"allowInterruptions\": \"!@itemTitle && #_Interruption.Score >= 0.9\"\n        },\n        {\n          \"$kind\": \"Microsoft.ChoiceInput\",\n          \"$designer\": {\n            \"id\": \"878594\",\n            \"name\": \"AskForListType\"\n          },\n          \"prompt\": \"${TextInput_Prompt_878594()}\",\n          \"maxTurnCount\": \"3\",\n          \"property\": \"dialog.listType\",\n          \"value\": \"=@listType\",\n          \"allowInterruptions\": \"!@listType\",\n          \"outputFormat\": \"value\",\n          \"choices\": [\n            {\n              \"value\": \"todo\",\n              \"synonyms\": [\n                \"to do\"\n              ]\n            },\n            {\n              \"value\": \"grocery\",\n              \"synonyms\": [\n                \"groceries\"\n              ]\n            },\n            {\n              \"value\": \"shopping\",\n              \"synonyms\": [\n                \"shoppers\"\n              ]\n            }\n          ],\n          \"appendChoices\": \"true\",\n          \"defaultLocale\": \"en-us\",\n          \"style\": \"Auto\",\n          \"choiceOptions\": {\n            \"inlineSeparator\": \", \",\n            \"inlineOr\": \" or \",\n            \"inlineOrMore\": \", or \",\n            \"includeNumbers\": true\n          },\n          \"recognizerOptions\": {\n            \"noValue\": false\n          }\n        },\n        {\n          \"$kind\": \"Microsoft.EditArray\",\n          \"$designer\": {\n            \"id\": \"733511\",\n            \"name\": \"Edit an Array property\"\n          },\n          \"changeType\": \"push\",\n          \"itemsProperty\": \"user.lists[dialog.listType]\",\n          \"value\": \"=$itemTitle\"\n        },\n        {\n          \"$kind\": \"Microsoft.SendActivity\",\n          \"$designer\": {\n            \"id\": \"139532\",\n            \"name\": \"Send a response\"\n          },\n          \"activity\": \"${SendActivity_139532()}\"\n        }\n      ]\n    }\n  ],\n  \"generator\": \"additem.lg\",\n  \"recognizer\": \"additem.lu\"\n}\n",
+    "path": "/Users/tester/Desktop/EmptyBot-1/additem.dialog",
+    "relativePath": "",
+    "lastModified": "Thu Jul 09 2020 10:19:09 GMT-0700 (Pacific Daylight Time)"
+  },
+  {
+    "name": "additem.en-us.lu",
+    "content": "\n# TextInput_Response_282825\n- Please remind me to {itemTitle=buy milk}\n- Please remember that I need to {itemTitle=buy milk}\n- shoppers",
+    "path": "/Users/tester/Desktop/EmptyBot-1/dialogs/additem/language-understanding/en-us/additem.en-us.lu",
+    "relativePath": "dialogs/additem/language-understanding/en-us/additem.en-us.lu",
+    "lastModified": "Thu Jul 09 2020 10:19:09 GMT-0700 (Pacific Daylight Time)"
+  }
+]

Composer/packages/server/src/models/bot/__tests__/orchestratorBuilder.test.ts

@@ -0,0 +1,85 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import * as path from 'path';
+
+import rimraf from 'rimraf';
+import { FileInfo } from '@bfc/shared';
+import { Utility } from '@microsoft/bf-orchestrator';
+
+import { IFileStorage } from '../../storage/interface';
+import { Builder } from '../builder';
+import mockLUInput from '../__mocks__/mockLUInput.json';
+
+const nlrId = 'pretrained.20200924.microsoft.dte.00.03.en.onnx';
+const nlrPath: string = path.resolve('./orchestrator_ut_model/');
+const downloadModelTimeoutMs = 90000;
+
+describe('Orchestrator Tests', () => {
+  beforeAll(async () => {
+    // disable Orchestrator Lib console logging across the board - interferes with Jest
+    Utility.toPrintDebuggingLogToConsole = false;
+
+    const progressStatusStub = jest.fn();
+    const successStatusStub = jest.fn();
+
+    const builder = new Builder('', {} as IFileStorage, 'en-us');
+
+    // download the UT NLR model once before all tests are run - build tests don't work without a valid model
+    await builder.runOrchestratorNlrGet(nlrPath, nlrId, progressStatusStub, successStatusStub);
+
+    expect(progressStatusStub).toBeCalled();
+    expect(successStatusStub).toBeCalledTimes(1);
+  }, downloadModelTimeoutMs);
+
+  afterAll(async () => {
+    const callbackStub = jest.fn();
+    rimraf(nlrPath, callbackStub);
+
+    expect(callbackStub).toBeCalledWith();
+  });
+
+  it('always lists DTE 3L model for FTs', async () => {
+    const builder = new Builder('', {} as IFileStorage, 'en-us');
+
+    const nlrList = await builder.runOrchestratorNlrList();
+    expect(Object.getOwnPropertyNames(nlrList.models)).toContain(nlrId);
+  });
+
+  it('throws if input empty', () => {
+    const builder = new Builder('', {} as IFileStorage, 'en-us');
+
+    expect(builder.runOrchestratorBuild([], nlrPath)).rejects.toThrow();
+  });
+
+  it('throws if NLR path invalid', () => {
+    const builder = new Builder('', {} as IFileStorage, 'en-us');
+
+    const data: FileInfo[] = [{ name: 'hello', content: 'test', lastModified: '', path: '', relativePath: '' }];
+    expect(builder.runOrchestratorBuild(data, 'invalidPath')).rejects.toThrow();
+  });
+
+  it('produces expected snapshot and recognizer shape', async () => {
+    const builder = new Builder('', {} as IFileStorage, 'en-us');
+
+    const buildOutput = await builder.runOrchestratorBuild(mockLUInput, nlrPath);
+
+    expect(buildOutput.outputs.map((o) => o.id)).toContain('additem.en-us.lu');
+
+    const addItemData = buildOutput.outputs.find((o) => o.id == 'additem.en-us.lu');
+    expect(addItemData?.snapshot).toBeTruthy();
+  });
+
+  it('produces expected recognizer shape', async () => {
+    const builder = new Builder('', {} as IFileStorage, 'en-us');
+    const buildOutput = await builder.runOrchestratorBuild(mockLUInput, nlrPath);
+
+    expect(buildOutput.outputs.map((o) => o.id)).toContain('additem.en-us.lu');
+
+    const addItemData = buildOutput.outputs.find((o) => o.id == 'additem.en-us.lu');
+    expect(addItemData?.recognizer).toBeTruthy();
+
+    expect(addItemData?.recognizer.orchestratorRecognizer).toBeTruthy();
+    expect(addItemData?.recognizer.orchestratorRecognizer.$kind).toBe('Microsoft.OrchestratorRecognizer');
+  });
+});

Composer/packages/server/src/models/bot/botProject.ts

@@ -818,7 +818,7 @@ export class BotProject implements IBotProject {
     try {
       const defaultBotProjectFile: any = await AssetService.manager.botProjectFileTemplate;
 
-      for (const [_, file] of files) {
+      for (const [, file] of files) {
         if (file.name.endsWith(FileExtensions.BotProject)) {
           return fileList;
         }

Composer/packages/server/src/models/bot/builder.ts

@@ -5,13 +5,16 @@
 import { FileInfo, IConfig } from '@bfc/shared';
 import { ComposerReservoirSampler } from '@microsoft/bf-dispatcher/lib/mathematics/sampler/ComposerReservoirSampler';
 import { ComposerBootstrapSampler } from '@microsoft/bf-dispatcher/lib/mathematics/sampler/ComposerBootstrapSampler';
+import { Orchestrator } from '@microsoft/bf-orchestrator';
 
 import { Path } from '../../utility/path';
 import { IFileStorage } from '../storage/interface';
 import log from '../../logger';
 
+import { IOrchestratorBuildOutput, IOrchestratorNLRList, IOrchestratorProgress } from './interface';
 import { luImportResolverGenerator, getLUFiles, getQnAFiles } from './luResolver';
 
+// eslint-disable-next-line @typescript-eslint/no-var-requires
 const crossTrainer = require('@microsoft/bf-lu/lib/parser/cross-train/crossTrainer.js');
 const luBuild = require('@microsoft/bf-lu/lib/parser/lubuild/builder.js');
 const qnaBuild = require('@microsoft/bf-lu/lib/parser/qnabuild/builder.js');
@@ -100,6 +103,59 @@ export class Builder {
     this._locale = v;
   }
 
+  /**
+   * Orchestrator: Get available list of NLR models
+   */
+  public async runOrchestratorNlrList(): Promise<IOrchestratorNLRList> {
+    return JSON.parse(await Orchestrator.nlrListAsync());
+  }
+
+  /**
+   * Orchestrator: Download an available NLR model.
+   *
+   * @remarks Available NLR models and VersionIds are obtained by running runOrchestratorNlrList first.
+   *
+   * @param modelPath - Folder path to save NLR model
+   * @param nlrId - VersionId of the model
+   * @param onProgress - Callback to notify of D/L progress
+   * @param onFinish - Callback to notify of D/L completed
+   */
+  public async runOrchestratorNlrGet(
+    modelPath: string,
+    nlrId: string,
+    onProgress: IOrchestratorProgress,
+    onFinish: IOrchestratorProgress
+  ): Promise<void> {
+    await Orchestrator.nlrGetAsync(modelPath, nlrId, onProgress, onFinish);
+  }
+
+  /**
+   * Orchestrator: Build command to compile .lu files into Binary LU (.blu) snapshots.
+   *
+   * A snapshot (.blu file) is created per .lu supplied
+   *
+   * @param files - Array of FileInfo
+   * @param modelPath - Path to NLR model folder
+   * @param isDialog - Flag to toggle creation of Recognizer Dialogs (default: true)
+   * @param fullEmbedding - Use larger embeddings and skip size optimization (default: false)
+   * @returns An object containing snapshot bytes and recognizer dialogs for each .lu file
+   */
+  public async runOrchestratorBuild(
+    files: FileInfo[],
+    modelPath: string,
+    isDialog = true,
+    fullEmbedding = false
+  ): Promise<IOrchestratorBuildOutput> {
+    const luObjects = files
+      .filter((fi) => fi.name.endsWith('.lu'))
+      .map((fi) => ({
+        id: fi.name,
+        content: fi.content,
+      }));
+
+    return await Orchestrator.buildAsync(modelPath, luObjects, isDialog, null, fullEmbedding);
+  }
+
   private async createGeneratedDir() {
     // clear previous folder
     await this.deleteDir(this.generatedFolderPath);

Composer/packages/server/src/models/bot/interface.ts

@@ -1,7 +1,7 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-import { ILuisConfig, IQnAConfig } from '@bfc/shared';
+import { BaseSchema, ILuisConfig, IQnAConfig } from '@bfc/shared';
 
 import { CrossTrainConfig } from './builder';
 import { RecognizerTypes } from './recognizer';
@@ -48,3 +48,33 @@ export interface IOperationLUFile {
 export interface ILuisStatusOperation {
   [key: string]: IOperationLUFile;
 }
+
+export interface IOrchestratorNLRList {
+  version: string;
+  readonly models: {
+    [versionId: string]: {
+      releaseDate: string;
+      description: string;
+    };
+  };
+}
+
+export interface IOrchestratorProgress {
+  (status: string): void;
+}
+
+export interface IOrchestratorRecognizer extends BaseSchema {
+  modelPath: string;
+  snapshotPath: string;
+  entityRecognizers: any[];
+}
+
+export interface IOrchestratorBuildOutput {
+  outputs: [{ id: string; snapshot: Uint8Array; recognizer: { [recog: string]: BaseSchema } }];
+  settings: {
+    orchestrator: {
+      modelPath: string;
+      snapshots: Map<string, string>;
+    };
+  };
+}