A harness for unit testing codegen (PolymerLabs#5755)

src/tools/schema2base.ts

@@ -15,6 +15,8 @@ import {Runtime} from '../runtime/runtime.js';
 import {SchemaGraph, SchemaNode} from './schema2graph.js';
 import {ParticleSpec} from '../runtime/particle-spec.js';
 
+Runtime.init('../..');
+
 export interface EntityGenerator {
   generate(): string;
 }
@@ -27,9 +29,7 @@ export class NodeAndGenerator {
 export abstract class Schema2Base {
   namespace: string;
 
-  constructor(readonly opts: minimist.ParsedArgs) {
-    Runtime.init('../..');
-  }
+  constructor(readonly opts: minimist.ParsedArgs) {}
 
   async call() {
     fs.mkdirSync(this.opts.outdir, {recursive: true});

src/tools/tests/codegen-unit-test-base.ts

@@ -0,0 +1,166 @@
+/**
+ * @license
+ * Copyright (c) 2020 Google Inc. All rights reserved.
+ * This code may only be used under the BSD style license found at
+ * http://polymer.github.io/LICENSE.txt
+ * Code distributed by Google as part of this project is also
+ * subject to an additional IP rights grant found at
+ * http://polymer.github.io/PATENTS.txt
+ */
+
+import fs from 'fs';
+import {Manifest} from '../../runtime/manifest.js';
+import {Flags} from '../../runtime/flags.js';
+
+type Test = {
+  name: string;
+  options: object;
+  input: string;
+  results: string[];
+};
+
+/**
+ * A base class for unit tests of codegen.
+ *
+ * To use this class one has to do 2 things:
+ * - Write a .cgtest in goldens/ directory that specifies a set of
+ *   inputs and expected outputs for codegen.
+ * - Add a CodegenUnitTest to a test suite (e.g. kotlin-codegen-test-suite.ts)
+ *   with the code that should run for each input.
+ *
+ * The main benefit of using this approach is programmatic updating of
+ * expectations in .cgtest files via ./tools/sigh updateCodegenUnitTests
+ *
+ * .cgtest files have one or more tests defined as following:
+ *
+ * [name]
+ * name of your test here
+ * [opts] // <- this is optional
+ * {"extraParam": 42} // JSON Object with extra arguments
+ * [results]
+ * expected output goes here
+ * [next] // <- this is optional, only use for multiple outputs
+ * another expected output
+ * [end]
+ */
+export abstract class CodegenUnitTest {
+
+  constructor(
+    readonly title: string,
+    readonly inputFileName: string
+  ) {}
+
+  get inputFilePath() {
+    return `./src/tools/tests/goldens/${this.inputFileName}`;
+  }
+
+  /**
+   * Calculates the codegen result for the given input.
+   */
+  abstract async compute(input: string, opts: object): Promise<string | string[]>;
+}
+
+/**
+ * Convenience base class for tests that take Arcs manifest as an input.
+ */
+export abstract class ManifestCodegenUnitTest extends CodegenUnitTest {
+
+  constructor(
+    title: string,
+    inputFileName: string,
+    private flags = {}
+  ) {
+    super(title, inputFileName);
+  }
+
+  async compute(input: string, opts: object): Promise<string | string[]> {
+    return Flags.withFlags(this.flags, async () => this.computeFromManifest(await Manifest.parse(input), opts))();
+  }
+
+  abstract async computeFromManifest(manifest: Manifest, opts: object): Promise<string | string[]>;
+}
+
+// Internal utilities below
+
+/**
+ * Run the computation for a given test with cleanups and data normalization.
+ */
+export async function runCompute(testCase: CodegenUnitTest, test: Test): Promise<string[]> {
+  Flags.reset();
+  const result = await testCase.compute(test.input, test.options);
+  return Array.isArray(result) ? result : [result];
+}
+
+/**
+ * Reads out Test data structure from the input .cgtest file.
+ */
+export function readTests(unitTest: CodegenUnitTest): Test[] {
+  let fileData = fs.readFileSync(unitTest.inputFilePath, 'utf-8');
+  const tests: Test[] = [];
+
+  fileData = fileData.replace(/\[header\].*\[end_header\]\n*/sm, '');
+
+  const caseStrings = (fileData).split('\n[end]');
+  for (const caseString of caseStrings) {
+    if (caseString.trim().length === 0) continue;
+    const matches = caseString.match(/\w*^\[name\]\n([^\n]*)(?:\n\[opts\]\n(.*))?\n\[input\]\n(.*)\n\[results\]\n?(.*)/sm);
+
+    if (!matches) {
+      throw Error(`Cound not parse a test case: ${caseString}`);
+    }
+
+    tests.push({
+      name: matches[1].trim(),
+      options: JSON.parse(matches[2] || '{}'),
+      input: matches[3],
+      results: matches[4].split('\n[next]\n')
+    });
+  }
+
+  return tests;
+}
+
+/**
+ * Updates expectations in the input .cgtest file.
+ */
+export async function regenerateInputFile(unit: CodegenUnitTest): Promise<number> {
+  const tests = readTests(unit);
+
+  let updatedCount = 0;
+
+  const newTests = await Promise.all(tests.map(async test => {
+    const results = await runCompute(unit, test);
+
+    if (JSON.stringify(results) !== JSON.stringify(test.results)) {
+      updatedCount++;
+    }
+
+    const optionsString = Object.entries(test.options).length === 0 ? '' : `\
+[opts]
+${JSON.stringify(test.options)}
+`;
+
+    return `\
+[name]
+${test.name}
+${optionsString}[input]
+${test.input}
+[results]
+${results.join('\n[next]\n')}
+[end]
+`;
+  }));
+
+  const content = `\
+[header]
+${unit.title}
+
+Expectations can be updated with:
+$ ./tools/sigh updateCodegenUnitTests
+[end_header]
+
+${newTests.join('\n')}`;
+
+  fs.writeFileSync(unit.inputFilePath, content);
+  return updatedCount;
+}

src/tools/tests/codegen-unit-tests-runner-test.ts

@@ -0,0 +1,27 @@
+/**
+ * @license
+ * Copyright (c) 2020 Google Inc. All rights reserved.
+ * This code may only be used under the BSD style license found at
+ * http://polymer.github.io/LICENSE.txt
+ * Code distributed by Google as part of this project is also
+ * subject to an additional IP rights grant found at
+ * http://polymer.github.io/PATENTS.txt
+ */
+
+import {testSuite} from './kotlin-codegen-test-suite.js';
+import {readTests, runCompute} from './codegen-unit-test-base.js';
+import {assert} from '../../platform/chai-web.js';
+
+/**
+ * Runs all the CodegenUnitTests.
+ *
+ * Note that this has to be in a separate file as the test logic has to be accessible
+ * in the CLI tool performing updating, where describe(...) and it(...) are not defined.
+ */
+for (const unit of testSuite) {
+  describe(unit.title, async () => {
+    for (const test of readTests(unit)) {
+      it(test.name, async () => assert.deepEqual(await runCompute(unit, test), test.results));
+    }
+  });
+}

src/tools/tests/codegen-unit-tests-updater.ts

@@ -0,0 +1,33 @@
+/**
+ * @license
+ * Copyright (c) 2020 Google Inc. All rights reserved.
+ * This code may only be used under the BSD style license found at
+ * http://polymer.github.io/LICENSE.txt
+ * Code distributed by Google as part of this project is also
+ * subject to an additional IP rights grant found at
+ * http://polymer.github.io/PATENTS.txt
+ */
+
+import {testSuite} from './kotlin-codegen-test-suite.js';
+import {regenerateInputFile} from './codegen-unit-test-base.js';
+
+/**
+ * Updates expectations in all .cgtest files.
+ *
+ * ./tools/sigh updateCodegenUnitTests
+ */
+let totalUpdateCount = 0;
+void Promise.all(testSuite.map(async testCase => {
+  const updateCount = await regenerateInputFile(testCase);
+  if (updateCount > 0) {
+    console.info(`${testCase.inputFileName}: ${updateCount} tests updated`);
+  }
+  totalUpdateCount += updateCount;
+})).then(() => {
+  if (totalUpdateCount === 0) {
+    console.info(`All tests up to date!`);
+  }
+}).catch(e => {
+  console.error(e.message);
+  process.exit(1);
+});

src/tools/tests/goldens/kotlin-connection-type-generator.cgtest

@@ -0,0 +1,56 @@
+[header]
+Kotlin Connection Type Generation
+
+Expectations can be updated with:
+$ ./tools/sigh updateCodegenUnitTests
+[end_header]
+
+[name]
+generates type for singleton entity
+[input]
+particle Module
+  data: reads Thing {name: Text}
+[results]
+SingletonType(EntityType(Module_Data.SCHEMA))
+[end]
+
+[name]
+generates type for singleton reference
+[input]
+particle Module
+  data: reads &Thing {name: Text}
+[results]
+SingletonType(ReferenceType(EntityType(Module_Data.SCHEMA)))
+[end]
+
+[name]
+generates type for collection of entities
+[input]
+particle Module
+  data: reads [Thing {name: Text}]
+[results]
+CollectionType(EntityType(Module_Data.SCHEMA))
+[end]
+
+[name]
+generates type for collection of references
+[input]
+particle Module
+  data: reads [&Thing {name: Text}]
+[results]
+CollectionType(ReferenceType(EntityType(Module_Data.SCHEMA)))
+[end]
+
+[name]
+generates type for collection of tuples
+[input]
+particle Module
+  data: reads [(&Thing {name: Text}, &Other {age: Number})]
+[results]
+CollectionType(
+    TupleType.of(
+        ReferenceType(EntityType(Module_Data_0.SCHEMA)),
+        ReferenceType(EntityType(Module_Data_1.SCHEMA))
+    )
+)
+[end]