Implement Visual Regression Tests / Runner

Including GitHub Action

.eslintrc.cjs

@@ -24,6 +24,7 @@ const config = {
       './tsconfig.vitest.json',
       './tsconfig.cfg.json',
       './examples/tsconfig.json',
+      './visual-regression/tsconfig.json',
     ],
     // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
     tsconfigRootDir: __dirname,

.github/workflows/tests.yml

@@ -0,0 +1,48 @@
+on:
+  - pull_request
+
+jobs:
+  run-all-tests:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Install Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: 20
+
+      - uses: pnpm/action-setup@v2
+        name: Install pnpm
+        with:
+          version: 8
+          run_install: false
+
+      - name: Get pnpm store directory
+        shell: bash
+        run: |
+          echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV
+
+      - uses: actions/cache@v3
+        name: Setup pnpm cache
+        with:
+          path: ${{ env.STORE_PATH }}
+          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-store-
+
+      - name: Install dependencies
+        run: pnpm install
+
+      - name: Run Unit Tests
+        run: pnpm test
+
+      - name: Install Playwright Browser
+        run: cd visual-regression && pnpm exec playwright install chromium
+
+      - name: Run Visual Regression Tests
+        run: pnpm run test:visual --color
+        env:
+          RUNTIME_ENV: ci

.gitignore

@@ -6,10 +6,13 @@ dist
 dist-cfg
 dist-vitest
 examples/dist-tsc
+visual-regression/failed-results
+visual-regression/certified-snapshots/*-local
 build
 releases
 .tmp
 .env
 *.tgz
+.pnpm-store
 # This project uses `pnpm` for package management
 package-lock.json

.husky/pre-commit

@@ -1,4 +1,10 @@
 #!/usr/bin/env sh
 . "$(dirname -- "$0")/_/husky.sh"
 
-npx lint-staged
+# If files exist in the `visual-regression/failed-results` directory, fail the commit
+if [ -n "$(ls -A visual-regression/failed-results)" ]; then
+  echo "Failed Visual Regression Test results found in \`visual-regression/failed-results\`. Please fix them before committing."
+  exit 1
+fi
+
+pnpm exec lint-staged

.npmrc

@@ -1 +1,2 @@
 engine-strict = true
+use-node-version = 20.9.0

Dockerfile

@@ -0,0 +1,18 @@
+# Use Playwright's base image
+FROM mcr.microsoft.com/playwright:v1.39.0-jammy
+
+# Set the working directory
+WORKDIR /work
+
+# Install PNPM
+RUN npm install -g pnpm
+
+# Copy the necessary files to the container
+COPY .npmrc .npmrc
+COPY package.json package.json
+
+# Get pnpm to install the version of Node declared in .npmrc
+RUN pnpm exec ls
+
+# Set the entry point command
+CMD ["/bin/bash", "-c", "echo 'Must run with Visual Regression Test Runner: `pnpm run test:visual --ci`'"]

README.md

@@ -25,21 +25,53 @@ pnpm watch
 # Run unit tests
 pnpm test
 
+# Run Visual Regression Tests
+pnpm test:visual
+
 # Build API Documentation (builds into ./docs folder)
 pnpm typedoc
 
-# Launch test examples in dev mode (includes Build Renderer (watch mode))
+# Launch Example Tests in dev mode (includes Build Renderer (watch mode))
 pnpm start
 
-# Launch test examples in production mode
+# Launch Example Tests in production mode
 # IMPORTANT: To run test examples on embedded devices that use older browser versions
 # you MUST run the examples in this mode.
 pnpm start:prod
 ```
 
-## Test Examples
+## Example Tests
+
+The Example Tests sub-project define a set of tests for various Renderer
+features. This is NOT an automated test. The command below will launch a
+web server which can be accessed by a web browser for manual testing. However,
+many of the Example Tests define Snapshots for the Visual Regression Test Runner
+(see below).
+
+The Example Tests can be launched with:
+
+```
+pnpm start
+```
+
+See [examples/README.md](./examples/README.md) for more info.
+
+## Visual Regression Tests
+
+In order to prevent bugs on existing Renderer features when new features or bug
+fixes are added, the Renderer includes a Visual Regression Test Runner along
+with a set of certified snapshot files that are checked into the repository.
+
+These tests can be launched with:
+
+```
+pnpm test:visual
+```
+
+The captured Snapshots of these tests are optionally defined in the individual
+Example Tests.
 
-See [examples/README.md](./examples/README.md)
+See [visual-regression/README.md](./visual-regression/README.md) for more info.
 
 ## Release Procedure
 

examples/.npmrc

@@ -0,0 +1 @@
+engine-strict = true

examples/README.md

@@ -1,13 +1,15 @@
-# Lightning 3 Renderer Test Examples
+# Example Tests
 
-This directory contains a set of independent examples which can be selected via
-URL parameters.
+This directory contains a set of independent Example Tests which can be selected
+via URL parameters for manual testing.
+
+Many of these Example Tests also define Snapshots for the Visual Regression Test
+Runner. See [visual-regression/README.md](../visual-regression/README.md) for
+more information on how those tests are run and how their Snapshots are defined.
 
 ## Setup
 
 ```
-pnpm install
-
 # Run code in dev mode (includes building Renderer in watch mode)
 pnpm start
 
@@ -33,10 +35,21 @@ pnpm watch
 
 ## URL Params
 
-- `test` - Test example to run
+- `test` (string, default: "test")
+  - Test example to run.
   - Can be any of the file names (minus extension) in the `tests` directory.
-- `driver` - Core driver to use
+- `driver` (string, default: "main")
+  - Core driver to use
   - Either `main` or `threadx`
+- `overlay` (boolean, default: "true")
+  - Whether or not to show the text overlay in the bottom-right corner that
+    displays the current test and driver being used.
+- `automation` (boolean, default: "false")
+  - Automation mode.
+  - Executes the exported `automation()` function for every Example Test
+    that defines one, one after the other.
+  - This is used by the Visual Regression Test Runner.
+  - See [visual-regression/README.md](../visual-regression/README.md) for more info.
 
 ## Note on imports
 

examples/common/ExampleSettings.ts

@@ -17,11 +17,43 @@
  * limitations under the License.
  */
 
-import type { Dimensions, RendererMain } from '@lightningjs/renderer';
+import type { INode, RendererMain } from '@lightningjs/renderer';
 
 export interface ExampleSettings {
+  /**
+   * Name of the test being run.
+   */
   testName: string;
+  /**
+   * Renderer instance
+   */
   renderer: RendererMain;
+  /**
+   * Core Driver being used by the test.
+   */
   driverName: 'main' | 'threadx';
-  canvas: HTMLCanvasElement;
+  /**
+   * The HTML Element that the Renderer's canvas is a child of
+   */
+  appElement: HTMLDivElement;
+  /**
+   * Renderer Node that all tests should use as their root node.
+   *
+   * @remarks
+   * Tests should NEVER use the `renderer.root` node as this will prevent the
+   * automation mode from being able to clean up after each test.
+   */
+  testRoot: INode;
+  /**
+   * Whether the test is being run in automation mode.
+   */
+  automation: boolean;
+  /**
+   * If the test is run in automation mode, this method will take a visual
+   * snapshot of the current state of the renderer's canvas for the Visual
+   * Regression Test Runner.
+   *
+   * This method will be a no-op if the test is not run in automation mode.
+   */
+  snapshot(): Promise<void>;
 }

examples/common/PageContainer.ts

@@ -20,6 +20,7 @@
 import type { INode, ITextNode, RendererMain } from '@lightningjs/renderer';
 import { Component } from './Component.js';
 import { loadStorage, saveStorage } from '../common/LocalStorage.js';
+import type { ExampleSettings } from './ExampleSettings.js';
 
 interface PageContainerLocalStorageData {
   curPage: number;
@@ -37,7 +38,6 @@ interface PageContainerProps {
   color?: number;
 
   //
-  testName?: string;
   title?: string;
 }
 
@@ -47,35 +47,36 @@ export class PageContainer extends Component {
   private curPageNode: INode | null = null;
   private curPageIndex = -1;
   private pageConstructors: ((page: INode) => Promise<void>)[] = [];
-  private testName?: string;
+  private settings: ExampleSettings;
 
-  constructor(renderer: RendererMain, props: PageContainerProps) {
+  constructor(settings: ExampleSettings, props: PageContainerProps) {
+    const { renderer } = settings;
     super(renderer, {
       x: props.x,
       y: props.y,
       color: props.color ?? 0x00000000,
       width: props.width,
       height: props.height,
-      parent: props.parent,
+      parent: props.parent ? props.parent : settings.testRoot,
     });
 
     this.titleNode = renderer.createTextNode({
       fontFamily: 'Ubuntu',
       fontSize: TITLE_FONT_SIZE,
       x: PADDING,
       y: PADDING,
-      parent: renderer.root,
+      parent: this.node,
       text: props.title ?? '',
     });
 
-    this.testName = props.testName;
+    this.settings = settings;
 
     this.pageNumberNode = renderer.createTextNode({
       fontFamily: 'Ubuntu',
       fontSize: 30,
       x: PADDING,
       y: this.node.height - 30 - PADDING,
-      parent: renderer.root,
+      parent: this.node,
     });
   }
 
@@ -85,9 +86,9 @@ export class PageContainer extends Component {
 
   finalizePages() {
     if (this.curPageIndex === -1 && this.pageConstructors.length > 0) {
-      const { testName } = this;
+      const { automation, testName } = this.settings;
       let pageNum = 0;
-      if (testName) {
+      if (!automation) {
         const savedState = loadStorage<PageContainerLocalStorageData>(
           `${testName}-PageContainer`,
         );
@@ -98,6 +99,7 @@ export class PageContainer extends Component {
         ) {
           pageNum = savedState.curPage;
         }
+        this.bindWindowKeys();
       }
       this.setPage(pageNum).catch(console.error);
     }
@@ -123,8 +125,8 @@ export class PageContainer extends Component {
       parent: this.node,
     });
 
-    const { testName } = this;
-    if (testName) {
+    const { automation, testName } = this.settings;
+    if (!automation) {
       saveStorage<PageContainerLocalStorageData>(`${testName}-PageContainer`, {
         curPage: pageIndex,
       });
@@ -134,7 +136,20 @@ export class PageContainer extends Component {
     await this.pageConstructors[pageIndex]!(this.curPageNode);
   }
 
-  bindWindowKeys() {
+  /**
+   * Performs an automation run of all the pages in this container.
+   */
+  async snapshotPages() {
+    if (!this.settings.automation) {
+      throw new Error('Cannot snapshot pages when not in automation mode');
+    }
+    for (let i = 0; i < this.pageConstructors.length; i++) {
+      await this.setPage(i);
+      await this.settings.snapshot();
+    }
+  }
+
+  private bindWindowKeys() {
     window.addEventListener('keydown', (e) => {
       const numPages = this.pageConstructors.length;
       if (e.key === 'ArrowLeft') {

examples/index.html

@@ -8,7 +8,8 @@
         overflow: hidden;
       }
 
-      canvas {
+      #app {
+        display: inline-block;
         background: #3677e0;
       }
     </style>