Add Khronos glTF sample models to fidelity suite

.gitignore

@@ -5,6 +5,7 @@ lib/*
 fidelity/*
 test/fidelity/results/*
 examples/built/*
+examples/assets/glTF-Sample-Models
 **/*.sw*
 .DS_Store
 .idea

.travis.yml

@@ -19,6 +19,7 @@ script:
 - ./scripts/ci-run-tests.sh
 cache:
   directories:
+  - examples/assets/glTF-Sample-Models
   - node_modules
 after_success:
 - npm run check-fidelity

README.md

@@ -275,6 +275,7 @@ The following npm scripts are available:
 * `npm test` - Runs tests.
 * `npm run check-fidelity` - Compare rendering to third-party renderers
 * `npm run install-renderers` - Install or update third-party renderers
+* `npm run checkout-samples` - Create or update a local copy of sample models. This is needed for running fidelity checks and updating screenshots.
 * `npm run update-screenshots` - Take screenshots of fidelity tests using third-party renderers
 
 ## Examples

package.json

@@ -32,6 +32,7 @@
     "clean": "rm -rf ./lib ./dist",
     "build": "tsc && rollup -c",
     "create-legacy-bundles": "node ./scripts/create-legacy-bundles.js",
+    "prepare": "npm run checkout-samples",
     "prepublishOnly": "npm run build && npm run create-legacy-bundles",
     "test": "npm run clean && npm run build && npm run wct-local-chrome",
     "wct-local-chrome": "wct --config-file test/wct-local.conf.json -l chrome",
@@ -41,6 +42,7 @@
     "compare-fidelity": "./scripts/compare-fidelity-to-ref.sh",
     "install-renderers": "./scripts/install-third-party-renderers.sh",
     "update-screenshots": "node ./scripts/update-screenshots.js",
+    "checkout-samples": "./scripts/checkout-khronos-gltf-samples.sh",
     "serve": "ws",
     "dev": "concurrently \"npm run watch\" \"npm run serve\"",
     "watch": "cross-os x-watch",

scripts/checkout-khronos-gltf-samples.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+##
+# Copyright 2019 Google Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the 'License');
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an 'AS IS' BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+# This script is responsible for cloning and / or updating the local repository
+# of Khronos glTF sample models
+
+REPO_URL=https://github.com/KhronosGroup/glTF-Sample-Models.git
+CLONE_PATH=./examples/assets/glTF-Sample-Models
+
+if [ ! -d "$CLONE_PATH/.git" ]; then
+  git clone $REPO_URL $CLONE_PATH
+fi
+
+pushd $CLONE_PATH
+ORIGIN_URL=`git config remote.origin.url`;
+
+if [ -z "$ORIGIN_URL" ]; then
+  git remote add origin $REPO_URL
+else
+  if [ ! "$REPO_URL" == "$ORIGIN_URL" ]; then
+    git remote rm origin
+    git remote add origin $REPO_URL
+  fi
+fi
+
+git fetch origin
+git reset --hard origin/master
+
+popd

scripts/install-third-party-renderers.sh

@@ -87,7 +87,7 @@ fi
 
 pushd $FILAMENT_DIR
 
-git fetch --depth=1 origin
+#git fetch --depth=1 origin
 git reset --hard origin/master
 
 git apply $FILAMENT_PATCH_PATH

scripts/model-viewer-screenshot.js

@@ -0,0 +1,56 @@
+/*
+ * Copyright 2019 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+require = require('esm')(module)
+
+const fs = require('fs').promises;
+const path = require('path');
+const LocalWebServer = require('local-web-server')
+const {ArtifactCreator} = require('../lib/test/fidelity/artifact-creator.js');
+const {ConfigReader} = require('../lib/test/fidelity/config-reader.js');
+
+const config = require('../test/fidelity/config.json');
+const configReader = new ConfigReader(config);
+
+const screenshotCreator =
+    new ArtifactCreator(config, 'http://localhost:9040/test/fidelity/');
+const localWebServer = new LocalWebServer()
+const server = localWebServer.listen({port: 9040, directory: './'});
+const slug = process.argv[2];
+const outputFile = process.argv[3];
+
+if (slug == null) {
+  console.error(' Test slug not specified!');
+  process.exit(1);
+}
+
+if (outputFile == null) {
+  console.error(' Output file not specified!');
+  process.exit(1);
+}
+
+screenshotCreator
+    .captureScreenshot(slug, configReader.dimensionsForSlug(slug), outputFile)
+    .then(() => {
+      return 0;
+    })
+    .catch(error => {
+      console.error(error)
+      return 1;
+    })
+    .then(code => {
+      server.close();
+      process.exit(code);
+    });

scripts/update-screenshots.js

@@ -29,14 +29,30 @@ const filamentScreenshotScript =
 const backgroundImageRe = /background-image\="([^"]+)"/;
 const modelSourceRe = /src\="([^"]+)"/
 
+const run = async (command, args) => new Promise((resolve, reject) => {
+  const childProcess = spawn(command, args, {
+    cwd: process.cwd(),
+    env: process.env,
+    stdio: ['ignore', 'inherit', 'inherit']
+  });
+
+  childProcess.once('error', (error) => {
+    warn(error);
+  });
+
+  childProcess.once('exit', (code) => {
+    if (code === 0) {
+      resolve();
+    } else {
+      reject(new Error('Command failed'));
+    }
+  });
+});
+
 const updateScreenshots = async (config) => {
   const {scenarios} = config;
 
-  console.log(`💡 Updating screenshots
-
-⏳ NOTE: The first time running this script can take a long time (5 - 10 minutes)
-because we have to build Filament from scratch. If you hear your CPU fan spin
-up, take a break and go make yourself a nice cup of tea!`);
+  console.log(`🆙 Updating screenshots`);
 
   for (const scenario of scenarios) {
     const {goldens, slug} = scenario;
@@ -78,22 +94,32 @@ up, take a break and go make yourself a nice cup of tea!`);
           console.log(
               `✋ Cannot automatically update ${name} screenshots (yet)`);
           break;
+        case '<model-viewer> (master)':
+          console.log(`💡 Rendering ${name} screenshot for ${slug}...`);
+
+          try {
+            await run(
+                'node',
+                ['./scripts/model-viewer-screenshot.js', slug, filePath]);
+          } catch (error) {
+            throw new Error(`Failed to capture <model-viewer> screenshot: ${
+                error.message}`);
+          }
+
+          break;
         case 'Filament':
           const {width, height} = scenario.dimensions;
-          // TODO(cdata): Figure out how to detect high-dpi here:
-          const scaledWidth = width;
-          const scaledHeight = height;
 
           await new Promise((resolve, reject) => {
-            console.log(`🖼 Rendering ${name} screenshot for ${slug}...`);
+            console.log(` Rendering ${name} screenshot for ${slug}...`);
 
             const childProcess = spawn(
                 filamentScreenshotScript,
                 [
                   '-w',
-                  `${scaledWidth}`,
+                  `${width}`,
                   '-h',
-                  `${scaledHeight}`,
+                  `${height}`,
                   '-i',
                   backgroundImagePath,
                   '-m',

src/test/fidelity/artifact-creator.ts

@@ -116,7 +116,10 @@ export class ArtifactCreator {
   }
 
 
-  protected async captureScreenshot(slug: string, dimensions: Dimensions) {
+  async captureScreenshot(
+      slug: string, dimensions: Dimensions,
+      outputPath: string =
+          path.join(this.config.outputDirectory, slug, 'model-viewer.png')) {
     const scaledWidth = dimensions.width / DEVICE_PIXEL_RATIO;
     const scaledHeight = dimensions.height / DEVICE_PIXEL_RATIO;
 
@@ -169,7 +172,7 @@ export class ArtifactCreator {
     console.log(`🖼  Capturing screenshot`);
 
     const screenshot = await page.screenshot({
-      path: path.join(this.config.outputDirectory, slug, 'model-viewer.png'),
+      path: outputPath,
       clip: {x: 0, y: 0, width: scaledWidth, height: scaledHeight}
     });
 

src/test/fidelity/common.ts

@@ -209,8 +209,11 @@ export class ImageComparator {
       }
     }
 
-    const mismatchingAverageDistanceRatio =
-        mismatchingSum / (this.imagePixels - matched) / MAX_COLOR_DISTANCE;
+    const mismatchingPixels = this.imagePixels - matched;
+
+    const mismatchingAverageDistanceRatio = mismatchingPixels > 0 ?
+        mismatchingSum / (this.imagePixels - matched) / MAX_COLOR_DISTANCE :
+        0;
     const averageDistanceRatio = sum / this.imagePixels / MAX_COLOR_DISTANCE;
 
     return {

src/test/fidelity/components/analysis-view.ts

@@ -167,7 +167,7 @@ export class AnalysisView extends LitElement {
       const width =
           Math.max(this.leftImage!.naturalWidth, this.rightImage!.naturalWidth);
       const height = Math.max(
-          this.rightImage!.naturalWidth, this.rightImage!.naturalHeight);
+          this.leftImage!.naturalHeight, this.rightImage!.naturalHeight);
 
       return {width, height};
     }

src/test/fidelity/config-reader.ts

@@ -0,0 +1,18 @@
+import {Dimensions, ImageComparisonConfig} from './common.js';
+
+export class ConfigReader {
+  constructor(public config: ImageComparisonConfig) {
+  }
+
+  dimensionsForSlug(slug: string): Dimensions {
+    const {scenarios} = this.config;
+
+    for (const scenario of scenarios) {
+      if (scenario.slug === slug) {
+        return scenario.dimensions;
+      }
+    }
+
+    return {width: 0, height: 0};
+  }
+}