Use Karma/BrowserStack to run <model-viewer> tests (google#1075)

* Use Karma to run <model-viewer> tests * All the tests pass in Karma/BrowserStack * Kill WCT w/ fire, add working dev script * Fix test:legacy script * Remove accidental global leak * Fix overlapping test runs * Give up running BrowserStack in parallel * Prevent fidelity tests from running in main build * Add ci test scripts to top-level package.json * Run fidelity tests in Github Actions * Try a conditional * Don't cd in xvfb command; more conditional * Move to GITHUB_WORKSPACE when running xvfb * Try macos for fidelity tests * Direct path to lerna * Stream fidelity test results * Try XVFB again * Split PR build into separate config * Fix job names * Try a different condition * Update package-lock.json to purge Sauce deps * Make sure USE_BROWSER_STACK is set * Remove Travis config * Remove vestigial WCT config
livelse · Mar 13, 2020 · 74c707f · 74c707f
1 parent 24a92bd
commit 74c707f
Show file tree

Hide file tree

Showing 41 changed files with 12,564 additions and 20,777 deletions.
diff --git a/.github/workflows/fidelity-tests.yml b/.github/workflows/fidelity-tests.yml
@@ -0,0 +1,43 @@
+name: Fidelity tests
+
+on: [pull_request]
+
+jobs:
+  compare_renders:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v1
+    - uses: actions/setup-node@v1
+      with:
+        node-version: '12.x'
+
+    - name: Get npm cache directory
+      id: npm-cache
+      run: |
+        echo "::set-output name=dir::$(npm config get cache)"
+
+    - name: Cache npm packages
+      uses: actions/cache@v1
+      with:
+        path: ${{ steps.npm-cache.outputs.dir }}
+        key: ${{ runner.os }}-npm-${{ hashFiles('**/package-lock.json') }}
+        restore-keys: |
+          ${{ runner.os }}-npm-
+
+    - name: NPM install
+      run: npm ci
+
+    - name: Lint TypeScript sources
+      run: npm run lint
+
+    - name: Bootstrap packages
+      run: npm run bootstrap
+
+    - name: Build packages
+      run: npm run build
+
+    - name: Fidelity tests
+      uses: GabrielBB/[email protected]
+      with:
+        run: ./node_modules/.bin/lerna run --scope @google/model-viewer-render-fidelity-tools test --stream
diff --git a/.github/workflows/unit-tests-third-party.yml b/.github/workflows/unit-tests-third-party.yml
@@ -0,0 +1,43 @@
+name: Unit tests (third-party PR)
+
+on: [pull_request]
+
+jobs:
+  minimal_test_run:
+    runs-on: ubuntu-latest
+
+    if: github.event.pull_request.base.repo.id != github.event.pull_request.head.repo.id
+
+    steps:
+    - uses: actions/checkout@v1
+    - uses: actions/setup-node@v1
+      with:
+        node-version: '12.x'
+
+    - name: Get npm cache directory
+      id: npm-cache
+      run: |
+        echo "::set-output name=dir::$(npm config get cache)"
+
+    - name: Cache npm packages
+      uses: actions/cache@v1
+      with:
+        path: ${{ steps.npm-cache.outputs.dir }}
+        key: ${{ runner.os }}-npm-${{ hashFiles('**/package-lock.json') }}
+        restore-keys: |
+          ${{ runner.os }}-npm-
+
+    - name: NPM install
+      run: npm ci
+
+    - name: Lint TypeScript sources
+      run: npm run lint
+
+    - name: Bootstrap packages
+      run: npm run bootstrap
+
+    - name: Build packages
+      run: npm run build
+
+    - name: Unit tests
+      run: npm run test:ci
diff --git a/.github/workflows/tests.yml → .github/workflows/unit-tests.yml b/.github/workflows/tests.yml → .github/workflows/unit-tests.yml
@@ -1,9 +1,13 @@
-name: tests
+name: Unit tests
 
-on: [push, pull_request]
+on: [push]
 
 jobs:
-  tests:
+  full_test_run:
+
+    env:
+      USE_BROWSER_STACK: true
+
     runs-on: ubuntu-latest
 
     steps:
@@ -39,15 +43,15 @@ jobs:
 
     - name: Unit tests
       env:
-        SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }}
-        SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }}
-      run: |
-        # Don't run Sauce from a forked repo, since the code may not be trusted
-        # and therefore secrets like the Sauce password are not available.
-        # GITHUB_HEAD_REF is only defined when building a fork.
-        if [ ! "$GITHUB_HEAD_REF" ]; then
-          export USE_SAUCE=true
-          export SAUCE_BUILD_ID=model-viewer-tests-${GITHUB_EVENT_NAME}-${GITHUB_ACTION}-${GITHUB_SHA}
-          export SAUCE_TUNNEL_ID=${SAUCE_BUILD_ID}-tunnel
-        fi
-        npm run test:ci
+        BROWSER_STACK_USERNAME: ${{ secrets.BROWSER_STACK_USERNAME }}
+        BROWSER_STACK_ACCESS_KEY: ${{ secrets.BROWSER_STACK_ACCESS_KEY }}
+      run: npm run test:ci
+
+    - name: Generate legacy bundles
+      run: npm run build:legacy-support
+
+    - name: Unit tests (legacy browsers)
+      env:
+        BROWSER_STACK_USERNAME: ${{ secrets.BROWSER_STACK_USERNAME }}
+        BROWSER_STACK_ACCESS_KEY: ${{ secrets.BROWSER_STACK_ACCESS_KEY }}
+      run: npm run test:ci:legacy
diff --git a/.travis.yml b/.travis.yml
diff --git a/package.json b/package.json
@@ -5,6 +5,7 @@
   "scripts": {
     "test": "./node_modules/.bin/lerna run test --stream",
     "test:ci": "./node_modules/.bin/lerna run test:ci --stream",
+    "test:ci:legacy": "./node_modules/.bin/lerna run test:ci:legacy --stream",
     "bootstrap": "./node_modules/.bin/lerna bootstrap --ci",
     "format": "find packages/**/src -name '*.ts' | grep -v .d.ts | xargs ./node_modules/.bin/clang-format --verbose -i",
     "lint": "./node_modules/.bin/eslint \"packages/**/*.ts\"",
@@ -46,4 +47,4 @@
     "lerna": "^3.20.2",
     "typescript": "^3.7.5"
   }
-}
+}
diff --git a/packages/3dom/karma.conf.js b/packages/3dom/karma.conf.js
@@ -60,65 +60,117 @@ module.exports = function(config) {
     ],
   });
 
-  if (process.env.USE_SAUCE) {
-    if (!process.env.SAUCE_USERNAME || !process.env.SAUCE_ACCESS_KEY) {
+  if (process.env.USE_BROWSER_STACK) {
+    if (!process.env.BROWSER_STACK_USERNAME ||
+        !process.env.BROWSER_STACK_ACCESS_KEY) {
       throw new Error(
-          'SAUCE_USERNAME and SAUCE_ACCESS_KEY must be set with USE_SAUCE')
+          'BROWSER_STACK_USERNAME and BROWSER_STACK_ACCESS_KEY must be set with USE_BROWSER_STACK');
     }
 
-    const sauceLaunchers = {
-      // 'IE11': {
-      //   base: 'SauceLabs',
-      //   browserName: 'internet explorer',
-      //   version: '11',
-      //   platform: 'Windows 8.1',
-      // },
+    // This terrible hack brought to you by a combination of two things:
+    //  1. BrowserStack drops the server port when redirecting from localhost
+    //     to bs-local.com on iOS
+    //  2. karma-browserstack-launcher drops the test-specific browser ID if
+    //     you configure the browser with a custom URL
+    // A support request to BrowserStack has been filed.
+    // A related bug has been filed againts karma-browserstack-launcher
+    // @see https://github.com/karma-runner/karma-browserstack-launcher/issues/172
+    const assign = Object.assign;
+    const newAssign = function(...args) {
+      // If we know this to be one very specific Object.assign call, then grab
+      // the test-specific browser ID and append it to our URL:
+      // @see https://github.com/karma-runner/karma-browserstack-launcher/blob/76dbfd0db6db46f4f85012cfe3c1f4c3accd2e44/index.js#L143
+      if (args[2] != null && args[2].url === 'http://bs-local.com:9876') {
+        console.warn('PATCHING URL TO ADD ID');
+        const config = args[0];
+        const browser = args[2];
+        const query = config.url.split('?')[1];
+        browser.url = `${browser.url}?${query}`;
+      }
+      return assign.apply(this, args);
+    };
+    // Something in Karma deps actually asserts the sub-keys of Object.assign,
+    // so make sure to copy those over too:
+    assign.call(Object, newAssign, assign);
+    Object.assign = newAssign;
+
+    const browserStackLaunchers = {
       'Edge (latest)': {
-        base: 'SauceLabs',
-        browserName: 'microsoftedge',
-        version: 'latest',
-        platform: 'Windows 10',
+        base: 'BrowserStack',
+        os: 'Windows',
+        os_version: '10',
+        browser: 'Edge',
+        browser_version: 'latest',
+      },
+      'Edge 79.0': {
+        base: 'BrowserStack',
+        os: 'Windows',
+        os_version: '10',
+        browser: 'Edge',
+        browser_version: '80.0',
+      },
+      'Firefox (latest)': {
+        base: 'BrowserStack',
+        os: 'Windows',
+        os_version: '10',
+        browser: 'Firefox',
+        browser_version: 'latest',
       },
-      'Safari (latest - 1)': {
-        base: 'SauceLabs',
-        browserName: 'safari',
-        version: 'latest-1',
-        platform: 'macOS 10.13',
+      'Firefox 72.0': {
+        base: 'BrowserStack',
+        os: 'Windows',
+        os_version: '10',
+        browser: 'Firefox',
+        browser_version: '72.0',
       },
       'Safari (latest)': {
-        base: 'SauceLabs',
-        browserName: 'safari',
-        version: 'latest',
-        platform: 'macOS 10.13',
+        base: 'BrowserStack',
+        os: 'OS X',
+        os_version: 'Catalina',
+        browser: 'safari',
+        browser_version: 'latest',
       },
-      'Firefox (latest)': {
-        base: 'SauceLabs',
-        browserName: 'firefox',
-        platform: 'Windows 10',
-        version: 'latest'
+      'Safari 12.1': {
+        base: 'BrowserStack',
+        os: 'OS X',
+        os_version: 'Mojave',
+        browser: 'safari',
+        browser_version: '12.1',
+      },
+      'iOS Safari (iOS 13)': {
+        base: 'BrowserStack',
+        os: 'iOS',
+        os_version: '12',
+        device: 'iPhone 8',
+        browser: 'iPhone',
+        real_mobile: 'true',
+        // BrowserStack seems to drop the port when redirecting to this special
+        // domain so we go there directly instead:
+        url: 'http://bs-local.com:9876'
+      },
+      'iOS Safari (iOS 12)': {
+        base: 'BrowserStack',
+        os: 'iOS',
+        os_version: '12',
+        device: 'iPhone 7',
+        browser: 'iPhone',
+        real_mobile: 'true',
+        // BrowserStack seems to drop the port when redirecting to this special
+        // domain so we go there directly instead:
+        url: 'http://bs-local.com:9876'
       },
-      'Firefox (latest - 1)': {
-        base: 'SauceLabs',
-        browserName: 'firefox',
-        platform: 'Windows 10',
-        version: 'latest-1'
-      }
     };
 
     config.set({
-      sauceLabs: {
+      browserStack: {
         idleTimeout: 600,
-        testName: '3DOM Unit Tests',
-        build: process.env.SAUCE_BUILD_ID,
-        tunnelIdentifier: process.env.SAUCE_TUNNEL_ID,
+        name: '3DOM Unit Tests',
       },
-      // Attempt to de-flake Sauce Labs tests on TravisCI.
-      transports: ['polling'],
-      browserDisconnectTolerance: 1,
-      reporters: ['saucelabs', 'mocha'],
 
-      customLaunchers: sauceLaunchers,
-      browsers: [...config.browsers, ...Object.keys(sauceLaunchers)],
+      reporters: ['BrowserStack', 'mocha'],
+
+      customLaunchers: browserStackLaunchers,
+      browsers: [...config.browsers, ...Object.keys(browserStackLaunchers)],
     });
   }
 };