[Fixed] Run CI on a single machine by generating cloudbuild files (#4186)

* Run CI on a single machine by generating cloudbuild files

Instead of running each CI test on a separate Google Cloud machine, generate a cloudbuild.yml file to test all affected packages on a single machine. The benifits of this strategy include the following:
* Fewer machines are required for a single test (2 instead of up to 12), so queuing should happen much less often.
* Packages only need to be built once, instead of once per machine.
* Multithreaded steps have access to more cores.

When CI is run, the root cloudbuild.yml is launched on Google Cloud. This file has only three steps: Install yarn dependencies, run `yarn generate-cloudbuild-for-packages`, and run `run-build.sh`. The second step finds what packages were affected by the change and combines their cloudbuild.yml files into a single `cloudbuild_generated.yml` file, and the third step launches the generated build on Google Cloud. The steps in the generated build are set to `waitFor` each other according to the dependency tree `scripts/package_dependencies.json`, and each package's `build-deps` step is removed in favor of waiting for its dependencies to be built.

Each package's cloudbuild file still works as expected, and there are two new yarn commands at the root:
* `yarn test-packages` tests all packages affected by a change. It performs the same steps as CI.
* `yarn generate-cloudbuild-for-packages [PACKAGE]...` generates the cloudbuild file to test each `PACKAGE` and packages it might affect. If given no packages, it detects what packages have been changed in the current PR.

* Verify the dependency graph is a valid graph

* Add dependencies to e2e

* Use the node:12 docker image for tfjs-node CI

tfjs-node builds N-API bindings that end-to-end tests need to use. End-to-end uses our custom docker image (gcr.io/learnjs-174218/release) since it has to build tfjs-backend-wasm when it's run by itself (which requires emscripten). Our custom image uses node 12, and is incompatible with node 10 N-API bindings. Eventually, we'll want to move everything that's using node:10 to node:12, but that's out of the scope of this commit.

* Remove TODO and allow changes to generate_cloudbuild.js to trigger a full rebuild

* Add tfjs-backend-cpu to tfjs-data in package_dependencies.json

tfjs-backend-cpu is listed in tfjs-data's package.json as a linked dependency, but was missing form package_dependencies.json.

* Don't have tfjs-backend-webgpu build its dependencies in CI

Move tfjs-backend-webgpu's dependency building to the 'build-deps' step so it can be removed during CI.

This commit also updates the test cloudbuild files that should have been updated in f9c1fdd. It would probably be best to amend that commit, but last time I tried force-pushing in GitHub, I broke the PR I was working on.

* Revert changes in tfjs-backend-webgpu to master

tfjs-backend-webgpu is now pinned at v2.7.0 for all its tfjs dependencies.

* Add the update-cloudbuild-tests script to package.json

update-cloudbuild-tests updates the change detection tests for scripts/generate_cloudbuild.js. It should be run iff a change affects cloudbuild files or the dependency tree.

* Make tfjs-backend-webgpu tests wait for yarn to run

* Add a destination argument to generate_cloudbuild_for_packages.js

Co-authored-by: Ping Yu <[email protected]>

Author: mattsoulanille

.gitignore

@@ -59,3 +59,4 @@ tfjs-backend-wasm/wasm-out/*.js
 tfjs-backend-wasm/wasm-out/*.wasm
 yalc.lock
 yarn-error.log
+cloudbuild_generated.yml

cloudbuild.yml

@@ -5,138 +5,25 @@ steps:
   id: 'yarn'
   args: ['install']
 
-# Run find-affected-packages to find affected files in each folder.
+# Generate cloudbuild_generated.yml,
+# which builds and tests all affected packages.
 - name: 'node:10'
   entrypoint: 'yarn'
-  id: 'find-affected-packages'
-  args: ['find-affected-packages']
+  id: 'generate-cloudbuild-for-packages'
+  args: ['generate-cloudbuild-for-packages']
   waitFor: ['yarn']
   env:
   - 'COMMIT_SHA=$COMMIT_SHA'
   - 'BRANCH_NAME=$BRANCH_NAME'
   - 'BASE_BRANCH=$_BASE_BRANCH'
+  - 'NIGHTLY=$_NIGHTLY'
 
-# Core.
+# Run the generated cloudbuild file
 - name: 'gcr.io/cloud-builders/gcloud'
   entrypoint: 'bash'
-  id: 'tfjs-core'
-  args: ['./scripts/run-build.sh', 'tfjs-core']
-  waitFor: ['find-affected-packages']
-  env: ['NIGHTLY=$_NIGHTLY']
-
-# CPU backend.
-- name: 'gcr.io/cloud-builders/gcloud'
-  entrypoint: 'bash'
-  id: 'tfjs-backend-cpu'
-  args: ['./scripts/run-build.sh', 'tfjs-backend-cpu']
-  waitFor: ['find-affected-packages']
-  env: ['NIGHTLY=$_NIGHTLY']
-
-# Webgl backend.
-- name: 'gcr.io/cloud-builders/gcloud'
-  entrypoint: 'bash'
-  id: 'tfjs-backend-webgl'
-  args: ['./scripts/run-build.sh', 'tfjs-backend-webgl']
-  waitFor: ['find-affected-packages']
-  env: ['NIGHTLY=$_NIGHTLY']
-
-# Converter.
-- name: 'gcr.io/cloud-builders/gcloud'
-  entrypoint: 'bash'
-  id: 'tfjs-converter'
-  args: ['./scripts/run-build.sh', 'tfjs-converter']
-  waitFor: ['find-affected-packages']
-  env: ['NIGHTLY=$_NIGHTLY']
-
-# Data.
-- name: 'gcr.io/cloud-builders/gcloud'
-  entrypoint: 'bash'
-  id: 'tfjs-data'
-  args: ['./scripts/run-build.sh', 'tfjs-data']
-  waitFor: ['find-affected-packages']
-  env: ['NIGHTLY=$_NIGHTLY']
-
-# Layers.
-- name: 'gcr.io/cloud-builders/gcloud'
-  entrypoint: 'bash'
-  id: 'tfjs-layers'
-  args: ['./scripts/run-build.sh', 'tfjs-layers']
-  waitFor: ['find-affected-packages']
-  env: ['NIGHTLY=$_NIGHTLY']
-
-# Union.
-- name: 'gcr.io/cloud-builders/gcloud'
-  entrypoint: 'bash'
-  id: 'tfjs'
-  args: ['./scripts/run-build.sh', 'tfjs']
-  waitFor: ['find-affected-packages']
-
-# Vis.
-- name: 'gcr.io/cloud-builders/gcloud'
-  entrypoint: 'bash'
-  id: 'tfjs-vis'
-  args: ['./scripts/run-build.sh', 'tfjs-vis']
-  waitFor: ['find-affected-packages']
-
-# WebGPU.
-- name: 'gcr.io/cloud-builders/gcloud'
-  entrypoint: 'bash'
-  id: 'tfjs-backend-webgpu'
-  args: ['./scripts/run-build.sh', 'tfjs-backend-webgpu']
-  waitFor: ['find-affected-packages']
-
-# WASM.
-- name: 'gcr.io/cloud-builders/gcloud'
-  entrypoint: 'bash'
-  id: 'tfjs-backend-wasm'
-  args: ['./scripts/run-build.sh', 'tfjs-backend-wasm']
-  waitFor: ['find-affected-packages']
-  env: ['NIGHTLY=$_NIGHTLY']
-
-# React Native.
-- name: 'gcr.io/cloud-builders/gcloud'
-  entrypoint: 'bash'
-  id: 'tfjs-react-native'
-  args: ['./scripts/run-build.sh', 'tfjs-react-native']
-  waitFor: ['find-affected-packages']
-
-# Node CPU.
-- name: 'gcr.io/cloud-builders/gcloud'
-  entrypoint: 'bash'
-  id: 'tfjs-node'
-  args: ['./scripts/run-build.sh', 'tfjs-node']
-  waitFor: ['find-affected-packages']
-  env: ['NIGHTLY=$_NIGHTLY']
-
-# Node GPU.
-- name: 'gcr.io/cloud-builders/gcloud'
-  entrypoint: 'bash'
-  id: 'tfjs-node-gpu'
-  args: ['./scripts/run-build.sh', 'tfjs-node-gpu']
-  waitFor: ['find-affected-packages']
-  env: ['NIGHTLY=$_NIGHTLY']
-
-# Release notes test
-- name: 'node:10'
-  id: 'test-monorepo'
-  entrypoint: 'yarn'
-  args: ['test-release-notes']
-  waitFor: ['yarn']
-
-# E2E test
-- name: 'gcr.io/cloud-builders/gcloud'
-  entrypoint: 'bash'
-  id: 'e2e'
-  args: ['./scripts/run-build.sh', 'e2e']
-  waitFor: ['find-affected-packages']
-  env: ['NIGHTLY=$_NIGHTLY']
-
-# Inference API
-- name: 'gcr.io/cloud-builders/gcloud'
-  entrypoint: 'bash'
-  id: 'tfjs-inference'
-  args: ['scripts/run-build.sh', 'tfjs-inference']
-  waitFor: ['find-affected-packages']
+  id: 'run-cloudbuild'
+  args: ['./scripts/run-build.sh']
+  waitFor: ['generate-cloudbuild-for-packages']
   env: ['NIGHTLY=$_NIGHTLY']
 
 # General settings.

e2e/cloudbuild.yml

@@ -21,8 +21,8 @@ steps:
   id: 'test'
   args: ['test-ci']
   env: ['BROWSERSTACK_USERNAME=deeplearnjs1', 'NIGHTLY=$_NIGHTLY']
-
   secretEnv: ['BROWSERSTACK_KEY']
+  waitFor: ['yarn', 'build-deps']
 
 secrets:
 - kmsKeyName: projects/learnjs-174218/locations/global/keyRings/tfjs/cryptoKeys/enc

package.json

@@ -3,26 +3,31 @@
     "@octokit/rest": "15.17.0",
     "@types/argparse": "^1.0.38",
     "@types/jasmine": "2.8.7",
-    "@types/node": "^12.7.5",
     "@types/mkdirp": "^0.5.2",
+    "@types/node": "^12.7.5",
     "@types/shelljs": "^0.8.7",
     "argparse": "^1.0.10",
     "chalk": "~2.4.2",
     "clang-format": "~1.2.4",
+    "console-table-printer": "^2.4.32",
     "jasmine": "3.1.0",
-    "shelljs": "~0.8.3",
+    "js-yaml": "^3.14.0",
     "mkdirp": "~0.5.1",
+    "shelljs": "~0.8.3",
     "ts-node": "~8.8.2",
     "tslint": "~5.20.0",
     "typescript": "3.5.3"
   },
   "scripts": {
-    "find-affected-packages": "./scripts/find-affected-packages.js",
+    "test-packages-ci": "yarn generate-cloudbuild-for-packages && ./scripts/run-build.sh",
+    "generate-cloudbuild-for-packages": "./scripts/generate_cloudbuild_for_packages.js",
+    "test-generate-cloudbuild": "jasmine run scripts/generate_cloudbuild_test.js",
     "release": "ts-node -s ./scripts/release.ts",
     "release-tfjs": "ts-node -s ./scripts/release-tfjs.ts",
     "publish-npm": "ts-node -s ./scripts/publish-npm.ts",
     "release-notes": "ts-node -s ./scripts/release_notes/release_notes.ts",
     "test-release-notes": "ts-node -s ./scripts/release_notes/run_tests.ts",
-    "update-tfjs-lockfiles": "ts-node -s ./scripts/update-tfjs-lockfiles"
+    "update-tfjs-lockfiles": "ts-node -s ./scripts/update-tfjs-lockfiles",
+    "update-cloudbuild-tests": "yarn generate-cloudbuild-for-packages tfjs-node -o scripts/cloudbuild_tfjs_node_expected.yml && yarn generate-cloudbuild-for-packages tfjs-core -o scripts/cloudbuild_tfjs_core_expected.yml"
   }
 }

scripts/cloudbuild.yml

@@ -0,0 +1,14 @@
+steps:
+# Install common dependencies.
+- name: 'node:10'
+  id: 'yarn-common'
+  entrypoint: 'yarn'
+  args: ['install']
+
+# Test generate_cloudbuild.js
+- name: 'node:10'
+  dir: 'scripts'
+  id: 'test-generate-cloudbuild'
+  entrypoint: 'yarn'
+  args: ['test-generate-cloudbuild']
+  waitFor: ['yarn-common']

scripts/cloudbuild_general_config.yml

@@ -0,0 +1,21 @@
+steps:
+# Install top-level deps.
+- name: 'node:10'
+  entrypoint: 'yarn'
+  id: 'yarn-common'
+  args: ['install']
+
+# General configuration
+secrets:
+- kmsKeyName: projects/learnjs-174218/locations/global/keyRings/tfjs/cryptoKeys/enc
+  secretEnv:
+    BROWSERSTACK_KEY: CiQAkwyoIW0LcnxymzotLwaH4udVTQFBEN4AEA5CA+a3+yflL2ASPQAD8BdZnGARf78MhH5T9rQqyz9HNODwVjVIj64CTkFlUCGrP1B2HX9LXHWHLmtKutEGTeFFX9XhuBzNExA=
+    FIREBASE_KEY: CiQAkwyoIXmET39jOD3ywloCIa6+WUpu3w49twpMmkMqy0vS+YsSUAAD8BdZQGOL8FKEBxr/1jl0G78OigwlNVHjD3usZobNtlOp8tV/9iacb8zPFqy0SwIO1gvz3HRr+VU7c7LS2qqaTCdacZF+dx3VJNewvdZu
+timeout: 3600s
+logsBucket: 'gs://tfjs-build-logs'
+substitutions:
+  _NIGHTLY: ''
+options:
+  logStreamingOption: 'STREAM_ON'
+  machineType: 'N1_HIGHCPU_32'
+  substitution_option: 'ALLOW_LOOSE'