Build execution environments with Browserify and LavaMoat (MetaMask#1262

)

* Browserify and LavaMoat experiment

* Build all execution environments with Browserify

* Try Browserify in CI

* More experimenting

* Proper cache

* Patch LavaMoat temporarily

* Support building Node.js bundles using LavaMoat

* Fix yarn.lock

* Fix patch on browser

* Use recommended lavamoat-browserify version

* Fix missing patch

* Cleaner solution for ts paths

* Remove lavamoat/aa patch

* Update LavaMoat policies

* Remove Webpack deps

* Add tree shaking

* Use node:module

* Remove bundle test

* Update coverage

* Enable scuttling in browser builds

* Inline LavaMoat runtime and use separate bundle for executor

* Attempt to run build script in LavaMoat node

* Patch babel/core

* Fix lint

* Update LavaMoat policies

* Address PR comments and fix CI

* Address PR comments

.eslintrc.js

@@ -37,6 +37,10 @@ module.exports = {
       files: ['**/*.js'],
       extends: ['@metamask/eslint-config-nodejs'],
 
+      parserOptions: {
+        ecmaVersion: 2020,
+      },
+
       rules: {
         // This prevents using Node.js and/or browser specific globals. We
         // currently use both in our codebase, so this rule is disabled.

.github/workflows/build-lint-test.yml

@@ -26,7 +26,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: |
-            packages/snaps-execution-environments/dist/webpack
+            packages/snaps-execution-environments/dist/browserify
           key: test-build-${{ runner.os }}-${{ matrix.node-version }}-${{ github.sha }}
       - name: Prepare test build
         if: steps.cache-test-build.outputs.cache-hit != 'true'
@@ -135,7 +135,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: |
-            packages/snaps-execution-environments/dist/webpack
+            packages/snaps-execution-environments/dist/browserify
           key: test-build-${{ runner.os }}-${{ matrix.node-version }}-${{ github.sha }}
       - run: yarn --immutable --immutable-cache
       - run: yarn workspace ${{ matrix.package-name }} run test:ci

.prettierignore

@@ -0,0 +1,79 @@
+**/dist
+
+# sed backup files
+*.*-e
+
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Coverage directory used by tools like istanbul
+coverage
+*.lcov
+
+# nyc test coverage
+.nyc_output
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+
+# Yarn directories
+.pnp.*
+.yarn/*
+!.yarn/patches
+!.yarn/plugins
+!.yarn/releases
+!.yarn/sdks
+!.yarn/versions
+
+# TypeScript v1 declaration files
+typings/
+
+# TypeScript cache
+*.tsbuildinfo
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variables file
+.env
+.env.test
+
+# Mac files
+.DS_Store
+
+# Webpack generated files
+packages/examples/examples/webpack/index.html
+
+packages/snaps-execution-environments/lavamoat/**/*.json

.yarn/patches/@babel-core-npm-7.21.0-fb3817b0e5.patch

@@ -0,0 +1,12 @@
+diff --git a/lib/index.js b/lib/index.js
+index 8b6dc73d1a71fb868cdd70c2ea98e543510ce80f..d32206166a2a9e33c20f1440876fe2dac2ad936a 100644
+--- a/lib/index.js
++++ b/lib/index.js
+@@ -231,7 +231,6 @@ function _module() {
+   };
+   return data;
+ }
+-var thisFile = require("./index");
+ const version = "7.21.0";
+ exports.version = version;
+ const DEFAULT_EXTENSIONS = Object.freeze([".js", ".jsx", ".es6", ".es", ".mjs", ".cjs"]);

.yarn/patches/@lavamoat-lavapack-npm-3.3.0-5a61b5374d.patch

@@ -0,0 +1,55 @@
+diff --git a/src/runtime.js b/src/runtime.js
+index 97c9ddb9b76a2fe31cf9b202cb491f15232232ff..f7f66db9e16d58e2f50bae9491eae88bc605a398 100644
+--- a/src/runtime.js
++++ b/src/runtime.js
+@@ -10831,6 +10831,10 @@ function makeGetEndowmentsForConfig ({ createFunctionWrapper }) {
+ function getPropertyDescriptorDeep (target, key) {
+   let receiver = target
+   while (true) {
++    // abort if this is the end of the prototype chain.
++    if (!receiver) {
++      return { prop: null, receiver: null }
++    }
+     // support lookup on objects and primitives
+     const typeofReceiver = typeof receiver
+     if (typeofReceiver === 'object' || typeofReceiver === 'function') {
+@@ -10845,10 +10849,6 @@ function getPropertyDescriptorDeep (target, key) {
+       // eslint-disable-next-line no-proto
+       receiver = receiver.__proto__
+     }
+-    // abort if this is the end of the prototype chain.
+-    if (!receiver) {
+-      return { prop: null, receiver: null }
+-    }
+   }
+ }
+
+@@ -11384,7 +11384,9 @@ module.exports = {
+         const descriptors = Object.getOwnPropertyDescriptors(source)
+         Object.values(descriptors).forEach(desc => {
+           if ('get' in desc) {
+-            Reflect.apply(desc.get, globalRef, [])
++            try {
++              Reflect.apply(desc.get, globalRef, [])
++            }catch{}
+           }
+         })
+       })
+@@ -11556,6 +11558,17 @@ module.exports = {
+
+
+   function loadModuleData (moduleId) {
++    if (typeof window === 'undefined' && require('node:module').isBuiltin(moduleId)) {
++      return {
++        type: 'builtin',
++        package: moduleId,
++        id: moduleId,
++        // Using unprotected require
++        moduleInitializer: (_, module) => {
++          module.exports = require(moduleId);
++        },
++      }
++    }
+     if (!moduleRegistry.has(moduleId)) {
+       throw new Error(`no module registered for "${moduleId}" (${typeof moduleId})`)
+     }

.yarn/patches/lavamoat-browserify-npm-15.5.0-ce0bbe5e4c.patch

@@ -0,0 +1,30 @@
+diff --git a/src/index.js b/src/index.js
+index 6943a606618d0841ad68eb0927f33b1267ab6ff2..b05229e20469c331508c64cdc8599a44acadbde6 100644
+--- a/src/index.js
++++ b/src/index.js
+@@ -8,6 +8,7 @@ const createLavaPack = require('@lavamoat/lavapack')
+ const { createSesWorkaroundsTransform } = require('./sesTransforms')
+ const { loadCanonicalNameMap } = require('@lavamoat/aa')
+ const browserResolve = require('browser-resolve')
++const { isBuiltin } = require('node:module')
+
+
+ // these are the reccomended arguments for lavaMoat to work well with browserify
+@@ -71,7 +72,7 @@ function plugin (browserify, pluginOpts) {
+       browserify.pipeline.get('emit-deps').push(createModuleInspectorSpy({
+         policyOverride,
+         // no builtins in the browser (yet!)
+-        isBuiltin: () => false,
++        isBuiltin: (name) => browserify._options.node && isBuiltin(name),
+         // should prepare debug info
+         includeDebugInfo: configuration.writeAutoPolicyDebug,
+         // write policy files to disk
+@@ -272,7 +273,7 @@ function validatePolicy (policy) {
+   Object.entries(policy.resources).forEach(([packageName, packageOpts], index) => {
+     const packageOptions = Object.keys(packageOpts)
+     const packageEntries = Object.values(packageOpts)
+-    const optionsWhitelist = ['globals', 'packages']
++    const optionsWhitelist = ['globals', 'packages', 'builtin']
+     const valuesWhitelist = [true, 'write']
+
+     if (!packageOptions.every(packageOpt => optionsWhitelist.includes(packageOpt))) {

.yarn/patches/luxon-npm-3.3.0-bdbae9bfd5.patch

@@ -0,0 +1,40 @@
+diff --git a/build/cjs-browser/luxon.js b/build/cjs-browser/luxon.js
+index d25aef4689e0144f13978735024a0c46823df1ac..496129e68bc51a36f13e875edfd4f17ce851d788 100644
+--- a/build/cjs-browser/luxon.js
++++ b/build/cjs-browser/luxon.js
+@@ -4263,7 +4263,7 @@ var Interval = /*#__PURE__*/function () {
+    * @example Interval.fromISO('2022-11-07T17:00Z/2022-11-07T19:00Z').toLocaleString({ weekday: 'short', month: 'short', day: '2-digit', hour: '2-digit', minute: '2-digit' }); //=> Mon, Nov 07, 6:00 – 8:00 p
+    * @return {string}
+    */;
+-  _proto.toLocaleString = function toLocaleString(formatOpts, opts) {
++   Reflect.defineProperty(_proto, 'toLocaleString', { value: function toLocaleString(formatOpts, opts) {
+     if (formatOpts === void 0) {
+       formatOpts = DATE_SHORT;
+     }
+@@ -4271,7 +4271,7 @@ var Interval = /*#__PURE__*/function () {
+       opts = {};
+     }
+     return this.isValid ? Formatter.create(this.s.loc.clone(opts), formatOpts).formatInterval(this) : INVALID$1;
+-  }
++  }})
+
+   /**
+    * Returns an ISO 8601-compliant string representation of this Interval.
+@@ -6640,7 +6640,7 @@ var DateTime = /*#__PURE__*/function () {
+    * @example DateTime.now().toLocaleString({ hour: '2-digit', minute: '2-digit', hourCycle: 'h23' }); //=> '11:32'
+    * @return {string}
+    */;
+-  _proto.toLocaleString = function toLocaleString(formatOpts, opts) {
++   Reflect.defineProperty(_proto, 'toLocaleString', { value: function toLocaleString(formatOpts, opts) {
+     if (formatOpts === void 0) {
+       formatOpts = DATE_SHORT;
+     }
+@@ -6648,7 +6648,7 @@ var DateTime = /*#__PURE__*/function () {
+       opts = {};
+     }
+     return this.isValid ? Formatter.create(this.loc.clone(opts), formatOpts).formatDateTime(this) : INVALID;
+-  }
++  }})
+
+   /**
+    * Returns an array of format "parts", meaning individual tokens along with metadata. This is allows callers to post-process individual sections of the formatted output.

package.json

@@ -15,7 +15,7 @@
     "publish:all": "./scripts/publish-all.sh",
     "link-packages": "./scripts/link-packages.sh",
     "lint:eslint": "eslint . --cache --ext js,ts",
-    "lint:misc": "prettier '**/*.json' '**/*.md' '!**/CHANGELOG.md' '**/*.yml' '**/*.html' --ignore-path .gitignore",
+    "lint:misc": "prettier '**/*.json' '**/*.md' '!**/CHANGELOG.md' '**/*.yml' '**/*.html'",
     "lint:changelogs": "yarn workspaces foreach --parallel --verbose run lint:changelog",
     "lint:tsconfig": "node scripts/verify-tsconfig.mjs",
     "lint": "yarn lint:tsconfig && yarn lint:eslint && yarn lint:misc --check",
@@ -44,8 +44,12 @@
     ]
   },
   "resolutions": {
+    "@babel/core": "patch:@babel/core@npm%3A7.21.0#./.yarn/patches/@babel-core-npm-7.21.0-fb3817b0e5.patch",
+    "@lavamoat/lavapack@^3.3.0": "patch:@lavamoat/lavapack@npm%3A3.3.0#./.yarn/patches/@lavamoat-lavapack-npm-3.3.0-5a61b5374d.patch",
     "@types/mocha@^10.0.1": "patch:@types/mocha@npm:10.0.1#.yarn/patches/@types-mocha-npm-10.0.1-7c94e9e170.patch",
-    "jest-fetch-mock@^3.0.3": "patch:jest-fetch-mock@npm:3.0.3#.yarn/patches/jest-fetch-mock-npm-3.0.3-ac072ca8af.patch"
+    "jest-fetch-mock@^3.0.3": "patch:jest-fetch-mock@npm:3.0.3#.yarn/patches/jest-fetch-mock-npm-3.0.3-ac072ca8af.patch",
+    "lavamoat-browserify@^15.5.0": "patch:lavamoat-browserify@npm%3A15.5.0#./.yarn/patches/lavamoat-browserify-npm-15.5.0-ce0bbe5e4c.patch",
+    "luxon@^3.2.1": "patch:luxon@npm%3A3.3.0#./.yarn/patches/luxon-npm-3.3.0-bdbae9bfd5.patch"
   },
   "devDependencies": {
     "@lavamoat/allow-scripts": "^2.0.3",

packages/snaps-controllers/src/services/node/NodeProcessExecutionService.ts

@@ -13,7 +13,7 @@ export class NodeProcessExecutionService extends AbstractExecutionService<ChildP
   }> {
     const worker = fork(
       require.resolve(
-        '@metamask/snaps-execution-environments/dist/webpack/node-process/bundle.js',
+        '@metamask/snaps-execution-environments/dist/browserify/node-process/bundle.js',
       ),
     );
 

packages/snaps-controllers/src/services/node/NodeThreadExecutionService.ts

@@ -14,7 +14,7 @@ export class NodeThreadExecutionService extends AbstractExecutionService<Worker>
   }> {
     const worker = new Worker(
       require.resolve(
-        '@metamask/snaps-execution-environments/dist/webpack/node-thread/bundle.js',
+        '@metamask/snaps-execution-environments/dist/browserify/node-thread/bundle.js',
       ),
     );
     const stream = new ThreadParentMessageStream({ thread: worker });

packages/snaps-controllers/wdio.config.ts

@@ -71,7 +71,7 @@ export const config: Options.Testrunner = {
             mount: '/',
             path: resolve(
               __dirname,
-              '../snaps-execution-environments/dist/webpack/iframe',
+              '../snaps-execution-environments/dist/browserify/iframe',
             ),
           },
 

packages/snaps-execution-environments/coverage.json

@@ -1,6 +1,6 @@
 {
   "branches": 78.62,
   "functions": 91.4,
-  "lines": 87.95,
-  "statements": 88.02
+  "lines": 88.6,
+  "statements": 88.66
 }