Skip to content

Commit

Permalink
🚧🚀 Add esbuild+babel caching to tests (ampproject#33119)
Browse files Browse the repository at this point in the history
* Add esbuild+babel caching to tests

* Cleanup

* Add build-system/common/esbuild-babel.js

* Fix path

* Fix type

* Optimize regex

* Fix digesting
  • Loading branch information
jridgewell authored Mar 6, 2021
1 parent 1b70807 commit 8d7baef
Show file tree
Hide file tree
Showing 7 changed files with 134 additions and 87 deletions.
123 changes: 123 additions & 0 deletions build-system/common/esbuild-babel.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
/**
* Copyright 2021 The AMP HTML Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS-IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

const babel = require('@babel/core');
const crypto = require('crypto');
const fs = require('fs');
const path = require('path');

/**
* Used to cache babel transforms done by esbuild.
* @private @const {!Map<string, {hash: string, promise: Promise<{contents: string}>}>}
*/
const transformCache = new Map();

/**
* Used to cache file reads done by esbuild, since it can issue multiple
* "loads" per file. This batches consecutive reads into a single, and then
* clears its cache item for the next load.
* @private @const {!Map<string, Promise<{hash: string, contents: string}>>}
*/
const readCache = new Map();

/**
* Creates a babel plugin for esbuild for the given caller. Optionally enables
* caching to speed up transforms.
* @param {string} callerName
* @param {boolean} enableCache
* @param {function()} preSetup
* @param {function()} postLoad
* @return {!Object}
*/
function getEsbuildBabelPlugin(
callerName,
enableCache,
preSetup = () => {},
postLoad = () => {}
) {
function sha256(contents) {
if (!enableCache) {
return '';
}
const hash = crypto.createHash('sha256');
hash.update(callerName);
hash.update(contents);
return hash.digest('hex');
}

function batchedRead(path) {
let read = readCache.get(path);
if (!read) {
read = fs.promises
.readFile(path)
.then((contents) => {
return {
contents,
hash: sha256(contents),
};
})
.finally(() => {
readCache.delete(path);
});
readCache.set(path, read);
}
return read;
}

function transformContents(filepath, contents, hash) {
if (enableCache) {
const cached = transformCache.get(filepath);
if (cached && cached.hash === hash) {
return cached.promise;
}
}

const babelOptions =
babel.loadOptions({
caller: {name: callerName},
filename: filepath,
sourceFileName: path.basename(filepath),
}) || undefined;
const promise = babel
.transformAsync(contents, babelOptions)
.then((result) => {
return {contents: result.code};
});

if (enableCache) {
transformCache.set(filepath, {hash, promise});
}

return promise.finally(postLoad);
}

return {
name: 'babel',
async setup(build) {
preSetup();

build.onLoad({filter: /\.[cm]?js$/, namespace: ''}, async (file) => {
const {path} = file;
const {contents, hash} = await batchedRead(path);
return transformContents(path, contents, hash);
});
},
};
}

module.exports = {
getEsbuildBabelPlugin,
};
5 changes: 2 additions & 3 deletions build-system/tasks/3p-vendor-helpers.js
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

const debounce = require('debounce');
const globby = require('globby');
const {compileJs, invalidateUnminifiedBabelCache} = require('./helpers');
const {compileJs} = require('./helpers');
const {endBuildStep} = require('./helpers');
const {VERSION} = require('../compile/internal-version');
const {watchDebounceDelay} = require('./helpers');
Expand All @@ -42,8 +42,7 @@ async function buildVendorConfigs(options) {
if (options.watch) {
// Do not set watchers again when we get called by the watcher.
const copyOptions = {...options, watch: false, calledByWatcher: true};
const watchFunc = (modifiedFile) => {
invalidateUnminifiedBabelCache(modifiedFile);
const watchFunc = () => {
buildVendorConfigs(copyOptions);
};
watch(srcPath).on('change', debounce(watchFunc, watchDebounceDelay));
Expand Down
5 changes: 2 additions & 3 deletions build-system/tasks/analytics-vendor-configs.js
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ const debounce = require('debounce');
const fs = require('fs-extra');
const globby = require('globby');
const jsonminify = require('jsonminify');
const {endBuildStep, invalidateUnminifiedBabelCache} = require('./helpers');
const {endBuildStep} = require('./helpers');
const {join, basename, dirname, extname} = require('path');
const {watchDebounceDelay} = require('./helpers');
const {watch} = require('chokidar');
Expand All @@ -45,8 +45,7 @@ async function analyticsVendorConfigs(opt_options) {
if (options.watch) {
// Do not set watchers again when we get called by the watcher.
const copyOptions = {...options, watch: false, calledByWatcher: true};
const watchFunc = (modifiedFile) => {
invalidateUnminifiedBabelCache(modifiedFile);
const watchFunc = () => {
analyticsVendorConfigs(copyOptions);
};
watch(srcPath).on('change', debounce(watchFunc, watchDebounceDelay));
Expand Down
2 changes: 1 addition & 1 deletion build-system/tasks/dep-check.js
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ const {
const {compileJison} = require('./compile-jison');
const {css} = require('./css');
const {cyan, green, red, yellow} = require('kleur/colors');
const {getEsbuildBabelPlugin} = require('./helpers');
const {getEsbuildBabelPlugin} = require('../common/esbuild-babel');
const {log, logLocalDev} = require('../common/logging');

const depCheckDir = '.amp-dep-check';
Expand Down
9 changes: 2 additions & 7 deletions build-system/tasks/extension-helpers.js
Original file line number Diff line number Diff line change
Expand Up @@ -18,17 +18,13 @@ const colors = require('kleur/colors');
const debounce = require('debounce');
const fs = require('fs-extra');
const wrappers = require('../compile/compile-wrappers');
const {
endBuildStep,
watchDebounceDelay,
invalidateUnminifiedBabelCache,
} = require('./helpers');
const {
extensionAliasBundles,
extensionBundles,
verifyExtensionBundles,
} = require('../compile/bundles.config');
const {analyticsVendorConfigs} = require('./analytics-vendor-configs');
const {endBuildStep, watchDebounceDelay} = require('./helpers');
const {isCiBuild} = require('../common/ci');
const {jsifyCssAsync} = require('./css/jsify-css');
const {log} = require('../common/logging');
Expand Down Expand Up @@ -386,8 +382,7 @@ async function doBuildExtension(extensions, extension, options) {
* @param {?Object} options
*/
function watchExtension(path, name, version, latestVersion, hasCss, options) {
const watchFunc = function (modifiedFile) {
invalidateUnminifiedBabelCache(modifiedFile);
const watchFunc = function () {
const bundleComplete = buildExtension(
name,
version,
Expand Down
73 changes: 2 additions & 71 deletions build-system/tasks/helpers.js
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
*/

const argv = require('minimist')(process.argv.slice(2));
const babel = require('@babel/core');
const debounce = require('debounce');
const del = require('del');
const esbuild = require('esbuild');
Expand All @@ -32,6 +31,7 @@ const {
} = require('../compile/internal-version');
const {applyConfig, removeConfig} = require('./prepend-global/index.js');
const {closureCompile} = require('../compile/compile');
const {getEsbuildBabelPlugin} = require('../common/esbuild-babel');
const {green, red, cyan} = require('kleur/colors');
const {isCiBuild} = require('../common/ci');
const {jsBundles} = require('../compile/bundles.config');
Expand Down Expand Up @@ -93,26 +93,12 @@ const hostname3p = argv.hostname3p || '3p.ampproject.net';
*/
const watchDebounceDelay = 1000;

/**
* Used to cache babel transforms done by esbuild.
* @private @const {!Map<string, Promise<File>>}
*/
const cache = new Map();

/**
* Stores esbuild's watch mode rebuilders.
* @private @const {!Map<string, {rebuild: function():!Promise<void>}>}
*/
const watchedTargets = new Map();

/*
* Used to remove a file from the babel cache after it is modified.
* @param {string} filepath relative to the project root.
*/
function invalidateUnminifiedBabelCache(filepath) {
cache.delete(path.resolve(filepath)); // Must be absolute path.
}

/**
* @param {!Object} jsBundles
* @param {string} name
Expand Down Expand Up @@ -173,11 +159,9 @@ async function bootstrapThirdPartyFrames(options) {
*/
async function compileCoreRuntime(options) {
/**
* @param {string} modifiedFile
* @return {Promise<void>}
*/
async function watchFunc(modifiedFile) {
invalidateUnminifiedBabelCache(modifiedFile);
async function watchFunc() {
const bundleComplete = await doBuildJs(jsBundles, 'amp.js', {
...options,
watch: false,
Expand Down Expand Up @@ -465,57 +449,6 @@ async function finishBundle(
}
}

/**
* Creates a babel plugin for esbuild for the given caller. Optionally enables
* caching to speed up transforms.
* @param {string} callerName
* @param {boolean} enableCache
* @param {function()} preSetup
* @param {function()} postLoad
* @return {!Object}
*/
function getEsbuildBabelPlugin(
callerName,
enableCache,
preSetup = () => {},
postLoad = () => {}
) {
return {
name: 'babel',
async setup(build) {
preSetup();
const transformContents = async (file) => {
const contents = await fs.promises.readFile(file.path, 'utf-8');
const babelOptions =
babel.loadOptions({
caller: {name: callerName},
filename: file.path,
sourceFileName: path.basename(file.path),
}) || undefined;
const result = await babel.transformAsync(contents, babelOptions);
return {contents: result.code};
};

build.onLoad({filter: /.*\.[cm]?js$/, namespace: ''}, async (file) => {
const {path} = file;
if (enableCache && cache.has(path)) {
return cache.get(path);
}
const promise = transformContents(file);
if (enableCache) {
// Cache needs to be set before awaiting, because esbuild can issue
// multiple "loads" to import a file while waiting for babel to
// transform.
cache.set(path, promise);
}
const transformed = await promise;
postLoad();
return transformed;
});
},
};
}

/**
* Transforms a given JavaScript file entry point with esbuild and babel, and
* watches it for changes (if required).
Expand Down Expand Up @@ -795,12 +728,10 @@ module.exports = {
compileTs,
doBuildJs,
endBuildStep,
getEsbuildBabelPlugin,
maybePrintCoverageMessage,
maybeToEsmName,
mkdirSync,
printConfigHelp,
printNobuildHelp,
watchDebounceDelay,
invalidateUnminifiedBabelCache,
};
4 changes: 2 additions & 2 deletions build-system/tasks/runtime-test/runtime-test-base.js
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ const {app} = require('../../server/test-server');
const {createKarmaServer, getAdTypes} = require('./helpers');
const {cyan, green, red, yellow} = require('kleur/colors');
const {dotWrappingWidth} = require('../../common/logging');
const {getEsbuildBabelPlugin} = require('../helpers');
const {getEsbuildBabelPlugin} = require('../../common/esbuild-babel');
const {getFilesFromArgv} = require('../../common/utils');
const {isCiBuild} = require('../../common/ci');
const {log} = require('../../common/logging');
Expand Down Expand Up @@ -297,7 +297,7 @@ function updateEsbuildConfig(config) {
};
const babelPlugin = getEsbuildBabelPlugin(
/* callerName */ 'test',
/* enableCache */ !argv.watch, // TODO(jridgewell): Make this true when unifiedJsFile goes away.
/* enableCache */ true,
/* preSetup */ logBabelStart,
/* postLoad */ printBabelDot
);
Expand Down

0 comments on commit 8d7baef

Please sign in to comment.