Merge pull request webpack#15515 from webpack/feat/concatenate-assets

add asset modules concatenation

lib/ChunkGraph.js

@@ -85,14 +85,17 @@ const getModuleRuntimes = chunks => {
 };
 
 /**
- * @param {SortableSet<Module>} set the set
- * @returns {Map<string, SortableSet<Module>>} modules by source type
+ * @param {WeakMap<Module, Set<string>> | undefined} sourceTypesByModule sourceTypesByModule
+ * @returns {function (SortableSet<Module>): Map<string, SortableSet<Module>>} modules by source type
  */
-const modulesBySourceType = set => {
+const modulesBySourceType = sourceTypesByModule => set => {
 	/** @type {Map<string, SortableSet<Module>>} */
 	const map = new Map();
 	for (const module of set) {
-		for (const sourceType of module.getSourceTypes()) {
+		const sourceTypes =
+			(sourceTypesByModule && sourceTypesByModule.get(module)) ||
+			module.getSourceTypes();
+		for (const sourceType of sourceTypes) {
 			let innerSet = map.get(sourceType);
 			if (innerSet === undefined) {
 				innerSet = new SortableSet();
@@ -110,6 +113,7 @@ const modulesBySourceType = set => {
 	}
 	return map;
 };
+const defaultModulesBySourceType = modulesBySourceType(undefined);
 
 /** @type {WeakMap<Function, any>} */
 const createOrderedArrayFunctionMap = new WeakMap();
@@ -201,6 +205,8 @@ class ChunkGraphChunk {
 	constructor() {
 		/** @type {SortableSet<Module>} */
 		this.modules = new SortableSet();
+		/** @type {WeakMap<Module, Set<string>> | undefined} */
+		this.sourceTypesByModule = undefined;
 		/** @type {Map<Module, Entrypoint>} */
 		this.entryModules = new Map();
 		/** @type {SortableSet<RuntimeModule>} */
@@ -213,6 +219,8 @@ class ChunkGraphChunk {
 		this.runtimeRequirements = undefined;
 		/** @type {Set<string>} */
 		this.runtimeRequirementsInTree = new Set();
+
+		this._modulesBySourceType = defaultModulesBySourceType;
 	}
 }
 
@@ -315,6 +323,8 @@ class ChunkGraph {
 		const cgm = this._getChunkGraphModule(module);
 		const cgc = this._getChunkGraphChunk(chunk);
 		cgc.modules.delete(module);
+		// No need to invalidate cgc._modulesBySourceType because we modified cgc.modules anyway
+		if (cgc.sourceTypesByModule) cgc.sourceTypesByModule.delete(module);
 		cgm.chunks.delete(chunk);
 	}
 
@@ -568,11 +578,84 @@ class ChunkGraph {
 	getChunkModulesIterableBySourceType(chunk, sourceType) {
 		const cgc = this._getChunkGraphChunk(chunk);
 		const modulesWithSourceType = cgc.modules
-			.getFromUnorderedCache(modulesBySourceType)
+			.getFromUnorderedCache(cgc._modulesBySourceType)
 			.get(sourceType);
 		return modulesWithSourceType;
 	}
 
+	/**
+	 * @param {Chunk} chunk chunk
+	 * @param {Module} module chunk module
+	 * @param {Set<string>} sourceTypes source types
+	 */
+	setChunkModuleSourceTypes(chunk, module, sourceTypes) {
+		const cgc = this._getChunkGraphChunk(chunk);
+		if (cgc.sourceTypesByModule === undefined) {
+			cgc.sourceTypesByModule = new WeakMap();
+		}
+		cgc.sourceTypesByModule.set(module, sourceTypes);
+		// Update cgc._modulesBySourceType to invalidate the cache
+		cgc._modulesBySourceType = modulesBySourceType(cgc.sourceTypesByModule);
+	}
+
+	/**
+	 * @param {Chunk} chunk chunk
+	 * @param {Module} module chunk module
+	 * @returns {Set<string>} source types
+	 */
+	getChunkModuleSourceTypes(chunk, module) {
+		const cgc = this._getChunkGraphChunk(chunk);
+		if (cgc.sourceTypesByModule === undefined) {
+			return module.getSourceTypes();
+		}
+		return cgc.sourceTypesByModule.get(module) || module.getSourceTypes();
+	}
+
+	/**
+	 * @param {Module} module module
+	 * @returns {Set<string>} source types
+	 */
+	getModuleSourceTypes(module) {
+		return (
+			this._getOverwrittenModuleSourceTypes(module) || module.getSourceTypes()
+		);
+	}
+
+	/**
+	 * @param {Module} module module
+	 * @returns {Set<string> | undefined} source types
+	 */
+	_getOverwrittenModuleSourceTypes(module) {
+		let newSet = false;
+		let sourceTypes;
+		for (const chunk of this.getModuleChunksIterable(module)) {
+			const cgc = this._getChunkGraphChunk(chunk);
+			if (cgc.sourceTypesByModule === undefined) return;
+			const st = cgc.sourceTypesByModule.get(module);
+			if (st === undefined) return;
+			if (!sourceTypes) {
+				sourceTypes = st;
+				continue;
+			} else if (!newSet) {
+				for (const type of st) {
+					if (!newSet) {
+						if (!sourceTypes.has(type)) {
+							newSet = true;
+							sourceTypes = new Set(sourceTypes);
+							sourceTypes.add(type);
+						}
+					} else {
+						sourceTypes.add(type);
+					}
+				}
+			} else {
+				for (const type of st) sourceTypes.add(type);
+			}
+		}
+
+		return sourceTypes;
+	}
+
 	/**
 	 * @param {Chunk} chunk the chunk
 	 * @param {function(Module, Module): -1|0|1} comparator comparator function
@@ -593,7 +676,7 @@ class ChunkGraph {
 	getOrderedChunkModulesIterableBySourceType(chunk, sourceType, comparator) {
 		const cgc = this._getChunkGraphChunk(chunk);
 		const modulesWithSourceType = cgc.modules
-			.getFromUnorderedCache(modulesBySourceType)
+			.getFromUnorderedCache(cgc._modulesBySourceType)
 			.get(sourceType);
 		if (modulesWithSourceType === undefined) return undefined;
 		modulesWithSourceType.sortWith(comparator);
@@ -1473,6 +1556,10 @@ Caller might not support runtime-dependent code generation (opt-out via optimiza
 		const graphHash = cgm.graphHashes.provide(runtime, () => {
 			const hash = createHash(this._hashFunction);
 			hash.update(`${cgm.id}${this.moduleGraph.isAsync(module)}`);
+			const sourceTypes = this._getOverwrittenModuleSourceTypes(module);
+			if (sourceTypes !== undefined) {
+				for (const type of sourceTypes) hash.update(type);
+			}
 			this.moduleGraph.getExportsInfo(module).updateHash(hash, runtime);
 			return BigInt(`0x${/** @type {string} */ (hash.digest("hex"))}`);
 		});

lib/Module.js

@@ -60,6 +60,7 @@ const makeSerializable = require("./util/makeSerializable");
  * @property {ConcatenationScope=} concatenationScope when in concatenated module, information about other concatenated modules
  * @property {CodeGenerationResults} codeGenerationResults code generation results of other modules (need to have a codeGenerationDependency to use that)
  * @property {Compilation=} compilation the compilation
+ * @property {ReadonlySet<string>=} sourceTypes source types
  */
 
 /**

lib/NormalModule.js

@@ -1176,7 +1176,8 @@ class NormalModule extends Module {
 		chunkGraph,
 		runtime,
 		concatenationScope,
-		codeGenerationResults
+		codeGenerationResults,
+		sourceTypes
 	}) {
 		/** @type {Set<string>} */
 		const runtimeRequirements = new Set();
@@ -1195,7 +1196,7 @@ class NormalModule extends Module {
 		};
 
 		const sources = new Map();
-		for (const type of this.generator.getTypes(this)) {
+		for (const type of sourceTypes || chunkGraph.getModuleSourceTypes(this)) {
 			const source = this.error
 				? new RawSource(
 						"throw new Error(" + JSON.stringify(this.error.message) + ");"

lib/asset/AssetGenerator.js

@@ -8,6 +8,7 @@
 const mimeTypes = require("mime-types");
 const path = require("path");
 const { RawSource } = require("webpack-sources");
+const ConcatenationScope = require("../ConcatenationScope");
 const Generator = require("../Generator");
 const RuntimeGlobals = require("../RuntimeGlobals");
 const createHash = require("../util/createHash");
@@ -23,6 +24,7 @@ const nonNumericOnlyHash = require("../util/nonNumericOnlyHash");
 /** @typedef {import("../Generator").GenerateContext} GenerateContext */
 /** @typedef {import("../Generator").UpdateHashContext} UpdateHashContext */
 /** @typedef {import("../Module")} Module */
+/** @typedef {import("../Module").ConcatenationBailoutReasonContext} ConcatenationBailoutReasonContext */
 /** @typedef {import("../NormalModule")} NormalModule */
 /** @typedef {import("../RuntimeTemplate")} RuntimeTemplate */
 /** @typedef {import("../util/Hash")} Hash */
@@ -145,6 +147,15 @@ class AssetGenerator extends Generator {
 		).replace(/^\.\//, "");
 	}
 
+	/**
+	 * @param {NormalModule} module module for which the bailout reason should be determined
+	 * @param {ConcatenationBailoutReasonContext} context context
+	 * @returns {string | undefined} reason why this module can't be concatenated, undefined when it can be concatenated
+	 */
+	getConcatenationBailoutReason(module, context) {
+		return undefined;
+	}
+
 	/**
 	 * @param {NormalModule} module module
 	 * @returns {string} mime type
@@ -198,14 +209,21 @@ class AssetGenerator extends Generator {
 	 */
 	generate(
 		module,
-		{ runtime, chunkGraph, runtimeTemplate, runtimeRequirements, type, getData }
+		{
+			runtime,
+			concatenationScope,
+			chunkGraph,
+			runtimeTemplate,
+			runtimeRequirements,
+			type,
+			getData
+		}
 	) {
 		switch (type) {
 			case "asset":
 				return module.originalSource();
 			default: {
-				runtimeRequirements.add(RuntimeGlobals.module);
-
+				let content;
 				const originalSource = module.originalSource();
 				if (module.buildInfo.dataUrl) {
 					let encodedSource;
@@ -255,11 +273,7 @@ class AssetGenerator extends Generator {
 					}
 					const data = getData();
 					data.set("url", Buffer.from(encodedSource));
-					return new RawSource(
-						`${RuntimeGlobals.module}.exports = ${JSON.stringify(
-							encodedSource
-						)};`
-					);
+					content = JSON.stringify(encodedSource);
 				} else {
 					const assetModuleFilename =
 						this.filename || runtimeTemplate.outputOptions.assetModuleFilename;
@@ -343,9 +357,22 @@ class AssetGenerator extends Generator {
 						data.set("filename", filename);
 						data.set("assetInfo", assetInfo);
 					}
+					content = assetPath;
+				}
 
+				if (concatenationScope) {
+					concatenationScope.registerNamespaceExport(
+						ConcatenationScope.NAMESPACE_OBJECT_EXPORT
+					);
+					return new RawSource(
+						`${runtimeTemplate.supportsConst() ? "const" : "var"} ${
+							ConcatenationScope.NAMESPACE_OBJECT_EXPORT
+						} = ${content};`
+					);
+				} else {
+					runtimeRequirements.add(RuntimeGlobals.module);
 					return new RawSource(
-						`${RuntimeGlobals.module}.exports = ${assetPath};`
+						`${RuntimeGlobals.module}.exports = ${content};`
 					);
 				}
 			}

lib/asset/AssetParser.js

@@ -31,6 +31,7 @@ class AssetParser extends Parser {
 		}
 		state.module.buildInfo.strict = true;
 		state.module.buildMeta.exportsType = "default";
+		state.module.buildMeta.defaultObject = false;
 
 		if (typeof this.dataUrlCondition === "function") {
 			state.module.buildInfo.dataUrl = this.dataUrlCondition(source, {

lib/asset/AssetSourceGenerator.js

@@ -6,11 +6,13 @@
 "use strict";
 
 const { RawSource } = require("webpack-sources");
+const ConcatenationScope = require("../ConcatenationScope");
 const Generator = require("../Generator");
 const RuntimeGlobals = require("../RuntimeGlobals");
 
 /** @typedef {import("webpack-sources").Source} Source */
 /** @typedef {import("../Generator").GenerateContext} GenerateContext */
+/** @typedef {import("../Module").ConcatenationBailoutReasonContext} ConcatenationBailoutReasonContext */
 /** @typedef {import("../NormalModule")} NormalModule */
 
 const TYPES = new Set(["javascript"]);
@@ -21,9 +23,10 @@ class AssetSourceGenerator extends Generator {
 	 * @param {GenerateContext} generateContext context for generate
 	 * @returns {Source} generated code
 	 */
-	generate(module, { chunkGraph, runtimeTemplate, runtimeRequirements }) {
-		runtimeRequirements.add(RuntimeGlobals.module);
-
+	generate(
+		module,
+		{ concatenationScope, chunkGraph, runtimeTemplate, runtimeRequirements }
+	) {
 		const originalSource = module.originalSource();
 
 		if (!originalSource) {
@@ -38,9 +41,31 @@ class AssetSourceGenerator extends Generator {
 		} else {
 			encodedSource = content.toString("utf-8");
 		}
-		return new RawSource(
-			`${RuntimeGlobals.module}.exports = ${JSON.stringify(encodedSource)};`
-		);
+
+		let sourceContent;
+		if (concatenationScope) {
+			concatenationScope.registerNamespaceExport(
+				ConcatenationScope.NAMESPACE_OBJECT_EXPORT
+			);
+			sourceContent = `${runtimeTemplate.supportsConst() ? "const" : "var"} ${
+				ConcatenationScope.NAMESPACE_OBJECT_EXPORT
+			} = ${JSON.stringify(encodedSource)};`;
+		} else {
+			runtimeRequirements.add(RuntimeGlobals.module);
+			sourceContent = `${RuntimeGlobals.module}.exports = ${JSON.stringify(
+				encodedSource
+			)};`;
+		}
+		return new RawSource(sourceContent);
+	}
+
+	/**
+	 * @param {NormalModule} module module for which the bailout reason should be determined
+	 * @param {ConcatenationBailoutReasonContext} context context
+	 * @returns {string | undefined} reason why this module can't be concatenated, undefined when it can be concatenated
+	 */
+	getConcatenationBailoutReason(module, context) {
+		return undefined;
 	}
 
 	/**

lib/asset/AssetSourceParser.js

@@ -23,6 +23,7 @@ class AssetSourceParser extends Parser {
 		const { module } = state;
 		module.buildInfo.strict = true;
 		module.buildMeta.exportsType = "default";
+		state.module.buildMeta.defaultObject = false;
 
 		return state;
 	}

lib/optimize/ConcatenatedModule.js

@@ -1669,7 +1669,8 @@ ${defineGetters}`
 					chunkGraph,
 					runtime,
 					concatenationScope,
-					codeGenerationResults
+					codeGenerationResults,
+					sourceTypes: TYPES
 				});
 				const source = codeGenResult.sources.get("javascript");
 				const data = codeGenResult.data;

lib/optimize/ModuleConcatenationPlugin.js

@@ -425,7 +425,21 @@ class ModuleConcatenationPlugin {
 										for (const chunk of chunkGraph.getModuleChunksIterable(
 											rootModule
 										)) {
-											chunkGraph.disconnectChunkAndModule(chunk, m);
+											const sourceTypes = chunkGraph.getChunkModuleSourceTypes(
+												chunk,
+												m
+											);
+											if (sourceTypes.size === 1) {
+												chunkGraph.disconnectChunkAndModule(chunk, m);
+											} else {
+												const newSourceTypes = new Set(sourceTypes);
+												newSourceTypes.delete("javascript");
+												chunkGraph.setChunkModuleSourceTypes(
+													chunk,
+													m,
+													newSourceTypes
+												);
+											}
 										}
 									}
 								}