Skip to content

Commit

Permalink
Add support for module groups to iOS Random Access Bundle format
Browse files Browse the repository at this point in the history 
Summary:
Adds the possibility to specify an array of files (group roots) that are used to bundle modules outside of the “startup section” into bigger groups by colocating their code.
A require call for any grouped module will load all modules in that group.
Files contained by multiple groups are deoptimized (i.e. bundled as individual script)

Reviewed By: martinbigio

Differential Revision: D3841780

fbshipit-source-id: 8d37782792efd66b5f557c7567489f68c9b229d8
  • Loading branch information
@davidaurelio
davidaurelio authored and Facebook Github Bot 5 committed Sep 16, 2016
1 parent 784b19b commit bb484bc
Show file tree
Hide file tree
Showing 3 changed files with 93 additions and 21 deletions.
75 changes: 59 additions & 16 deletions bundle/output/unbundle/as-indexed-file.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,19 +32,20 @@ function saveAsIndexedFile(bundle, options, log) {
} = options;

log('start');
const {startupModules, lazyModules} = bundle.getUnbundle();
const {startupModules, lazyModules, groups} = bundle.getUnbundle();
log('finish');

const moduleGroups = ModuleGroups(groups, lazyModules);
const startupCode = joinModules(startupModules);

log('Writing unbundle output to:', bundleOutput);
const writeUnbundle = writeBuffers(
fs.createWriteStream(bundleOutput),
buildTableAndContents(startupCode, lazyModules, encoding)
buildTableAndContents(startupCode, lazyModules, moduleGroups, encoding)
).then(() => log('Done writing unbundle output'));

const sourceMap =
buildSourceMapWithMetaData({startupModules, lazyModules});
buildSourceMapWithMetaData({startupModules, lazyModules, moduleGroups});

return Promise.all([
writeUnbundle,
Expand Down Expand Up @@ -84,7 +85,7 @@ function entryOffset(n) {
return (2 + n * 2) * SIZEOF_UINT32;
}

function buildModuleTable(startupCode, buffers) {
function buildModuleTable(startupCode, buffers, moduleGroups) {
// table format:
// - num_entries: uint_32 number of entries
// - startup_code_len: uint_32 length of the startup section
Expand All @@ -94,7 +95,8 @@ function buildModuleTable(startupCode, buffers) {
// - module_offset: uint_32 offset into the modules blob
// - module_length: uint_32 length of the module code in bytes

const maxId = buffers.reduce((max, {id}) => Math.max(max, id), 0);
const moduleIds = Array.from(moduleGroups.modulesById.keys());
const maxId = moduleIds.reduce((max, id) => Math.max(max, id));
const numEntries = maxId + 1;
const table = new Buffer(entryOffset(numEntries)).fill(0);

Expand All @@ -107,32 +109,59 @@ function buildModuleTable(startupCode, buffers) {
// entries
let codeOffset = startupCode.length;
buffers.forEach(({id, buffer}) => {
const offset = entryOffset(id);
// module_offset
table.writeUInt32LE(codeOffset, offset);
// module_length
table.writeUInt32LE(buffer.length, offset + SIZEOF_UINT32);
const idsInGroup = moduleGroups.groups.has(id)
? [id].concat(Array.from(moduleGroups.groups.get(id)))
: [id];

idsInGroup.forEach(moduleId => {
const offset = entryOffset(moduleId);
// module_offset
table.writeUInt32LE(codeOffset, offset);
// module_length
table.writeUInt32LE(buffer.length, offset + SIZEOF_UINT32);
});
codeOffset += buffer.length;
});

return table;
}

function buildModuleBuffers(modules, encoding) {
return modules.map(
module => moduleToBuffer(module.id, module.code, encoding));
function groupCode(rootCode, moduleGroup, modulesById) {
if (!moduleGroup || !moduleGroup.size) {
return rootCode;
}
const code = [rootCode];
for (const id of moduleGroup) {
code.push(modulesById.get(id).code);
}

return code.join('\n');
}

function buildModuleBuffers(modules, moduleGroups, encoding) {
return modules
.filter(m => !moduleGroups.modulesInGroups.has(m.id))
.map(({id, code}) => moduleToBuffer(
id,
groupCode(
code,
moduleGroups.groups.get(id),
moduleGroups.modulesById,
),
encoding
));
}

function buildTableAndContents(startupCode, modules, encoding) {
function buildTableAndContents(startupCode, modules, moduleGroups, encoding) {
// file contents layout:
// - magic number char[4] 0xE5 0xD1 0x0B 0xFB (0xFB0BD1E5 uint32 LE)
// - offset table table see `buildModuleTables`
// - code blob char[] null-terminated code strings, starting with
// the startup code

const startupCodeBuffer = nullTerminatedBuffer(startupCode, encoding);
const moduleBuffers = buildModuleBuffers(modules, encoding);
const table = buildModuleTable(startupCodeBuffer, moduleBuffers);
const moduleBuffers = buildModuleBuffers(modules, moduleGroups, encoding);
const table = buildModuleTable(startupCodeBuffer, moduleBuffers, moduleGroups);

return [
fileHeader,
Expand All @@ -141,4 +170,18 @@ function buildTableAndContents(startupCode, modules, encoding) {
].concat(moduleBuffers.map(({buffer}) => buffer));
}

function ModuleGroups(groups, modules) {
return {
groups,
modulesById: new Map(modules.map(m => [m.id, m])),
modulesInGroups: new Set(concat(groups.values())),
};
}

function * concat(iterators) {
for (const it of iterators) {
yield * it;
}
}

module.exports = saveAsIndexedFile;
3 changes: 2 additions & 1 deletion bundle/output/unbundle/build-unbundle-sourcemap-with-metadata.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,14 @@

const {combineSourceMaps, joinModules} = require('./util');

module.exports = ({startupModules, lazyModules}) => {
module.exports = ({startupModules, lazyModules, moduleGroups}) => {
const startupModule = {
code: joinModules(startupModules),
map: combineSourceMaps({modules: startupModules}),
};
return combineSourceMaps({
modules: [startupModule].concat(lazyModules),
moduleGroups,
withCustomOffsets: true,
});
};
36 changes: 32 additions & 4 deletions bundle/output/unbundle/util.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ const wrapperEnd = wrappedCode => wrappedCode.indexOf('{') + 1;

const Section = (line, column, map) => ({map, offset: {line, column}});

function combineSourceMaps({modules, withCustomOffsets}) {
function combineSourceMaps({modules, withCustomOffsets, moduleGroups}) {
let offsets;
const sections = [];
const sourceMap = {
Expand All @@ -45,13 +45,41 @@ function combineSourceMaps({modules, withCustomOffsets}) {

let line = 0;
modules.forEach(({code, id, map, name}) => {
const hasOffset = withCustomOffsets && id != null;
const column = hasOffset ? wrapperEnd(code) : 0;
let column = 0;
let hasOffset = false;
let group;
let groupLines = 0;

if (withCustomOffsets) {
if (moduleGroups && moduleGroups.modulesInGroups.has(id)) {
// this is a module appended to another module
return;
}

if (moduleGroups && moduleGroups.groups.has(id)) {
group = moduleGroups.groups.get(id);
const otherModules = Array.from(group).map(
moduleId => moduleGroups.modulesById.get(moduleId));
otherModules.forEach(m => {
groupLines += countLines(m.code);
});
map = combineSourceMaps({
modules: [{code, id, map, name}].concat(otherModules),
});
}

hasOffset = id != null;
column = wrapperEnd(code);
}

sections.push(Section(line, column, map || lineToLineSourceMap(code, name)));
if (hasOffset) {
offsets[id] = line;
for (const moduleId of group || []) {
offsets[moduleId] = line;
}
}
line += countLines(code);
line += countLines(code) + groupLines;
});

return sourceMap;
Expand Down

0 comments on commit bb484bc

Please sign in to comment.