From fc1093ff1560b47611293bf71f8074030116edcc Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Mon, 4 Jul 2022 16:07:05 -0500 Subject: [PATCH] Add more comments explaining package (de)serialization (#45891) --- src/dump.c | 98 ++++++++++++++++++++++++++++++++++++++++++++-------- src/method.c | 45 ++++++++++++++++++++++++ 2 files changed, 128 insertions(+), 15 deletions(-) diff --git a/src/dump.c b/src/dump.c index 63c504d5813c7..6a1bd2ea582b8 100644 --- a/src/dump.c +++ b/src/dump.c @@ -37,21 +37,76 @@ extern "C" { // careful to match the sequence, if necessary reserving space for something that will // be updated later. -// It is also necessary to save & restore references to externally-defined objects, -// e.g., for package methods that call methods defined in Base or elsewhere. -// Consequently during deserialization there's a distinction between "reference" -// types, methods, and method instances (essentially like a GlobalRef), -// and "recached" version that refer to the actual entity in the running session. -// We complete deserialization before beginning the process of recaching, -// because we need the backreferences during deserialization and the actual -// objects during recaching. +// It is also necessary to save & restore references to externally-defined +// objects, e.g., for package methods that call methods defined in Base or +// elsewhere. Consequently during deserialization there's a distinction between +// "reference" types, methods, and method instances (essentially like a +// GlobalRef), and "recached" version that refer to the actual entity in the +// running session. As a concrete example, types have a module in which they are +// defined, but once defined those types can be used by any dependent package. +// We don't store the full type definition again in that dependent package, we +// just encode a reference to that type. In the running session, such references +// are merely pointers to the type-cache, but the specific address is obviously +// not likely to be reproducible across sessions (it will differ between the +// session in which you precompile and the session in which you're using the +// package). Hence, during serialization we recode them as "verbose" references +// (that follow Julia syntax to allow them to be reconstructed), but on +// deserialization we have to replace those verbose references with the +// appropriate pointer in the user's running session. We complete +// deserialization before beginning the process of recaching, because we need +// the backreferences during deserialization and the actual objects during +// recaching. // Finally, because our backedge graph is not bidirectional, special handling is // required to identify backedges from external methods that call internal methods. // These get set aside and restored at the end of deserialization. +// In broad terms, the major steps in serialization are: +// - starting from a "worklist" of modules, write the header. This stores things +// like the Julia build this was precompiled for, the package dependencies, +// the list of include files, file modification times, etc. +// - gather the collection of items to be written to this precompile file. This +// includes accessible from the module's binding table (if they are owned by a +// worklist module), but also includes things like methods added to external +// functions, instances of external methods that were newly type-inferred +// while precompiling a worklist module, and backedges of callees that were +// called by methods in this package. By and large, these latter items are not +// referenced by the module(s) in the package, and so these have to be +// extracted by traversing the entire system searching for things that do link +// back to a module in the worklist. +// - serialize all the items. The first time we encounter an item, we serialized +// it, and on future references (pointers) to that item we replace them with +// with a backreference. `jl_serialize_*` functions handle this work. +// - write source text for the files that defined the package. This is primarily +// to support Revise.jl. + +// Deserialization is the mirror image of serialization, but in some ways is +// trickier: +// - we have to merge items into the running session (recaching as described +// above) and handle cases like having two dependent packages caching the same +// MethodInstance of a dependency +// - we have to check for invalidation---the user might have loaded other +// packages that define methods that supersede some of the dispatches chosen +// when the package was precompiled, or this package might define methods that +// supercede dispatches for previously-loaded packages. These two +// possibilities are checked during backedge and method insertion, +// respectively. +// Both of these mean that deserialization requires one to look up a lot of +// things in the running session; for example, for invalidation checks we have +// to do type-intersection between signatures used for MethodInstances and the +// current session's full MethodTable. In practice, such steps dominate package +// loading time (it has very little to do with I/O or deserialization +// performance). Paradoxically, sometimes storing more code in a package can +// lead to faster performance: references to things in the same .ji file can be +// precomputed, but external references have to be looked up. You can see this +// effect in the benchmarks for #43990, where storing external MethodInstances +// and CodeInstances (more code than was stored previously) actually decreased +// load times for many packages. + // Note that one should prioritize deserialization performance over serialization performance, // since deserialization may be performed much more often than serialization. +// Certain items are preprocessed during serialization to save work when they are +// later deserialized. // TODO: put WeakRefs on the weak_refs list during deserialization @@ -69,9 +124,11 @@ static jl_value_t *deser_symbols[256]; // (the order in the serializer stream). the low // bit is reserved for flagging certain entries and pos is // left shift by 1 -static htable_t backref_table; +static htable_t backref_table; // pos = backref_table[obj] static int backref_table_numel; -static arraylist_t backref_list; +static arraylist_t backref_list; // obj = backref_list[pos] + +// set of all CodeInstances yet to be (in)validated static htable_t new_code_instance_validate; // list of (jl_value_t **loc, size_t pos) entries @@ -83,16 +140,20 @@ static arraylist_t flagref_list; // like types, methods, and method instances static htable_t uniquing_table; -// list of (size_t pos, (void *f)(jl_value_t*)) entries -// for the serializer to mark values in need of rework by function f +// list of (size_t pos, itemkey) entries +// for the serializer to mark values in need of rework // during deserialization later +// This includes items that need rehashing (IdDict, TypeMapLevels) +// and modules. static arraylist_t reinit_list; -// list of stuff that is being serialized +// list of modules being serialized // This is not quite globally rooted, but we take care to only // ever assigned rooted values here. static jl_array_t *serializer_worklist JL_GLOBALLY_ROOTED; -// external MethodInstances we want to serialize +// The set of external MethodInstances we want to serialize +// (methods owned by other modules that were first inferred for a +// module currently being serialized) static htable_t external_mis; // Inference tracks newly-inferred MethodInstances during precompilation // and registers them by calling jl_set_newly_inferred @@ -100,7 +161,14 @@ static jl_array_t *newly_inferred JL_GLOBALLY_ROOTED; // New roots to add to Methods. These can't be added until after // recaching is complete, so we have to hold on to them separately -// Stored as method => (worklist_key, roots) +// Stored as method => (worklist_key, newroots) +// The worklist_key is the uuid of the module that triggered addition +// of `newroots`. This is needed because CodeInstances reference +// their roots by "index", and we use a bipartite index +// (module_uuid, integer_index) to make indexes "relocatable" +// (meaning that users can load modules in different orders and +// so the absolute integer index of a root is not reproducible). +// See the "root blocks" section of method.c for more detail. static htable_t queued_method_roots; // inverse of backedges graph (caller=>callees hash) diff --git a/src/method.c b/src/method.c index 33abedcfdb62e..89eead6515bc5 100644 --- a/src/method.c +++ b/src/method.c @@ -1016,6 +1016,46 @@ JL_DLLEXPORT jl_method_t* jl_method_def(jl_svec_t *argdata, // root blocks +// This section handles method roots. Roots are GC-preserved items needed to +// represent lowered, type-inferred, and/or compiled code. These items are +// stored in a flat list (`m.roots`), and during serialization and +// deserialization of code we replace C-pointers to these items with a +// relocatable reference. We use a bipartite reference, `(key, index)` pair, +// where `key` identifies the module that added the root and `index` numbers +// just those roots with the same `key`. +// +// During precompilation (serialization), we save roots that were added to +// methods that are tagged with this package's module-key, even for "external" +// methods not owned by a module currently being precompiled. During +// deserialization, we load the new roots and append them to the method. When +// code is deserialized (see ircode.c), we replace the bipartite reference with +// the pointer to the memory address in the current session. The bipartite +// reference allows us to cache both roots and references in precompilation .ji +// files using a naming scheme that is independent of which packages are loaded +// in arbitrary order. +// +// To track the module-of-origin for each root, methods also have a +// `root_blocks` field that uses run-length encoding (RLE) storing `key` and the +// (absolute) integer index within `roots` at which a block of roots with that +// key begins. This makes it possible to look up an individual `(key, index)` +// pair fairly efficiently. A given `key` may possess more than one block; the +// `index` continues to increment regardless of block boundaries. +// +// Roots with `key = 0` are considered to be of unknown origin, and +// CodeInstances referencing such roots will remain unserializable unless all +// such roots were added at the time of system image creation. To track this +// additional data, we use two fields: +// +// - methods have an `nroots_sysimg` field to count the number of roots defined +// at the time of writing the system image (such occur first in the list of +// roots). These are the cases with `key = 0` that do not prevent +// serialization. +// - CodeInstances have a `relocatability` field which when 1 indicates that +// every root is "safe," meaning it was either added at sysimg creation or is +// tagged with a non-zero `key`. Even a single unsafe root will cause this to +// have value 0. + +// Get the key of the current (final) block of roots static uint64_t current_root_id(jl_array_t *root_blocks) { if (!root_blocks) @@ -1028,6 +1068,7 @@ static uint64_t current_root_id(jl_array_t *root_blocks) return blocks[nx2-2]; } +// Add a new block of `len` roots with key `modid` (module id) static void add_root_block(jl_array_t *root_blocks, uint64_t modid, size_t len) { assert(jl_is_array(root_blocks)); @@ -1038,6 +1079,7 @@ static void add_root_block(jl_array_t *root_blocks, uint64_t modid, size_t len) blocks[nx2-1] = len; } +// Allocate storage for roots static void prepare_method_for_roots(jl_method_t *m, uint64_t modid) { if (!m->roots) { @@ -1050,6 +1092,7 @@ static void prepare_method_for_roots(jl_method_t *m, uint64_t modid) } } +// Add a single root with owner `mod` to a method JL_DLLEXPORT void jl_add_method_root(jl_method_t *m, jl_module_t *mod, jl_value_t* root) { JL_GC_PUSH2(&m, &root); @@ -1066,6 +1109,7 @@ JL_DLLEXPORT void jl_add_method_root(jl_method_t *m, jl_module_t *mod, jl_value_ JL_GC_POP(); } +// Add a list of roots with key `modid` to a method void jl_append_method_roots(jl_method_t *m, uint64_t modid, jl_array_t* roots) { JL_GC_PUSH2(&m, &roots); @@ -1105,6 +1149,7 @@ jl_value_t *lookup_root(jl_method_t *m, uint64_t key, int index) return jl_array_ptr_ref(m->roots, i); } +// Count the number of roots added by module with id `key` int nroots_with_key(jl_method_t *m, uint64_t key) { size_t nroots = 0;