Add serializeVM() to trigger Serialization from js code.

Summary:
To be able to trigger Serialization after user code, add serializeVM to allow
user to call `serializeVM(string filename, function() {/*resumed;*/})` to start a Serialization. When deserialize, we will resume to execute the function closure after reconstructing the Runtime.

Also allow Serializer/Deserializer to map external pointers to relocation ids.

Reviewed By: avp

Differential Revision: D16523696

fbshipit-source-id: 31a8bad51e98ae569c4b713653db560346800f01

include/hermes/VM/Deserializer.h

@@ -55,9 +55,12 @@ class Deserializer {
  public:
   /// Note: it is the caller's responsibility to make sure that MemoryBuffer
   /// can outlive the Runtime that we are deserializing.
-  Deserializer(std::shared_ptr<llvm::MemoryBuffer> buffer, Runtime *runtime)
+  Deserializer(
+      std::shared_ptr<llvm::MemoryBuffer> buffer,
+      Runtime *runtime,
+      ExternalPointersVectorFunction *externalPointersVectorCallBack)
       : runtime_(runtime), buffer_(std::move(buffer)) {
-    init();
+    init(externalPointersVectorCallBack);
   }
 
   /// Read data from the MemoryBuffer. We use memcpy for unaligned read. We also
@@ -225,7 +228,7 @@ class Deserializer {
 
   /// Initialize data structures for deserialization. Read map size and resize
   /// relocation map. Reconstruct string literal buffers.
-  void init();
+  void init(ExternalPointersVectorFunction *externalPointersVectorCallBack);
 
   /// Check if an id is materialized. If so update value at \p address
   /// according to \p kind. Otherwise recode in relocationQueue_.

include/hermes/VM/Runtime.h

@@ -713,7 +713,16 @@ class Runtime : public HandleRootOwner,
   void checkHeaderRuntimeConfig(SerializeHeader &header) const;
 
   /// Serialize the VM state.
-  void serialize(llvm::raw_ostream &O);
+  void serialize(Serializer &s);
+
+  /// Set the closure function to execute after deserialization.
+  void setSerializeClosure(Handle<JSFunction> function) {
+    serializeClosure = function.getHermesValue();
+  }
+
+  HermesValue getSerializeClosure() {
+    return serializeClosure;
+  }
 #endif
 
  protected:
@@ -851,9 +860,7 @@ class Runtime : public HandleRootOwner,
   /// \param currentlyInYoung Whether we are allocating from the young gen
   /// before deserialization starts and should we go back to allocating from the
   /// young gen after deserialziation.
-  void deserializeImpl(
-      std::shared_ptr<llvm::MemoryBuffer> inputFile,
-      bool currentlyInYoung);
+  void deserializeImpl(Deserializer &d, bool currentlyInYoung);
 #endif
 
  private:

include/hermes/VM/RuntimeHermesValueFields.def

@@ -82,6 +82,11 @@ RUNTIME_HV_FIELD_INSTANCE(thrownValue_)
 RUNTIME_HV_FIELD_INSTANCE(debuggerInternalObject_)
 #endif // HERMES_ENABLE_DEBUGGER
 
+#ifdef HERMESVM_SERIALIZE
+RUNTIME_HV_FIELD_INSTANCE(serializeClosure)
+#endif
+
 #undef RUNTIME_HV_FIELD_PROTOTYPE
 #undef RUNTIME_HV_FIELD_INSTANCE
 #undef RUNTIME_HV_FIELD_RUNTIMEMODULE
+#undef RUNTIME_HV_FIELD

include/hermes/VM/SerializeHeader.h

@@ -59,6 +59,8 @@ static_assert(
     IsTriviallyCopyable<SerializeHeader, true>::value,
     "SerializeHeader should be trivially copyable");
 
+using ExternalPointersVectorFunction = std::vector<void *>();
+
 } // namespace vm
 } // namespace hermes
 

include/hermes/VM/Serializer.h

@@ -67,7 +67,10 @@ class Runtime;
 /// perform this check, we check SerializeHeader before deserializing anything.
 class Serializer {
  public:
-  Serializer(llvm::raw_ostream &OS, Runtime *runtime);
+  Serializer(
+      llvm::raw_ostream &OS,
+      Runtime *runtime,
+      ExternalPointersVectorFunction *externalPointersVectorCallBack);
 
   Runtime *getRuntime() {
     return runtime_;

lib/ConsoleHost/ConsoleHost.cpp

@@ -107,6 +107,64 @@ loadSegment(void *ctx, vm::Runtime *runtime, vm::NativeArgs args) {
   return HermesValue::encodeUndefinedValue();
 }
 
+#ifdef HERMESVM_SERIALIZE
+static std::vector<void *> getNativeFunctionPtrs();
+
+/// serializeVM(filename, funciton() {/*resumed*/}) will serialize the VM state
+/// to file \p filename. When deserialize from the file, we will continue to
+/// execute the closure funciton provided.
+static vm::CallResult<vm::HermesValue>
+serializeVM(void *, vm::Runtime *runtime, vm::NativeArgs args) {
+  using namespace vm;
+  std::string fileName;
+  if (!args.getArg(0).isString()) {
+    return runtime->raiseTypeError(
+        "Missing filename argument or filename argument not a string");
+  }
+
+  if (!args.getArg(1).isObject()) {
+    return runtime->raiseTypeError("Invalid/Missing function argument");
+  }
+
+  // In the rare events where we have a UTF16 string, convert it to ASCII.
+  auto str = Handle<StringPrimitive>::vmcast(args.getArgHandle(runtime, 0));
+  auto jsFileName = StringPrimitive::createStringView(runtime, str);
+  llvm::SmallVector<char16_t, 16> buf;
+  convertUTF16ToUTF8WithReplacements(fileName, jsFileName.getUTF16Ref(buf));
+
+  if (fileName.empty()) {
+    return runtime->raiseTypeError("Filename must not be empty");
+  }
+
+  std::error_code EC;
+  std::unique_ptr<llvm::raw_ostream> serializeStream =
+      std::make_unique<llvm::raw_fd_ostream>(llvm::StringRef(fileName), EC);
+  if (EC) {
+    return runtime->raiseTypeError(
+        TwineChar16("Could not write to file located at ") +
+        llvm::StringRef(fileName));
+  }
+
+  auto closureFunction =
+      Handle<JSFunction>::vmcast(args.getArgHandle(runtime, 1));
+
+  Serializer s(*serializeStream, runtime, getNativeFunctionPtrs);
+  runtime->setSerializeClosure(closureFunction);
+  runtime->serialize(s);
+  return HermesValue::encodeUndefinedValue();
+}
+
+/// Gather function pointers of native functions and put them in \p vec.
+static std::vector<void *> getNativeFunctionPtrs() {
+  std::vector<void *> res;
+  res.push_back((void *)quit);
+  res.push_back((void *)createHeapSnapshot);
+  res.push_back((void *)serializeVM);
+  res.push_back((void *)loadSegment);
+  return res;
+}
+#endif
+
 void installConsoleBindings(
     vm::Runtime *runtime,
     vm::StatSamplingThread *statSampler,
@@ -120,10 +178,21 @@ void installConsoleBindings(
   normalDPF.writable = 1;
   normalDPF.configurable = 1;
 
+#if defined HERMESVM_SERIALIZE && !defined NDEBUG
+  // Verify that all native pointers can be captured by getNativeFunctionPtrs.
+  std::vector<void *> pointers = getNativeFunctionPtrs();
+#endif
+
   auto defineGlobalFunc = [&](vm::SymbolID name,
                               vm::NativeFunctionPtr functionPtr,
                               void *context,
                               unsigned paramCount) {
+#ifdef HERMESVM_SERIALIZE
+    assert(
+        (std::find(pointers.begin(), pointers.end(), (void *)functionPtr) !=
+         pointers.end()) &&
+        "All function pointers must be added in getNativeFunctionPtrs");
+#endif
     auto func = vm::NativeFunction::createWithoutPrototype(
         runtime, context, functionPtr, name, paramCount);
     auto res = vm::JSObject::defineOwnProperty(
@@ -142,6 +211,17 @@ void installConsoleBindings(
       createHeapSnapshot,
       nullptr,
       2);
+#ifdef HERMESVM_SERIALIZE
+  defineGlobalFunc(
+      runtime
+          ->ignoreAllocationFailure(
+              runtime->getIdentifierTable().getSymbolHandle(
+                  runtime, llvm::createASCIIRef("serializeVM")))
+          .get(),
+      serializeVM,
+      nullptr,
+      2);
+#endif
 
   // Define the 'loadSegment' function.
   defineGlobalFunc(
@@ -223,10 +303,12 @@ bool executeHBCBytecodeImpl(
 
   std::unique_ptr<vm::StatSamplingThread> statSampler;
 #ifdef HERMESVM_SERIALIZE
-  auto runtime = vm::Runtime::create(options.runtimeConfig.rebuild()
-                                         .withSerializeFile(serializeFile)
-                                         .withDeserializeFile(deserializeFile)
-                                         .build());
+  auto runtime = vm::Runtime::create(
+      options.runtimeConfig.rebuild()
+          .withSerializeFile(serializeFile)
+          .withDeserializeFile(deserializeFile)
+          .withExternalPointersVectorCallBack(getNativeFunctionPtrs)
+          .build());
 #else
   auto runtime = vm::Runtime::create(options.runtimeConfig);
 #endif

lib/VM/Deserializer.cpp

@@ -51,7 +51,8 @@ void Deserializer::flushRelocationQueue() {
   }
 }
 
-void Deserializer::init() {
+void Deserializer::init(
+    ExternalPointersVectorFunction *externalPointersVectorCallBack) {
   // Do the sanity check of the header first.
   readHeader();
 
@@ -137,6 +138,13 @@ void Deserializer::init() {
   idx++;
 #include "hermes/VM/NativeFunctions.def"
 #undef NATIVE_CONSTRUCTOR
+
+  // Map external function pointers.
+  for (auto *ptr : externalPointersVectorCallBack()) {
+    assert(!objectTable_[idx] && "External pointer should only be mapped once");
+    objectTable_[idx] = ptr;
+    idx++;
+  }
 }
 
 void Deserializer::readHeader() {
@@ -186,8 +194,8 @@ void Deserializer::readHeader() {
 }
 
 void Deserializer::readAndCheckOffset() {
-  uint32_t currentOffset = offset_;
-  uint32_t bytes = readInt<uint32_t>();
+  size_t currentOffset = offset_;
+  size_t bytes = readInt<size_t>();
   if (currentOffset != bytes) {
     hermes_fatal("Deserializer sanity check failed: offset don't match");
   }

lib/VM/Runtime.cpp

@@ -243,11 +243,16 @@ Runtime::Runtime(StorageProvider *provider, const RuntimeConfig &runtimeConfig)
 
 #ifdef HERMESVM_SERIALIZE
   if (runtimeConfig.getDeserializeFile()) {
+    assert(
+        runtimeConfig.getExternalPointersVectorCallBack() &&
+        "missing function pointer to map external pointers.");
     // If there is a serialized heap file available, use that to initialize
     // Runtime instead of re-creating the Runtime.
-    deserializeImpl(
+    Deserializer d(
         runtimeConfig.getDeserializeFile(),
-        runtimeConfig.getGCConfig().getAllocInYoung());
+        this,
+        runtimeConfig.getExternalPointersVectorCallBack());
+    deserializeImpl(d, runtimeConfig.getGCConfig().getAllocInYoung());
 
     LLVM_DEBUG(llvm::dbgs() << "Runtime initialized\n");
 
@@ -315,7 +320,14 @@ Runtime::Runtime(StorageProvider *provider, const RuntimeConfig &runtimeConfig)
 
 #ifdef HERMESVM_SERIALIZE
   if (runtimeConfig.getSerializeFile()) {
-    serialize(*runtimeConfig.getSerializeFile());
+    assert(
+        runtimeConfig.getExternalPointersVectorCallBack() &&
+        "missing function pointer to map external pointers.");
+    Serializer s(
+        *runtimeConfig.getSerializeFile(),
+        this,
+        runtimeConfig.getExternalPointersVectorCallBack());
+    serialize(s);
   }
 #endif // HERMESVM_SERIALIZE
 
@@ -656,6 +668,25 @@ CallResult<HermesValue> Runtime::runBytecode(
     Handle<> thisArg) {
   clearThrownValue();
 
+#ifdef HERMESVM_SERIALIZE
+  // If we are constructed from serialize data with a ClosureFunction, execute
+  // the function.
+  if (!serializeClosure.isUndefined()) {
+    ScopedNativeCallFrame newFrame{this,
+                                   0,
+                                   serializeClosure,
+                                   HermesValue::encodeUndefinedValue(),
+                                   *thisArg};
+    if (LLVM_UNLIKELY(newFrame.overflowed()))
+      return raiseStackOverflow(StackOverflowKind::NativeStack);
+    return shouldRandomizeMemoryLayout_
+        ? interpretFunctionWithRandomStack(
+              this, vmcast<JSFunction>(serializeClosure)->getCodeBlock())
+        : interpretFunction(
+              vmcast<JSFunction>(serializeClosure)->getCodeBlock());
+  }
+#endif
+
   auto globalFunctionIndex = bytecode->getGlobalFunctionIndex();
 
 // TODO(T35544739): clean up the experiment after we are done.
@@ -1540,8 +1571,7 @@ std::string Runtime::getCallStackNoAlloc(const Inst *ip) {
 }
 
 #ifdef HERMESVM_SERIALIZE
-void Runtime::serialize(llvm::raw_ostream &O) {
-  Serializer s(O, this);
+void Runtime::serialize(Serializer &s) {
   // Full GC here.
   heap_.collect();
 
@@ -1565,7 +1595,6 @@ void Runtime::serialize(llvm::raw_ostream &O) {
   // beginning and record there.
   s.writeCurrentOffset();
   s.writeEpilogue();
-  return;
 }
 
 void Runtime::serializeIdentifierTable(Serializer &s) {
@@ -1737,17 +1766,13 @@ void Runtime::deserializeRuntimeFields(Deserializer &d) {
 #endif // HERMES_ENABLE_DEBUGGER
 }
 
-void Runtime::deserializeImpl(
-    std::shared_ptr<llvm::MemoryBuffer> inputFile,
-    bool currentlyInYoung) {
+void Runtime::deserializeImpl(Deserializer &d, bool currentlyInYoung) {
   if (currentlyInYoung) {
     heap_.deserializeStart();
   }
 
   GCScope scope(this);
 
-  Deserializer d(std::move(inputFile), this);
-
   d.readAndCheckOffset();
   heap_.deserializeWeakRefs(d);
 

lib/VM/Serializer.cpp

@@ -33,7 +33,10 @@ static SerializeCallBack *serializeImpl[] = {
 #undef CELL_KIND
 };
 
-Serializer::Serializer(llvm::raw_ostream &OS, Runtime *runtime)
+Serializer::Serializer(
+    llvm::raw_ostream &OS,
+    Runtime *runtime,
+    ExternalPointersVectorFunction *externalPointersVectorCallBack)
     : os_(OS), runtime_(runtime) {
   //  Write the header here.
   writeHeader();
@@ -95,6 +98,15 @@ Serializer::Serializer(llvm::raw_ostream &OS, Runtime *runtime)
   currentId_++;
 
 #include "hermes/VM/NativeFunctions.def"
+
+  // Map external pointers.
+  for (auto *ptr : externalPointersVectorCallBack()) {
+    assert(
+        relocationMap_.count(ptr) == 0 &&
+        "External pointer should only be mapped once");
+    relocationMap_[ptr] = currentId_;
+    currentId_++;
+  }
 }
 
 void Serializer::flushCharBufs() {
@@ -147,7 +159,7 @@ void Serializer::writeHeader() {
 }
 
 void Serializer::writeCurrentOffset() {
-  writeInt<uint32_t>(writtenBytes_);
+  writeInt<size_t>(writtenBytes_);
 }
 
 } // namespace vm