Internal bytecode round 2

Summary:
The previous attempt to allow internal bytecode had a bug that caused an assertion failure in some
builds.
Bring the code back, but add an ifdef for `HERMESVM_USE_JS_LIBRARY_IMPLEMENTATION` to avoid
enabling it always.

For now, it is off by default in CMake.

Reviewed By: ridiculousfish

Differential Revision: D17621584

fbshipit-source-id: 7967bcac08acc40a4975cf9a8f8f98c892e01340

.circleci/config.yml

@@ -47,13 +47,21 @@ jobs:
           name: Build LLVM
           command: |
             cd "$HERMES_WS_DIR"
-            hermes/utils/build/build_llvm.py --target llvm-tblgen llvm llvm_build
+            hermes/utils/build/build_llvm.py llvm llvm_build
       - run:
           name: Crosscompile LLVM
           command: |
             export ANDROID_SDK="$ANDROID_HOME"
             export ANDROID_NDK="$ANDROID_HOME/ndk-bundle"
             cd "$HERMES_WS_DIR" && hermes/utils/crosscompile_llvm.sh
+      - run:
+          name: Build Hermes Compiler
+          command: |
+            cd "$HERMES_WS_DIR"
+            hermes/utils/build/configure.py ./build ./llvm_build ./llvm
+            # Build the Hermes compiler so that the cross compiler build can
+            # access it to build the VM
+            cmake --build ./build --target hermesc
       - run:
           name: Build Hermes for Android
           command: |

CMakeLists.txt

@@ -95,6 +95,9 @@ set(HERMESVM_PLATFORM_LOGGING OFF CACHE BOOL
 set(HERMESVM_JIT OFF CACHE BOOL
   "Enable the JIT")
 
+set(HERMESVM_USE_JS_LIBRARY_IMPLEMENTATION OFF CACHE BOOL
+  "Enable using JS to implement parts of Hermes")
+
 set(HERMESVM_JIT_DISASSEMBLER OFF CACHE BOOL
   "Enable the JIT disassembler")
 
@@ -234,6 +237,9 @@ endif()
 if(HERMESVM_JIT_DISASSEMBLER)
   add_definitions(-DHERMESVM_JIT_DISASSEMBLER)
 endif()
+if(HERMESVM_USE_JS_LIBRARY_IMPLEMENTATION)
+  add_definitions(-DHERMESVM_USE_JS_LIBRARY_IMPLEMENTATION)
+endif()
 if (NOT (ANDROID_LINUX_PERF_PATH STREQUAL ""))
   add_definitions(-DANDROID_LINUX_PERF_PATH="${ANDROID_LINUX_PERF_PATH}")
 endif()
@@ -515,6 +521,7 @@ set(HERMES_LIT_TEST_PARAMS
   exception_on_oom_enabled=${HERMESVM_EXCEPTION_ON_OOM}
   serialize_enabled=${HERMESVM_SERIALIZE}
   profiler=${HERMES_PROFILER_MODE_IN_LIT_TEST}
+  use_js_library_implementation=${HERMESVM_USE_JS_LIBRARY_IMPLEMENTATION}
   )
 
 set(LLVM_LIT_ARGS "-sv")

android/hermes/build.gradle

@@ -7,6 +7,7 @@
 apply plugin: 'com.android.library'
 
 def facebookBuild = System.getenv("FACEBOOK") ?: "0"
+def hermesHostBuild = System.getenv("HERMES_WS_DIR")
 
 // For Facebook internal use:
 def fbsource = System.getenv("FBSOURCE_DIR") ?:
@@ -15,6 +16,13 @@ def fbsource = System.getenv("FBSOURCE_DIR") ?:
 buildDir = "${rootProject.ext.hermes_ws}/build_android/hermes"
 buildDir.mkdirs()
 
+preBuild.doFirst {
+  // The Hermes compiler cmake file is required to exist before
+  // cross-compilation can happen.
+  assert hermesHostBuild != null
+  assert file("${hermesHostBuild}/build/ImportHermesc.cmake").exists()
+}
+
 android {
   compileSdkVersion = rootProject.ext.compileSdkVersion
 
@@ -26,6 +34,7 @@ android {
         arguments "-DHERMES_FACEBOOK_BUILD=${facebookBuild}"
         arguments "-DANDROID_STL=c++_shared"
         arguments "-DANDROID_PIE=True"
+        arguments "-DIMPORT_HERMESC=${hermesHostBuild}/build/ImportHermesc.cmake"
         arguments "-DLLVM_BUILD_BASE=${hermes_ws}/llvm"
         arguments "-DLLVM_SRC_DIR=${hermes_ws}/llvm"
         arguments "-DFBSOURCE_DIR=${fbsource}"

include/hermes/InternalBytecode/InternalBytecode.h

@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the LICENSE
+ * file in the root directory of this source tree.
+ */
+#ifndef HERMES_INTERNALBYTECODE_INTERNALBYTECODE_H
+#define HERMES_INTERNALBYTECODE_INTERNALBYTECODE_H
+
+/// \file
+/// InternalBytecode is a way to embed JS source into the VM at build time.
+/// The purpose of this is to implement parts of Hermes in JS.
+/// The build will concatenate multiple JS files together to form a single file,
+/// which will be compiled and turned into a static C array of bytes.
+/// Concatenating the files has multiple benefits:
+///   * Explicit control of the initialization order
+///   * Single string table and bytecode file
+///   * Explicit control over scoping and sharing
+///   * Cross-file inlining
+///   * Cross-file inlining
+///   * Better optimization because sharing occurs in local variables,
+///     not in object properties.
+
+#include "llvm/ADT/ArrayRef.h"
+
+#include <cstdint>
+#include <vector>
+
+namespace hermes {
+namespace vm {
+
+/// Get a pre-compiled bytecode module to be included with the VM upon
+/// construction. This module must be run before any user code can be run.
+/// \return A list of bytes that can be turned into a Hermes bytecode module.
+llvm::ArrayRef<uint8_t> getInternalBytecode();
+
+} // namespace vm
+} // namespace hermes
+
+#endif // HERMES_INTERNALBYTECODE_INTERNALBYTECODE_H

include/hermes/VM/Runtime.h

@@ -199,6 +199,9 @@ class Runtime : public HandleRootOwner,
       Handle<RequireContext> requireContext,
       RuntimeModuleFlags flags = {});
 
+  /// Runs the internal bytecode. This is called once during initialization.
+  void runInternalBytecode();
+
   /// A convenience function to print an exception to a stream.
   void printException(llvm::raw_ostream &os, Handle<> valueHandle);
 

lib/CMakeLists.txt

@@ -60,3 +60,4 @@ add_subdirectory(BCGen)
 add_subdirectory(ConsoleHost)
 add_subdirectory(Regex)
 add_subdirectory(Platform)
+add_subdirectory(InternalBytecode)

lib/InternalBytecode/CMakeLists.txt

@@ -0,0 +1,73 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the LICENSE
+# file in the root directory of this source tree.
+
+add_definitions(-DHERMES_CMAKE_BUILD)
+
+if(CMAKE_CROSSCOMPILING)
+  set(IMPORT_HERMESC "IMPORTFILE-NOTFOUND" CACHE FILEPATH "hermesc export file")
+  file(TO_CMAKE_PATH "${IMPORT_HERMESC}" IMPORT_HERMESC_CMAKE)
+  include(${IMPORT_HERMESC_CMAKE})
+  set(hermesc_EXE native-hermesc)
+else()
+  set(hermesc_EXE hermesc)
+endif()
+
+add_llvm_library(hermesInternalBytecode
+    InternalBytecode.cpp
+)
+
+# Only enable optimized libraries if the Hermes debugger isn't on.
+# This allows the user to debug the JS libraries.
+if(HERMES_ENABLE_DEBUGGER)
+  set(JS_COMPILER_FLAGS "")
+else()
+  set(JS_COMPILER_FLAGS "-O")
+endif()
+
+# Concatenate all JS files into one source file for compilation.
+# This way there is only one RuntimeModule made for them.
+# The concatenation order is set specifically so that order is controlled here.
+if(NOT WIN32)
+  set(concatenate "cat")
+else()
+  set(concatenate "type")
+endif()
+
+add_custom_command(
+  OUTPUT JSLib.js
+  COMMAND ${concatenate} Number.js > ${CMAKE_CURRENT_BINARY_DIR}/JSLib.js
+  DEPENDS Number.js
+  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+  VERBATIM
+)
+
+add_custom_command(
+  OUTPUT JSLib.hbc
+  COMMAND ${hermesc_EXE} ${JS_COMPILER_FLAGS} -emit-binary -out=${CMAKE_CURRENT_BINARY_DIR}/JSLib.hbc ${CMAKE_CURRENT_BINARY_DIR}/JSLib.js
+  DEPENDS ${hermesc_EXE} JSLib.js
+  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+  VERBATIM
+)
+
+# X.inc is a "C array" source file containing Hermes bytecode.
+# It's built by compiling X with hermesc, and then piping that through xxd -i.
+
+add_custom_command(
+  OUTPUT InternalBytecode.inc
+  COMMAND python3 xxd.py ${CMAKE_CURRENT_BINARY_DIR}/JSLib.hbc > ${CMAKE_CURRENT_BINARY_DIR}/InternalBytecode.inc
+  DEPENDS xxd.py JSLib.hbc
+  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+  VERBATIM
+)
+
+set_source_files_properties(InternalBytecode.cpp
+  PROPERTIES
+  OBJECT_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/InternalBytecode.inc
+)
+
+# Allow InternalBytecode to find its .inc file
+target_include_directories(hermesInternalBytecode
+  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}
+)

lib/InternalBytecode/InternalBytecode.cpp

@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the LICENSE
+ * file in the root directory of this source tree.
+ */
+#include "hermes/InternalBytecode/InternalBytecode.h"
+
+namespace hermes {
+namespace vm {
+
+llvm::ArrayRef<uint8_t> getInternalBytecode() {
+#ifdef HERMESVM_USE_JS_LIBRARY_IMPLEMENTATION
+  static const uint8_t InternalBytecode[] = {
+#ifdef HERMES_CMAKE_BUILD
+#include "InternalBytecode.inc"
+#else
+#include "hermes/InternalBytecode/InternalBytecode.inc"
+#endif
+  };
+
+  return llvm::makeArrayRef(InternalBytecode, sizeof(InternalBytecode));
+#else
+  return llvm::ArrayRef<uint8_t>{};
+#endif
+}
+} // namespace vm
+} // namespace hermes

lib/InternalBytecode/Number.js

@@ -0,0 +1,23 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// This source code is licensed under the MIT license found in the LICENSE
+// file in the root directory of this source tree.
+
+(function() {
+  'use strict';
+
+  var desc = {
+    enumerable: false,
+    writable: false,
+    configurable: false,
+    value: 9007199254740991,
+  };
+
+  // ES6.0 20.1.2.6
+  Object.defineProperty(Number, 'MAX_SAFE_INTEGER', desc);
+
+  desc.value = -desc.value;
+
+  // ES6.0 20.1.2.8
+  Object.defineProperty(Number, 'MIN_SAFE_INTEGER', desc);
+})();

lib/InternalBytecode/xxd.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the LICENSE
+# file in the root directory of this source tree.
+
+"""
+This module exists so that a Windows build can have access to functionality
+like xxd -i
+"""
+
+import argparse
+from os import path
+
+
+# This is about 80 characters long (4 characters per byte + comma + space)
+BYTES_PER_LINE = 12
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("file")
+    args = parser.parse_args()
+
+    # Ensure the file exists before writing out anything
+    if not path.exists(args.file):
+        raise Exception(f'File "{args.file}" doesn\'t exist')
+
+    with open(args.file, "rb") as f:
+        # Could read in chunks instead for extra performance, but this script
+        # isn't meant to be used on gigantic files.
+        file_as_bytes = f.read()
+
+    # Make groups of bytes that fit in a single line
+    lines = [
+        file_as_bytes[i : i + BYTES_PER_LINE]
+        for i in range(0, len(file_as_bytes), BYTES_PER_LINE)
+    ]
+    print(",\n".join(", ".join("0x{:02x}".format(b) for b in l) for l in lines))
+
+
+if __name__ == "__main__":
+    main()

lib/VM/CMakeLists.txt

@@ -180,6 +180,7 @@ add_llvm_library(hermesVMRuntime STATIC ${source_files}
     hermesParser
     hermesPlatform
     hermesHBCBackend
+    hermesInternalBytecode
     dtoa
     ${CORE_FOUNDATION}
 )

lib/VM/Debugger/Debugger.cpp

@@ -1056,7 +1056,15 @@ bool Debugger::resolveBreakpointLocation(Breakpoint &breakpoint) const {
   }
 #endif
 
-  for (auto &runtimeModule : runtime_->getRuntimeModules()) {
+  // Iterate backwards through runtime modules, under the assumption that
+  // modules at the end of the list were added more recently, and are more
+  // likely to match the user's intention.
+  // Specifically, this will check any user source before runtime modules loaded
+  // by the VM.
+  for (auto it = runtime_->getRuntimeModules().rbegin();
+       it != runtime_->getRuntimeModules().rend();
+       ++it) {
+    auto &runtimeModule = *it;
     GCScope gcScope{runtime_};
 
     if (!runtimeModule.isInitialized()) {

lib/VM/JSLib/Number.cpp

@@ -116,13 +116,15 @@ Handle<JSObject> createNumberConstructor(Runtime *runtime) {
   setNumberValueProperty(
       Predefined::getSymbolID(Predefined::EPSILON),
       std::numeric_limits<double>::epsilon());
+#ifndef HERMESVM_USE_JS_LIBRARY_IMPLEMENTATION
   // ES6.0 20.1.2.6
   constexpr double kMaxSafeInteger = 9007199254740991;
   setNumberValueProperty(
       Predefined::getSymbolID(Predefined::MAX_SAFE_INTEGER), kMaxSafeInteger);
   // ES6.0 20.1.2.8
   setNumberValueProperty(
       Predefined::getSymbolID(Predefined::MIN_SAFE_INTEGER), -kMaxSafeInteger);
+#endif
 
   defineMethod(
       runtime,
@@ -162,7 +164,6 @@ Handle<JSObject> createNumberConstructor(Runtime *runtime) {
       cons,
       Predefined::getSymbolID(Predefined::parseFloat),
       Handle<>(&runtime->parseFloatFunction));
-
   return cons;
 }