add NodeString, define NodeKind as 32-bit

Summary:
Changes needed for the Juno Rust API:
- Add new NodeLabel type NodeString. It is the same data type, but it
  is used for things like JS strings, which are not guaranteed to be
  valid Unicode. This is needed since Rust requires its strings to be
  valid utf-8, so we can't represent NodeString as a Rust string. Our
  Rust API generator knows the differences and does the right thing.
- Define NodeKind to be a fixed size (32-bit) for cleaner FFI
  compatibility with Rust.

Reviewed By: avp

Differential Revision: D30124042

fbshipit-source-id: 435086208c69aa6e5dca27e966741c47759d2924

include/hermes/AST/ESTree.def

@@ -175,7 +175,7 @@ ESTREE_NODE_2_ARGS(
     NodePtr,
     expression,
     false,
-    NodeLabel,
+    NodeString,
     directive,
     true)
 
@@ -209,7 +209,7 @@ ESTREE_LAST(Statement)
 
 ESTREE_NODE_0_ARGS(NullLiteral, Base)
 ESTREE_NODE_1_ARGS(BooleanLiteral, Base, NodeBoolean, value, false)
-ESTREE_NODE_1_ARGS(StringLiteral, Base, NodeLabel, value, false)
+ESTREE_NODE_1_ARGS(StringLiteral, Base, NodeString, value, false)
 ESTREE_NODE_1_ARGS(NumericLiteral, Base, NodeNumber, value, false)
 ESTREE_NODE_2_ARGS(
     RegExpLiteral,
@@ -362,7 +362,7 @@ ESTREE_NODE_3_ARGS(
     false)
 
 ESTREE_NODE_1_ARGS(Directive, Base, NodePtr, value, false)
-ESTREE_NODE_1_ARGS(DirectiveLiteral, Base, NodeLabel, value, false)
+ESTREE_NODE_1_ARGS(DirectiveLiteral, Base, NodeString, value, false)
 
 ESTREE_NODE_3_ARGS(
     Identifier, Base,
@@ -441,8 +441,8 @@ ESTREE_NODE_2_ARGS(TaggedTemplateExpression, Base,
 // cooked will be null.
 ESTREE_NODE_3_ARGS(TemplateElement, Base,
                    NodeBoolean, tail, false,
-                   NodeLabel, cooked, true,
-                   NodeLabel, raw, false)
+                   NodeString, cooked, true,
+                   NodeString, raw, false)
 
 ESTREE_NODE_6_ARGS(
     Property, Base,
@@ -645,8 +645,8 @@ ESTREE_NODE_1_ARGS(
 
 ESTREE_NODE_2_ARGS(
     JSXText, Base,
-    NodeLabel, value, false,
-    NodeLabel, raw, false)
+    NodeString, value, false,
+    NodeString, raw, false)
 
 ESTREE_NODE_3_ARGS(
     JSXElement, Base,
@@ -678,7 +678,7 @@ ESTREE_NODE_0_ARGS(StringTypeAnnotation, Base)
 ESTREE_NODE_0_ARGS(NumberTypeAnnotation, Base)
 ESTREE_NODE_1_ARGS(
     StringLiteralTypeAnnotation, Base,
-    NodeLabel, value, false)
+    NodeString, value, false)
 ESTREE_NODE_2_ARGS(
     NumberLiteralTypeAnnotation, Base,
     NodeNumber, value, false,

include/hermes/AST/ESTree.h

@@ -35,13 +35,24 @@ using llvh::SMLoc;
 using llvh::SMRange;
 
 class Node;
+/// This is a string which is guaranteed to contain only valid Unicode
+/// characters when decoded. In particular no mismatched surrogate pairs.
+/// It is encoded with our "modified" utf-8 encoding, where parts of surrogate
+/// pairs are encoded as separate characters. So, it does NOT represent valid
+/// utf-8. To turn it into valid utf-8 it must be reencoded.
 using NodeLabel = UniqueString *;
+/// This is a JS string, which is a sequence of arbitrary 16-bit values, which
+/// may or may not represent a valid utf-16 string.
+/// It is encoded with our "modified" utf-8 encoding, where each separate 16-bit
+/// value is encoded as a separate character. There are no guarantees about the
+/// validity.
+using NodeString = UniqueString *;
 using NodeBoolean = bool;
 using NodeNumber = double;
 using NodePtr = Node *;
 using NodeList = llvh::simple_ilist<Node>;
 
-enum class NodeKind {
+enum class NodeKind : uint32_t {
 #define ESTREE_FIRST(NAME, ...) _##NAME##_First,
 #define ESTREE_LAST(NAME) _##NAME##_Last,
 #define ESTREE_NODE_0_ARGS(NAME, ...) NAME,
@@ -252,6 +263,7 @@ class FunctionLikeDecoration {
   /// Whether this function was a method definition rather than using
   /// 'function'. Note that getters and setters are also considered method
   /// definitions, as they do not use the keyword 'function'.
+  /// This is used for lazy reparsing of the function.
   bool isMethodDefinition{false};
 
   void setSemInfo(sem::FunctionInfo *semInfo) {

lib/AST/ASTBuilder.cpp

@@ -62,6 +62,12 @@ class ASTBuilder {
       const parser::JSONObject *jsObj,
       StringRef name,
       UniqueString *&result);
+  bool extractNodeString(
+      const parser::JSONObject *jsObj,
+      StringRef name,
+      UniqueString *&result) {
+    return extractNodeLabel(jsObj, name, result);
+  }
   bool extractNodeBoolean(
       const parser::JSONObject *jsObj,
       StringRef name,

tools/hermes-parser/js/scripts/templates/HermesESLintVisitorKeys.template

@@ -7,6 +7,7 @@
 
 // String, boolean, and number children are leaf nodes in AST and do not need to be visited
 #define NodeLabel(ARG)
+#define NodeString(ARG)
 #define NodeBoolean(ARG)
 #define NodeNumber(ARG)
 

tools/hermes-parser/js/scripts/templates/HermesParserNodeDeserializers.template

@@ -6,6 +6,7 @@
  */
 
 #define NodeLabel this.deserializeString()
+#define NodeString this.deserializeString()
 #define NodeBoolean this.deserializeBoolean()
 #define NodeNumber this.deserializeNumber()
 #define NodePtr this.deserializeNode()

tools/hermes-parser/js/scripts/templates/HermesParserVisitorKeys.template

@@ -7,6 +7,7 @@
 
 // String, boolean, and number children are leaf nodes in AST and do not need to be visited
 #define NodeLabel(ARG)
+#define NodeString(ARG)
 #define NodeBoolean(ARG)
 #define NodeNumber(ARG)