Compute source visibility from directives

Summary:
This diff introduced the notion of "source visibility" and
1. add it to ESTree FunctionLikeNode
2. compute the visibility in semantic validator according to
   the override rule.

This is the fundation to implemetn the Stage2 proposal
"Function Implementation Hiding":
https://github.com/tc39/proposal-function-implementation-hiding,
as well as the `'show source'` extension that Hermes is proposing
to accomendate the needs of preserving source for `toString` at
facebook#114

Note that previously by computing the visibility in semantic
validation phase, we can scale this to support external AST
and lazy compilation properly.

Reviewed By: avp

Differential Revision: D26269664

fbshipit-source-id: d4eba4a78c0e41c0cd7d2320fe2a469ba7928d95

include/hermes/AST/Context.h

@@ -93,6 +93,30 @@ enum class ParseFlowSetting {
   UNAMBIGUOUS,
 };
 
+/// An enum to track the "source visibility" of functions. This notion is coined
+/// to implement "directives" such as 'hide source' and 'sensitive' defined by
+/// https://github.com/tc39/proposal-function-implementation-hiding, as well as
+/// 'show source' Hermes proposed to explicitly preserve source for `toString`.
+///
+/// Members are ordered in an increasingly stronger manner, where only later
+/// source visibility can override the earlier but not vice versa.
+enum class SourceVisibility {
+  /// The implementation-default behavior, e.g. `toString` prints
+  /// `{ [bytecode] }` in Hermes.
+  Default,
+
+  /// Enforce the source code text to be available for the `toString` use.
+  ShowSource,
+
+  /// Enforce to have the syntax of NativeFunction, e.g. `toString` prints
+  /// `{ [native code] }`.
+  HideSource,
+
+  /// Considered security-sensitive, e.g. `toString` printed as NativeFunction;
+  /// hidden from error stack trace to protect from leaking its existence.
+  Sensitive,
+};
+
 /// Holds shared dependencies and state.
 class Context {
  public:

include/hermes/AST/ESTree.h

@@ -247,9 +247,11 @@ class FunctionLikeDecoration {
 
  public:
   Strictness strictness{Strictness::NotSet};
-  /// Whether this function was a method definiton rather than using 'function'.
-  /// Note that getters and setters are also considered method definitions,
-  /// as they do not use the keyword 'function'.
+  SourceVisibility sourceVisibility{SourceVisibility::Default};
+
+  /// Whether this function was a method definition rather than using
+  /// 'function'. Note that getters and setters are also considered method
+  /// definitions, as they do not use the keyword 'function'.
   bool isMethodDefinition{false};
 
   void setSemInfo(sem::FunctionInfo *semInfo) {

lib/AST/SemanticValidator.cpp

@@ -36,6 +36,12 @@ Keywords::Keywords(Context &astContext)
       identThis(astContext.getIdentifier("this").getUnderlyingPointer()),
       identUseStrict(
           astContext.getIdentifier("use strict").getUnderlyingPointer()),
+      identShowSource(
+          astContext.getIdentifier("show source").getUnderlyingPointer()),
+      identHideSource(
+          astContext.getIdentifier("hide source").getUnderlyingPointer()),
+      identSensitive(
+          astContext.getIdentifier("sensitive").getUnderlyingPointer()),
       identVar(astContext.getIdentifier("var").getUnderlyingPointer()),
       identLet(astContext.getIdentifier("let").getUnderlyingPointer()),
       identConst(astContext.getIdentifier("const").getUnderlyingPointer()) {}
@@ -73,7 +79,7 @@ void SemanticValidator::visit(ProgramNode *node) {
   FunctionContext newFuncCtx{this, astContext_.isStrictMode(), node};
 
   scanDirectivePrologue(node->_body);
-  updateNodeStrictness(node);
+  setDirectiveDerivedInfo(node);
 
   visitESTreeChildren(*this, node);
 }
@@ -619,7 +625,11 @@ void SemanticValidator::visitFunction(
     NodeList &params,
     Node *body) {
   FunctionContext newFuncCtx{
-      this, haveActiveContext() && curFunction()->strictMode, node};
+      this,
+      haveActiveContext() && curFunction()->strictMode,
+      node,
+      haveActiveContext() ? curFunction()->sourceVisibility
+                          : SourceVisibility::Default};
 
   // It is a Syntax Error if UniqueFormalParameters Contains YieldExpression
   // is true.
@@ -640,7 +650,7 @@ void SemanticValidator::visitFunction(
     } else {
       useStrictNode = scanDirectivePrologue(bodyNode->_body);
     }
-    updateNodeStrictness(node);
+    setDirectiveDerivedInfo(node);
   }
 
   if (id)
@@ -737,6 +747,13 @@ void SemanticValidator::visitParamsAndBody(FunctionLikeNode *node) {
   }
 }
 
+void SemanticValidator::tryOverrideSourceVisibility(
+    SourceVisibility newSourceVisibility) {
+  if (newSourceVisibility > curFunction()->sourceVisibility) {
+    curFunction()->sourceVisibility = newSourceVisibility;
+  }
+}
+
 Node *SemanticValidator::scanDirectivePrologue(NodeList &body) {
   Node *result = nullptr;
   for (auto &nodeRef : body) {
@@ -751,6 +768,15 @@ Node *SemanticValidator::scanDirectivePrologue(NodeList &body) {
       if (!result)
         result = &nodeRef;
     }
+    if (directive == kw_.identShowSource) {
+      tryOverrideSourceVisibility(SourceVisibility::ShowSource);
+    }
+    if (directive == kw_.identHideSource) {
+      tryOverrideSourceVisibility(SourceVisibility::HideSource);
+    }
+    if (directive == kw_.identSensitive) {
+      tryOverrideSourceVisibility(SourceVisibility::Sensitive);
+    }
   }
 
   return result;
@@ -894,9 +920,9 @@ void SemanticValidator::validateAssignmentTarget(const Node *node) {
   sm_.error(node->getSourceRange(), "invalid assignment left-hand side");
 }
 
-void SemanticValidator::updateNodeStrictness(FunctionLikeNode *node) {
-  auto strictness = ESTree::makeStrictness(curFunction()->strictMode);
-  node->strictness = strictness;
+void SemanticValidator::setDirectiveDerivedInfo(FunctionLikeNode *node) {
+  node->strictness = ESTree::makeStrictness(curFunction()->strictMode);
+  node->sourceVisibility = curFunction()->sourceVisibility;
 }
 
 LabelDecorationBase *SemanticValidator::getLabelDecorationBase(
@@ -926,11 +952,13 @@ void SemanticValidator::recursionDepthExceeded(Node *n) {
 FunctionContext::FunctionContext(
     SemanticValidator *validator,
     bool strictMode,
-    FunctionLikeNode *node)
+    FunctionLikeNode *node,
+    SourceVisibility sourceVisibility)
     : validator_(validator),
       oldContextValue_(validator->funcCtx_),
       semInfo(validator->semCtx_.createFunction()),
-      strictMode(strictMode) {
+      strictMode(strictMode),
+      sourceVisibility(sourceVisibility) {
   validator->funcCtx_ = this;
 
   if (node)

lib/AST/SemanticValidator.h

@@ -36,6 +36,12 @@ class Keywords {
   const UniqueString *const identThis;
   /// Identifier for "use strict".
   const UniqueString *const identUseStrict;
+  /// Identifier for "show source ".
+  const UniqueString *const identShowSource;
+  /// Identifier for "hide source ".
+  const UniqueString *const identHideSource;
+  /// Identifier for "sensitive".
+  const UniqueString *const identSensitive;
   /// Identifier for "var".
   const UniqueString *const identVar;
   /// Identifier for "let".
@@ -228,8 +234,7 @@ class SemanticValidator {
   /// Scan a list of directives in the beginning of a program or function
   /// (see ES5.1 4.1 - a directive is a statement consisting of a single
   /// string literal).
-  /// Update the flags in the function context to reflect the directives. (We
-  /// currently only recognize "use strict".)
+  /// Update the flags in the function context to reflect the directives.
   /// \return the node containing "use strict" or nullptr.
   Node *scanDirectivePrologue(NodeList &body);
 
@@ -255,8 +260,15 @@ class SemanticValidator {
   /// (used by elision).
   void validateAssignmentTarget(const Node *node);
 
-  /// Set the strictness of a function-like node to the current strictness.
-  void updateNodeStrictness(FunctionLikeNode *node);
+  /// Set directives derived information (e.g. strictness, source visibility)
+  /// to a function-like node.
+  /// Data is retrieved from \c curFunction().
+  void setDirectiveDerivedInfo(FunctionLikeNode *node);
+
+  /// Called when the any of the source visibility directives are seen.
+  /// Only a stronger source visibility from inner function scope can override
+  /// the current source visibility set by outer function scope.
+  void tryOverrideSourceVisibility(SourceVisibility newSourceVisibility);
 
   /// Get the LabelDecorationBase depending on the node type.
   static LabelDecorationBase *getLabelDecorationBase(StatementNode *node);
@@ -297,14 +309,17 @@ class FunctionContext {
   StatementNode *activeSwitchOrLoop = nullptr;
   /// Is this function in strict mode.
   bool strictMode = false;
+  /// Source visibility of this function.
+  SourceVisibility sourceVisibility{SourceVisibility::Default};
 
   /// The currently active labels in the function.
   llvh::DenseMap<NodeLabel, Label> labelMap;
 
   explicit FunctionContext(
       SemanticValidator *validator,
       bool strictMode,
-      FunctionLikeNode *node);
+      FunctionLikeNode *node,
+      SourceVisibility sourceVisibility = SourceVisibility::Default);
 
   ~FunctionContext();
 

lib/Parser/JSParserImpl.cpp

@@ -54,9 +54,12 @@ void JSParserImpl::initializeIdentifiers() {
   setIdent_ = lexer_.getIdentifier("set");
   initIdent_ = lexer_.getIdentifier("init");
   useStrictIdent_ = lexer_.getIdentifier("use strict");
+  showSourceIdent_ = lexer_.getIdentifier("show source");
+  hideSourceIdent_ = lexer_.getIdentifier("hide source");
+  sensitiveIdent_ = lexer_.getIdentifier("sensitive");
+  useStaticBuiltinIdent_ = lexer_.getIdentifier("use static builtin");
   letIdent_ = lexer_.getIdentifier("let");
   ofIdent_ = lexer_.getIdentifier("of");
-  useStaticBuiltinIdent_ = lexer_.getIdentifier("use static builtin");
   fromIdent_ = lexer_.getIdentifier("from");
   asIdent_ = lexer_.getIdentifier("as");
   implementsIdent_ = lexer_.getIdentifier("implements");

lib/Parser/JSParserImpl.h

@@ -244,9 +244,12 @@ class JSParserImpl {
   UniqueString *setIdent_;
   UniqueString *initIdent_;
   UniqueString *useStrictIdent_;
+  UniqueString *showSourceIdent_;
+  UniqueString *hideSourceIdent_;
+  UniqueString *sensitiveIdent_;
+  UniqueString *useStaticBuiltinIdent_;
   UniqueString *letIdent_;
   UniqueString *ofIdent_;
-  UniqueString *useStaticBuiltinIdent_;
   UniqueString *fromIdent_;
   UniqueString *asIdent_;
   UniqueString *implementsIdent_;

unittests/AST/ValidatorTest.cpp

@@ -147,4 +147,140 @@ TEST(ValidatorTest, NamedBreakLabelTest) {
   ASSERT_FALSE(validateAST(ctx, semCtx, *parsed));
   ASSERT_EQ(2, diag.getErrCountClear());
 }
+
+void assertFunctionLikeSourceVisibility(
+    llvh::Optional<ESTree::FunctionLikeNode *> funcLikeNode,
+    SourceVisibility sourceVisibility) {
+  ASSERT_TRUE(funcLikeNode.hasValue());
+  ASSERT_EQ((*funcLikeNode)->sourceVisibility, sourceVisibility);
+}
+
+void assertFirstNodeAsFunctionLikeWithSourceVisibility(
+    llvh::Optional<ESTree::ProgramNode *> parsed,
+    SourceVisibility sourceVisibility) {
+  ASSERT_TRUE(parsed.hasValue());
+  auto *programNode = llvh::cast<ESTree::ProgramNode>(parsed.getValue());
+  ASSERT_TRUE(llvh::isa<ESTree::FunctionLikeNode>(programNode->_body.front()));
+  auto *funcLikeNode =
+      llvh::cast<ESTree::FunctionLikeNode>(&programNode->_body.front());
+
+  ASSERT_EQ(funcLikeNode->sourceVisibility, sourceVisibility);
+}
+
+void assertSecondNodeAsFunctionLikeWithSourceVisibility(
+    llvh::Optional<ESTree::ProgramNode *> parsed,
+    SourceVisibility sourceVisibility) {
+  ASSERT_TRUE(parsed.hasValue());
+  auto *programNode = llvh::cast<ESTree::ProgramNode>(parsed.getValue());
+  auto it = programNode->_body.begin();
+  // Step to the 2nd node.
+  it++;
+  ASSERT_TRUE(llvh::isa<ESTree::FunctionLikeNode>(*it));
+  auto *funcLikeNode = llvh::cast<ESTree::FunctionLikeNode>(it);
+
+  ASSERT_EQ(funcLikeNode->sourceVisibility, sourceVisibility);
+}
+
+TEST(ValidatorTest, SourceVisibilityTest) {
+  Context context;
+  sem::SemContext semCtx{};
+  // Top-level program node.
+  {
+    JSParser parser(context, "");
+    auto parsed = parser.parse();
+    validateAST(context, semCtx, *parsed);
+    assertFunctionLikeSourceVisibility(*parsed, SourceVisibility::Default);
+  }
+  {
+    JSParser parser(context, "'show source'");
+    auto parsed = parser.parse();
+    // source visibility is set to default before semantic validation.
+    assertFunctionLikeSourceVisibility(*parsed, SourceVisibility::Default);
+    validateAST(context, semCtx, *parsed);
+    // source visibility is correctly updated after semantic validation.
+    assertFunctionLikeSourceVisibility(*parsed, SourceVisibility::ShowSource);
+  }
+  // Singleton function node.
+  {
+    JSParser parser(context, "function func (a, b) { return 10 }");
+    auto parsed = parser.parse();
+    validateAST(context, semCtx, *parsed);
+    assertFirstNodeAsFunctionLikeWithSourceVisibility(
+        parsed, SourceVisibility::Default);
+  }
+  {
+    JSParser parser(context, "function func (a, b) { 'sensitive'; return 10 }");
+    auto parsed = parser.parse();
+    validateAST(context, semCtx, *parsed);
+    assertFirstNodeAsFunctionLikeWithSourceVisibility(
+        parsed, SourceVisibility::Sensitive);
+  }
+  {
+    JSParser parser(
+        context, "function func (a, b) { 'hide source'; return 10 }");
+    auto parsed = parser.parse();
+    validateAST(context, semCtx, *parsed);
+    assertFirstNodeAsFunctionLikeWithSourceVisibility(
+        parsed, SourceVisibility::HideSource);
+  }
+  {
+    JSParser parser(
+        context, "function func (a, b) { 'show source'; return 10 }");
+    auto parsed = parser.parse();
+    validateAST(context, semCtx, *parsed);
+    assertFirstNodeAsFunctionLikeWithSourceVisibility(
+        parsed, SourceVisibility::ShowSource);
+  }
+  // Visibility is correctly restored.
+  {
+    JSParser parser(context, "function foo(x) { 'sensitive' }function bar(){}");
+    auto parsed = parser.parse();
+    validateAST(context, semCtx, *parsed);
+    assertFirstNodeAsFunctionLikeWithSourceVisibility(
+        parsed, SourceVisibility::Sensitive);
+    assertSecondNodeAsFunctionLikeWithSourceVisibility(
+        parsed, SourceVisibility::Default);
+  }
+  // Overriding.
+  {
+    // ShowSource > Default
+    JSParser parser(
+        context, "'show source'; function func (a, b) { return 10 }");
+    auto parsed = parser.parse();
+    validateAST(context, semCtx, *parsed);
+    assertSecondNodeAsFunctionLikeWithSourceVisibility(
+        parsed, SourceVisibility::ShowSource);
+  }
+  {
+    // HideSource > ShowSource
+    JSParser parser(
+        context,
+        "'show source'; function func (a, b) { 'hide source'; return 10 }");
+    auto parsed = parser.parse();
+    validateAST(context, semCtx, *parsed);
+    assertSecondNodeAsFunctionLikeWithSourceVisibility(
+        parsed, SourceVisibility::HideSource);
+  }
+  {
+    // ShowSource < HideSource
+    JSParser parser(
+        context,
+        "'hide source'; function func (a, b) { 'show source'; return 10 }");
+    auto parsed = parser.parse();
+    validateAST(context, semCtx, *parsed);
+    assertSecondNodeAsFunctionLikeWithSourceVisibility(
+        parsed, SourceVisibility::HideSource);
+  }
+  {
+    // Sensitive > HideSource
+    JSParser parser(
+        context,
+        "'hide source'; function func (a, b) { 'sensitive'; return 10 }");
+    auto parsed = parser.parse();
+    validateAST(context, semCtx, *parsed);
+    assertSecondNodeAsFunctionLikeWithSourceVisibility(
+        parsed, SourceVisibility::Sensitive);
+  }
+}
+
 } // anonymous namespace