Replace the deprecated esprima-fb parser with flow-parser, on the RN …

…website

Summary:
(I changed a ton from when I previously submitted this PR so please take another look if you already did.)

PROBLEM: the no-longer-maintained `esprima-fb` parser does not support class properties, leading our website docgen to die if we use class properties, which we're gonna do real soon now
SOLUTION: use `flow-parser` instead, which the flow team is maintaining including all the fancy-pants ES? stuff that FB uses internally.

This removes the `esprima-fb` parser from jsdocs and replaces it with `flow-parser`. It's almost the same, I checked by diffing all the parser json output and it only had a few irrelevant differences. I had to add a file of constants so that we could remove esprima-fb altogether, too.

This also adds a couple unit tests, so that we can test that jsDocs works programmatically. They don't run if you run the regular RN tests, you have to run `npm test` from the `/website/` subdirectory.
Closes facebook#9890

Differential Revision: D3865629

Pulled By: bestander

fbshipit-source-id: 8f561b78ca4a02f3f7b45e55904ec2fa911e3bb6

circle.yml

@@ -79,7 +79,8 @@ test:
     # Android e2e test
     - source scripts/circle-ci-android-setup.sh && retry3 node ./scripts/run-ci-e2e-tests.js --android --js --retries 2
 
-    # testing docs generation is not broken
+    # testing docs generation
+    - cd website && npm test
     - cd website && node ./server/generate.js
   post:
     # copy test report for Circle CI to display

package.json

@@ -24,6 +24,7 @@
     },
     "testPathIgnorePatterns": [
       "/node_modules/",
+      "/website/",
       "local-cli/generator/templates/"
     ],
     "haste": {

website/jsdocs/TypeExpressionParser.js

@@ -10,7 +10,7 @@
 /*global exports:true*/
 'use strict';
 
-var Syntax = require('esprima-fb').Syntax;
+var Syntax = require('./syntax');
 
 function toObject(/*array*/ array) /*object*/ {
   var object = {};

website/jsdocs/__tests__/jsdocs-test.js

@@ -0,0 +1,39 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ *
+ */
+'use strict';
+
+const fs = require('fs');
+const jsDocs = require('../jsdocs');
+const libs = __dirname + '/../../../Libraries/';
+
+function checkWeCanParse(library) {
+  let path = libs + library;
+  let code = fs.readFileSync(path).toString();
+  let json = jsDocs(code);
+  expect(json).toBeTruthy();
+}
+
+describe('parseSource', () => {
+  it('should parse Vibration.js', () => {
+    checkWeCanParse('Vibration/Vibration.js');
+  });
+
+  it('should parse AsyncStorage.js', () => {
+    checkWeCanParse('Storage/AsyncStorage.js');
+  });
+
+  it('should not parse invalid code', () => {
+    let code = `
+    for x in range(10):
+      print 'oops this isnt python'
+    `;
+    expect(jsDocs('fakepath', code)).toBeFalsy();
+  });
+});

website/jsdocs/findExportDefinition.js

@@ -10,8 +10,7 @@
 /*jslint node: true */
 'use strict';
 
-var esprima = require('esprima-fb');
-var Syntax = esprima.Syntax;
+var Syntax = require('./syntax');
 var traverseFlat = require('./traverseFlat');
 
 

website/jsdocs/generic-function-visitor.js

@@ -13,7 +13,7 @@
 
 var util = require('util');
 
-var Syntax = require('esprima-fb').Syntax;
+var Syntax = require('./syntax');
 var utils = require('jstransform/src/utils');
 
 // Transforms

website/jsdocs/jsdocs.js

@@ -10,9 +10,9 @@
 /*jslint node: true */
 'use strict';
 
-var esprima = require('esprima-fb');
+var flowParser = require('flow-parser');
 var fs = require('fs');
-var Syntax = esprima.Syntax;
+var Syntax = require('./syntax');
 
 var findExportDefinition = require('./findExportDefinition');
 var genericTransform = require('./generic-function-visitor');
@@ -76,7 +76,7 @@ function stripStaticUpstreamWarning(docblock) {
   if (!docblock) {
     return docblock;
   }
-  // Esprima strips out the starting and ending tokens, so add them back
+  // The parser strips out the starting and ending tokens, so add them back
   docblock = '/*' + docblock + '*/\n';
   return docblock;
 }
@@ -196,8 +196,8 @@ function getModuleName(commentsForFile) {
 }
 
 /**
- * Esprima includes the leading colon (and possibly spaces) as part of the
- * typehint, so we have to strip those out.
+ * The parser includes the leading colon (and possibly spaces) as part of
+ * the typehint, so we have to strip those out.
  */
 function sanitizeTypehint(string) {
   for (var i = 0; i < string.length; i++) {
@@ -229,7 +229,7 @@ function getFunctionData(
   var typechecks = commentsForFile.typechecks;
   var typehintsFromBlock = null;
   if (typechecks) {
-    // esprima has trouble with some params so ignore them (e.g. $__0)
+    // The parser has trouble with some params so ignore them (e.g. $__0)
     if (!node.params.some(function(param) { return !param.name; })) {
       try {
         typehintsFromBlock = genericTransform.getTypeHintsFromDocBlock(
@@ -497,7 +497,7 @@ function getRequireData(node) {
  */
 function parseSource(source) {
   var lines = source.split('\n');
-  var ast = esprima.parse(source, {
+  var ast = flowParser.parse(source, {
     loc: true,
     comment: true,
     range: true,

website/jsdocs/meta.js

@@ -13,7 +13,7 @@
 
 var util = require('util');
 
-var Syntax = require('esprima-fb').Syntax;
+var Syntax = require('./syntax');
 var utils = require('jstransform/src/utils');
 
 // Top level file pragmas that must not exist for the meta transform to

website/jsdocs/syntax.js

@@ -0,0 +1,121 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+ // This Syntax object defines a number of constants that flow-parser uses.
+
+module.exports = {
+  AnyTypeAnnotation: 'AnyTypeAnnotation',
+  ArrayExpression: 'ArrayExpression',
+  ArrayPattern: 'ArrayPattern',
+  ArrayTypeAnnotation: 'ArrayTypeAnnotation',
+  ArrowFunctionExpression: 'ArrowFunctionExpression',
+  AssignmentExpression: 'AssignmentExpression',
+  BinaryExpression: 'BinaryExpression',
+  BlockStatement: 'BlockStatement',
+  BooleanTypeAnnotation: 'BooleanTypeAnnotation',
+  BreakStatement: 'BreakStatement',
+  CallExpression: 'CallExpression',
+  CatchClause: 'CatchClause',
+  ClassBody: 'ClassBody',
+  ClassDeclaration: 'ClassDeclaration',
+  ClassExpression: 'ClassExpression',
+  ClassImplements: 'ClassImplements',
+  ClassProperty: 'ClassProperty',
+  ComprehensionBlock: 'ComprehensionBlock',
+  ComprehensionExpression: 'ComprehensionExpression',
+  ConditionalExpression: 'ConditionalExpression',
+  ContinueStatement: 'ContinueStatement',
+  DebuggerStatement: 'DebuggerStatement',
+  DeclareClass: 'DeclareClass',
+  DeclareFunction: 'DeclareFunction',
+  DeclareModule: 'DeclareModule',
+  DeclareVariable: 'DeclareVariable',
+  DoWhileStatement: 'DoWhileStatement',
+  EmptyStatement: 'EmptyStatement',
+  ExportDeclaration: 'ExportDeclaration',
+  ExportBatchSpecifier: 'ExportBatchSpecifier',
+  ExportSpecifier: 'ExportSpecifier',
+  ExpressionStatement: 'ExpressionStatement',
+  ForInStatement: 'ForInStatement',
+  ForOfStatement: 'ForOfStatement',
+  ForStatement: 'ForStatement',
+  FunctionDeclaration: 'FunctionDeclaration',
+  FunctionExpression: 'FunctionExpression',
+  FunctionTypeAnnotation: 'FunctionTypeAnnotation',
+  FunctionTypeParam: 'FunctionTypeParam',
+  GenericTypeAnnotation: 'GenericTypeAnnotation',
+  Identifier: 'Identifier',
+  IfStatement: 'IfStatement',
+  ImportDeclaration: 'ImportDeclaration',
+  ImportDefaultSpecifier: 'ImportDefaultSpecifier',
+  ImportNamespaceSpecifier: 'ImportNamespaceSpecifier',
+  ImportSpecifier: 'ImportSpecifier',
+  InterfaceDeclaration: 'InterfaceDeclaration',
+  InterfaceExtends: 'InterfaceExtends',
+  IntersectionTypeAnnotation: 'IntersectionTypeAnnotation',
+  LabeledStatement: 'LabeledStatement',
+  Literal: 'Literal',
+  LogicalExpression: 'LogicalExpression',
+  MemberExpression: 'MemberExpression',
+  MethodDefinition: 'MethodDefinition',
+  NewExpression: 'NewExpression',
+  NullableTypeAnnotation: 'NullableTypeAnnotation',
+  NumberTypeAnnotation: 'NumberTypeAnnotation',
+  ObjectExpression: 'ObjectExpression',
+  ObjectPattern: 'ObjectPattern',
+  ObjectTypeAnnotation: 'ObjectTypeAnnotation',
+  ObjectTypeCallProperty: 'ObjectTypeCallProperty',
+  ObjectTypeIndexer: 'ObjectTypeIndexer',
+  ObjectTypeProperty: 'ObjectTypeProperty',
+  Program: 'Program',
+  Property: 'Property',
+  QualifiedTypeIdentifier: 'QualifiedTypeIdentifier',
+  ReturnStatement: 'ReturnStatement',
+  SequenceExpression: 'SequenceExpression',
+  SpreadElement: 'SpreadElement',
+  SpreadProperty: 'SpreadProperty',
+  StringLiteralTypeAnnotation: 'StringLiteralTypeAnnotation',
+  StringTypeAnnotation: 'StringTypeAnnotation',
+  SwitchCase: 'SwitchCase',
+  SwitchStatement: 'SwitchStatement',
+  TaggedTemplateExpression: 'TaggedTemplateExpression',
+  TemplateElement: 'TemplateElement',
+  TemplateLiteral: 'TemplateLiteral',
+  ThisExpression: 'ThisExpression',
+  ThrowStatement: 'ThrowStatement',
+  TupleTypeAnnotation: 'TupleTypeAnnotation',
+  TryStatement: 'TryStatement',
+  TypeAlias: 'TypeAlias',
+  TypeAnnotation: 'TypeAnnotation',
+  TypeCastExpression: 'TypeCastExpression',
+  TypeofTypeAnnotation: 'TypeofTypeAnnotation',
+  TypeParameterDeclaration: 'TypeParameterDeclaration',
+  TypeParameterInstantiation: 'TypeParameterInstantiation',
+  UnaryExpression: 'UnaryExpression',
+  UnionTypeAnnotation: 'UnionTypeAnnotation',
+  UpdateExpression: 'UpdateExpression',
+  VariableDeclaration: 'VariableDeclaration',
+  VariableDeclarator: 'VariableDeclarator',
+  VoidTypeAnnotation: 'VoidTypeAnnotation',
+  WhileStatement: 'WhileStatement',
+  WithStatement: 'WithStatement',
+  JSXIdentifier: 'JSXIdentifier',
+  JSXNamespacedName: 'JSXNamespacedName',
+  JSXMemberExpression: 'JSXMemberExpression',
+  JSXEmptyExpression: 'JSXEmptyExpression',
+  JSXExpressionContainer: 'JSXExpressionContainer',
+  JSXElement: 'JSXElement',
+  JSXClosingElement: 'JSXClosingElement',
+  JSXOpeningElement: 'JSXOpeningElement',
+  JSXAttribute: 'JSXAttribute',
+  JSXSpreadAttribute: 'JSXSpreadAttribute',
+  JSXText: 'JSXText',
+  YieldExpression: 'YieldExpression',
+  AwaitExpression: 'AwaitExpression'
+}

website/jsdocs/traverseFlat.js

@@ -11,7 +11,7 @@
 /*jslint node:true*/
 'use strict';
 
-var Syntax = require('esprima-fb').Syntax;
+var Syntax = require('./syntax');
 
 /**
  * Executes visitor on the object and its children (recursively).

website/jsdocs/type.js

@@ -12,7 +12,7 @@
 
 var util = require('util');
 
-var Syntax = require('esprima-fb').Syntax;
+var Syntax = require('./syntax');
 var utils = require('jstransform/src/utils');
 
 var parse = require('./TypeExpressionParser').parse;

website/package.json

@@ -1,29 +1,33 @@
 {
   "scripts": {
     "start": "RN_VERSION=next node server/server.js",
+    "test": "jest",
     "gh-pages": "node publish-gh-pages.js"
   },
   "dependencies": {
+    "babel-core": "^6.6.4",
+    "babel-plugin-external-helpers": "^6.5.0",
+    "babel-polyfill": "^6.6.1",
+    "babel-preset-react-native": "~1.6.0",
+    "babel-register": "^6.6.0",
+    "babel-types": "^6.6.4",
     "bluebird": "^2.9.21",
     "connect": "2.8.3",
-    "esprima-fb": "15001.1001.0-dev-harmony-fb",
+    "deep-assign": "^2.0.0",
+    "flow-parser": "^0.32.0",
     "fs.extra": "1.3.2",
     "glob": "6.0.4",
+    "jsdoc-api": "^1.1.0",
     "jstransform": "11.0.3",
     "mkdirp": "^0.5.1",
     "optimist": "0.6.0",
     "react": "~0.13.0",
     "react-docgen": "^2.9.0",
     "react-page-middleware": "git://github.com/facebook/react-page-middleware.git",
     "request": "^2.69.0",
-    "semver-compare": "^1.0.0",
-    "babel-core": "^6.6.4",
-    "babel-plugin-external-helpers": "^6.5.0",
-    "babel-polyfill": "^6.6.1",
-    "babel-preset-react-native": "~1.6.0",
-    "babel-register": "^6.6.0",
-    "babel-types": "^6.6.4",
-    "jsdoc-api": "^1.1.0",
-    "deep-assign": "^2.0.0"
+    "semver-compare": "^1.0.0"
+  },
+  "devDependencies": {
+    "jest": "^15.1.1"
   }
 }

website/server/extractDocs.js

@@ -9,6 +9,8 @@
 
 'use strict';
 
+const assert = require('assert');
+
 const docgen = require('react-docgen');
 const docgenHelpers = require('./docgenHelpers');
 const fs = require('fs');
@@ -298,12 +300,12 @@ function parseAPIJsDocFormat(filepath, fileContent) {
   };
   // Babel transform
   const code = babel.transform(fileContent, babelRC).code;
-  // Parse via jsdocs
+  // Parse via jsdoc-api
   let jsonParsed = jsdocApi.explainSync({
     source: code,
     configure: './jsdocs/jsdoc-conf.json'
   });
-  // Cleanup jsdocs return
+  // Clean up jsdoc-api return
   jsonParsed = jsonParsed.filter(i => {
     return !i.undocumented && !/package|file/.test(i.kind);
   });
@@ -336,7 +338,7 @@ function parseAPIInferred(filepath, fileContent) {
   try {
     json = jsDocs(fileContent);
     if (!json) {
-      throw new Error('jsDocs returned falsy');
+      throw new Error('parseSource returned falsy');
     }
   } catch (e) {
     console.error('Cannot parse file', filepath, e);
@@ -595,7 +597,7 @@ const styleDocs = stylesForEmbed.reduce(function(docs, filepath) {
   return docs;
 }, {});
 
-module.exports = function() {
+function extractDocs() {
   componentCount = 0;
   return [].concat(
     components.map(renderComponent),
@@ -604,4 +606,6 @@ module.exports = function() {
     }),
     stylesWithPermalink.map(renderStyle)
   );
-};
+}
+
+module.exports = extractDocs;