Merge pull request oracle#2957 from idodeclare/feature/typescript

Feature/typescript

opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/AnalyzerGuru.java

@@ -100,6 +100,7 @@
 import org.opengrok.indexer.analysis.sql.SQLAnalyzerFactory;
 import org.opengrok.indexer.analysis.swift.SwiftAnalyzerFactory;
 import org.opengrok.indexer.analysis.tcl.TclAnalyzerFactory;
+import org.opengrok.indexer.analysis.typescript.TypeScriptAnalyzerFactory;
 import org.opengrok.indexer.analysis.uue.UuencodeAnalyzerFactory;
 import org.opengrok.indexer.analysis.vb.VBAnalyzerFactory;
 import org.opengrok.indexer.analysis.verilog.VerilogAnalyzerFactory;
@@ -294,7 +295,8 @@ public class AnalyzerGuru {
                 new AdaAnalyzerFactory(),
                 new RubyAnalyzerFactory(),
                 new EiffelAnalyzerFactory(),
-                new VerilogAnalyzerFactory()
+                new VerilogAnalyzerFactory(),
+                new TypeScriptAnalyzerFactory()
             };
 
             for (AnalyzerFactory analyzer : analyzers) {
@@ -328,15 +330,15 @@ public class AnalyzerGuru {
      * {@link FileAnalyzerFactory} subclasses are revised to target more or
      * different files.
      * @return a value whose lower 32-bits are a static value
-     * 20190211_00
+     * 20191006_00
      * for the current implementation and whose higher-32 bits are non-zero if
      * {@link #addExtension(java.lang.String, AnalyzerFactory)}
      * or
      * {@link #addPrefix(java.lang.String, AnalyzerFactory)}
      * has been called.
      */
     public static long getVersionNo() {
-        final int ver32 = 20190211_00; // Edit comment above too!
+        final int ver32 = 20191006_00; // Edit comment above too!
         long ver = ver32;
         if (customizationHashCode != 0) {
             ver |= (long) customizationHashCode << 32;

opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/Ctags.java

@@ -19,7 +19,7 @@
 
  /*
  * Copyright (c) 2005, 2018, Oracle and/or its affiliates. All rights reserved.
- * Portions Copyright (c) 2017-2018, Chris Fraire <[email protected]>.
+ * Portions Copyright (c) 2017-2019, Chris Fraire <[email protected]>.
  */
 package org.opengrok.indexer.analysis;
 
@@ -137,9 +137,6 @@ private void initialize() throws IOException {
         command.add(binary);
         command.add("--c-kinds=+l");
 
-        command.add("--langmap=clojure:+.cljs");
-        command.add("--langmap=clojure:+.cljx");
-
         // Workaround for bug #14924: Don't get local variables in Java
         // code since that creates many false positives.
         // CtagsTest : bug14924 "too many methods" guards for this
@@ -163,8 +160,6 @@ private void initialize() throws IOException {
         command.add("--langmap=sql:+.pkb"); // # 1763
         command.add("--langmap=sql:+.pck"); // # 1763
 
-        command.add("--langmap=javascript:+.ts");
-
         //Ideally all below should be in ctags, or in outside config file,
         //we might run out of command line SOON
         //Also note, that below ctags definitions HAVE to be in POSIX
@@ -319,6 +314,8 @@ private void addKotlinSupport(List<String> command) {
     private void addClojureSupport(List<String> command) {
         command.add("--langdef=clojure"); // clojure support (patterns are from https://gist.github.com/kul/8704283)
         command.add("--langmap=clojure:+.clj");
+        command.add("--langmap=clojure:+.cljs");
+        command.add("--langmap=clojure:+.cljx");
 
         command.add("--regex-clojure=/\\([[:space:]]*create-ns[[:space:]]+([-[:alnum:]*+!_:\\/.?]+)/\\1/n,namespace/");
         command.add("--regex-clojure=/\\([[:space:]]*def[[:space:]]+([-[:alnum:]*+!_:\\/.?]+)/\\1/d,definition/");

opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/javascript/Consts.java

@@ -19,24 +19,26 @@
 
 /*
  * Copyright (c) 2006, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Portions Copyright (c) 2019, Chris Fraire <[email protected]>.
  */
 
 package org.opengrok.indexer.analysis.javascript;
 
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
 
 /**
- * Holds static hash set containing the JavaScript keywords.
- * 
- * ECMA-262 5.1 Edition June 2011
+ * Holds JavaScript keywords from ECMA-262 10th Edition, June 2019.
  */
-//TODO update to latest ecmascript ... (for angular support)
 public class Consts {
 
-    public static final Set<String> kwd = new HashSet<>();
+    private static final Set<String> kwd = new HashSet<>();
+
+    public static final Set<String> KEYWORDS = Collections.unmodifiableSet(kwd);
+
     static {
-        //constants
+        // literals
         kwd.add("true");
         kwd.add("false");
         kwd.add("null");
@@ -45,29 +47,37 @@ public class Consts {
         kwd.add("Boolean");
         kwd.add("Date");
         kwd.add("Function");
+        kwd.add("Infinity"); // ECMA-262, 10th edition, June 2019
         kwd.add("Math");
         kwd.add("Number");
         kwd.add("Object");
         kwd.add("RegExp");
         kwd.add("String");                
         //keywords
+        kwd.add("await"); // ECMA-262, 10th edition, June 2019
         kwd.add("break");
         kwd.add("case");
         kwd.add("catch");
+        kwd.add("class");
+        kwd.add("const");
         kwd.add("continue");
         kwd.add("debugger");
         kwd.add("default");
         kwd.add("delete");
         kwd.add("do");
         kwd.add("else");
+        kwd.add("export");
+        kwd.add("extends");
         kwd.add("finally");
         kwd.add("for");
         kwd.add("function");
         kwd.add("if");
         kwd.add("in");
         kwd.add("instanceof");
+        kwd.add("import");
         kwd.add("new");
         kwd.add("return");
+        kwd.add("super");
         kwd.add("switch");
         kwd.add("this");
         kwd.add("throw");
@@ -77,14 +87,9 @@ public class Consts {
         kwd.add("void");
         kwd.add("while");
         kwd.add("with");
+        kwd.add("yield");
         //future reserved
-        kwd.add("class");
-        kwd.add("const");
         kwd.add("enum");
-        kwd.add("export");
-        kwd.add("extends");
-        kwd.add("import");
-        kwd.add("super");
         //strict future reserved
         kwd.add("implements");
         kwd.add("interface");
@@ -94,10 +99,8 @@ public class Consts {
         kwd.add("protected");
         kwd.add("public");
         kwd.add("static");
-        kwd.add("yield");
     }
 
-    private Consts() {
+    protected Consts() {
     }
-
 }

opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/javascript/JavaScriptAnalyzer.java

@@ -19,7 +19,7 @@
 
 /*
  * Copyright (c) 2006, 2019, Oracle and/or its affiliates. All rights reserved.
- * Portions Copyright (c) 2017-2018, Chris Fraire <[email protected]>.
+ * Portions Copyright (c) 2017-2019, Chris Fraire <[email protected]>.
  */
 package org.opengrok.indexer.analysis.javascript;
 
@@ -50,11 +50,11 @@ protected JavaScriptAnalyzer(AnalyzerFactory factory) {
      * Gets a version number to be used to tag processed documents so that
      * re-analysis can be re-done later if a stored version number is different
      * from the current implementation.
-     * @return 20190217_00
+     * @return 20191006_00
      */
     @Override
     protected int getSpecializedVersionNo() {
-        return 20190217_00; // Edit comment above too!
+        return 20191006_00; // Edit comment above too!
     }
 
     /**

opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/javascript/JavaScriptAnalyzerFactory.java

@@ -19,7 +19,7 @@
 
 /*
  * Copyright (c) 2007, 2018, Oracle and/or its affiliates. All rights reserved.
- * Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
+ * Portions Copyright (c) 2017, 2019, Chris Fraire <[email protected]>.
  */
 
 package org.opengrok.indexer.analysis.javascript;
@@ -31,10 +31,7 @@ public class JavaScriptAnalyzerFactory extends FileAnalyzerFactory {
 
     private static final String name = "JavaScript";
 
-    private static final String[] SUFFIXES = {
-        "JS",
-        "TS"
-    };
+    private static final String[] SUFFIXES = {"JS"};
 
     public JavaScriptAnalyzerFactory() {
         super(null, null, SUFFIXES, null, null, "text/plain", AbstractAnalyzer.Genre.PLAIN, name);

opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/javascript/JavaScriptLexer.java

@@ -0,0 +1,146 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * See LICENSE.txt included in this distribution for the specific
+ * language governing permissions and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at LICENSE.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2017, 2019, Chris Fraire <[email protected]>.
+ */
+
+package org.opengrok.indexer.analysis.javascript;
+
+import org.opengrok.indexer.analysis.JFlexJointLexer;
+import org.opengrok.indexer.analysis.JFlexSymbolMatcher;
+import org.opengrok.indexer.analysis.Resettable;
+import org.opengrok.indexer.web.HtmlConsts;
+
+import java.io.IOException;
+import java.util.Stack;
+
+/**
+ * Represents an abstract base class for JavaScript lexers.
+ */
+@SuppressWarnings("Duplicates")
+public abstract class JavaScriptLexer extends JFlexSymbolMatcher
+        implements JFlexJointLexer, Resettable {
+
+    private ECMAScriptLexerData data;
+
+    /**
+     * Represents the stack of data if substitution is nested.
+     */
+    private Stack<ECMAScriptLexerData> dataStack;
+
+    public JavaScriptLexer() {
+        data = new ECMAScriptLexerData();
+        // dataStack is null to begin.
+    }
+
+    /**
+     * Resets the instance to an initial state.
+     */
+    @Override
+    public void reset() {
+        super.reset();
+        data = new ECMAScriptLexerData();
+        if (dataStack != null) {
+            dataStack.clear();
+        }
+    }
+
+    /**
+     * Calls {@link #phLOC()} if the yystate is not COMMENT or SCOMMENT.
+     */
+    public void chkLOC() {
+        if (yystate() != COMMENT() && yystate() != SCOMMENT()) {
+            phLOC();
+        }
+    }
+
+    /**
+     * Resets the substitution brace counter to 1.
+     */
+    protected void substitutionOp() {
+        data.nEndBrace = 1;
+    }
+
+    /**
+     * Determine if template substitution should end based on the first
+     * character of {@code capture}, and also recognizing tokens that increase
+     * the nesting level alternatively.
+     * <p>
+     * Calling this method has side effects to possibly modify
+     * {@code nEndBrace}.
+     * @return {@code true} if the substitution state does not end
+     */
+    protected boolean notInTemplateOrSubstitutionDoesNotEnd(String capture) throws IOException {
+        if (data.nEndBrace <= 0) {
+            return true;
+        }
+        if (capture.startsWith("}")) {
+            if (--data.nEndBrace <= 0) {
+                int nRemaining = capture.length() - 1;
+                String opener = capture.substring(0, 1);
+                popData();
+                yypop();
+                disjointSpan(HtmlConsts.STRING_CLASS);
+                offer(opener);
+                if (nRemaining > 0) {
+                    yypushback(nRemaining);
+                }
+                return false;
+            }
+        }
+        if (capture.startsWith("{")) {
+            ++data.nEndBrace;
+        }
+        return true;
+    }
+
+    protected void pushData() {
+        if (dataStack == null) {
+            dataStack = new Stack<>();
+        }
+        dataStack.push(data);
+        data = new ECMAScriptLexerData();
+    }
+
+    private void popData() {
+        data = dataStack.pop();
+    }
+
+    /**
+     * Subclasses must override to get the constant value created by JFlex to
+     * represent COMMENT.
+     */
+    protected abstract int COMMENT();
+
+    /**
+     * Subclasses must override to get the constant value created by JFlex to
+     * represent SCOMMENT.
+     */
+    protected abstract int SCOMMENT();
+
+    private static class ECMAScriptLexerData {
+        /**
+         * When interpolating inside `` with ${, the number of remaining '}'
+         * characters is stored. It starts at 1, and any nesting increases the
+         * value.
+         */
+        int nEndBrace;
+    }
+}