Upgrade ANTLRv4 Runtime to 4.11.1 (apache#4845)

* Upgrade ANTLR Runtime 4.11.1

* Moved the Json parser to ANTLRv4 4.11.1

* Added release version 2 to ANTLR v4 runtime.

* Re-generated ANTLR v3 and v4 grammars with 4.11

* TOML Suppoer update for TOML 1.0 (tomlj-1.1.0)

* Updated ANTLR License for 4.11.1

* Updated release version of libs.tomlj

* Generated new signature file for javascript2.json

ide/languages.toml/nbproject/project.xml

@@ -39,15 +39,17 @@
                     <build-prerequisite/>
                     <compile-dependency/>
                     <run-dependency>
-                        <specification-version>1.18</specification-version>
+                        <release-version>2</release-version>
+                        <specification-version>1.20</specification-version>
                     </run-dependency>
                 </dependency>
                 <dependency>
                     <code-name-base>org.netbeans.libs.tomlj</code-name-base>
                     <build-prerequisite/>
                     <compile-dependency/>
                     <run-dependency>
-                        <specification-version>1.0.0.0</specification-version>
+                        <release-version>1</release-version>
+                        <specification-version>1.1</specification-version>
                     </run-dependency>
                 </dependency>
                 <dependency>

ide/languages.toml/src/org/netbeans/modules/languages/toml/Bundle.properties

@@ -30,6 +30,7 @@ number=Number
 operator=Operator
 separator=Separator
 string=String
+string-quote=String Quote
 string-escape=String Escape Sequence
 table-mark=Table Marking
 whitespace=Whitespace

ide/languages.toml/src/org/netbeans/modules/languages/toml/FontAndColors.xml

@@ -31,6 +31,7 @@
     <fontcolor name="operator" default="operator"/>
     <fontcolor name="separator" default="separator"/>
     <fontcolor name="string" default="string"/>
+    <fontcolor name="string-quote" default="string"/>
     <fontcolor name="string-escape" default="string">
         <font style="bold" />
     </fontcolor>

ide/languages.toml/src/org/netbeans/modules/languages/toml/LexerInputCharStream.java

@@ -19,6 +19,7 @@
 package org.netbeans.modules.languages.toml;
 
 import org.antlr.v4.runtime.CharStream;
+import static org.antlr.v4.runtime.IntStream.UNKNOWN_SOURCE_NAME;
 import org.antlr.v4.runtime.misc.Interval;
 import org.netbeans.spi.lexer.*;
 
@@ -39,11 +40,19 @@ public LexerInputCharStream(LexerInput input) {
     @Override
     public String getText(Interval intrvl) {
         if (intrvl.a < tokenMark) {
-            throw new UnsupportedOperationException("Read before the current token start is not supported: " + intrvl.a + " < " + tokenMark);
+            throw new UnsupportedOperationException("Can't read before the last token end: " + tokenMark);
         }
         int start = intrvl.a - tokenMark;
         int end = intrvl.b - tokenMark + 1;
-        return String.valueOf(input.readText(start, end));
+        int toread = end - start - input.readLength();
+        for (int i = 0; i < toread; i++) {
+            input.read();
+        }
+        String ret = String.valueOf(input.readText(start, end));
+        if (toread > 0) {
+            input.backup(toread);
+        }
+        return ret;
     }
 
     @Override
@@ -76,6 +85,10 @@ public int mark() {
         return -1;
     }
 
+    public void markToken() {
+        tokenMark = index;
+    }
+
     @Override
     public void release(int marker) {
     }
@@ -110,10 +123,6 @@ private void backup(int count) {
         input.backup(count);
     }
 
-    public final void markToken() {
-        tokenMark = index;
-    }
-
     @Override
     public int size() {
         throw new UnsupportedOperationException("Stream size is unknown.");

ide/languages.toml/src/org/netbeans/modules/languages/toml/TomlLexer.java

@@ -18,7 +18,6 @@
  */
 package org.netbeans.modules.languages.toml;
 
-import java.lang.reflect.Field;
 import org.antlr.v4.runtime.misc.IntegerStack;
 import org.netbeans.api.lexer.Token;
 import org.netbeans.spi.lexer.Lexer;
@@ -41,31 +40,43 @@ public final class TomlLexer implements Lexer<TomlTokenId> {
     public TomlLexer(LexerRestartInfo<TomlTokenId> info) {
         this.tokenFactory = info.tokenFactory();
         this.input = new LexerInputCharStream(info.input());
-        this.lexer = new org.tomlj.internal.TomlLexer(input);
-        if (info.state() != null) {
-            ((LexerState) info.state()).restore(lexer);
+        try {
+            this.lexer = new org.tomlj.internal.TomlLexer(input);
+            if (info.state() != null) {
+                ((LexerState) info.state()).restore(lexer);
+            }
+            input.markToken();
+        } catch (Throwable ex) {
+            ex.printStackTrace();
+            throw ex;
         }
     }
 
+    private org.antlr.v4.runtime.Token preFetchedToken = null;
+
     @Override
-    public Token<TomlTokenId> nextToken() {
-        org.antlr.v4.runtime.Token nextToken = lexer.nextToken();
-        if (nextToken.getType() == EOF) {
-            return null;
+    public org.netbeans.api.lexer.Token<TomlTokenId> nextToken() {
+        org.antlr.v4.runtime.Token nextToken;
+        if (preFetchedToken != null) {
+            nextToken = preFetchedToken;
+            lexer.getInputStream().seek(preFetchedToken.getStopIndex() + 1);
+            preFetchedToken = null;
+        } else {
+            nextToken = lexer.nextToken();
         }
-        switch (nextToken.getType()) {
-            case TripleQuotationMark:
-            case TripleApostrophe:
-                return token(ML_STRING_START);
+        int tokenType = nextToken.getType();
+        switch (tokenType) {
+            case EOF:
+                return null;
 
             case StringChar:
+                return collate(StringChar, STRING);
+
+            case TripleQuotationMark:
+            case TripleApostrophe:
             case QuotationMark:
             case Apostrophe:
-                return token(STRING);
-
-            case MLBasicStringEnd:
-            case MLLiteralStringEnd:
-                return token(ML_STRING_END);
+                return token(STRING_QUOTE);
 
             case Comma:
             case ArrayStart:
@@ -93,6 +104,7 @@ public Token<TomlTokenId> nextToken() {
                 return token(TomlTokenId.WHITESPACE);
             case Error:
                 return token(ERROR);
+
             case DecimalInteger:
             case HexInteger:
             case OctalInteger:
@@ -115,13 +127,21 @@ public Token<TomlTokenId> nextToken() {
             case Z:
             case TimeDelimiter:
             case DateDigits:
-            case DateComma:
                 return token(DATE);
             default:
                 return token(ERROR);
         }
     }
 
+    protected org.netbeans.api.lexer.Token<TomlTokenId> collate(int tokenType, TomlTokenId tokenId) {
+        preFetchedToken = lexer.nextToken();
+        while (preFetchedToken.getType() == tokenType) {
+            preFetchedToken = lexer.nextToken();
+        }
+        lexer.getInputStream().seek(preFetchedToken.getStartIndex());
+        return token(tokenId);
+    }
+
     @Override
     public Object state() {
         return new LexerState(lexer);
@@ -137,53 +157,29 @@ private Token<TomlTokenId> token(TomlTokenId id) {
     }
 
     private static class LexerState {
-        private static Field ARRAY_DEPTH;
-        private static Field ARRAY_DEPTH_STACK;
-
         final int state;
         final int mode;
         final IntegerStack modes;
 
         final int arrayDepth;
         final IntegerStack arrayDepthStack;
 
-        static {
-            try {
-                // Hack accessing private state parts of TomlLexer
-                // See: https://github.com/tomlj/tomlj/pull/42
-                ARRAY_DEPTH = org.tomlj.internal.TomlLexer.class.getDeclaredField("arrayDepth");
-                ARRAY_DEPTH.setAccessible(true);
-                ARRAY_DEPTH_STACK = org.tomlj.internal.TomlLexer.class.getDeclaredField("arrayDepthStack");
-                ARRAY_DEPTH_STACK.setAccessible(true);
-            } catch (ReflectiveOperationException ex) {
-            }
-        }
-
         LexerState(org.tomlj.internal.TomlLexer lexer) {
             this.state= lexer.getState();
 
             this.mode = lexer._mode;
             this.modes = new IntegerStack(lexer._modeStack);
-
-            try {
-                this.arrayDepth = ARRAY_DEPTH.getInt(lexer);
-                this.arrayDepthStack = new IntegerStack((IntegerStack)ARRAY_DEPTH_STACK.get(lexer));
-            } catch (IllegalAccessException ex) {
-                throw new RuntimeException(ex);
-            }
+            this.arrayDepth = lexer.arrayDepth;
+            this.arrayDepthStack = new IntegerStack(lexer.arrayDepthStack);
         }
 
         public void restore(org.tomlj.internal.TomlLexer lexer) {
             lexer.setState(state);
             lexer._modeStack.addAll(modes);
             lexer._mode = mode;
 
-            try {
-                ARRAY_DEPTH.setInt(lexer, arrayDepth);
-                ((IntegerStack) ARRAY_DEPTH_STACK.get(lexer)).addAll(arrayDepthStack);
-            } catch (IllegalAccessException ex) {
-                throw new RuntimeException(ex);
-            }
+            lexer.arrayDepth = arrayDepth;
+            lexer.arrayDepthStack.addAll(arrayDepthStack);
         }
 
         @Override

ide/languages.toml/src/org/netbeans/modules/languages/toml/TomlTokenId.java

@@ -43,8 +43,7 @@ public enum TomlTokenId implements TokenId {
     DOT("separator"),
     TABLE_MARK("table-mark"),
     STRING("string"),
-    ML_STRING_START("string"),
-    ML_STRING_END("string"),
+    STRING_QUOTE("string"),
     ESCAPE_SEQUENCE("string-escape"),
     WHITESPACE("whitespace");
 

ide/languages.toml/src/org/netbeans/modules/languages/toml/TomlTypedTextInterceptor.java

@@ -132,7 +132,9 @@ private static boolean isInMultilineString(Context context){
         while ((ts.token() != null) && STRING_OR_WS.contains(ts.token().id())) {
             ts.movePrevious();
         }
-        return (ts.token() != null) && TomlTokenId.ML_STRING_START == ts.token().id();
+        return (ts.token() != null) 
+                && (TomlTokenId.STRING_QUOTE == ts.token().id()) 
+                && ts.token().text().length() == 3;
     }
 
     private static int quotesInLine(Context context, char quote) throws BadLocationException {

ide/libs.antlr4.runtime/external/antlr4-runtime-4.11.1-license.txt

@@ -0,0 +1,39 @@
+Name: Antlr
+Description: ANother Tool for Language Recognition, is a language tool that provides a framework for constructing recognizers, interpreters, compilers, and translators from grammatical descriptions.
+Version: 4.11.1
+License: BSD-antlr-runtime4-2
+Origin: Antlr
+URL: https://www.antlr.org
+Files: antlr4-runtime-4.11.1.jar, antlr4-4.11.1.jar, ST4-4.3.4.jar
+
+Use of Antlr version 4.11.1 is governed by the terms of the license below:
+
+ [The "BSD license"]
+Copyright (c) 2012-2022 The ANTLR Project. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither name of copyright holders nor the names of its contributors
+may be used to endorse or promote products derived from this software
+without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR
+CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

ide/libs.antlr4.runtime/external/binaries-list

@@ -14,6 +14,6 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-E27D8AB4F984F9D186F54DA984A6AB1CCCAC755E org.antlr:antlr4-runtime:4.7.2
-93058C41A6CBB7F5CA63EDAC837F42C002DBC556 org.antlr:antlr4:4.7.2
-467D508BE07A542AD0A68FFCAED2D561C5FB2E0C org.antlr:ST4:4.1
+069214C1DE1960040729702EB58DEAC8827135E7 org.antlr:antlr4-runtime:4.11.1
+844C603E04AB201B769849EE9D3CCE67BA7A1337 org.antlr:antlr4:4.11.1
+BF68D049DD4E6E104055A79AC3BF9E6307D29258 org.antlr:ST4:4.3.4

ide/libs.antlr4.runtime/manifest.mf

@@ -1,5 +1,5 @@
 Manifest-Version: 1.0
-OpenIDE-Module: org.netbeans.libs.antlr4.runtime
+OpenIDE-Module: org.netbeans.libs.antlr4.runtime/2
 OpenIDE-Module-Implementation-Version: 1
 OpenIDE-Module-Localizing-Bundle: org/netbeans/libs/antlr4/runtime/Bundle.properties
 AutoUpdate-Show-In-Client: false

ide/libs.antlr4.runtime/nbproject/project.properties

@@ -19,9 +19,9 @@
 is.autoload=true
 javac.compilerargs=-Xlint -Xlint:-serial
 javac.source=1.8
-release.external/antlr4-runtime-4.7.2.jar=modules/ext/antlr4-runtime-4.7.2.jar
+release.external/antlr4-runtime-4.11.1.jar=modules/ext/antlr4-runtime-4.11.1.jar
 
-license.file=../external/antlr4-runtime-4.7.2-license.txt
+license.file=../external/antlr4-runtime-4.11.1-license.txt
 nbm.homepage=https://www.antlr.org/
 sigtest.gen.fail.on.error=false
 spec.version.base=1.20.0

ide/libs.antlr4.runtime/nbproject/project.xml

@@ -33,8 +33,8 @@
                 <package>org.antlr.v4.runtime.tree</package>
             </public-packages>
             <class-path-extension>
-                <runtime-relative-path>ext/antlr4-runtime-4.7.2.jar</runtime-relative-path>
-                <binary-origin>external/antlr4-runtime-4.7.2.jar</binary-origin>
+                <runtime-relative-path>ext/antlr4-runtime-4.11.1.jar</runtime-relative-path>
+                <binary-origin>external/antlr4-runtime-4.11.1.jar</binary-origin>
             </class-path-extension>
         </data>
     </configuration>

ide/libs.tomlj/external/binaries-list

@@ -14,4 +14,4 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-D1922FB5E672E04D8E8D8DAC0B058B1897E6E940 org.tomlj:tomlj:1.0.0
+8B3AAD16EA96BA9B22C1C8E77820CBC901B19C05 org.tomlj:tomlj:1.1.0

ide/libs.tomlj/external/tomlj-1.0.0-license.txt → ide/libs.tomlj/external/tomlj-1.1.0-license.txt

@@ -1,5 +1,5 @@
 Name: TOML parser in Java
-Version: 1.0.0
+Version: 1.1.0
 Description: A parser for Tom's Obvious, Minimal Language (TOML).
 License: Apache-2.0
 Origin: Apache Software Foundation