Flyweight Token support for ANTLR4 Lexer Bridge

manojetic · Jun 28, 2023 · aee53d5 · aee53d5
1 parent 868f28a
commit aee53d5
Show file tree

Hide file tree

Showing 5 changed files with 197 additions and 5 deletions.
diff --git a/ide/lexer.antlr4/apichanges.xml b/ide/lexer.antlr4/apichanges.xml
@@ -0,0 +1,144 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+    Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+-->
+<?xml-stylesheet type="text/xml" href="../../nbbuild/javadoctools/apichanges.xsl"?>
+<!DOCTYPE apichanges PUBLIC "-//NetBeans//DTD API changes list 1.0//EN" "../../nbbuild/javadoctools/apichanges.dtd">
+
+<!--
+
+INFO FOR PEOPLE ADDING CHANGES:
+
+Check the DTD (apichanges.dtd) for details on the syntax. You do not
+need to regenerate the HTML, as this is part of Javadoc generation; just
+change the XML. Rough syntax of a change (several parts optional):
+
+<change>
+    <api name="compiler"/>
+    <summary>Some brief description here, can use <b>XHTML</b></summary>
+    <version major="1" minor="99"/>
+    <date day="13" month="6" year="2001"/>
+    <author login="jrhacker"/>
+    <compatibility addition="yes"/>
+    <description>
+        The main description of the change here.
+        Again can use full <b>XHTML</b> as needed.
+    </description>
+    <class package="org.openide.compiler" name="DoWhatIWantCompiler"/>
+    <issue number="14309"/>
+</change>
+
+Also permitted elements: <package>, <branch>. <version> is API spec
+version, recommended for all new changes. <compatibility> should say
+if things were added/modified/deprecated/etc. and give all information
+related to upgrading old code. List affected top-level classes and
+link to issue numbers if applicable. See the DTD for more details.
+
+Changes need not be in any particular order, they are sorted in various
+ways by the stylesheet anyway.
+
+Dates are assumed to mean "on the trunk". If you *also* make the same
+change on a stabilization branch, use the <branch> tag to indicate this
+and explain why the change was made on a branch in the <description>.
+
+Please only change this file on the trunk! Rather: you can change it
+on branches if you want, but these changes will be ignored; only the
+trunk version of this file is important.
+
+Deprecations do not count as incompatible, assuming that code using the
+deprecated calls continues to see their documented behavior. But do
+specify deprecation="yes" in <compatibility>.
+
+This file is not a replacement for Javadoc: it is intended to list changes,
+not describe the complete current behavior, for which ordinary documentation
+is the proper place.
+
+-->
+
+<apichanges>
+
+    <!-- First, a list of API names you may use: -->
+    <apidefs>
+        <apidef name="general">ANTLR4 Lexer Bridge</apidef>
+        <!-- etc. -->
+    </apidefs>
+
+    <!-- ACTUAL CHANGES BEGIN HERE: -->
+
+    <changes>
+        <change id="flyweight-token-support">
+            <api name="general"/>
+            <summary>Adding Flyweight Token Support</summary>
+            <version major="1" minor="3"/>
+            <date day="25" month="6" year="2023"/>
+            <author login="lkishalmi"/>
+            <compatibility semantic="compatible" addition="yes"/>
+            <description>
+                <p>
+                    Introduced a new overrideable method <code>flyweightText</code> to
+                    <code>
+                        <a href="@TOP@/org/netbeans/spi/lexer/antlr4/AbstractAntlrLexerBridge.html">AbstractAntlrLexerBridge</a>
+                    </code> so it can create a FlyweightToken for tokens which represents static text.
+                </p>
+            </description>
+            <class package="org.netbeans.spi.lexer.antl4" name="AbstractAntlrLexerBridge"/>
+        </change>
+    </changes>
+
+    <!-- Now the surrounding HTML text and document structure: -->
+
+    <htmlcontents>
+        <!--
+
+                            NO NO NO NO NO!
+
+         ==============>    DO NOT EDIT ME!  <==============
+
+          AUTOMATICALLY GENERATED FROM APICHANGES.XML, DO NOT EDIT
+
+                SEE projects/projectuiapi/apichanges.xml
+
+        -->
+        <head>
+            <title>Change History for the ANTLR4 Lexer Bridge</title>
+            <link rel="stylesheet" href="prose.css" type="text/css"/>
+        </head>
+        <body>
+
+            <p class="overviewlink">
+                <a href="overview-summary.html">Overview</a>
+            </p>
+
+            <h1>Introduction</h1>
+
+            <p>This document lists changes made to the <a href="@org-netbeans-modules-lexer-antlr4@/index.html">ANTLR4 Lexer Bridge</a>.</p>
+
+            <!-- The actual lists of changes, as summaries and details: -->
+            <hr/>
+            <standard-changelists module-code-name="org.netbeans.modules.lexer.antlr4/1"/>
+
+            <hr/>
+            <p>@FOOTER@</p>
+
+        </body>
+    </htmlcontents>
+
+</apichanges>
+
diff --git a/ide/lexer.antlr4/nbproject/project.properties b/ide/lexer.antlr4/nbproject/project.properties
@@ -18,5 +18,7 @@
 is.autoload=true
 javac.compilerargs=-Xlint:unchecked
 javac.source=1.8
-spec.version.base=1.2.0
+javadoc.apichanges=${basedir}/apichanges.xml
+
+spec.version.base=1.3.0
 
diff --git a/ide/lexer.antlr4/src/org/netbeans/spi/lexer/antlr4/AbstractAntlrLexerBridge.java b/ide/lexer.antlr4/src/org/netbeans/spi/lexer/antlr4/AbstractAntlrLexerBridge.java
@@ -140,10 +140,30 @@ protected final Token<T> groupToken(T id, int antlrTokenType) {
         input.seek(preFetchedToken.getStartIndex());
         return token(id);
     }
-
+
+    /**
+     * Implementations can overwrite this method to help the {@linkplain #token(org.netbeans.api.lexer.TokenId)}
+     * method return Flyweight tokens, that could improve the lexer performance.
+     *
+     * The default implementation simply returns {@code null} that means no
+     * flyweight token creation.
+     *
+     * @param id the token id.
+     * @return the static text shall be used for flyweight tokens, if applicable
+     *         {@code null} otherwise.
+     *
+     * @since 1.3
+     */
+    protected String flyweightText(T id) {
+        return null;
+    }
+
     protected final Token<T> token(T id) {
         input.markToken();
-        return tokenFactory.createToken(id);
+        String ft = flyweightText(id);
+        return (ft != null) && (ft.length() == input.readLength())
+                ? tokenFactory.getFlyweightToken(id, ft)
+                : tokenFactory.createToken(id);
     }
 
     private org.antlr.v4.runtime.Token nextRealToken() {

diff --git a/ide/lexer.antlr4/src/org/netbeans/spi/lexer/antlr4/LexerInputCharStream.java b/ide/lexer.antlr4/src/org/netbeans/spi/lexer/antlr4/LexerInputCharStream.java
@@ -47,11 +47,11 @@ public String getText(Interval intrvl) {
         int end = intrvl.b - tokenMark + 1;
         int readCount = 0;
         int next = 0;
-        while ((end > input.readLength()) && (next != EOF)) {
+        while ((end > readLength()) && (next != EOF)) {
             next = input.read();
             readCount++;
         }
-        String ret = String.valueOf(input.readText(start, Math.min(end, input.readLength())));
+        String ret = String.valueOf(input.readText(start, Math.min(end, readLength())));
         input.backup(readCount);
         return ret;
     }
@@ -129,6 +129,10 @@ public int size() {
         throw new UnsupportedOperationException("Stream size is unknown.");
     }
 
+    int readLength() {
+        return input.readLength();
+    }
+
     @Override
     public String getSourceName() {
         return UNKNOWN_SOURCE_NAME;

diff --git a/ide/lexer.antlr4/src/org/netbeans/spi/lexer/antlr4/package-info.java b/ide/lexer.antlr4/src/org/netbeans/spi/lexer/antlr4/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/**
+ * ANTLR4 to NetBeans Lexer Bridge Support
+ */
+package org.netbeans.spi.lexer.antlr4;