LUCENE-4291: reduce jflex buffer sizes

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/branch_4x@1369892 13f79535-47bb-0310-9956-ffa450edef68
andyfowler · Aug 6, 2012 · 777f25e · 777f25e
1 parent f689567
commit 777f25e
Show file tree

Hide file tree

Showing 22 changed files with 2,027 additions and 2,020 deletions.
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
@@ -119,6 +119,10 @@ Optimizations
   making them substantially more lightweight. Behavior is unchanged. 
   (Robert Muir)
 
+* LUCENE-4291: Reduced internal buffer size for Jflex-based tokenizers 
+  such as StandardTokenizer from 32kb to 8kb.  
+  (Raintung Li, Steven Rowe, Robert Muir)
+
 Bug Fixes
 
 * LUCENE-4109: BooleanQueries are not parsed correctly with the 

diff --git a/...e/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.java b/...e/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.java
diff --git a/.../analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.jflex b/.../analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.jflex
@@ -141,9 +141,9 @@ InlineElment = ( [aAbBiIqQsSuU]                   |
                  [vV][aA][rR]                     )
 
 
-%include src/java/org/apache/lucene/analysis/charfilter/HTMLCharacterEntities.jflex
+%include HTMLCharacterEntities.jflex
 
-%include src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.SUPPLEMENTARY.jflex-macro
+%include HTMLStripCharFilter.SUPPLEMENTARY.jflex-macro
 
 %{
   private static final int INITIAL_INPUT_SEGMENT_SIZE = 1024;

diff --git a/...ne/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.java b/...ne/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.java
diff --git a/...e/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex b/...e/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex
@@ -36,6 +36,7 @@ import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 %function getNextToken
 %pack
 %char
+%buffer 4096
 
 %{
 

diff --git a/...ne/analysis/common/src/java/org/apache/lucene/analysis/standard/SUPPLEMENTARY.jflex-macro b/...ne/analysis/common/src/java/org/apache/lucene/analysis/standard/SUPPLEMENTARY.jflex-macro
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-// Generated using ICU4J 49.1.0.0 on Friday, July 27, 2012 6:24:21 AM UTC
+// Generated using ICU4J 49.1.0.0 on Monday, August 6, 2012 5:23:08 PM UTC
 // by org.apache.lucene.analysis.icu.GenerateJFlexSupplementaryMacros
 
 

diff --git a/...e/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java b/...e/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java
diff --git a/.../analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex b/.../analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex
@@ -44,8 +44,9 @@ import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 %implements StandardTokenizerInterface
 %function getNextToken
 %char
+%buffer 4096
 
-%include src/java/org/apache/lucene/analysis/standard/SUPPLEMENTARY.jflex-macro
+%include SUPPLEMENTARY.jflex-macro
 ALetter = ([\p{WB:ALetter}] | {ALetterSupp})
 Format =  ([\p{WB:Format}] | {FormatSupp})
 Numeric = ([\p{WB:Numeric}] | {NumericSupp})

diff --git a/...lysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.java b/...lysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.java
diff --git a/...ysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.jflex b/...ysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.jflex
@@ -47,8 +47,9 @@ import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 %implements StandardTokenizerInterface
 %function getNextToken
 %char
+%buffer 4096
 
-%include src/java/org/apache/lucene/analysis/standard/SUPPLEMENTARY.jflex-macro
+%include SUPPLEMENTARY.jflex-macro
 ALetter = ([\p{WB:ALetter}] | {ALetterSupp})
 Format =  ([\p{WB:Format}] | {FormatSupp})
 Numeric = ([\p{WB:Numeric}] | {NumericSupp})
@@ -88,7 +89,7 @@ HiraganaEx = {Hiragana} ({Format} | {Extend})*
 //     RFC-5321: Simple Mail Transfer Protocol
 //     RFC-5322: Internet Message Format
 
-%include src/java/org/apache/lucene/analysis/standard/ASCIITLD.jflex-macro
+%include ASCIITLD.jflex-macro
 
 DomainLabel = [A-Za-z0-9] ([-A-Za-z0-9]* [A-Za-z0-9])?
 DomainNameStrict = {DomainLabel} ("." {DomainLabel})* {ASCIITLD}

diff --git a/...is/common/src/java/org/apache/lucene/analysis/standard/std31/StandardTokenizerImpl31.java b/...is/common/src/java/org/apache/lucene/analysis/standard/std31/StandardTokenizerImpl31.java
diff --git a/...s/common/src/java/org/apache/lucene/analysis/standard/std31/StandardTokenizerImpl31.jflex b/...s/common/src/java/org/apache/lucene/analysis/standard/std31/StandardTokenizerImpl31.jflex
@@ -38,8 +38,9 @@ import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 %implements StandardTokenizerInterface
 %function getNextToken
 %char
+%buffer 4096
 
-%include src/java/org/apache/lucene/analysis/standard/std31/SUPPLEMENTARY.jflex-macro
+%include SUPPLEMENTARY.jflex-macro
 ALetter = ([\p{WB:ALetter}] | {ALetterSupp})
 Format =  ([\p{WB:Format}] | {FormatSupp})
 Numeric = ([\p{WB:Numeric}] | {NumericSupp})

diff --git a/...mmon/src/java/org/apache/lucene/analysis/standard/std31/UAX29URLEmailTokenizerImpl31.java b/...mmon/src/java/org/apache/lucene/analysis/standard/std31/UAX29URLEmailTokenizerImpl31.java
diff --git a/...mon/src/java/org/apache/lucene/analysis/standard/std31/UAX29URLEmailTokenizerImpl31.jflex b/...mon/src/java/org/apache/lucene/analysis/standard/std31/UAX29URLEmailTokenizerImpl31.jflex
@@ -38,8 +38,9 @@ import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 %implements StandardTokenizerInterface
 %function getNextToken
 %char
+%buffer 4096
 
-%include src/java/org/apache/lucene/analysis/standard/std31/SUPPLEMENTARY.jflex-macro
+%include SUPPLEMENTARY.jflex-macro
 ALetter = ([\p{WB:ALetter}] | {ALetterSupp})
 Format =  ([\p{WB:Format}] | {FormatSupp})
 Numeric = ([\p{WB:Numeric}] | {NumericSupp})
@@ -77,7 +78,7 @@ ExtendNumLetEx = {ExtendNumLet}                ({Format} | {Extend})*
 //     RFC-5321: Simple Mail Transfer Protocol
 //     RFC-5322: Internet Message Format
 
-%include src/java/org/apache/lucene/analysis/standard/std31/ASCIITLD.jflex-macro
+%include ASCIITLD.jflex-macro
 
 DomainLabel = [A-Za-z0-9] ([-A-Za-z0-9]* [A-Za-z0-9])?
 DomainNameStrict = {DomainLabel} ("." {DomainLabel})* {ASCIITLD}

diff --git a/...is/common/src/java/org/apache/lucene/analysis/standard/std34/StandardTokenizerImpl34.java b/...is/common/src/java/org/apache/lucene/analysis/standard/std34/StandardTokenizerImpl34.java
diff --git a/...s/common/src/java/org/apache/lucene/analysis/standard/std34/StandardTokenizerImpl34.jflex b/...s/common/src/java/org/apache/lucene/analysis/standard/std34/StandardTokenizerImpl34.jflex
@@ -36,8 +36,9 @@ import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 %implements StandardTokenizerInterface
 %function getNextToken
 %char
+%buffer 4096
 
-%include src/java/org/apache/lucene/analysis/standard/std34/SUPPLEMENTARY.jflex-macro
+%include SUPPLEMENTARY.jflex-macro
 ALetter = ([\p{WB:ALetter}] | {ALetterSupp})
 Format =  ([\p{WB:Format}] | {FormatSupp})
 Numeric = ([\p{WB:Numeric}] | {NumericSupp})
-Original file line number
+Diff line change
@@ Expand Up @@
     %function getNextToken
     %pack
     %char
+    %buffer 4096
     %{
@@ Expand Down @@