increase default batchSize from 100 to 1000 for better throughput

yzygenuine · Jun 24, 2013 · d1baf4e · d1baf4e
1 parent 58242f1
commit d1baf4e
Show file tree

Hide file tree

Showing 7 changed files with 7 additions and 7 deletions.
diff --git a/cdk-morphlines/cdk-morphlines-core/src/test/resources/test-morphlines/loadSolrBasic.conf b/cdk-morphlines/cdk-morphlines-core/src/test/resources/test-morphlines/loadSolrBasic.conf
@@ -35,7 +35,7 @@ SOLR_LOCATOR : {
   collection : ${SOLR_COLLECTION}
   zkHost : ${ZK_HOST}
   solrHomeDir : ${SOLR_HOME_DIR}
-  # batchSize : 100
+  # batchSize : 1000
 }
 SOLR_LOCATOR : ${?ENV_SOLR_LOCATOR}
 

diff --git a/cdk-morphlines/cdk-morphlines-core/src/test/resources/test-morphlines/parseVariables.conf b/cdk-morphlines/cdk-morphlines-core/src/test/resources/test-morphlines/parseVariables.conf
@@ -38,7 +38,7 @@ SOLR_LOCATOR : {
   zkHost : ${ZK_HOST}
   solrUrl : ${SOLR_URL}
   solrHomeDir : ${SOLR_HOME_DIR}
-  # batchSize : 100
+  # batchSize : 1000
 }
 SOLR_LOCATOR : ${?ENV_SOLR_LOCATOR}
 

diff --git a/cdk-morphlines/cdk-morphlines-core/src/test/resources/test-morphlines/solrLocator.conf b/cdk-morphlines/cdk-morphlines-core/src/test/resources/test-morphlines/solrLocator.conf
@@ -35,6 +35,6 @@ SOLR_LOCATOR : {
   collection : ${SOLR_COLLECTION}
   zkHost : ${ZK_HOST}
   solrHomeDir : ${SOLR_HOME_DIR}
-  # batchSize : 100
+  # batchSize : 1000
 }
 SOLR_LOCATOR : ${?ENV_SOLR_LOCATOR}
diff --git a/...nes/cdk-morphlines-core/src/test/resources/test-morphlines/tutorialReadAvroContainer.conf b/...nes/cdk-morphlines-core/src/test/resources/test-morphlines/tutorialReadAvroContainer.conf
@@ -29,7 +29,7 @@ SOLR_LOCATOR : {
   zkHost : "127.0.0.1:2181/solr"
 
   # The maximum number of documents to send to Solr per network batch (throughput knob)
-  # batchSize : 100
+  # batchSize : 1000
 }
 
 # Specify an array of one or more morphlines, each of which defines an ETL 

diff --git a/...es/cdk-morphlines-core/src/test/resources/test-morphlines/tutorialReadJsonTestTweets.conf b/...es/cdk-morphlines-core/src/test/resources/test-morphlines/tutorialReadJsonTestTweets.conf
@@ -15,7 +15,7 @@
 SOLR_LOCATOR : {
   collection : collection1
   zkHost : "127.0.0.1:2181/solr"
-  # batchSize : 100
+  # batchSize : 1000
 }
 
 morphlines : [

diff --git a/...s/cdk-morphlines-solr-core/src/main/java/com/cloudera/cdk/morphline/solr/SolrLocator.java b/...s/cdk-morphlines-solr-core/src/main/java/com/cloudera/cdk/morphline/solr/SolrLocator.java
@@ -57,7 +57,7 @@ public class SolrLocator {
   private String zkHost;
   private String solrUrl;
   private String solrHomeDir;
-  private int batchSize = 100;
+  private int batchSize = 1000;
 
   private static final String SOLR_HOME_PROPERTY_NAME = "solr.solr.home";
 

diff --git a/search-flume/src/main/java/org/apache/flume/sink/solr/morphline/MorphlineSink.java b/search-flume/src/main/java/org/apache/flume/sink/solr/morphline/MorphlineSink.java
@@ -37,7 +37,7 @@
  */
 public class MorphlineSink extends AbstractSink implements Configurable {
 
-  private int maxBatchSize = 100;
+  private int maxBatchSize = 1000;
   private long maxBatchDurationMillis = 1000;
   private String handlerClass;
   private MorphlineHandler handler;