Add spark log ext conf param (linkedin#79)

jeesim2 · Jun 23, 2016 · e387110 · e387110
1 parent a616a69
commit e387110
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 1 deletion.
diff --git a/app-conf/FetcherConf.xml b/app-conf/FetcherConf.xml
@@ -40,6 +40,7 @@
     <params>
       <event_log_size_limit_in_mb>100</event_log_size_limit_in_mb>
       <event_log_dir>/system/spark-history</event_log_dir>
+      <spark_log_ext>_1.snappy</spark_log_ext>
 
       #the values specified in namenode_addresses will be used for obtaining spark logs. The cluster configuration will be ignored.
       <namenode_addresses>address1,address2</namenode_addresses>

diff --git a/app/org/apache/spark/deploy/history/SparkFSFetcher.scala b/app/org/apache/spark/deploy/history/SparkFSFetcher.scala
@@ -237,7 +237,8 @@ class SparkFSFetcher(fetcherConfData: FetcherConfigurationData) extends Elephant
               null
             }
           } else {
-            val logFilePath = new Path(logPath + "_1.snappy")
+            val sparkLogExt = fetcherConfData.getParamMap.getOrDefault(SPARK_LOG_EXT, defSparkLogExt)
+            val logFilePath = new Path(logPath + sparkLogExt)
             if (!shouldThrottle(logFilePath)) {
               EventLoggingListener.openEventLog(logFilePath, fs)
             } else {
@@ -338,6 +339,7 @@ private object SparkFSFetcher {
 
   var defEventLogDir = "/system/spark-history"
   var defEventLogSizeInMb = 100d; // 100MB
+  var defSparkLogExt = "_1.snappy"
 
   val LOG_SIZE_XML_FIELD = "event_log_size_limit_in_mb"
   val LOG_DIR_XML_FIELD = "event_log_dir"
@@ -346,5 +348,7 @@ private object SparkFSFetcher {
   val LOG_PREFIX = "EVENT_LOG_"
   val COMPRESSION_CODEC_PREFIX = EventLoggingListener.COMPRESSION_CODEC_KEY + "_"
 
+  // Param map property names that allow users to configer various aspects of the fetcher
   val NAMENODE_ADDRESSES = "namenode_addresses"
+  val SPARK_LOG_EXT = "spark_log_ext"
 }