Revert "Peak JVM used memory heuristic - (Depends on Custom SHS - Req…

…uires peakJvmUsedMemory metric) (linkedin#283)" (linkedin#317) This reverts commit 6b2f7e8.
Coderlxl · Jan 10, 2018 · 5500aad · 5500aad
1 parent 6b2f7e8
commit 5500aad
Show file tree

Hide file tree

Showing 10 changed files with 7 additions and 276 deletions.
diff --git a/app-conf/FetcherConf.xml b/app-conf/FetcherConf.xml
@@ -81,7 +81,7 @@
   -->
   <fetcher>
     <applicationtype>spark</applicationtype>
-    <classname>com.linkedin.drelephant.spark.fetchers.SparkFetcher</classname>
+    <classname>com.linkedin.drelephant.spark.fetchers.FSFetcher</classname>
   </fetcher>
 
   <!--

diff --git a/app-conf/HeuristicConf.xml b/app-conf/HeuristicConf.xml
@@ -193,12 +193,6 @@
     <classname>com.linkedin.drelephant.spark.heuristics.StagesHeuristic</classname>
     <viewname>views.html.help.spark.helpStagesHeuristic</viewname>
   </heuristic>
-  <heuristic>
-    <applicationtype>spark</applicationtype>
-    <heuristicname>JVM Used Memory</heuristicname>
-    <classname>com.linkedin.drelephant.spark.heuristics.JvmUsedMemoryHeuristic</classname>
-    <viewname>views.html.help.spark.helpJvmUsedMemoryHeuristic</viewname>
-  </heuristic>
   <heuristic>
     <applicationtype>spark</applicationtype>
     <heuristicname>Executor GC</heuristicname>

diff --git a/app/com/linkedin/drelephant/spark/fetchers/statusapiv1/statusapiv1.scala b/app/com/linkedin/drelephant/spark/fetchers/statusapiv1/statusapiv1.scala
@@ -87,9 +87,7 @@ trait ExecutorSummary{
   def totalShuffleWrite: Long
   def maxMemory: Long
   def totalGCTime: Long
-  def executorLogs: Map[String, String]
-  def peakJvmUsedMemory: Map[String, Long]
-}
+  def executorLogs: Map[String, String]}
 
 trait JobData{
   def jobId: Int
@@ -295,8 +293,7 @@ class ExecutorSummaryImpl(
   var totalShuffleWrite: Long,
   var maxMemory: Long,
   var totalGCTime: Long,
-  var executorLogs: Map[String, String],
-  var peakJvmUsedMemory: Map[String, Long]) extends ExecutorSummary
+  var executorLogs: Map[String, String]) extends ExecutorSummary
 
 class JobDataImpl(
   var jobId: Int,

diff --git a/app/com/linkedin/drelephant/spark/heuristics/JvmUsedMemoryHeuristic.scala b/app/com/linkedin/drelephant/spark/heuristics/JvmUsedMemoryHeuristic.scala
diff --git a/app/com/linkedin/drelephant/spark/legacydata/LegacyDataConverters.scala b/app/com/linkedin/drelephant/spark/legacydata/LegacyDataConverters.scala
@@ -174,8 +174,7 @@ object LegacyDataConverters {
         executorInfo.shuffleWrite,
         executorInfo.maxMem,
         executorInfo.totalGCTime,
-        executorLogs = Map.empty,
-        peakJvmUsedMemory = Map.empty
+        executorLogs = Map.empty
       )
     }
 

diff --git a/app/views/help/spark/helpJvmUsedMemoryHeuristic.scala.html b/app/views/help/spark/helpJvmUsedMemoryHeuristic.scala.html
diff --git a/test/com/linkedin/drelephant/spark/SparkMetricsAggregatorTest.scala b/test/com/linkedin/drelephant/spark/SparkMetricsAggregatorTest.scala
@@ -195,7 +195,6 @@ object SparkMetricsAggregatorTest {
     totalShuffleWrite = 0,
     maxMemory = 0,
     totalGCTime = 0,
-    executorLogs = Map.empty,
-    peakJvmUsedMemory = Map.empty
+    executorLogs = Map.empty
   )
 }
diff --git a/test/com/linkedin/drelephant/spark/heuristics/ExecutorGcHeuristicTest.scala b/test/com/linkedin/drelephant/spark/heuristics/ExecutorGcHeuristicTest.scala
@@ -119,8 +119,7 @@ object ExecutorGcHeuristicTest {
     totalShuffleWrite= 0,
     maxMemory= 0,
     totalGCTime,
-    executorLogs = Map.empty,
-    peakJvmUsedMemory = Map.empty
+    executorLogs = Map.empty
   )
 
   def newFakeSparkApplicationData(

diff --git a/test/com/linkedin/drelephant/spark/heuristics/ExecutorsHeuristicTest.scala b/test/com/linkedin/drelephant/spark/heuristics/ExecutorsHeuristicTest.scala
@@ -250,8 +250,7 @@ object ExecutorsHeuristicTest {
     totalShuffleWrite,
     maxMemory,
     totalGCTime = 0,
-    executorLogs = Map.empty,
-    peakJvmUsedMemory = Map.empty
+    executorLogs = Map.empty
   )
 
   def newFakeSparkApplicationData(executorSummaries: Seq[ExecutorSummaryImpl]): SparkApplicationData = {

diff --git a/test/com/linkedin/drelephant/spark/heuristics/JvmUsedMemoryHeuristicTest.scala b/test/com/linkedin/drelephant/spark/heuristics/JvmUsedMemoryHeuristicTest.scala