LIHADOOP-20347: Fix Mapper Speed Heuristic's Disk Speed Severity Thre…

…sholds
adbadb · May 27, 2016 · 91cf4a5 · 91cf4a5
1 parent f69a0cd
commit 91cf4a5
Show file tree

Hide file tree

Showing 2 changed files with 21 additions and 8 deletions.
diff --git a/app/com/linkedin/drelephant/mapreduce/heuristics/MapperSpeedHeuristic.java b/app/com/linkedin/drelephant/mapreduce/heuristics/MapperSpeedHeuristic.java
@@ -61,7 +61,7 @@ private void loadParameters() {
     logger.info(heuristicName + " will use " + DISK_SPEED_SEVERITY + " with the following threshold settings: "
         + Arrays.toString(diskSpeedLimits));
     for (int i = 0; i < diskSpeedLimits.length; i++) {
-      diskSpeedLimits[i] = diskSpeedLimits[i] * HDFSContext.HDFS_BLOCK_SIZE;
+      diskSpeedLimits[i] = diskSpeedLimits[i] * HDFSContext.DISK_READ_SPEED;
     }
 
     double[] confRuntimeThreshold = Utils.getParam(paramMap.get(RUNTIME_SEVERITY), runtimeLimits.length);

diff --git a/test/com/linkedin/drelephant/mapreduce/heuristics/MapperSpeedHeuristicTest.java b/test/com/linkedin/drelephant/mapreduce/heuristics/MapperSpeedHeuristicTest.java
@@ -32,39 +32,52 @@
 import java.util.HashMap;
 import java.util.Map;
 import junit.framework.TestCase;
+import org.apache.commons.io.FileUtils;
 
 
 public class MapperSpeedHeuristicTest extends TestCase {
   private static Map<String, String> paramsMap = new HashMap<String, String>();
   private static Heuristic _heuristic = new MapperSpeedHeuristic(new HeuristicConfigurationData("test_heuristic",
       "test_class", "test_view", new ApplicationType("test_apptype"), paramsMap));
 
-  private static final long UNITSIZE = HDFSContext.HDFS_BLOCK_SIZE / 64;
+  private static final long MB_IN_BYTES = FileUtils.ONE_MB;
   private static final long MINUTE_IN_MS = Statistics.MINUTE_IN_MS;
   private static final int NUMTASKS = 100;
 
   public void testCritical() throws IOException {
-    assertEquals(Severity.CRITICAL, analyzeJob(120 * MINUTE_IN_MS, 10000 * UNITSIZE));
+    long runtime = 120 * MINUTE_IN_MS;
+    long speed_factor = (runtime * MB_IN_BYTES) / 1000;
+    assertEquals(Severity.CRITICAL, analyzeJob(runtime, 1 * speed_factor));
   }
 
   public void testSevere() throws IOException {
-    assertEquals(Severity.SEVERE, analyzeJob(120 * MINUTE_IN_MS, 50000 * UNITSIZE));
+    long runtime = 120 * MINUTE_IN_MS;
+    long speed_factor = (runtime * MB_IN_BYTES) / 1000;
+    assertEquals(Severity.SEVERE, analyzeJob(runtime, 4 * speed_factor));
   }
 
   public void testModerate() throws IOException {
-    assertEquals(Severity.MODERATE, analyzeJob(120 * MINUTE_IN_MS, 100000 * UNITSIZE));
+    long runtime = 120 * MINUTE_IN_MS;
+    long speed_factor = (runtime * MB_IN_BYTES) / 1000;
+    assertEquals(Severity.MODERATE, analyzeJob(runtime, 13 * speed_factor));
   }
 
   public void testLow() throws IOException {
-    assertEquals(Severity.LOW, analyzeJob(120 * MINUTE_IN_MS, 200000 * UNITSIZE));
+    long runtime = 120 * MINUTE_IN_MS;
+    long speed_factor = (runtime * MB_IN_BYTES) / 1000;
+    assertEquals(Severity.LOW, analyzeJob(runtime, 50 * speed_factor));
   }
 
   public void testNone() throws IOException {
-    assertEquals(Severity.NONE, analyzeJob(120 * MINUTE_IN_MS, 500000 * UNITSIZE));
+    long runtime = 120 * MINUTE_IN_MS;
+    long speed_factor = (runtime * MB_IN_BYTES) / 1000;
+    assertEquals(Severity.NONE, analyzeJob(runtime, 51 * speed_factor));
   }
 
   public void testShortTask() throws IOException {
-    assertEquals(Severity.NONE, analyzeJob(2 * MINUTE_IN_MS, 10 * UNITSIZE));
+    long runtime = 2 * MINUTE_IN_MS;
+    long speed_factor = (runtime * MB_IN_BYTES) / 1000;
+    assertEquals(Severity.NONE, analyzeJob(runtime, 1 * speed_factor));
   }
 
   private Severity analyzeJob(long runtimeMs, long readBytes) throws IOException {