Skip to content

Commit

Permalink
Fix linkedin#599, linkedin#600: Set score for TaskMemoryHeuristic, al…
Browse files Browse the repository at this point in the history 
…low for grace memory headroom
  • Loading branch information
Anthony Hsu committed Jun 20, 2019
1 parent f280a85 commit 12f40ab
Show file tree
Hide file tree
Showing 2 changed files with 51 additions and 20 deletions.
19 changes: 16 additions & 3 deletions app/com/linkedin/drelephant/tony/heuristics/TaskMemoryHeuristic.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import java.util.Set;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.log4j.Logger;


Expand All @@ -52,6 +53,10 @@ public class TaskMemoryHeuristic implements Heuristic<TonyApplicationData> {
// Initialized to default max memory thresholds
private double[] maxMemoryLimits = {0.8, 0.7, 0.6, 0.5};

// If the requested memory is within this amount of the max memory usage, automatic pass.
// This is to prevent Dr. Elephant flagging container sizes of 3 GB when max memory usage is 2 GB.
private long graceMemoryHeadroomBytes;

/**
* Constructor for {@link TaskMemoryHeuristic}.
* @param heuristicConfData the configuration for this heuristic
Expand All @@ -68,6 +73,11 @@ public TaskMemoryHeuristic(HeuristicConfigurationData heuristicConfData) {
if (params.containsKey(TASK_MEMORY_THRESHOLDS_CONF)) {
maxMemoryLimits = Utils.getParam(params.get(TASK_MEMORY_THRESHOLDS_CONF), maxMemoryLimits.length);
}

Configuration yarnConf = new YarnConfiguration();
int mininumMBAllocation = yarnConf.getInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
graceMemoryHeadroomBytes = 2 * mininumMBAllocation * FileUtils.ONE_MB;
}

@Override
Expand All @@ -79,6 +89,7 @@ public HeuristicResult apply(TonyApplicationData data) {
Set<String> taskTypes = com.linkedin.tony.util.Utils.getAllJobTypes(conf);
Severity finalSeverity = Severity.NONE;
List<HeuristicResultDetails> details = new ArrayList<>();
int severityScore = 0;

for (String taskType : taskTypes) {
int taskInstances = conf.getInt(TonyConfigurationKeys.getInstancesKey(taskType), 0);
Expand All @@ -105,18 +116,20 @@ public HeuristicResult apply(TonyApplicationData data) {
Long.toString((long) maxMemoryBytesUsed / FileUtils.ONE_MB)));

// compare to threshold and update severity
if (taskBytesRequested <= defaultContainerMemoryBytes) {
// If using default container memory, automatic pass
if (taskBytesRequested <= defaultContainerMemoryBytes
|| taskBytesRequested <= maxMemoryBytesUsed + graceMemoryHeadroomBytes) {
// If using default container memory or within grace headroom, automatic pass
continue;
}
double maxMemoryRatio = maxMemoryBytesUsed / taskBytesRequested;
Severity taskMemorySeverity = Severity.getSeverityDescending(maxMemoryRatio, maxMemoryLimits[0],
maxMemoryLimits[1], maxMemoryLimits[2], maxMemoryLimits[3]);
severityScore = Utils.getHeuristicScore(taskMemorySeverity, taskInstances);
finalSeverity = Severity.max(finalSeverity, taskMemorySeverity);
}

return new HeuristicResult(_heuristicConfData.getClassName(), _heuristicConfData.getHeuristicName(), finalSeverity,
0, details);
severityScore, details);
}

@Override
Expand Down
52 changes: 35 additions & 17 deletions test/com/linkedin/drelephant/tony/heuristics/TaskMemoryHeuristicTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@
public class TaskMemoryHeuristicTest {

/**
* 3g workers requested, max worker memory < 50%
* 10g workers requested, max worker memory < 50%
*/
@Test
public void testCritical() {
Expand All @@ -53,67 +53,67 @@ public void testCritical() {
1e9,
1.3e9
}, Constants.PS_JOB_NAME, new double[]{0.5e9}),
ImmutableMap.of(Constants.WORKER_JOB_NAME, "3g", Constants.PS_JOB_NAME, "2g"),
ImmutableMap.of(Constants.WORKER_JOB_NAME, "10g", Constants.PS_JOB_NAME, "2g"),
Severity.CRITICAL
);
}

/**
* 3g ps requested, max ps memory < 60%
* 10g ps requested, max ps memory < 60%
*/
@Test
public void testSevere() {
testHelper(
ImmutableMap.of(Constants.WORKER_JOB_NAME, new double[]{
1.5e9,
1.6e9,
}, Constants.PS_JOB_NAME, new double[]{1.84e9}),
ImmutableMap.of(Constants.WORKER_JOB_NAME, "2g", Constants.PS_JOB_NAME, "3g"),
}, Constants.PS_JOB_NAME, new double[]{5.84e9}),
ImmutableMap.of(Constants.WORKER_JOB_NAME, "2g", Constants.PS_JOB_NAME, "10g"),
Severity.SEVERE
);
}

/**
* 3g workers requested, max worker memory < 70%
* 10g workers requested, max worker memory < 70%
*/
@Test
public void testModerate() {
testHelper(
ImmutableMap.of(Constants.WORKER_JOB_NAME, new double[]{
2.14e9,
2e9,
6.5e9,
6.6e9,
}),
ImmutableMap.of(Constants.WORKER_JOB_NAME, "3g"),
ImmutableMap.of(Constants.WORKER_JOB_NAME, "10g"),
Severity.MODERATE
);
}

/**
* 3g workers requested, max worker memory < 80%
* 10g workers requested, max worker memory < 80%
*/
@Test
public void testLow() {
testHelper(
ImmutableMap.of(Constants.WORKER_JOB_NAME, new double[]{
2e9,
2.45e9,
7.56e9,
7.45e9,
}),
ImmutableMap.of(Constants.WORKER_JOB_NAME, "3g"),
ImmutableMap.of(Constants.WORKER_JOB_NAME, "10g"),
Severity.LOW
);
}

/**
* 3g workers requested, max worker memory > 80%
* 10g workers requested, max worker memory > 80%
*/
@Test
public void testNone() {
testHelper(
ImmutableMap.of(Constants.WORKER_JOB_NAME, new double[]{
2.5e9,
2.6e9,
8.5e9,
8.6e9,
}),
ImmutableMap.of(Constants.WORKER_JOB_NAME, "3g"),
ImmutableMap.of(Constants.WORKER_JOB_NAME, "10g"),
Severity.NONE
);
}
Expand All @@ -133,6 +133,23 @@ public void testLowUtilizationDefaultContainerSize() {
);
}

/**
* Though memory utilization is about 50%, severity should be none
* because requested memory is within the default 2 GB grace headroom of the
* max used memory.
*/
@Test
public void testRequestedSizeWithinGraceHeadroomSeverity() {
testHelper(
ImmutableMap.of(Constants.WORKER_JOB_NAME, new double[]{
1.5e9,
1.6e9,
}),
ImmutableMap.of(Constants.WORKER_JOB_NAME, "3g"),
Severity.NONE
);
}

/**
* Verifies that no exception is thrown when the task map is empty.
*/
Expand All @@ -145,6 +162,7 @@ public void testEmptyTaskMap() {
appType, conf, Collections.EMPTY_LIST);
new TaskMemoryHeuristic(new HeuristicConfigurationData("ignored",
"ignored", "ignored", appType, Collections.EMPTY_MAP)).apply(data);

}

public void testHelper(Map<String, double[]> memUsed, Map<String, String> memRequested, Severity expectedSeverity) {
Expand Down

0 comments on commit 12f40ab

Please sign in to comment.