Skip to content

Commit

Permalink
LIHADOOP-23923: Add rest end points for search results and search opt…
Browse files Browse the repository at this point in the history 
…ions
  • Loading branch information
@nntnag17
nntnag17 committed Oct 11, 2016
1 parent e41025e commit 113c9d3
Show file tree
Hide file tree
Showing 6 changed files with 474 additions and 4 deletions.
4 changes: 2 additions & 2 deletions app/controllers/Application.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -289,7 +289,7 @@ public static Result search() {
*
* @return URL Encoded String of Parameter Value Pair
*/
private static String getQueryString() {
public static String getQueryString() {
List<BasicNameValuePair> fields = new LinkedList<BasicNameValuePair>();
final Set<Map.Entry<String, String[]>> entries = request().queryString().entrySet();
for (Map.Entry<String, String[]> entry : entries) {
Expand All @@ -306,7 +306,7 @@ private static String getQueryString() {
}
}

private static Map<String, String> getSearchParams() {
public static Map<String, String> getSearchParams() {
Map<String, String> searchParams = new HashMap<String, String>();

DynamicForm form = Form.form().bindFromRequest(request());
Expand Down
11 changes: 10 additions & 1 deletion app/controllers/api/v1/JsonKeys.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,16 @@ public class JsonKeys {
public static final String COUNT = "count";
public static final String VALUE = "value";
public static final String QUEUE = "queue";

public static final String SEARCH_OPTS = "search-options";
public static final String START = "start";
public static final String END = "end";
public static final String SEARCH_RESULTS = "search-results";
public static final String JOB_TYPES = "jobtypes";
public static final String HEURISTICS = "heuristics";
public static final String SEVERITIES = "severities";
public static final String JOB_CATEGORY = "jobcategory";
public static final String TOTAL = "total";
public static final String SUMMARIES = "summaries";

// Workflows
public static final String WORKFLOW_SUMMARIES = "workflow-summaries";
Expand Down
282 changes: 282 additions & 0 deletions app/controllers/api/v1/Web.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,15 @@

package controllers.api.v1;

import com.avaje.ebean.Query;
import com.google.common.collect.Lists;
import com.google.gson.Gson;
import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
import com.linkedin.drelephant.ElephantContext;
import com.linkedin.drelephant.analysis.ApplicationType;
import com.linkedin.drelephant.analysis.Heuristic;
import com.linkedin.drelephant.analysis.JobType;
import com.linkedin.drelephant.analysis.Severity;
import com.linkedin.drelephant.util.Utils;
import controllers.ControllerUtil;
Expand All @@ -34,8 +39,11 @@
import models.AppHeuristicResultDetails;
import models.AppResult;
import org.apache.log4j.Logger;
import play.data.DynamicForm;
import play.data.Form;
import play.mvc.Controller;
import play.mvc.Result;
import controllers.Application;


/**
Expand All @@ -49,6 +57,9 @@ public class Web extends Controller {
private static final int MAX_APPLICATIONS_IN_JOB = 5000;
private static final int MAX_FLOW_LIMIT = 25;
private static final int MAX_JOB_LIMIT = 25;
private static final int SEARCH_DEFAULT_PAGE_OFFSET = 0;
private static final int SEARCH_DEFAULT_PAGE_LIMIT = 25;
private static final int SEARCH_APPLICATION_MAX_OFFSET = 500;

/**
* Returns the list of AppResults for the given username limit by maxApplications
Expand Down Expand Up @@ -1060,6 +1071,277 @@ public static Result restApplicationFromApplicationId(String applicationid) {
return ok(new Gson().toJson(parent));
}

/**
* This returns the rest search options which are filled in the forms for the search page.
* @return Returns the json object which should be filled in the search form.
* return object:
* <pre>
* *{
* "search-options": {
* "jobcategory": [
* {
* "name": "SPARK",
* "jobtypes": [
* {
* "name": "Spark"
* }
* ],
* "heuristics": [
* {
* "name": "Spark Configuration Best Practice"
* },
* {
* "name": "Spark Memory Limit"
* },
* {
* "name": "Spark Stage Runtime"
* },
* {
* "name": "Spark Job Runtime"
* },
* {
* "name": "Spark Executor Load Balance"
* },
* {
* "name": "Spark Event Log Limit"
* }
* ]
* },
* {
* "name": "MAPREDUCE",
* "jobtypes": [
* {
* "name": "Pig"
* },
* {
* "name": "Hive"
* },
* {
* "name": "Cascading"
* },
* {
* "name": "Voldemort"
* },
* {
* "name": "Kafka"
* },
* {
* "name": "HadoopJava"
* }
* ],
* "heuristics": [
* {
* "name": "Mapper Data Skew"
* },
* {
* "name": "Mapper GC"
* },
* {
* "name": "Mapper Time"
* },
* {
* "name": "Mapper Speed"
* },
* {
* "name": "Mapper Spill"
* },
* {
* "name": "Mapper Memory"
* },
* {
* "name": "Reducer Data Skew"
* },
* {
* "name": "Reducer GC"
* },
* {
* "name": "Reducer Time"
* },
* {
* "name": "Reducer Memory"
* },
* {
* "name": "Shuffle & Sort"
* },
* {
* "name": "Exception"
* }
* ]
* }
* ],
* "severities": [
* {
* "name": "Critical",
* "value": 4
* },
* {
* "name": "Severe",
* "value": 3
* },
* {
* "name": "Moderate",
* "value": 2
* },
* {
* "name": "Low",
* "value": 1
* },
* {
* "name": "None",
* "value": 0
* }
* ],
* "id": "search"
* }
*}
* </pre>
*/
public static Result restSearchOptions() {
JsonObject searchOptions = new JsonObject();
JsonArray jobCategory = new JsonArray();
JsonArray severities = new JsonArray();

Map<ApplicationType, List<JobType>> applicationTypeListMap = ElephantContext.instance().getAppTypeToJobTypes();

for (ApplicationType key : applicationTypeListMap.keySet()) {
JsonObject applicationType = new JsonObject();
JsonArray jobTypes = new JsonArray();
JsonArray heuristics = new JsonArray();

for (JobType jobtype : applicationTypeListMap.get(key)) {
JsonObject jobTypeNode = new JsonObject();
jobTypeNode.addProperty(JsonKeys.NAME, jobtype.getName());
jobTypes.add(jobTypeNode);
}

for (Heuristic heuristic : ElephantContext.instance().getHeuristicsForApplicationType(key)) {
JsonObject heuristicNode = new JsonObject();
heuristicNode.addProperty(JsonKeys.NAME, heuristic.getHeuristicConfData().getHeuristicName());
heuristics.add(heuristicNode);
}

applicationType.addProperty(JsonKeys.NAME, key.getName());
applicationType.add(JsonKeys.JOB_TYPES, jobTypes);
applicationType.add(JsonKeys.HEURISTICS, heuristics);
jobCategory.add(applicationType);
}

for (Severity severity : Severity.values()) {
JsonObject severityObject = new JsonObject();
severityObject.addProperty(JsonKeys.NAME, severity.getText());
severityObject.addProperty(JsonKeys.VALUE, severity.getValue());
severities.add(severityObject);
}

searchOptions.add(JsonKeys.JOB_CATEGORY, jobCategory);
searchOptions.add(JsonKeys.SEVERITIES, severities);
searchOptions.addProperty(JsonKeys.ID, "search");
JsonObject parent = new JsonObject();
parent.add(JsonKeys.SEARCH_OPTS, searchOptions);
return ok(new Gson().toJson(parent));
}

/**
* Returns the search results for the given query
* @return
* JsonObject:
*
* <pre>
* {
* search-results: {
* id: "id"
* start: 0,
* end: 20,
* total: 0,
* summaries: [
* {
* application_summary_object
* }
* ]
* }
* }
* </pre>
*/
public static Result search() {
DynamicForm form = Form.form().bindFromRequest(request());
JsonObject parent = new JsonObject();

int offset = SEARCH_DEFAULT_PAGE_OFFSET;
int limit = SEARCH_DEFAULT_PAGE_LIMIT;
int end = 0;
int total = 0;

if (form.get("offset") != null && form.get("offset") != "") {
offset = Integer.valueOf(form.get("offset"));
}

if (form.get("limit") != null && form.get("limit") != "") {
limit = Integer.valueOf(form.get("limit"));
}

if (offset < 0) {
offset = 0;
}

if (limit > SEARCH_APPLICATION_MAX_OFFSET) {
limit = SEARCH_APPLICATION_MAX_OFFSET;
} else if (limit <= 0) {
return ok(new Gson().toJson(parent));
}

Query<AppResult> query =
Application.generateSearchQuery(AppResult.getSearchFields(), Application.getSearchParams());

total = query.findRowCount();

if (offset > total) {
offset = total;
}

List<AppResult> results = query.setFirstRow(offset).setMaxRows(limit)
.fetch(AppResult.TABLE.APP_HEURISTIC_RESULTS, AppHeuristicResult.getSearchFields()).findList();

end = offset + results.size();

JsonArray applicationSummaryArray = new JsonArray();

for (AppResult application : results) {
JsonObject applicationObject = new JsonObject();
JsonArray heuristicsArray = new JsonArray();
List<AppHeuristicResult> appHeuristicResult = application.yarnAppHeuristicResults;

for (AppHeuristicResult heuristic : appHeuristicResult) {
JsonObject heuristicObject = new JsonObject();
heuristicObject.addProperty(JsonKeys.NAME, heuristic.heuristicName);
heuristicObject.addProperty(JsonKeys.SEVERITY, heuristic.severity.getText());
heuristicsArray.add(heuristicObject);
}

applicationObject.addProperty(JsonKeys.ID, application.id);
applicationObject.addProperty(JsonKeys.USERNAME, application.username);
applicationObject.addProperty(JsonKeys.START_TIME, application.startTime);
applicationObject.addProperty(JsonKeys.FINISH_TIME, application.finishTime);
applicationObject.addProperty(JsonKeys.RUNTIME, application.finishTime - application.startTime);
applicationObject.addProperty(JsonKeys.WAITTIME, application.totalDelay);
applicationObject.addProperty(JsonKeys.RESOURCE_USED, application.resourceUsed);
applicationObject.addProperty(JsonKeys.RESOURCE_WASTED, application.resourceWasted);
applicationObject.addProperty(JsonKeys.SEVERITY, application.severity.getText());
applicationObject.addProperty(JsonKeys.QUEUE, application.queueName);

applicationObject.add(JsonKeys.HEURISTICS_SUMMARY, heuristicsArray);
applicationSummaryArray.add(applicationObject);
}

JsonObject searchResults = new JsonObject();
searchResults.addProperty(JsonKeys.ID, query.toString());
searchResults.addProperty(JsonKeys.START, offset);
searchResults.addProperty(JsonKeys.END, end);
searchResults.addProperty(JsonKeys.TOTAL, total);
searchResults.add(JsonKeys.SUMMARIES, applicationSummaryArray);
parent.add(JsonKeys.SEARCH_RESULTS, searchResults);
return ok(new Gson().toJson(parent));
}

/**
* This utility method is used to sort the jsonArray based on FinishTime
* @param jsonArray The jsonArray to be sorted
Expand Down
2 changes: 2 additions & 0 deletions conf/routes
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@ GET /rest/application-summaries controllers.api.v1.Web.restAppli
GET /rest/workflows controllers.api.v1.Web.restWorkflowFromFlowId(workflowid: String)
GET /rest/jobs controllers.api.v1.Web.restJobFromJobId(jobid: String)
GET /rest/applications controllers.api.v1.Web.restApplicationFromApplicationId(applicationid: String)
GET /rest/search-options controllers.api.v1.Web.restSearchOptions()
GET /rest/search-results controllers.api.v1.Web.search()
GET /rest/resourceusagedatabyuser controllers.Application.restResourceUsageDataByUser(startTime: String, endTime: String)

# Metrics calls
Expand Down
1 change: 1 addition & 0 deletions test/common/TestConstants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ public class TestConstants {
public static final String REST_WORKFLOWS_PATH = "/rest/workflows";
public static final String REST_JOBS_PATH = "/rest/jobs";
public static final String REST_APPLICATIONS_PATH = "/rest/applications";
public static final String REST_SEARCH_RESULTS = "/rest/search-results";

// Sample mapreduce constants
public static final String FILENAME_JOBCOUNTER = "mrdata/sampleJobCounter.properties";
Expand Down
Loading

0 comments on commit 113c9d3

Please sign in to comment.