Integration of the ember app with play framework

Other fixes: * Additional information to workflow, workflow-summaries rest end point * Added an rest end point for the dashboard summary
kimoonkim · Oct 24, 2016 · a8ec16d · a8ec16d
1 parent 39888ff
commit a8ec16d
Show file tree

Hide file tree

Showing 6 changed files with 167 additions and 11 deletions.
diff --git a/app/controllers/Application.java b/app/controllers/Application.java
@@ -55,6 +55,7 @@
 import views.html.help.metrics.helpWaittime;
 import views.html.help.metrics.helpUsedResources;
 import views.html.help.metrics.helpWastedResources;
+import views.html.index;
 import views.html.page.comparePage;
 import views.html.page.flowHistoryPage;
 import views.html.page.helpPage;
@@ -122,6 +123,14 @@ private enum Version {OLD,NEW};
   private static int _numJobsCritical = 0;
   private static int _numJobsSevere = 0;
 
+
+  /**
+  * Serves the initial index.html page for the new user interface. This page contains the whole web app
+  */
+  public static Result serveAsset(String path) {
+    return ok(index.render());
+  }
+
   /**
    * Controls the Home page of Dr. Elephant.
    *
@@ -607,13 +616,8 @@ private static Result getFlowHistory(Version version) {
         return ok(flowHistoryPage.render(flowDefPair.getId(), graphType,
             flowHistoryResults.render(flowDefPair, executionMap, idPairToJobNameMap, flowExecTimeList)));
       } else if (graphType.equals("resources") || graphType.equals("time")) {
-        if (hasSparkJob) {
-          return notFound("Cannot plot graph for " + graphType + " since it contains a spark job. " + graphType
-              + " graphs are not supported for spark right now");
-        } else {
           return ok(flowHistoryPage.render(flowDefPair.getId(), graphType, flowMetricsHistoryResults
               .render(flowDefPair, graphType, executionMap, idPairToJobNameMap, flowExecTimeList)));
-        }
       }
     } else {
       if (graphType.equals("heuristics")) {
@@ -743,12 +747,8 @@ private static Result getJobHistory(Version version) {
         return ok(jobHistoryPage.render(jobDefPair.getId(), graphType,
             jobHistoryResults.render(jobDefPair, executionMap, maxStages, flowExecTimeList)));
       } else if (graphType.equals("resources") || graphType.equals("time")) {
-        if (hasSparkJob) {
-          return notFound("Resource and time graph are not supported for spark right now");
-        } else {
           return ok(jobHistoryPage.render(jobDefPair.getId(), graphType,
               jobMetricsHistoryResults.render(jobDefPair, graphType, executionMap, maxStages, flowExecTimeList)));
-        }
       }
     } else {
       if (graphType.equals("heuristics")) {

diff --git a/app/controllers/api/v1/JsonKeys.java b/app/controllers/api/v1/JsonKeys.java
@@ -42,6 +42,13 @@ public class JsonKeys {
   public static final String JOB_CATEGORY = "jobcategory";
   public static final String TOTAL = "total";
   public static final String SUMMARIES = "summaries";
+  public static final String SCHEDULER = "scheduler";
+  public static final String CRITICAL = "critical";
+  public static final String SEVERE = "severe";
+  public static final String MODERATE = "moderate";
+  public static final String LOW = "low";
+  public static final String NONE = "none";
+  public static final String DASHBOARD_SUMMARIES = "dashboard-summaries";
 
   // Workflows
   public static final String WORKFLOW_SUMMARIES = "workflow-summaries";

diff --git a/app/controllers/api/v1/Web.java b/app/controllers/api/v1/Web.java
@@ -52,6 +52,10 @@
 public class Web extends Controller {
 
   private static final Logger logger = Logger.getLogger(Web.class);
+
+  private static final long DAY = 24 * 60 * 60 * 1000;
+  private static final long FETCH_DELAY = 60 * 1000;
+
   private static final int MAX_APPLICATIONS = 50;
   private static final int MAX_APPLICATIONS_IN_WORKFLOW = 5000;
   private static final int MAX_APPLICATIONS_IN_JOB = 5000;
@@ -61,6 +65,63 @@ public class Web extends Controller {
   private static final int SEARCH_DEFAULT_PAGE_LIMIT = 25;
   private static final int SEARCH_APPLICATION_MAX_OFFSET = 500;
 
+  private static long _lastFetch = 0;
+  private static int _numJobsAnalyzed = 0;
+  private static int _numJobsCritical = 0;
+  private static int _numJobsSevere = 0;
+  private static int _numJobsModerate = 0;
+  private static int _numJobsLow = 0;
+  private static int _numJobsNone = 0;
+
+  /**
+  * Returns the json object for the dashboard summaries of jobs analzyed in last day.
+  */
+  public static Result restDashboardSummaries() {
+
+    long now = System.currentTimeMillis();
+    long finishDate = now - DAY;
+
+    //Update statistics only after FETCH_DELAY
+    if (now - _lastFetch > FETCH_DELAY) {
+      _numJobsAnalyzed = AppResult.find.where().gt(AppResult.TABLE.FINISH_TIME, finishDate).findRowCount();
+      _numJobsCritical = AppResult.find.where()
+        .gt(AppResult.TABLE.FINISH_TIME, finishDate)
+        .eq(AppResult.TABLE.SEVERITY, Severity.CRITICAL.getValue())
+        .findRowCount();
+      _numJobsSevere = AppResult.find.where()
+        .gt(AppResult.TABLE.FINISH_TIME, finishDate)
+        .eq(AppResult.TABLE.SEVERITY, Severity.SEVERE.getValue())
+        .findRowCount();
+      _numJobsModerate = AppResult.find.where()
+        .gt(AppResult.TABLE.FINISH_TIME, finishDate)
+        .eq(AppResult.TABLE.SEVERITY, Severity.MODERATE.getValue())
+        .findRowCount();
+      _numJobsLow = AppResult.find.where()
+        .gt(AppResult.TABLE.FINISH_TIME, finishDate)
+        .eq(AppResult.TABLE.SEVERITY, Severity.LOW.getValue())
+        .findRowCount();
+      _numJobsNone = AppResult.find.where()
+        .gt(AppResult.TABLE.FINISH_TIME, finishDate)
+        .eq(AppResult.TABLE.SEVERITY, Severity.NONE.getValue())
+        .findRowCount();
+      _lastFetch = now;
+    }
+
+    JsonObject dashboard = new JsonObject();
+    dashboard.addProperty(JsonKeys.ID, "dashboard");
+    dashboard.addProperty(JsonKeys.TOTAL, _numJobsAnalyzed);
+    dashboard.addProperty(JsonKeys.CRITICAL, _numJobsCritical);
+    dashboard.addProperty(JsonKeys.SEVERE, _numJobsSevere);
+    dashboard.addProperty(JsonKeys.MODERATE, _numJobsModerate);
+    dashboard.addProperty(JsonKeys.LOW, _numJobsLow);
+    dashboard.addProperty(JsonKeys.NONE, _numJobsNone);
+    JsonObject parent = new JsonObject();
+    parent.add(JsonKeys.DASHBOARD_SUMMARIES, dashboard);
+
+    return ok(new Gson().toJson(parent));
+  }
+
+
   /**
    * Returns the list of AppResults for the given username limit by maxApplications
    * @param username The username for which applications need to be fetched.
@@ -520,6 +581,9 @@ public static Result restWorkflowSummariesForUser(String username) {
       dataset.addProperty(JsonKeys.RESOURCE_WASTED, totalFlowMemoryWasted);
       dataset.addProperty(JsonKeys.QUEUE, mrJobsList.get(0).queueName);
       dataset.addProperty(JsonKeys.SEVERITY, flowSeverity.getText());
+      dataset.addProperty(JsonKeys.SCHEDULER, mrJobsList.get(0).scheduler);
+      dataset.addProperty(JsonKeys.FLOW_EXEC_ID, mrJobsList.get(0).flowExecId);
+      dataset.addProperty(JsonKeys.FLOW_DEF_ID, mrJobsList.get(0).flowDefId);
       dataset.add(JsonKeys.JOBS_SEVERITY, jobSeverity);
       workflowSummaryArray.add(dataset);
     }
@@ -597,6 +661,7 @@ public static Result restWorkflowFromFlowId(String flowId) {
     String flowDefinitionId = "";
     Map<Severity, Long> jobSeverityCount = new HashMap<Severity, Long>();
     String wfQueueName = "";
+    String wfSchedulerName = "";
 
     List<AppResult> results = getRestFlowResultsFromFlowExecutionId(flowId);
 
@@ -621,6 +686,7 @@ public static Result restWorkflowFromFlowId(String flowId) {
       String jobId = jobDefPair.getId();
       String jobName = "";
       String queueName = "";
+      String schedulerName = "";
 
       Map<Severity, Long> taskSeverityCount = new HashMap<Severity, Long>();
 
@@ -632,6 +698,7 @@ public static Result restWorkflowFromFlowId(String flowId) {
         jobName = task.jobName;
         flowDefinitionId = task.flowDefId;
         queueName = task.queueName;
+	schedulerName = task.scheduler;
 
         if (task.startTime < jobStartTime) {
           jobStartTime = task.startTime;
@@ -667,6 +734,7 @@ public static Result restWorkflowFromFlowId(String flowId) {
       }
 
       wfQueueName = queueName;
+      wfSchedulerName = schedulerName;
       totalJobDelay = Utils.getTotalWaittime(jobExecIdToJobsMap.get(jobDefPair));
       totalJobRuntime = Utils.getTotalRuntime(jobExecIdToJobsMap.get(jobDefPair));
 
@@ -682,6 +750,7 @@ public static Result restWorkflowFromFlowId(String flowId) {
       jobObject.addProperty(JsonKeys.RESOURCE_USED, totalJobMemoryUsed);
       jobObject.addProperty(JsonKeys.RESOURCE_WASTED, totalJobMemoryWasted);
       jobObject.addProperty(JsonKeys.QUEUE, queueName);
+      jobObject.addProperty(JsonKeys.SCHEDULER, schedulerName);
       jobObject.addProperty(JsonKeys.SEVERITY, jobSeverity.getText());
       jobObject.add(JsonKeys.TASKS_SEVERITY, taskSeverity);
 
@@ -723,6 +792,7 @@ public static Result restWorkflowFromFlowId(String flowId) {
     data.addProperty(JsonKeys.FLOW_EXEC_ID, flowExecId);
     data.addProperty(JsonKeys.FLOW_DEF_ID, flowDefinitionId);
     data.addProperty(JsonKeys.QUEUE, wfQueueName);
+    data.addProperty(JsonKeys.SCHEDULER, wfSchedulerName);
     data.add(JsonKeys.JOBSSUMMARIES, jobSummaryArray);
     data.add(JsonKeys.JOBS_SEVERITY, jobSeverityArray);
     JsonObject parent = new JsonObject();

diff --git a/app/views/index.scala.html b/app/views/index.scala.html
@@ -0,0 +1,42 @@
+<!--
+  Copyright 2016 LinkedIn Corp.
+
+  Licensed under the Apache License, Version 2.0 (the "License"); you may not
+  use this file except in compliance with the License. You may obtain a copy of
+  the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+  License for the specific language governing permissions and limitations under
+  the License.
+-->
+
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="utf-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <title>DrElephant</title>
+    <meta name="description" content="">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+
+
+
+    <link rel="stylesheet" href="assets/assets/ember/vendor.css">
+    <link rel="stylesheet" href="assets/assets/ember/dr-elephant.css">
+
+
+  </head>
+  <body>
+
+
+    <script src="assets/assets/ember/vendor.js"></script>
+    <script src="assets/assets/ember/dr-elephant.js"></script>
+
+
+  </body>
+</html>
+
diff --git a/compile.sh b/compile.sh
@@ -76,12 +76,46 @@ OPTS+=" -Dhadoopversion=$HADOOP_VERSION"
 OPTS+=" -Dsparkversion=$SPARK_VERSION"
 OPTS+=" $PLAY_OPTS"
 
-set -x
-trap "exit" SIGINT SIGTERM
 
 project_root=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 cd ${project_root}
 
+cd ${project_root}
+
+
+#if npm is installed, install bower,ember-cli and other components for new UI
+
+if hash npm 2>/dev/null; then
+  echo "############################################################################"
+  echo "npm installation found, we'll compile with the new user interface"
+  echo "############################################################################"
+  sleep 3
+  ember_resources_dir=${project_root}/public/assets/ember
+  ember_web_directory=${project_root}/web
+
+  # cd to the ember directory
+  cd ${ember_web_directory}
+
+  npm install
+  node_modules/bower/bin/bower install
+  node_modules/ember-cli/bin/ember build --prod
+  rm -r ${ember_resources_dir} 2> /dev/null
+  mkdir ${ember_resources_dir}
+  cp dist/assets/dr-elephant.css ${ember_resources_dir}/
+  cp dist/assets/dr-elephant.js ${ember_resources_dir}/
+  cp dist/assets/vendor.js ${ember_resources_dir}/
+  cp dist/assets/vendor.css ${ember_resources_dir}/
+  cd ${project_root}
+else
+  echo "############################################################################"
+  echo "npm installation not found. Please install npm in order to compile with new user interface"
+  echo "############################################################################"
+  sleep 3
+fi
+
+set -x
+trap "exit" SIGINT SIGTERM
+
 start_script=${project_root}/scripts/start.sh
 stop_script=${project_root}/scripts/stop.sh
 app_conf=${project_root}/app-conf

diff --git a/conf/routes b/conf/routes
@@ -42,6 +42,7 @@ GET        /rest/flowgraphdata                  controllers.Application.restFlow
 GET        /rest/jobgraphdata                   controllers.Application.restJobGraphData(id: String)
 GET        /rest/jobmetricsgraphdata            controllers.Application.restJobMetricsGraphData(id: String)
 GET        /rest/flowmetricsgraphdata           controllers.Application.restFlowMetricsGraphData(id: String)
+GET        /rest/dashboard-summaries            controllers.api.v1.Web.restDashboardSummaries()
 GET        /rest/workflow-summaries             controllers.api.v1.Web.restWorkflowSummariesForUser(username: String)
 GET        /rest/job-summaries                  controllers.api.v1.Web.restJobSummariesForUser(username: String)
 GET        /rest/application-summaries          controllers.api.v1.Web.restApplicationSummariesForUser(username: String)
@@ -59,3 +60,5 @@ GET        /healthcheck                         controllers.MetricsController.he
 
 # Map static resources from the /public folder to the /assets URL path
 GET        /assets/*file                        controllers.Assets.at(path="/public", file)
+GET        /new                                 controllers.Application.serveAsset(path="index.html")
+GET        /new/*path                           controllers.Application.serveAsset(path)