LIHADOOP-17485: Refactor and cleanup Dr. Elephant code

RB=660706

G=superfriends-reviewers
R=annag,fli,rratti,shanm,viramach
A=annag

README.md

@@ -122,7 +122,7 @@ jvm\_props="... -Devolutionplugin=enabled -DapplyEvolutions.default=true"
 ```
 * To start dr-elephant, run the start script specifying a path to the application's configuration files.  
 ```
-$DR\_RELEASE/bin/start.sh -appconf $DR\_RELEASE/../../app-conf
+$DR\_RELEASE/bin/start.sh $DR\_RELEASE/../../app-conf
 ```
 * To stop dr-elephant run,  
 ```
@@ -209,6 +209,41 @@ cd /export/apps/elephant/
 ```
 * Run Doctor Elephant, it should now include the new heuristics.
 
+## Project Structure
+
+    app                             → Contains all the source files
+     └ com.linkedin.drelepahnt      → Application Daemons
+     └ org.apache.spark             → Spark Support
+     └ controllers                  → Controller logic
+     └ models                       → Includes models that Map to DB
+     └ views                        → Page templates
+
+    app-conf                        → Application Configurations
+     └ elephant.conf                → Port, DB, Keytab and other JVM Configurations (Overrides application.conf)
+     └ FetcherConf.xml              → Fetcher Configurations
+     └ HeuristicConf.xml            → Heuristic Configurations
+     └ JobTypeConf.xml              → JobType Configurations
+
+    conf                            → Configurations files
+     └ evolutions                   → DB Schema
+     └ application.conf             → Main configuration file
+     └ log4j.properties             → log configuration file
+     └ routes                       → Routes definition
+
+    public                          → Public assets
+     └ assets                       → Library files
+     └ css                          → CSS files
+     └ images                       → Image files
+     └ js                           → Javascript files
+
+    scripts
+     └ start.sh                     → Starts Dr. Elephant
+     └ stop.sh                      → Stops Dr. Elephant
+
+    test                            → Source folder for unit tests
+
+    compile.sh                      → Compiles the application
+
 ## License
 
     Copyright 2015 LinkedIn Corp.

app/Global.java

@@ -13,6 +13,7 @@
  * License for the specific language governing permissions and limitations under
  * the License.
  */
+
 import com.linkedin.drelephant.DrElephant;
 import com.sun.security.sasl.util.AbstractSaslImpl;
 
@@ -26,12 +27,15 @@
 import java.util.logging.Level;
 
 
+/**
+ * This class manages all the global settings
+ */
 public class Global extends GlobalSettings {
 
   DrElephant _drElephant;
 
   public void onStart(Application app) {
-    Logger.info("Application has started");
+    Logger.info("Starting Application...");
 
     fixJavaKerberos();
 
@@ -44,7 +48,7 @@ public void onStart(Application app) {
   }
 
   public void onStop(Application app) {
-    Logger.info("Application shutdown...");
+    Logger.info("Stopping application...");
     if (_drElephant != null) {
       _drElephant.kill();
     }

app/com/linkedin/drelephant/DrElephant.java

@@ -13,11 +13,15 @@
  * License for the specific language governing permissions and limitations under
  * the License.
  */
+
 package com.linkedin.drelephant;
 
 import java.io.IOException;
 
 
+/**
+ * The main class which starts Dr. Elephant
+ */
 public class DrElephant extends Thread {
   private ElephantRunner _elephant;
 

app/com/linkedin/drelephant/ElephantContext.java

@@ -13,6 +13,7 @@
  * License for the specific language governing permissions and limitations under
  * the License.
  */
+
 package com.linkedin.drelephant;
 
 import com.google.common.collect.ImmutableMap;
@@ -23,11 +24,11 @@
 import com.linkedin.drelephant.analysis.Heuristic;
 import com.linkedin.drelephant.analysis.HeuristicResult;
 import com.linkedin.drelephant.analysis.JobType;
-import com.linkedin.drelephant.util.FetcherConfiguration;
-import com.linkedin.drelephant.util.FetcherConfigurationData;
-import com.linkedin.drelephant.util.HeuristicConfiguration;
-import com.linkedin.drelephant.util.HeuristicConfigurationData;
-import com.linkedin.drelephant.util.JobTypeConf;
+import com.linkedin.drelephant.configurations.fetcher.FetcherConfiguration;
+import com.linkedin.drelephant.configurations.fetcher.FetcherConfigurationData;
+import com.linkedin.drelephant.configurations.heuristic.HeuristicConfiguration;
+import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData;
+import com.linkedin.drelephant.configurations.jobtype.JobTypeConf;
 import com.linkedin.drelephant.util.Utils;
 import java.lang.reflect.InvocationTargetException;
 import java.util.ArrayList;
@@ -63,8 +64,7 @@ public class ElephantContext {
   private List<FetcherConfigurationData> _fetchersConfData;
 
   private final Map<String, ApplicationType> _nameToType = new HashMap<String, ApplicationType>();
-  private final Map<ApplicationType, List<Heuristic>> _typeToHeuristics =
-      new HashMap<ApplicationType, List<Heuristic>>();
+  private final Map<ApplicationType, List<Heuristic>> _typeToHeuristics = new HashMap<ApplicationType, List<Heuristic>>();
   private final Map<ApplicationType, ElephantFetcher> _typeToFetcher = new HashMap<ApplicationType, ElephantFetcher>();
   private Map<ApplicationType, List<JobType>> _appTypeToJobTypes = new HashMap<ApplicationType, List<JobType>>();
 
@@ -205,7 +205,7 @@ private void configureSupportedApplicationTypes() {
     _typeToHeuristics.keySet().retainAll(supportedTypes);
     _appTypeToJobTypes.keySet().retainAll(supportedTypes);
 
-    logger.info("ElephantContext configured:");
+    logger.info("Configuring ElephantContext...");
     for (ApplicationType type : supportedTypes) {
       _nameToType.put(type.getName(), type);
 
@@ -216,12 +216,15 @@ private void configureSupportedApplicationTypes() {
       }
 
       List<JobType> jobTypes = _appTypeToJobTypes.get(type);
-      logger.info("Supports " + type.getName() + " application type, using " + _typeToFetcher.get(type)
+      logger.info("Supports " + type.getName() + " application type, using " + _typeToFetcher.get(type).toString()
           + " fetcher class with Heuristics [" + StringUtils.join(classes, ", ") + "] and following JobTypes ["
           + StringUtils.join(jobTypes, ", ") + "].");
     }
   }
 
+  /**
+   * Load all the job types configured in JobTypeConf.xml
+   */
   private void loadJobTypes() {
     JobTypeConf conf = new JobTypeConf(JOB_TYPES_CONF);
     _appTypeToJobTypes = conf.getAppTypeToJobTypeList();

app/com/linkedin/drelephant/ElephantRunner.java

@@ -13,6 +13,7 @@
  * License for the specific language governing permissions and limitations under
  * the License.
  */
+
 package com.linkedin.drelephant;
 
 import com.linkedin.drelephant.analysis.AnalyticJob;
@@ -21,6 +22,7 @@
 import com.linkedin.drelephant.analysis.HadoopSystemContext;
 import com.linkedin.drelephant.analysis.AnalyticJobGeneratorHadoop2;
 
+import com.linkedin.drelephant.security.HadoopSecurity;
 import java.io.IOException;
 import java.security.PrivilegedAction;
 import java.util.List;
@@ -37,10 +39,15 @@
 import org.apache.log4j.Logger;
 
 
+/**
+ * The class that runs the Dr. Elephant daemon
+ */
 public class ElephantRunner implements Runnable {
-  private static final long WAIT_INTERVAL = 60 * 1000;
-  private static final int EXECUTOR_NUM = 5;
   private static final Logger logger = Logger.getLogger(ElephantRunner.class);
+
+  private static final long WAIT_INTERVAL = 60 * 1000;      // Interval between fetches and retries
+  private static final int EXECUTOR_NUM = 5;                // The number of executor threads to analyse the jobs
+
   private AtomicBoolean _running = new AtomicBoolean(true);
   private long lastRun;
   private HadoopSecurity _hadoopSecurity;
@@ -86,7 +93,7 @@ public Void run() {
           while (_running.get() && !Thread.currentThread().isInterrupted()) {
             lastRun = System.currentTimeMillis();
 
-            logger.info("Fetching analytic job list.....");
+            logger.info("Fetching analytic job list...");
 
             try {
               _hadoopSecurity.checkLogin();