LIHADOOP-16687 Update Dr. Elephant to use the new Hadoop API

RB=636803 G=superfriends-reviewers R=annag,fli,rratti,viramach A=rratti
dog-sunFlower · Jan 14, 2016 · ba92a86 · ba92a86
1 parent ce4abb4
commit ba92a86
Show file tree

Hide file tree

Showing 5 changed files with 8 additions and 27 deletions.
diff --git a/app/com/linkedin/drelephant/ElephantRunner.java b/app/com/linkedin/drelephant/ElephantRunner.java
@@ -33,7 +33,7 @@
 import model.JobResult;
 
 import org.apache.commons.lang.exception.ExceptionUtils;
-import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.log4j.Logger;
 
 
@@ -49,7 +49,7 @@ public class ElephantRunner implements Runnable {
   private AnalyticJobGenerator _analyticJobGenerator;
 
   private void loadAnalyticJobGenerator() {
-    JobConf configuration = new JobConf();
+    Configuration configuration = new Configuration();
     if (HadoopSystemContext.isHadoop2Env()) {
       _analyticJobGenerator = new AnalyticJobGeneratorHadoop2();
     } else {

diff --git a/app/com/linkedin/drelephant/HadoopSecurity.java b/app/com/linkedin/drelephant/HadoopSecurity.java
@@ -16,7 +16,6 @@
 package com.linkedin.drelephant;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.log4j.Logger;
 
@@ -36,7 +35,7 @@ public class HadoopSecurity {
   private boolean _securityEnabled = false;
 
   public HadoopSecurity() throws IOException {
-    Configuration conf = new JobConf();
+    Configuration conf = new Configuration();
     UserGroupInformation.setConfiguration(conf);
     _securityEnabled = UserGroupInformation.isSecurityEnabled();
     if (_securityEnabled) {

diff --git a/app/com/linkedin/drelephant/analysis/HadoopSystemContext.java b/app/com/linkedin/drelephant/analysis/HadoopSystemContext.java
@@ -16,36 +16,20 @@
 package com.linkedin.drelephant.analysis;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapred.JobConf;
 
 public final class HadoopSystemContext {
 
   private static final String MAPREDUCE_FRAMEWORK_NAME_PROP = "mapreduce.framework.name";
   private static final String MAPRED_JOB_TRACKER_PROP = "mapred.job.tracker.http.address";
   private static final String YARN = "yarn";
-  private static final String CLASSIC = "classic";
-
-  /**
-   * Detect if the current Hadoop environment is 1.x
-   *
-   * @return true if it is Hadoop 1 env, else false
-   */
-  public static boolean isHadoop1Env() {
-    Configuration hadoopConf = new JobConf();
-    String hadoopVersion = hadoopConf.get(MAPREDUCE_FRAMEWORK_NAME_PROP);
-    if (hadoopVersion == null) {
-      return hadoopConf.get(MAPRED_JOB_TRACKER_PROP) != null;
-    }
-    return hadoopVersion.equals(CLASSIC);
-  }
 
   /**
    * Detect if the current Hadoop environment is 2.x
    *
    * @return true if it is Hadoop 2 env, else false
    */
   public static boolean isHadoop2Env() {
-    Configuration hadoopConf = new JobConf();
+    Configuration hadoopConf = new Configuration();
     String hadoopVersion = hadoopConf.get(MAPREDUCE_FRAMEWORK_NAME_PROP);
     return hadoopVersion != null && hadoopVersion.equals(YARN);
   }
@@ -57,6 +41,6 @@ public static boolean isHadoop2Env() {
    * @return true if we have a major version match else false
    */
   public static boolean matchCurrentHadoopVersion(int majorVersion) {
-    return majorVersion == 2 && isHadoop2Env() || majorVersion == 1 && isHadoop1Env();
+    return majorVersion == 2 && isHadoop2Env();
   }
 }
diff --git a/app/com/linkedin/drelephant/mapreduce/MapReduceFetcherHadoop2.java b/app/com/linkedin/drelephant/mapreduce/MapReduceFetcherHadoop2.java
@@ -33,7 +33,7 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
 import org.apache.hadoop.security.authentication.client.AuthenticationException;
 import org.apache.log4j.Logger;
@@ -51,7 +51,7 @@ public class MapReduceFetcherHadoop2 implements ElephantFetcher<MapReduceApplica
 
   public MapReduceFetcherHadoop2() throws IOException {
     logger.info("Connecting to the job history server...");
-    final String jhistoryAddr = new JobConf().get("mapreduce.jobhistory.webapp.address");
+    final String jhistoryAddr = new Configuration().get("mapreduce.jobhistory.webapp.address");
     _urlFactory = new URLFactory(jhistoryAddr);
     _jsonFactory = new JSONFactory();
     _jhistoryWebAddr = "http://" + jhistoryAddr + "/jobhistory/job/";

diff --git a/build.sbt b/build.sbt
@@ -53,10 +53,8 @@ unmanagedClasspath in Compile ++= update.value.select(configurationFilter("compi
 libraryDependencies ++= Seq(
   "com.linkedin.hadoop" % "hadoop-common" % "2.3.0.+" % "compileonly",
   "com.linkedin.hadoop" % "hadoop-hdfs" % "2.3.0.+" % "compileonly",
-  "com.linkedin.hadoop" % "hadoop-mapreduce-client-core" % "2.3.0.+" % "compileonly",
   "com.linkedin.hadoop" % "hadoop-common" % "2.3.0.+" % "test",
-  "com.linkedin.hadoop" % "hadoop-hdfs" % "2.3.0.+" % "test",
-  "com.linkedin.hadoop" % "hadoop-mapreduce-client-core" % "2.3.0.+" % "test"
+  "com.linkedin.hadoop" % "hadoop-hdfs" % "2.3.0.+" % "test"
 )
 
 val LinkedInPatterns = Patterns(