Merge pull request #163 from ibm-et/AddHadoopVersionToBuild

Added optional APACHE_HADOOP_VERSION environment variable to allow different versions of Hadoop/YARN (defaults to 2.3.0)
pdendek · Oct 16, 2015 · 8817f8c · 8817f8c
2 parents d8e4a55 + 48885e2
commit 8817f8c
Show file tree

Hide file tree

Showing 2 changed files with 25 additions and 1 deletion.
diff --git a/kernel-api/build.sbt b/kernel-api/build.sbt
@@ -65,7 +65,7 @@ libraryDependencies ++= Seq(
 // HADOOP DEPENDENCIES
 //
 libraryDependencies ++= Seq(
-  "org.apache.hadoop" % "hadoop-client" % "2.3.0" excludeAll
+  "org.apache.hadoop" % "hadoop-client" % hadoopVersion.value excludeAll
     ExclusionRule(organization = "javax.servlet")
 )
 

diff --git a/project/Common.scala b/project/Common.scala
@@ -51,6 +51,8 @@ object Common {
 
   lazy val sparkVersion = settingKey[String]("The Apache Spark version to use")
 
+  lazy val hadoopVersion = settingKey[String]("The Apache Hadoop version to use")
+
   // The prefix used for our custom artifact names
   private val artifactPrefix = "ibm-spark"
 
@@ -83,6 +85,28 @@ object Common {
         version
       }
     },
+    hadoopVersion := {
+      val hadoopEnvironmentVariable = "APACHE_HADOOP_VERSION"
+      val defaultHadoopVersion = "2.3.0"
+
+      val _hadoopVersion = Properties.envOrNone(hadoopEnvironmentVariable)
+
+      if (_hadoopVersion.isEmpty) {
+        scala.Console.out.println(
+          s"""
+             |[INFO] Using default Apache Hadoop $defaultHadoopVersion!
+           """.stripMargin.trim.replace('\n', ' '))
+        defaultHadoopVersion
+      } else {
+        val version = _hadoopVersion.get
+        scala.Console.out.println(
+          s"""
+             |[INFO] Using Apache Hadoop $version provided from
+             |$hadoopEnvironmentVariable!
+           """.stripMargin.trim.replace('\n', ' '))
+        version
+      }
+    },
 
     scalacOptions in (Compile, doc) ++= Seq(
       // Ignore packages (for Scaladoc) not from our project