Skip to content

Commit

Permalink
LIVY-229. Spark version check fails when it's used with HDP built Spark.
Browse files Browse the repository at this point in the history
Fixed Spark version regex to support version like 2.0.0.2.5.1.0-56.

Closes apache#209
  • Loading branch information
alex-the-man authored Oct 20, 2016
1 parent f1aaa14 commit f3d6cd5
Show file tree
Hide file tree
Showing 2 changed files with 6 additions and 2 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ object LivySparkUtils {
* @return Two element tuple, one is major version and the other is minor version
*/
def formatSparkVersion(version: String): (Int, Int) = {
val versionPattern = """(\d)+\.(\d)+(?:\.\d*)?""".r
val versionPattern = """(\d)+\.(\d)+(?:[\.-]\d*)*""".r
version match {
case versionPattern(major, minor) =>
(major.toInt, minor.toInt)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,10 +47,10 @@ class LivySparkUtilsSuite extends FunSuite with LivyBaseUnitTestSuite {
testSparkVersion("2.0.0")
testSparkVersion("2.0.1")
testSparkVersion("2.0.2")
testSparkVersion("2.0.0.2.5.1.0-56") // LIVY-229
}

test("should not support Spark older than 1.6 or newer than 2.0") {
val s = new LivyServer()
intercept[IllegalArgumentException] { testSparkVersion("1.4.0") }
intercept[IllegalArgumentException] { testSparkVersion("1.5.0") }
intercept[IllegalArgumentException] { testSparkVersion("1.5.1") }
Expand All @@ -61,6 +61,10 @@ class LivySparkUtilsSuite extends FunSuite with LivyBaseUnitTestSuite {
intercept[IllegalArgumentException] { testSparkVersion("2.2.1") }
}

test("should fail on bad version") {
intercept[IllegalArgumentException] { testSparkVersion("not a version") }
}

test("should error out if recovery is turned on but master isn't yarn") {
val livyConf = new LivyConf()
livyConf.set(LivyConf.LIVY_SPARK_MASTER, "local")
Expand Down

0 comments on commit f3d6cd5

Please sign in to comment.