Skip to content

Commit

Permalink
Revert "[MINOR] [BUILD] Use custom temp directory during build."
Browse files Browse the repository at this point in the history
This reverts commit b16b543.
  • Loading branch information
Andrew Or committed Jun 5, 2015
1 parent 12f5eae commit 4036d05
Show file tree
Hide file tree
Showing 3 changed files with 11 additions and 16 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -28,12 +28,9 @@ import org.apache.ivy.plugins.resolver.IBiblioResolver

import org.apache.spark.SparkFunSuite
import org.apache.spark.deploy.SparkSubmitUtils.MavenCoordinate
import org.apache.spark.util.Utils

class SparkSubmitUtilsSuite extends SparkFunSuite with BeforeAndAfterAll {

private var tempIvyPath: String = _

private val noOpOutputStream = new OutputStream {
def write(b: Int) = {}
}
Expand All @@ -50,7 +47,6 @@ class SparkSubmitUtilsSuite extends SparkFunSuite with BeforeAndAfterAll {
super.beforeAll()
// We don't want to write logs during testing
SparkSubmitUtils.printStream = new BufferPrintStream
tempIvyPath = Utils.createTempDir(namePrefix = "ivy").getAbsolutePath()
}

test("incorrect maven coordinate throws error") {
Expand Down Expand Up @@ -94,20 +90,21 @@ class SparkSubmitUtilsSuite extends SparkFunSuite with BeforeAndAfterAll {
}

test("ivy path works correctly") {
val ivyPath = "dummy" + File.separator + "ivy"
val md = SparkSubmitUtils.getModuleDescriptor
val artifacts = for (i <- 0 until 3) yield new MDArtifact(md, s"jar-$i", "jar", "jar")
var jPaths = SparkSubmitUtils.resolveDependencyPaths(artifacts.toArray, new File(tempIvyPath))
var jPaths = SparkSubmitUtils.resolveDependencyPaths(artifacts.toArray, new File(ivyPath))
for (i <- 0 until 3) {
val index = jPaths.indexOf(tempIvyPath)
val index = jPaths.indexOf(ivyPath)
assert(index >= 0)
jPaths = jPaths.substring(index + tempIvyPath.length)
jPaths = jPaths.substring(index + ivyPath.length)
}
val main = MavenCoordinate("my.awesome.lib", "mylib", "0.1")
IvyTestUtils.withRepository(main, None, None) { repo =>
// end to end
val jarPath = SparkSubmitUtils.resolveMavenCoordinates(main.toString, Option(repo),
Option(tempIvyPath), true)
assert(jarPath.indexOf(tempIvyPath) >= 0, "should use non-default ivy path")
Option(ivyPath), true)
assert(jarPath.indexOf(ivyPath) >= 0, "should use non-default ivy path")
}
}

Expand All @@ -126,12 +123,13 @@ class SparkSubmitUtilsSuite extends SparkFunSuite with BeforeAndAfterAll {
assert(jarPath.indexOf("mylib") >= 0, "should find artifact")
}
// Local ivy repository with modified home
val dummyIvyLocal = new File(tempIvyPath, "local" + File.separator)
val dummyIvyPath = "dummy" + File.separator + "ivy"
val dummyIvyLocal = new File(dummyIvyPath, "local" + File.separator)
IvyTestUtils.withRepository(main, None, Some(dummyIvyLocal), true) { repo =>
val jarPath = SparkSubmitUtils.resolveMavenCoordinates(main.toString, None,
Some(tempIvyPath), true)
Some(dummyIvyPath), true)
assert(jarPath.indexOf("mylib") >= 0, "should find artifact")
assert(jarPath.indexOf(tempIvyPath) >= 0, "should be in new ivy path")
assert(jarPath.indexOf(dummyIvyPath) >= 0, "should be in new ivy path")
}
}

Expand Down
4 changes: 1 addition & 3 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,7 @@
<parquet.deps.scope>compile</parquet.deps.scope>

<!--
Overridable test home. So that you can call individual pom files directly without
Overridable test home. So that you can call individual pom files directory without
things breaking.
-->
<spark.test.home>${session.executionRootDirectory}</spark.test.home>
Expand Down Expand Up @@ -1256,7 +1256,6 @@
<systemProperties>
<derby.system.durability>test</derby.system.durability>
<java.awt.headless>true</java.awt.headless>
<java.io.tmpdir>${project.build.directory}/tmp</java.io.tmpdir>
<spark.test.home>${spark.test.home}</spark.test.home>
<spark.testing>1</spark.testing>
<spark.ui.enabled>false</spark.ui.enabled>
Expand Down Expand Up @@ -1290,7 +1289,6 @@
<systemProperties>
<derby.system.durability>test</derby.system.durability>
<java.awt.headless>true</java.awt.headless>
<java.io.tmpdir>${project.build.directory}/tmp</java.io.tmpdir>
<spark.test.home>${spark.test.home}</spark.test.home>
<spark.testing>1</spark.testing>
<spark.ui.enabled>false</spark.ui.enabled>
Expand Down
1 change: 0 additions & 1 deletion project/SparkBuild.scala
Original file line number Diff line number Diff line change
Expand Up @@ -496,7 +496,6 @@ object TestSettings {
"SPARK_DIST_CLASSPATH" ->
(fullClasspath in Test).value.files.map(_.getAbsolutePath).mkString(":").stripSuffix(":"),
"JAVA_HOME" -> sys.env.get("JAVA_HOME").getOrElse(sys.props("java.home"))),
javaOptions in Test += s"-Djava.io.tmpdir=$sparkHome/target/tmp",
javaOptions in Test += "-Dspark.test.home=" + sparkHome,
javaOptions in Test += "-Dspark.testing=1",
javaOptions in Test += "-Dspark.port.maxRetries=100",
Expand Down

0 comments on commit 4036d05

Please sign in to comment.