[SPARK-37951][MLLIB][K8S] Move test file from ../data/ to correspondi…

…ng module's resource folder ### What changes were proposed in this pull request? Move test file form `data/` dir to corresponding module's resource folder 1. move `../data/mllib/images/partitioned` to mllib's `resources/inages/partitioned` 2. move `../data/mllib/iris_libsvm.txt` to mllib's `resources/iris_libsvm.txt` 3. copy `data/mllib/pagerank_data.txt` to kubenets-integration-test's `resources/pagerank_data.txt` ### Why are the changes needed? Refactor code to avoid test failure ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Existed UT Closes apache#35237 from AngersZhuuuu/SPARK-37951. Authored-by: Angerszhuuuu <[email protected]> Signed-off-by: Wenchen Fan <[email protected]>
bruno0o · Jan 19, 2022 · 6fba9e2 · 6fba9e2
1 parent 71af5b9
commit 6fba9e2
Show file tree

Hide file tree

Showing 15 changed files with 13 additions and 8 deletions.
diff --git a/mllib/src/main/scala/org/apache/spark/ml/source/image/ImageDataSource.scala b/mllib/src/main/scala/org/apache/spark/ml/source/image/ImageDataSource.scala
@@ -37,12 +37,12 @@ package org.apache.spark.ml.source.image
  *   // Scala
  *   val df = spark.read.format("image")
  *     .option("dropInvalid", true)
- *     .load("data/mllib/images/partitioned")
+ *     .load("/path/to/images")
  *
  *   // Java
  *   Dataset<Row> df = spark.read().format("image")
  *     .option("dropInvalid", true)
- *     .load("data/mllib/images/partitioned");
+ *     .load("/path/to/images");
  * }}}
  *
  * Image data source supports the following options:

diff --git a/...tens/date=2018-01/29.5.a_b_EGDP022204.jpg → ...tens/date=2018-01/29.5.a_b_EGDP022204.jpg b/...tens/date=2018-01/29.5.a_b_EGDP022204.jpg → ...tens/date=2018-01/29.5.a_b_EGDP022204.jpg
diff --git a/...ed/cls=kittens/date=2018-01/not-image.txt → ...ed/cls=kittens/date=2018-01/not-image.txt b/...ed/cls=kittens/date=2018-01/not-image.txt → ...ed/cls=kittens/date=2018-01/not-image.txt
diff --git a/...tioned/cls=kittens/date=2018-02/54893.jpg → ...tioned/cls=kittens/date=2018-02/54893.jpg b/...tioned/cls=kittens/date=2018-02/54893.jpg → ...tioned/cls=kittens/date=2018-02/54893.jpg
diff --git a/...ned/cls=kittens/date=2018-02/DP153539.jpg → ...ned/cls=kittens/date=2018-02/DP153539.jpg b/...ned/cls=kittens/date=2018-02/DP153539.jpg → ...ned/cls=kittens/date=2018-02/DP153539.jpg
diff --git a/...ned/cls=kittens/date=2018-02/DP802813.jpg → ...ned/cls=kittens/date=2018-02/DP802813.jpg b/...ned/cls=kittens/date=2018-02/DP802813.jpg → ...ned/cls=kittens/date=2018-02/DP802813.jpg
diff --git a/...ed/cls=multichannel/date=2018-01/BGRA.png → ...ed/cls=multichannel/date=2018-01/BGRA.png b/...ed/cls=multichannel/date=2018-01/BGRA.png → ...ed/cls=multichannel/date=2018-01/BGRA.png
diff --git a/...ltichannel/date=2018-01/BGRA_alpha_60.png → ...ltichannel/date=2018-01/BGRA_alpha_60.png b/...ltichannel/date=2018-01/BGRA_alpha_60.png → ...ltichannel/date=2018-01/BGRA_alpha_60.png
diff --git a/...multichannel/date=2018-02/chr30.4.184.jpg → ...multichannel/date=2018-02/chr30.4.184.jpg b/...multichannel/date=2018-02/chr30.4.184.jpg → ...multichannel/date=2018-02/chr30.4.184.jpg
diff --git a/...s=multichannel/date=2018-02/grayscale.jpg → ...s=multichannel/date=2018-02/grayscale.jpg b/...s=multichannel/date=2018-02/grayscale.jpg → ...s=multichannel/date=2018-02/grayscale.jpg
diff --git a/data/mllib/iris_libsvm.txt → mllib/src/test/resources/iris_libsvm.txt b/data/mllib/iris_libsvm.txt → mllib/src/test/resources/iris_libsvm.txt
diff --git a/mllib/src/test/scala/org/apache/spark/ml/evaluation/ClusteringEvaluatorSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/evaluation/ClusteringEvaluatorSuite.scala
@@ -40,7 +40,7 @@ class ClusteringEvaluatorSuite
 
   override def beforeAll(): Unit = {
     super.beforeAll()
-    irisDataset = spark.read.format("libsvm").load("../data/mllib/iris_libsvm.txt")
+    irisDataset = spark.read.format("libsvm").load(getTestResourcePath("iris_libsvm.txt"))
     val datasets = MLTestingUtils.generateArrayFeatureDataset(irisDataset)
     newIrisDataset = datasets._1
     newIrisDatasetD = datasets._2

diff --git a/mllib/src/test/scala/org/apache/spark/ml/source/image/ImageFileFormatSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/source/image/ImageFileFormatSuite.scala
@@ -29,8 +29,7 @@ import org.apache.spark.sql.functions.{col, substring_index}
 class ImageFileFormatSuite extends SparkFunSuite with MLlibTestSparkContext {
 
   // Single column of images named "image"
-  private lazy val imagePath = "../data/mllib/images/partitioned"
-  private lazy val recursiveImagePath = "../data/mllib/images"
+  private lazy val imagePath = getTestResourcePath("images/partitioned")
 
   test("Smoke test: create basic ImageSchema dataframe") {
     val origin = "path"

diff --git a/resource-managers/kubernetes/integration-tests/src/test/resources/pagerank_data.txt b/resource-managers/kubernetes/integration-tests/src/test/resources/pagerank_data.txt
@@ -0,0 +1,6 @@
+1 2
+1 3
+1 4
+2 1
+3 1
+4 1
diff --git a/...on-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala b/...on-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala
@@ -22,7 +22,7 @@ import io.fabric8.kubernetes.api.model.Pod
 import org.scalatest.concurrent.Eventually
 import org.scalatest.matchers.should.Matchers._
 
-import org.apache.spark.TestUtils
+import org.apache.spark.{SparkFunSuite, TestUtils}
 import org.apache.spark.launcher.SparkLauncher
 
 private[spark] trait BasicTestsSuite { k8sSuite: KubernetesSuite =>
@@ -126,11 +126,11 @@ private[spark] trait BasicTestsSuite { k8sSuite: KubernetesSuite =>
   }
 }
 
-private[spark] object BasicTestsSuite {
+private[spark] object BasicTestsSuite extends SparkFunSuite {
   val SPARK_PAGE_RANK_MAIN_CLASS: String = "org.apache.spark.examples.SparkPageRank"
   val CONTAINER_LOCAL_FILE_DOWNLOAD_PATH = "/var/spark-data/spark-files"
   val CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE =
      s"$CONTAINER_LOCAL_FILE_DOWNLOAD_PATH/pagerank_data.txt"
-  val REMOTE_PAGE_RANK_DATA_FILE = "data/mllib/pagerank_data.txt"
+  val REMOTE_PAGE_RANK_DATA_FILE = getTestResourcePath("pagerank_data.txt")
   val REMOTE_PAGE_RANK_FILE_NAME = "pagerank_data.txt"
 }
-Original file line number
+Diff line change
@@ -0,0 +1,6 @@
+2
+3
+4
+1
+1
+1