Skip to content

Commit

Permalink
LIVY-219. Support Scala-2.11 integration test for Livy IT.
Browse files Browse the repository at this point in the history 
- Added Maven submodules (minicluster-dependencies) to write Spark classpath of different Scala version to files for MiniCluster.
- Writing classpath to a file instead of Maven properties to maintain ability to run integration tests from IDEs.
- MiniCluster picks and reads the correct Spark classpath and pass it to MiniLivy via spark-default.conf.
- Dependency changes to make sure both repl is built when Maven runs integration-test.
- Verify "ensure Livy internal configurations are not exposed" in InteractiveIT instead of JobsApiIT to avoid job incompatibility issue between Scala 2.10 and 2.11.
- Changed Maven pom.xml to start integration tests in 2.10 and 2.11.
- MiniCluster sets env. var. to tell LivyServer what Spark Scala version it should use.
- Changed Travis to launch 2 builds for Spark 1.6 and Spark 2.0 with Scala 2.10 and 2.11.
- During development, to skip integration test for a Scala version, use `-DskipITs-2.10` or `-DskipITs-2.11`. `-DskipITs` will skip both.
- Fixed JaCoCo EOFException by disabling JaCoCo for `InteractiveIT.should kill RSCDriver if it doesn't respond to end session`.

Closes apache#211
  • Loading branch information
@alex-the-man
alex-the-man authored Oct 29, 2016
1 parent e7e929f commit 8ebc5bd
Show file tree
Hide file tree
Showing 12 changed files with 411 additions and 65 deletions.
8 changes: 6 additions & 2 deletions .travis.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@ sudo: required
dist: trusty
language: scala

env:
- MVN_FLAG=-Pspark-1.6
- MVN_FLAG=-Pspark-2.0

jdk:
- oraclejdk7

Expand All @@ -26,14 +30,14 @@ before_install:
- pip3 install --user cloudpickle

install:
- mvn install -Dskip -DskipTests -DskipITs -Dmaven.javadoc.skip=true -B -V
- mvn $MVN_FLAG install -Dskip -DskipTests -DskipITs -Dmaven.javadoc.skip=true -B -V

before_script:
- pip install --user requests pytest flaky flake8
- pip3 install --user requests pytest flaky

script:
- mvn verify -e
- mvn $MVN_FLAG verify -e

after_success:
- bash <(curl -s https://codecov.io/bash)
136 changes: 136 additions & 0 deletions integration-test/minicluster-dependencies/pom.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
<?xml version="1.0" encoding="utf-8"?>
<!--
~ Licensed to Cloudera, Inc. under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. Cloudera, Inc. licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<!-- This module is created to generate classpath for MiniCluster to run integration test with different
Scala version -->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.cloudera.livy</groupId>
<artifactId>multi-scala-project-root</artifactId>
<relativePath>../../scala/pom.xml</relativePath>
<version>0.3.0-SNAPSHOT</version>
</parent>
<artifactId>minicluster-dependencies-parent</artifactId>
<version>0.3.0-SNAPSHOT</version>
<packaging>pom</packaging>
<properties>
<skipDeploy>true</skipDeploy>
</properties>
<dependencies>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.module</groupId>
<artifactId>jackson-module-scala_${scala.binary.version}</artifactId>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.5</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-client</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<exclusions>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-hive_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-repl_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-yarn_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
</dependency>
<dependency>
<groupId>org.objenesis</groupId>
<artifactId>objenesis</artifactId>
<version>1.2</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<phase>package</phase>
<goals><goal>build-classpath</goal></goals>
<configuration>
<outputFile>target/classpath</outputFile>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
37 changes: 37 additions & 0 deletions integration-test/minicluster-dependencies/scala-2.10/pom.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to Cloudera, Inc. under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. Cloudera, Inc. licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.cloudera.livy</groupId>
<artifactId>minicluster-dependencies_2.10</artifactId>
<version>0.3.0-SNAPSHOT</version>
<packaging>jar</packaging>

<parent>
<groupId>com.cloudera.livy</groupId>
<artifactId>minicluster-dependencies-parent</artifactId>
<version>0.3.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

<properties>
<scala.version>${scala-2.10.version}</scala.version>
<scala.binary.version>2.10</scala.binary.version>
</properties>
</project>
37 changes: 37 additions & 0 deletions integration-test/minicluster-dependencies/scala-2.11/pom.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to Cloudera, Inc. under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. Cloudera, Inc. licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.cloudera.livy</groupId>
<artifactId>minicluster-dependencies_2.11</artifactId>
<version>0.3.0-SNAPSHOT</version>
<packaging>jar</packaging>

<parent>
<groupId>com.cloudera.livy</groupId>
<artifactId>minicluster-dependencies-parent</artifactId>
<version>0.3.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

<properties>
<scala.version>${scala-2.11.version}</scala.version>
<scala.binary.version>2.11</scala.binary.version>
</properties>
</project>
63 changes: 62 additions & 1 deletion integration-test/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@
-->
<cluster.spec>default</cluster.spec>
<skipDeploy>true</skipDeploy>
<skipITs-2.10>false</skipITs-2.10>
<skipITs-2.11>false</skipITs-2.11>
</properties>

<dependencies>
Expand All @@ -62,6 +64,32 @@
<scope>test</scope>
</dependency>

<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>minicluster-dependencies_2.10</artifactId>
<version>${project.version}</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>*</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>minicluster-dependencies_2.11</artifactId>
<version>${project.version}</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>*</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>livy-test-lib</artifactId>
Expand Down Expand Up @@ -207,11 +235,30 @@
<phase>none</phase>
</execution>
<execution>
<id>integration-test</id>
<id>integration-test-scala-2.10</id>
<phase>integration-test</phase>
<goals>
<goal>test</goal>
</goals>
<configuration>
<environmentVariables>
<LIVY_SPARK_SCALA_VERSION>2.10</LIVY_SPARK_SCALA_VERSION>
</environmentVariables>
<skipTests>${skipITs-2.10}</skipTests>
</configuration>
</execution>
<execution>
<id>integration-test-scala-2.11</id>
<phase>integration-test</phase>
<goals>
<goal>test</goal>
</goals>
<configuration>
<environmentVariables>
<LIVY_SPARK_SCALA_VERSION>2.11</LIVY_SPARK_SCALA_VERSION>
</environmentVariables>
<skipTests>${skipITs-2.11}</skipTests>
</configuration>
</execution>
</executions>
<configuration>
Expand Down Expand Up @@ -241,6 +288,20 @@
<spark.home>${real.spark.home}</spark.home>
</properties>
</profile>

<profile>
<id>skip-integration-test</id>
<activation>
<property>
<name>skipITs</name>
<value>true</value>
</property>
</activation>
<properties>
<skipITs-2.10>true</skipITs-2.10>
<skipITs-2.11>true</skipITs-2.11>
</properties>
</profile>
</profiles>

</project>
22 changes: 19 additions & 3 deletions integration-test/src/main/scala/com/cloudera/livy/test/framework/MiniCluster.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
package com.cloudera.livy.test.framework

import java.io._
import java.nio.charset.Charset
import java.nio.file.{Files, Paths}
import javax.servlet.http.HttpServletResponse

Expand Down Expand Up @@ -53,6 +54,13 @@ private class MiniClusterConfig(val config: Map[String, String]) {
}

sealed trait MiniClusterUtils extends ClusterUtils {
private val livySparkScalaVersionEnvVarName = "LIVY_SPARK_SCALA_VERSION"

protected def getSparkScalaVersion(): String = {
sys.env.getOrElse(livySparkScalaVersionEnvVarName, {
throw new RuntimeException(s"Please specify env var $livySparkScalaVersionEnvVarName.")
})
}

protected def saveConfig(conf: Configuration, dest: File): Unit = {
val redacted = new Configuration(conf)
Expand Down Expand Up @@ -148,6 +156,7 @@ object MiniLivyMain extends MiniClusterBase {
var livyConf = Map(
LivyConf.LIVY_SPARK_MASTER.key -> "yarn",
LivyConf.LIVY_SPARK_DEPLOY_MODE.key -> "cluster",
LivyConf.LIVY_SPARK_SCALA_VERSION.key -> getSparkScalaVersion(),
LivyConf.YARN_POLL_INTERVAL.key -> "500ms",
LivyConf.RECOVERY_MODE.key -> "recovery",
LivyConf.RECOVERY_STATE_STORE.key -> "filesystem",
Expand Down Expand Up @@ -188,7 +197,6 @@ private case class ProcessInfo(process: Process, logFile: File)
* TODO: add support for MiniKdc.
*/
class MiniCluster(config: Map[String, String]) extends Cluster with MiniClusterUtils with Logging {

private val tempDir = new File(s"${sys.props("java.io.tmpdir")}/livy-int-test")
private var sparkConfDir: File = _
private var _configDir: File = _
Expand Down Expand Up @@ -229,12 +237,20 @@ class MiniCluster(config: Map[String, String]) extends Cluster with MiniClusterU
assert(tempDir.mkdir(), "Cannot create temp test dir.")
sparkConfDir = mkdir("spark-conf")

val sparkScalaVersion = getSparkScalaVersion()
val classPathFile =
new File(s"minicluster-dependencies/scala-$sparkScalaVersion/target/classpath")
assert(classPathFile.isFile,
s"Cannot read MiniCluster classpath file: ${classPathFile.getCanonicalPath}")
val sparkClassPath =
FileUtils.readFileToString(classPathFile, Charset.defaultCharset())

// When running a real Spark cluster, don't set the classpath.
val extraCp = if (!isRealSpark()) {
val dummyJar = Files.createTempFile(Paths.get(tempDir.toURI), "dummy", "jar").toFile
Map(
SparkLauncher.DRIVER_EXTRA_CLASSPATH -> childClasspath,
SparkLauncher.EXECUTOR_EXTRA_CLASSPATH -> childClasspath,
SparkLauncher.DRIVER_EXTRA_CLASSPATH -> sparkClassPath,
SparkLauncher.EXECUTOR_EXTRA_CLASSPATH -> sparkClassPath,
// Used for Spark 2.0. Spark 2.0 will upload specified jars to distributed cache in yarn
// mode, if not specified it will check jars folder. Here since jars folder is not
// existed, so it will throw exception.
Expand Down
Loading

0 comments on commit 8ebc5bd

Please sign in to comment.