LIVY-162. Livy to support Spark 2.0 + Scala 2.11 cross build

This PR adds support to build against Spark 2.0 and Scala 2.11, what this PR addressed: 1. Support building against different Spark version with profile like -Pspark-1.6, -Pspark-2.0. 2. Support building against different version of Scala through profile: -Pscala-2.10 -Pscala-2.11. 3. Refactor SparkInterpret to support different Scala version + Spark version. What this PR doesn't address: 1. Batch API to support SparkSession. Currently batch API still uses old API for `SparkContext`, `SQLContext` and `HiveContext`. This PR is tested with following scenarios: 1. Spark 1.6 + Scala 2.10 2. Spark 1.6 + Scala 2.11 3. Spark 2.0 + Scala 2.10 3. Spark 2.0 + Scala 2.11 Please suggest and comment, any feedback is greatly appreciated. Closes apache#186
ksakellis · Oct 3, 2016 · 0fa2120 · 0fa2120
1 parent c9a7a89
commit 0fa2120
Show file tree

Hide file tree

Showing 32 changed files with 1,054 additions and 301 deletions.
diff --git a/assembly/assembly.xml b/assembly/assembly.xml
@@ -29,7 +29,7 @@
       </includes>
     </fileSet>
     <fileSet>
-      <directory>${project.parent.basedir}/repl/target/jars</directory>
+      <directory>${project.parent.basedir}/${repl.dir}/target/jars</directory>
       <outputDirectory>${assembly.name}/repl-jars</outputDirectory>
       <includes>
         <include>*</include>

diff --git a/assembly/pom.xml b/assembly/pom.xml
@@ -33,28 +33,16 @@
     <assembly.name>livy-server-${project.version}</assembly.name>
     <assembly.format>zip</assembly.format>
     <skipDeploy>true</skipDeploy>
+    <repl.dir>repl/scala-2.10</repl.dir>
   </properties>
 
-  <profiles>
-    <profile>
-      <id>targz</id>
-      <properties>
-        <assembly.format>tar.gz</assembly.format>
-      </properties>
-    </profile>
-  </profiles>
-
   <dependencies>
     <dependency>
       <groupId>${project.groupId}</groupId>
       <artifactId>livy-rsc</artifactId>
       <version>${project.version}</version>
     </dependency>
-    <dependency>
-      <groupId>${project.groupId}</groupId>
-      <artifactId>livy-repl</artifactId>
-      <version>${project.version}</version>
-    </dependency>
+
     <dependency>
       <groupId>${project.groupId}</groupId>
       <artifactId>livy-server</artifactId>
@@ -89,4 +77,48 @@
     </plugins>
   </build>
 
+  <profiles>
+    <profile>
+      <id>targz</id>
+      <properties>
+        <assembly.format>tar.gz</assembly.format>
+      </properties>
+    </profile>
+
+    <profile>
+      <id>scala-2.10</id>
+      <activation>
+        <property>
+          <name>!scala-2.11</name>
+        </property>
+      </activation>
+      <dependencies>
+        <dependency>
+          <groupId>${project.groupId}</groupId>
+          <artifactId>livy-repl_2.10</artifactId>
+          <version>${project.version}</version>
+        </dependency>
+      </dependencies>
+    </profile>
+
+    <profile>
+      <id>scala-2.11</id>
+      <activation>
+        <property>
+          <name>scala-2.11</name>
+        </property>
+      </activation>
+      <properties>
+        <repl.dir>repl/scala-2.11</repl.dir>
+      </properties>
+      <dependencies>
+        <dependency>
+          <groupId>${project.groupId}</groupId>
+          <artifactId>livy-repl_2.11</artifactId>
+          <version>${project.version}</version>
+        </dependency>
+      </dependencies>
+    </profile>
+  </profiles>
+
 </project>
diff --git a/client-http/pom.xml b/client-http/pom.xml
@@ -50,7 +50,7 @@
 
     <dependency>
       <groupId>com.cloudera.livy</groupId>
-      <artifactId>livy-core</artifactId>
+      <artifactId>livy-core_${scala.binary.version}</artifactId>
       <version>${project.version}</version>
       <type>test-jar</type>
       <scope>test</scope>

diff --git a/core/pom.xml b/core/pom.xml
@@ -22,14 +22,14 @@
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>com.cloudera.livy</groupId>
-    <artifactId>livy-main</artifactId>
+    <artifactId>multi-scala-project-root</artifactId>
     <version>0.3.0-SNAPSHOT</version>
-    <relativePath>../pom.xml</relativePath>
+    <relativePath>../scala/pom.xml</relativePath>
   </parent>
 
-  <artifactId>livy-core</artifactId>
+  <artifactId>livy-core-parent</artifactId>
   <version>0.3.0-SNAPSHOT</version>
-  <packaging>jar</packaging>
+  <packaging>pom</packaging>
 
   <properties>
     <skipDeploy>true</skipDeploy>

diff --git a/core/scala-2.10/pom.xml b/core/scala-2.10/pom.xml
@@ -0,0 +1,37 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to Cloudera, Inc. under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  Cloudera, Inc. licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>com.cloudera.livy</groupId>
+  <artifactId>livy-core_2.10</artifactId>
+  <version>0.3.0-SNAPSHOT</version>
+  <packaging>jar</packaging>
+
+  <parent>
+    <groupId>com.cloudera.livy</groupId>
+    <artifactId>livy-core-parent</artifactId>
+    <version>0.3.0-SNAPSHOT</version>
+    <relativePath>../pom.xml</relativePath>
+  </parent>
+
+  <properties>
+    <scala.version>${scala-2.10.version}</scala.version>
+    <scala.binary.version>2.10</scala.binary.version>
+  </properties>
+</project>
diff --git a/core/scala-2.11/pom.xml b/core/scala-2.11/pom.xml
@@ -0,0 +1,37 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to Cloudera, Inc. under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  Cloudera, Inc. licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>com.cloudera.livy</groupId>
+  <artifactId>livy-core_2.11</artifactId>
+  <version>0.3.0-SNAPSHOT</version>
+  <packaging>jar</packaging>
+
+  <parent>
+    <groupId>com.cloudera.livy</groupId>
+    <artifactId>livy-core-parent</artifactId>
+    <version>0.3.0-SNAPSHOT</version>
+    <relativePath>../pom.xml</relativePath>
+  </parent>
+
+  <properties>
+    <scala.version>${scala-2.11.version}</scala.version>
+    <scala.binary.version>2.11</scala.binary.version>
+  </properties>
+</project>
diff --git a/coverage/pom.xml b/coverage/pom.xml
@@ -53,13 +53,13 @@
 
     <dependency>
       <groupId>${project.groupId}</groupId>
-      <artifactId>livy-core</artifactId>
+      <artifactId>livy-core_${scala.binary.version}</artifactId>
       <version>${project.version}</version>
     </dependency>
 
     <dependency>
       <groupId>${project.groupId}</groupId>
-      <artifactId>livy-repl</artifactId>
+      <artifactId>livy-repl_${scala.binary.version}</artifactId>
       <version>${project.version}</version>
     </dependency>
 

diff --git a/integration-test/pom.xml b/integration-test/pom.xml
@@ -52,7 +52,7 @@
 
     <dependency>
       <groupId>${project.groupId}</groupId>
-      <artifactId>livy-core</artifactId>
+      <artifactId>livy-core_${scala.binary.version}</artifactId>
       <version>${project.version}</version>
     </dependency>
 

diff --git a/pom.xml b/pom.xml
@@ -57,9 +57,11 @@
     <mockito.version>1.9.5</mockito.version>
     <netty.version>4.0.23.Final</netty.version>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-    <py4j.version>0.8.2.1</py4j.version>
+    <py4j.version>0.9</py4j.version>
+    <scala-2.10.version>2.10.4</scala-2.10.version>
+    <scala-2.11.version>2.11.8</scala-2.11.version>
     <scala.binary.version>2.10</scala.binary.version>
-    <scala.version>2.10.4</scala.version>
+    <scala.version>${scala-2.10.version}</scala.version>
     <scalatest.version>2.2.4</scalatest.version>
     <scalatra.version>2.3.0</scalatra.version>
     <java.version>1.7</java.version>
@@ -175,11 +177,13 @@
     <module>assembly</module>
     <module>client-common</module>
     <module>client-http</module>
-    <module>core</module>
+    <module>core/scala-2.10</module>
+    <module>core/scala-2.11</module>
     <module>coverage</module>
     <module>examples</module>
     <module>python-api</module>
-    <module>repl</module>
+    <module>repl/scala-2.10</module>
+    <module>repl/scala-2.11</module>
     <module>rsc</module>
     <module>scala-api</module>
     <module>server</module>
@@ -595,6 +599,12 @@
           <version>2.4.0</version>
         </plugin>
 
+        <plugin>
+          <groupId>org.codehaus.mojo</groupId>
+          <artifactId>build-helper-maven-plugin</artifactId>
+          <version>1.10</version>
+        </plugin>
+
         <plugin>
           <groupId>net.alchim31.maven</groupId>
           <artifactId>scala-maven-plugin</artifactId>
@@ -963,4 +973,37 @@
     </snapshotRepository>
   </distributionManagement>
 
+  <profiles>
+    <!-- Spark version profile -->
+    <profile>
+      <id>spark-1.6</id>
+      <activation>
+        <property>
+          <name>!spark-2.0</name>
+        </property>
+      </activation>
+      <properties>
+        <spark.version>1.6.2</spark.version>
+        <py4j.version>0.9</py4j.version>
+        <netty.version>4.0.23.Final</netty.version>
+        <json4s.version>3.2.10</json4s.version>
+      </properties>
+    </profile>
+
+    <profile>
+      <id>spark-2.0</id>
+      <activation>
+        <property>
+          <name>spark-2.0</name>
+        </property>
+      </activation>
+      <properties>
+        <spark.version>2.0.0</spark.version>
+        <py4j.version>0.10.1</py4j.version>
+        <netty.version>4.0.29.Final</netty.version>
+        <json4s.version>3.2.11</json4s.version>
+      </properties>
+    </profile>
+  </profiles>
+
 </project>
diff --git a/repl/pom.xml b/repl/pom.xml
@@ -22,14 +22,14 @@
     <modelVersion>4.0.0</modelVersion>
     <parent>
         <groupId>com.cloudera.livy</groupId>
-        <artifactId>livy-main</artifactId>
-        <relativePath>../pom.xml</relativePath>
+        <artifactId>multi-scala-project-root</artifactId>
+        <relativePath>../scala/pom.xml</relativePath>
         <version>0.3.0-SNAPSHOT</version>
     </parent>
 
-    <artifactId>livy-repl</artifactId>
+    <artifactId>livy-repl-parent</artifactId>
     <version>0.3.0-SNAPSHOT</version>
-    <packaging>jar</packaging>
+    <packaging>pom</packaging>
 
     <properties>
         <skipDeploy>true</skipDeploy>
@@ -45,21 +45,21 @@
         </dependency>
 
         <dependency>
-            <groupId>${project.groupId}</groupId>
-            <artifactId>livy-core</artifactId>
+            <groupId>com.cloudera.livy</groupId>
+            <artifactId>livy-core_${scala.binary.version}</artifactId>
             <version>${project.version}</version>
             <exclusions>
-              <!-- Provided and shaded by livy-rsc already. -->
-              <exclusion>
-                <groupId>com.cloudera.livy</groupId>
-                <artifactId>livy-client-common</artifactId>
-              </exclusion>
+                <!-- Provided and shaded by livy-rsc already. -->
+                <exclusion>
+                    <groupId>com.cloudera.livy</groupId>
+                    <artifactId>livy-client-common</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
 
         <dependency>
             <groupId>${project.groupId}</groupId>
-            <artifactId>livy-core</artifactId>
+            <artifactId>livy-core_${scala.binary.version}</artifactId>
             <version>${project.version}</version>
             <type>test-jar</type>
             <scope>test</scope>
@@ -89,9 +89,9 @@
         </dependency>
 
         <dependency>
-          <groupId>log4j</groupId>
-          <artifactId>log4j</artifactId>
-          <scope>provided</scope>
+            <groupId>log4j</groupId>
+            <artifactId>log4j</artifactId>
+            <scope>provided</scope>
         </dependency>
 
         <dependency>
@@ -178,7 +178,7 @@
                 </executions>
             </plugin>
 
-             <plugin>
+            <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-jar-plugin</artifactId>
                 <configuration>