[FLINK-20845] Drop Scala 2.11 support

changzhiwin · Sep 30, 2021 · d80d487 · d80d487
1 parent 6b58562
commit d80d487
Show file tree

Hide file tree

Showing 27 changed files with 49 additions and 175 deletions.
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -76,7 +76,7 @@ stages:
             vmImage: 'ubuntu-20.04'
           e2e_pool_definition:
             vmImage: 'ubuntu-20.04'
-          environment: PROFILE="-Dhadoop.version=2.8.3 -Dinclude_hadoop_aws -Dscala-2.11"
+          environment: PROFILE="-Dhadoop.version=2.8.3 -Dinclude_hadoop_aws -Dscala-2.12"
           run_end_to_end: false
           container: flink-build-container
           jdk: 8
@@ -93,5 +93,5 @@ stages:
       - template: tools/azure-pipelines/build-python-wheels.yml
         parameters:
           stage_name: cron_python_wheels
-          environment: PROFILE="-Dhadoop.version=2.8.3 -Dinclude_hadoop_aws -Dscala-2.11"
+          environment: PROFILE="-Dhadoop.version=2.8.3 -Dinclude_hadoop_aws -Dscala-2.12"
           container: flink-build-container
diff --git a/docs/README.md b/docs/README.md
@@ -110,7 +110,7 @@ This will be replaced by the maven artifact for flink-streaming-java that users
 ```xml
 <dependency>
     <groupId>org.apache.flink</groupId>
-    <artifactId>flink-streaming-java_2.11</artifactId>
+    <artifactId>flink-streaming-java_2.12</artifactId>
     <version><!-- current flink version --></version>
 </dependency>
 ```

diff --git a/docs/config.toml b/docs/config.toml
@@ -52,7 +52,7 @@ pygmentsUseClasses = true
   TrainingExercises = "//github.com/apache/flink-training"
 
   # This suffix is appended to the Scala-dependent Maven artifact names
-  ScalaVersion = "_2.11"
+  ScalaVersion = "_2.12"
 
   ProjectHomepage = "//flink.apache.org"
 

diff --git a/docs/content.zh/docs/deployment/resource-providers/standalone/docker.md b/docs/content.zh/docs/deployment/resource-providers/standalone/docker.md
@@ -232,7 +232,7 @@ Images for each supported combination of Flink and Scala versions are available,
 For example, you can use the following aliases:
 
 * `flink:latest` → `flink:<latest-flink>-scala_<latest-scala>`
-* `flink:1.11` → `flink:1.11.<latest-flink-1.11>-scala_2.11`
+* `flink:1.11` → `flink:1.11.<latest-flink-1.11>-scala_2.12`
 
 <span class="label label-info">Note</span> It is recommended to always use an explicit version tag of the docker image that specifies both the needed Flink and Scala
 versions (for example `flink:1.11-scala_2.12`).

diff --git a/docs/content.zh/docs/dev/datastream/project-configuration.md b/docs/content.zh/docs/dev/datastream/project-configuration.md
@@ -151,7 +151,7 @@ For that reason, Flink for Scala 2.11 cannot be used with an application that us
 Scala 2.12.
 
 All Flink dependencies that (transitively) depend on Scala are suffixed with the
-Scala version that they are built for, for example `flink-streaming-scala_2.11`.
+Scala version that they are built for, for example `flink-streaming-scala_2.12`.
 
 Developers that only use Java can pick any Scala version, Scala developers need to
 pick the Scala version that matches their application's Scala version.

diff --git a/docs/content/docs/deployment/resource-providers/standalone/docker.md b/docs/content/docs/deployment/resource-providers/standalone/docker.md
@@ -232,7 +232,7 @@ Images for each supported combination of Flink and Scala versions are available,
 For example, you can use the following aliases:
 
 * `flink:latest` → `flink:<latest-flink>-scala_<latest-scala>`
-* `flink:1.11` → `flink:1.11.<latest-flink-1.11>-scala_2.11`
+* `flink:1.11` → `flink:1.11.<latest-flink-1.11>-scala_2.12`
 
 <span class="label label-info">Note</span> It is recommended to always use an explicit version tag of the docker image that specifies both the needed Flink and Scala
 versions (for example `flink:1.11-scala_2.12`).

diff --git a/docs/content/docs/dev/datastream/project-configuration.md b/docs/content/docs/dev/datastream/project-configuration.md
@@ -150,7 +150,7 @@ For that reason, Flink for Scala 2.11 cannot be used with an application that us
 Scala 2.12.
 
 All Flink dependencies that (transitively) depend on Scala are suffixed with the
-Scala version that they are built for, for example `flink-streaming-scala_2.11`.
+Scala version that they are built for, for example `flink-streaming-scala_2.12`.
 
 Developers that only use Java can pick any Scala version, Scala developers need to
 pick the Scala version that matches their application's Scala version.

diff --git a/flink-dist/pom.xml b/flink-dist/pom.xml
@@ -424,15 +424,6 @@ under the License.
 	</dependencies>
 
 	<profiles>
-		<profile>
-			<id>scala-2.11</id>
-			<activation>
-				<property>
-					<name>!scala-2.12</name>
-				</property>
-			</activation>
-		</profile>
-
 		<profile>
 			<!-- Creates/Removes the 'build-target' symlink in the root directory (only Unix systems) -->
 			<id>symlink-build-target</id>

diff --git a/flink-dist/src/main/resources/META-INF/NOTICE b/flink-dist/src/main/resources/META-INF/NOTICE
@@ -9,7 +9,7 @@ This project bundles the following dependencies under the Apache Software Licens
 - com.ververica:frocksdbjni:6.20.3-ververica-1.0
 - com.google.code.findbugs:jsr305:1.3.9
 - com.twitter:chill-java:0.7.6
-- com.twitter:chill_2.11:0.7.6
+- com.twitter:chill_2.12:0.7.6
 - commons-cli:commons-cli:1.4
 - commons-collections:commons-collections:3.2.2
 - commons-io:commons-io:2.11.0
@@ -29,11 +29,10 @@ See bundled license files for details.
 
 The following dependencies all share the same BSD license which you find under licenses/LICENSE.scala.
 
-- org.scala-lang:scala-compiler:2.11.12
-- org.scala-lang:scala-library:2.11.12
-- org.scala-lang:scala-reflect:2.11.12
-- org.scala-lang.modules:scala-parser-combinators_2.11:1.1.1
-- org.scala-lang.modules:scala-xml_2.11:1.0.5
+- org.scala-lang:scala-compiler:2.12.7
+- org.scala-lang:scala-library:2.12.7
+- org.scala-lang:scala-reflect:2.12.7
+- org.scala-lang.modules:scala-xml_2.12:1.0.6
 
 This project bundles the following dependencies under the MIT/X11 license.
 See bundled license files for details.

diff --git a/flink-end-to-end-tests/pom.xml b/flink-end-to-end-tests/pom.xml
@@ -88,6 +88,8 @@ under the License.
 		<module>flink-python-test</module>
 		<module>flink-end-to-end-tests-hbase</module>
 		<module>flink-end-to-end-tests-pulsar</module>
+		<module>flink-glue-schema-registry-avro-test</module>
+		<module>flink-glue-schema-registry-json-test</module>
 	</modules>
 
 	<dependencyManagement>
@@ -171,23 +173,6 @@ under the License.
 				<e2e.exclude.org.apache.flink.testutils.junit.FailsOnJava11>true</e2e.exclude.org.apache.flink.testutils.junit.FailsOnJava11>
 			</properties>
 		</profile>
-
-		<profile>
-			<id>scala-2.12</id>
-			<properties>
-				<scala.version>2.12.7</scala.version>
-				<scala.binary.version>2.12</scala.binary.version>
-			</properties>
-			<activation>
-				<property>
-					<name>scala-2.12</name>
-				</property>
-			</activation>
-			<modules>
-				<module>flink-glue-schema-registry-avro-test</module>
-				<module>flink-glue-schema-registry-json-test</module>
-			</modules>
-		</profile>
 	</profiles>
 
 	<build>

diff --git a/flink-end-to-end-tests/run-nightly-tests.sh b/flink-end-to-end-tests/run-nightly-tests.sh
@@ -253,11 +253,7 @@ LOG4J_PROPERTIES=${END_TO_END_DIR}/../tools/ci/log4j.properties
 
 MVN_LOGGING_OPTIONS="-Dlog.dir=${DEBUG_FILES_OUTPUT_DIR} -DlogBackupDir=${DEBUG_FILES_OUTPUT_DIR} -Dlog4j.configurationFile=file://$LOG4J_PROPERTIES"
 MVN_COMMON_OPTIONS="-Dflink.forkCount=2 -Dflink.forkCountTestPackage=2 -Dfast -Pskip-webui-build"
-if [[ ${PROFILE} != *"scala-2.12"* ]]; then
-    e2e_modules=$(find flink-end-to-end-tests -mindepth 2 -maxdepth 5 -name 'pom.xml' -not -path '*glue-schema-registry*' -printf '%h\n' | sort -u | tr '\n' ',')
-else
-    e2e_modules=$(find flink-end-to-end-tests -mindepth 2 -maxdepth 5 -name 'pom.xml' -printf '%h\n' | sort -u | tr '\n' ',')
-fi
+e2e_modules=$(find flink-end-to-end-tests -mindepth 2 -maxdepth 5 -name 'pom.xml' -printf '%h\n' | sort -u | tr '\n' ',')
 e2e_modules="${e2e_modules},$(find flink-walkthroughs -mindepth 2 -maxdepth 2 -name 'pom.xml' -printf '%h\n' | sort -u | tr '\n' ',')"
 
 PROFILE="$PROFILE -Pe2e-travis1 -Pe2e-travis2 -Pe2e-travis3 -Pe2e-travis4 -Pe2e-travis5 -Pe2e-travis6"

diff --git a/flink-end-to-end-tests/test-scripts/test_quickstarts.sh b/flink-end-to-end-tests/test-scripts/test_quickstarts.sh
@@ -70,12 +70,6 @@ sed -i -e "s/org.apache.flink.quickstart.StreamingJob/org.apache.flink.quickstar
 
 case $PROFILE in
 *"scala-2.12"*)
-  echo "Changing scala version"
-  sed -i -e "s/scala.binary.version>2.11<\/scala.binary/scala.binary.version>2.12<\/scala.binary/" pom.xml
-  # for flink-quickstart-scala, also change scala version
-  sed -i -e "s/scala.version>2.11.12<\/scala.ver/scala.version>2.12.7<\/scala.ver/" pom.xml
-  ;;
-*"scala-2.11"*)
   # all good
   ;;
 *"scala-"*)

diff --git a/...flink-avro/src/test/java/org/apache/flink/formats/avro/AvroSplittableInputFormatTest.java b/...flink-avro/src/test/java/org/apache/flink/formats/avro/AvroSplittableInputFormatTest.java
@@ -353,7 +353,7 @@ public void testAvroRecovery() throws Exception {
 
     	<dependency>
     		<groupId>org.apache.flink</groupId>
-    		<artifactId>flink-hadoop-compatibility_2.11</artifactId>
+    		<artifactId>flink-hadoop-compatibility_2.12</artifactId>
     		<version>1.6-SNAPSHOT</version>
     	</dependency>
 

diff --git a/flink-formats/pom.xml b/flink-formats/pom.xml
@@ -50,6 +50,8 @@ under the License.
 		<module>flink-orc-nohive</module>
 		<module>flink-hadoop-bulk</module>
 		<module>flink-format-common</module>
+		<module>flink-avro-glue-schema-registry</module>
+		<module>flink-json-glue-schema-registry</module>
     </modules>
 
 	<!-- override these root dependencies as 'provided', so they don't end up
@@ -86,23 +88,6 @@ under the License.
 				<module>flink-sql-avro-confluent-registry</module>
 			</modules>
 		</profile>
-
-		<profile>
-			<id>scala-2.12</id>
-				<properties>
-					<scala.version>2.12.7</scala.version>
-					<scala.binary.version>2.12</scala.binary.version>
-				</properties>
-			<activation>
-				<property>
-					<name>scala-2.12</name>
-				</property>
-			</activation>
-			<modules>
-				<module>flink-avro-glue-schema-registry</module>
-				<module>flink-json-glue-schema-registry</module>
-			</modules>
-		</profile>
 	</profiles>
 
 </project>
diff --git a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
@@ -31,7 +31,7 @@ under the License.
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 		<flink.version>@project.version@</flink.version>
 		<target.java.version>@target.java.version@</target.java.version>
-		<scala.binary.version>2.11</scala.binary.version>
+		<scala.binary.version>2.12</scala.binary.version>
 		<maven.compiler.source>${target.java.version}</maven.compiler.source>
 		<maven.compiler.target>${target.java.version}</maven.compiler.target>
 		<log4j.version>@log4j.version@</log4j.version>

diff --git a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
@@ -46,8 +46,8 @@ under the License.
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 		<flink.version>@project.version@</flink.version>
 		<target.java.version>@target.java.version@</target.java.version>
-		<scala.binary.version>2.11</scala.binary.version>
-		<scala.version>2.11.12</scala.version>
+		<scala.binary.version>2.12</scala.binary.version>
+		<scala.version>2.12.7</scala.version>
 		<log4j.version>@log4j.version@</log4j.version>
 	</properties>
 

diff --git a/flink-rpc/flink-rpc-akka/pom.xml b/flink-rpc/flink-rpc-akka/pom.xml
@@ -152,6 +152,11 @@ under the License.
 				<artifactId>scala-reflect</artifactId>
 				<version>${akka.scala.version}</version>
 			</dependency>
+			<dependency>
+				<groupId>org.scala-lang.modules</groupId>
+				<artifactId>scala-parser-combinators_${scala.binary.version}</artifactId>
+				<version>1.1.2</version>
+			</dependency>
 		</dependencies>
 	</dependencyManagement>
 

diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/ClosureCleaner.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/ClosureCleaner.scala
@@ -39,8 +39,6 @@ object ClosureCleaner {
 
   val LOG = LoggerFactory.getLogger(this.getClass)
 
-  private val isScala2_11 = scala.util.Properties.versionString.contains("2.11")
-
   // Get an ASM class reader for a given class from the JAR that loaded it
   private[scala] def getClassReader(cls: Class[_]): ClassReader = {
     // Copy data over, before delegating to ClassReader - else we can run out of open file handles.
@@ -174,9 +172,6 @@ object ClosureCleaner {
    * @param closure the closure to check.
    */
   private def getSerializedLambda(closure: AnyRef): Option[SerializedLambda] = {
-    if (isScala2_11) {
-      return None
-    }
     val isClosureCandidate =
       closure.getClass.isSynthetic &&
         closure

diff --git a/flink-table/flink-table-planner/src/main/resources/META-INF/NOTICE b/flink-table/flink-table-planner/src/main/resources/META-INF/NOTICE
@@ -16,7 +16,7 @@ This project bundles the following dependencies under the Apache Software Licens
 - org.apache.calcite:calcite-core:1.26.0
 - org.apache.calcite:calcite-linq4j:1.26.0
 - org.apache.calcite.avatica:avatica-core:1.17.0
-- org.scala-lang.modules:scala-parser-combinators_2.11:1.1.1
+- org.scala-lang.modules:scala-parser-combinators_2.12:1.1.1
 - commons-codec:commons-codec:1.15
 - commons-io:commons-io:2.11.0
 

diff --git a/...planner/src/main/scala/org/apache/flink/table/planner/utils/JavaScalaConversionUtil.scala b/...planner/src/main/scala/org/apache/flink/table/planner/utils/JavaScalaConversionUtil.scala
@@ -30,42 +30,28 @@ import scala.collection.JavaConverters._
   */
 object JavaScalaConversionUtil {
 
-  // most of these methods are not necessary once we upgraded to Scala 2.12
-
   def toJava[T](option: Option[T]): Optional[T] = option match {
     case Some(v) => Optional.of(v)
     case None => Optional.empty()
   }
 
   def toScala[T](option: Optional[T]): Option[T] = Option(option.orElse(null.asInstanceOf[T]))
 
-  def toJava[T](func: (T) => Unit): Consumer[T] = new Consumer[T] {
-    override def accept(t: T): Unit = {
-      func.apply(t)
-    }
-  }
+  def toJava[T](func: T => Unit): Consumer[T] =
+    func(_)
 
-  def toJava[K, V](func: (K, V) => Unit): BiConsumer[K, V] = new BiConsumer[K, V] {
-    override def accept(k: K, v: V): Unit = {
-      func.apply(k ,v)
-    }
-  }
+  def toJava[K, V](func: (K, V) => Unit): BiConsumer[K, V] =
+    func(_, _)
 
-  def toJava[I, O](func: (I) => O): Function[I, O] = new Function[I, O] {
-    override def apply(in: I): O = {
-      func.apply(in)
-    }
-  }
+  def toJava[I, O](func: I => O): Function[I, O] =
+    func(_)
 
-  def toJava[T0, T1](tuple: (T0, T1)): JTuple2[T0, T1] = {
+  def toJava[T0, T1](tuple: (T0, T1)): JTuple2[T0, T1] =
     new JTuple2[T0, T1](tuple._1, tuple._2)
-  }
 
-  def toJava[T](seq: Seq[T]): JList[T] = {
+  def toJava[T](seq: Seq[T]): JList[T] =
     seq.asJava
-  }
 
-  def toScala[T](list: JList[T]): Seq[T] = {
+  def toScala[T](list: JList[T]): Seq[T] =
     list.asScala
-  }
 }
diff --git a/...throughs/flink-walkthrough-datastream-java/src/main/resources/archetype-resources/pom.xml b/...throughs/flink-walkthrough-datastream-java/src/main/resources/archetype-resources/pom.xml
@@ -32,7 +32,7 @@ under the License.
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 		<flink.version>@project.version@</flink.version>
 		<target.java.version>1.8</target.java.version>
-		<scala.binary.version>2.11</scala.binary.version>
+		<scala.binary.version>2.12</scala.binary.version>
 		<maven.compiler.source>${target.java.version}</maven.compiler.source>
 		<maven.compiler.target>${target.java.version}</maven.compiler.target>
 		<log4j.version>@log4j.version@</log4j.version>

diff --git a/...hroughs/flink-walkthrough-datastream-scala/src/main/resources/archetype-resources/pom.xml b/...hroughs/flink-walkthrough-datastream-scala/src/main/resources/archetype-resources/pom.xml
@@ -46,8 +46,8 @@ under the License.
 	<properties>
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 		<flink.version>@project.version@</flink.version>
-		<scala.binary.version>2.11</scala.binary.version>
-		<scala.version>2.11.12</scala.version>
+		<scala.binary.version>2.12</scala.binary.version>
+		<scala.version>2.12.7</scala.version>
 		<log4j.version>@log4j.version@</log4j.version>
 	</properties>