Build: Refactor Flink tests into separate CI (apache#3310)

.github/workflows/flink-ci.yml

@@ -0,0 +1,77 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: "Flink CI"
+on:
+  push:
+    paths-ignore:
+    - '.github/workflows/python-ci.yml'
+    - '.github/workflows/spark-ci.yml'
+    - '.github/workflows/hive-ci.yml'
+    - 'mr/**'
+    - 'hive3/**'
+    - 'hive3-orc-bundle/**'
+    - 'hive-runtime/**'
+    - 'spark/**'
+    - 'pig/**'
+    - 'python/**'
+    - 'python_legacy/**'
+    - 'site/**'
+  pull_request:
+    paths-ignore:
+    - '.github/workflows/python-ci.yml'
+    - '.github/workflows/spark-ci.yml'
+    - '.github/workflows/hive-ci.yml'
+    - 'mr/**'
+    - 'hive3/**'
+    - 'hive3-orc-bundle/**'
+    - 'hive-runtime/**'
+    - 'spark/**'
+    - 'pig/**'
+    - 'python/**'
+    - 'python_legacy/**'
+    - 'site/**'
+
+jobs:
+  flink-tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        jvm: [8, 11]
+        flink: ['1.13']
+    env:
+      SPARK_LOCAL_IP: localhost
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-java@v1
+      with:
+        java-version: ${{ matrix.jvm }}
+    - uses: actions/cache@v2
+      with:
+        path: ~/.gradle/caches
+        key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
+        restore-keys: ${{ runner.os }}-gradle
+    - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
+    - run: ./gradlew -DsparkVersions= -DhiveVersions= -DflinkVersions=${{ matrix.flink }} :iceberg-flink:check :iceberg-flink-runtime:check -Pquick=true -x javadoc 
+    - uses: actions/upload-artifact@v2
+      if: failure()
+      with:
+        name: test logs
+        path: |
+          **/build/testlogs

.github/workflows/hive-ci.yml

@@ -23,6 +23,7 @@ on:
     paths-ignore:
     - '.github/workflows/python-ci.yml'
     - '.github/workflows/spark-ci.yml'
+    - '.github/workflows/flink-ci.yml'
     - 'arrow/**'
     - 'spark/**'
     - 'flink/**'
@@ -34,6 +35,7 @@ on:
     paths-ignore:
     - '.github/workflows/python-ci.yml'
     - '.github/workflows/spark-ci.yml'
+    - '.github/workflows/flink-ci.yml'
     - 'arrow/**'
     - 'spark/**'
     - 'flink/**'
@@ -61,7 +63,7 @@ jobs:
         key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
         restore-keys: ${{ runner.os }}-gradle
     - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
-    - run: ./gradlew -DsparkVersions= -DhiveVersions=2 -Pquick=true :iceberg-mr:check :iceberg-hive-runtime:check -x javadoc
+    - run: ./gradlew -DsparkVersions= -DhiveVersions=2 -DflinkVersions= -Pquick=true :iceberg-mr:check :iceberg-hive-runtime:check -x javadoc
     - uses: actions/upload-artifact@v2
       if: failure()
       with:
@@ -84,7 +86,7 @@ jobs:
         key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
         restore-keys: ${{ runner.os }}-gradle
     - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
-    - run: ./gradlew -DsparkVersions= -DhiveVersions=3 -Pquick=true :iceberg-hive3-orc-bundle:check :iceberg-hive3:check :iceberg-hive-runtime:check -x javadoc
+    - run: ./gradlew -DsparkVersions= -DhiveVersions=3 -DflinkVersions= -Pquick=true :iceberg-hive3-orc-bundle:check :iceberg-hive3:check :iceberg-hive-runtime:check -x javadoc
     - uses: actions/upload-artifact@v2
       if: failure()
       with:

.github/workflows/java-ci.yml

@@ -22,12 +22,30 @@ on:
   push:
     paths-ignore:
     - '.github/workflows/python-ci.yml'
+    - '.github/workflows/spark-ci.yml'
+    - '.github/workflows/flink-ci.yml'
+    - '.github/workflows/hive-ci.yml'
+    - 'mr/**'
+    - 'hive3/**'
+    - 'hive3-orc-bundle/**'
+    - 'hive-runtime/**'
+    - 'spark/**'
+    - 'flink/**'
     - 'python/**'
     - 'python_legacy/**'
     - 'site/**'
   pull_request:
     paths-ignore:
     - '.github/workflows/python-ci.yml'
+    - '.github/workflows/spark-ci.yml'
+    - '.github/workflows/flink-ci.yml'
+    - '.github/workflows/hive-ci.yml'
+    - 'mr/**'
+    - 'hive3/**'
+    - 'hive3-orc-bundle/**'
+    - 'hive-runtime/**'
+    - 'spark/**'
+    - 'flink/**'
     - 'python/**'
     - 'python_legacy/**'
     - 'site/**'
@@ -51,7 +69,7 @@ jobs:
         key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
         restore-keys: ${{ runner.os }}-gradle
     - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
-    - run: ./gradlew check -DsparkVersions= -DhiveVersions= -Pquick=true -x javadoc 
+    - run: ./gradlew check -DsparkVersions= -DhiveVersions= -DflinkVersions= -Pquick=true -x javadoc 
     - uses: actions/upload-artifact@v2
       if: failure()
       with:

.github/workflows/spark-ci.yml

@@ -22,6 +22,7 @@ on:
   push:
     paths-ignore:
     - '.github/workflows/python-ci.yml'
+    - '.github/workflows/flink-ci.yml'
     - '.github/workflows/hive-ci.yml'
     - 'mr/**'
     - 'hive3/**'
@@ -35,6 +36,7 @@ on:
   pull_request:
     paths-ignore:
     - '.github/workflows/python-ci.yml'
+    - '.github/workflows/flink-ci.yml'
     - '.github/workflows/hive-ci.yml'
     - 'mr/**'
     - 'hive3/**'
@@ -62,7 +64,7 @@ jobs:
         key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
         restore-keys: ${{ runner.os }}-gradle
     - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
-    - run: ./gradlew -DsparkVersions=2.4 -DhiveVersions= :iceberg-spark:check :iceberg-spark:iceberg-spark2:check :iceberg-spark:iceberg-spark-runtime:check -Pquick=true -x javadoc
+    - run: ./gradlew -DsparkVersions=2.4 -DhiveVersions= -DflinkVersions= :iceberg-spark:check :iceberg-spark:iceberg-spark2:check :iceberg-spark:iceberg-spark-runtime:check -Pquick=true -x javadoc
     - uses: actions/upload-artifact@v2
       if: failure()
       with:
@@ -89,7 +91,7 @@ jobs:
         key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
         restore-keys: ${{ runner.os }}-gradle
     - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
-    - run: ./gradlew -DsparkVersions=${{ matrix.spark }} -DhiveVersions= :iceberg-spark:check :iceberg-spark:iceberg-spark3:check :iceberg-spark:iceberg-spark3-extensions:check :iceberg-spark:iceberg-spark3-runtime:check -Pquick=true -x javadoc
+    - run: ./gradlew -DsparkVersions=${{ matrix.spark }} -DhiveVersions= -DflinkVersions= :iceberg-spark:check :iceberg-spark:iceberg-spark3:check :iceberg-spark:iceberg-spark3-extensions:check :iceberg-spark:iceberg-spark3-runtime:check -Pquick=true -x javadoc
     - uses: actions/upload-artifact@v2
       if: failure()
       with:

build.gradle

@@ -359,153 +359,6 @@ project(':iceberg-aws') {
   }
 }
 
-project(':iceberg-flink') {
-  dependencies {
-    implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
-    api project(':iceberg-api')
-    implementation project(':iceberg-common')
-    implementation project(':iceberg-core')
-    api project(':iceberg-data')
-    implementation project(':iceberg-orc')
-    implementation project(':iceberg-parquet')
-    implementation project(':iceberg-hive-metastore')
-
-    compileOnly "org.apache.flink:flink-streaming-java_2.12"
-    compileOnly "org.apache.flink:flink-streaming-java_2.12::tests"
-    compileOnly "org.apache.flink:flink-table-api-java-bridge_2.12"
-    compileOnly "org.apache.flink:flink-table-planner-blink_2.12"
-    compileOnly "org.apache.flink:flink-table-planner_2.12"
-    compileOnly "org.apache.hadoop:hadoop-hdfs"
-    compileOnly "org.apache.hadoop:hadoop-common"
-    compileOnly("org.apache.hadoop:hadoop-minicluster") {
-      exclude group: 'org.apache.avro', module: 'avro'
-    }
-
-    implementation("org.apache.parquet:parquet-avro") {
-      exclude group: 'org.apache.avro', module: 'avro'
-      // already shaded by Parquet
-      exclude group: 'it.unimi.dsi'
-      exclude group: 'org.codehaus.jackson'
-    }
-
-    compileOnly "org.apache.avro:avro"
-
-    implementation("org.apache.orc:orc-core::nohive") {
-      exclude group: 'org.apache.hadoop'
-      exclude group: 'commons-lang'
-      // These artifacts are shaded and included in the orc-core fat jar
-      exclude group: 'com.google.protobuf', module: 'protobuf-java'
-      exclude group: 'org.apache.hive', module: 'hive-storage-api'
-    }
-
-    testImplementation "org.apache.flink:flink-core"
-    testImplementation "org.apache.flink:flink-runtime_2.12"
-    testImplementation "org.apache.flink:flink-table-planner-blink_2.12"
-    testImplementation ("org.apache.flink:flink-test-utils-junit") {
-      exclude group: 'junit'
-    }
-    testImplementation("org.apache.flink:flink-test-utils_2.12") {
-      exclude group: "org.apache.curator", module: 'curator-test'
-      exclude group: 'junit'
-    }
-
-    testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts')
-    testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
-    testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
-    testImplementation project(path: ':iceberg-data', configuration: 'testArtifacts')
-
-    // By default, hive-exec is a fat/uber jar and it exports a guava library
-    // that's really old. We use the core classifier to be able to override our guava
-    // version. Luckily, hive-exec seems to work okay so far with this version of guava
-    // See: https://github.com/apache/hive/blob/master/ql/pom.xml#L911 for more context.
-    testImplementation("org.apache.hive:hive-exec::core") {
-      exclude group: 'org.apache.avro', module: 'avro'
-      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
-      exclude group: 'org.pentaho' // missing dependency
-      exclude group: 'org.apache.hive', module: 'hive-llap-tez'
-      exclude group: 'org.apache.logging.log4j'
-      exclude group: 'com.google.protobuf', module: 'protobuf-java'
-      exclude group: 'org.apache.calcite'
-      exclude group: 'org.apache.calcite.avatica'
-      exclude group: 'com.google.code.findbugs', module: 'jsr305'
-    }
-
-    testImplementation("org.apache.hive:hive-metastore") {
-      exclude group: 'org.apache.avro', module: 'avro'
-      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
-      exclude group: 'org.pentaho' // missing dependency
-      exclude group: 'org.apache.hbase'
-      exclude group: 'org.apache.logging.log4j'
-      exclude group: 'co.cask.tephra'
-      exclude group: 'com.google.code.findbugs', module: 'jsr305'
-      exclude group: 'org.eclipse.jetty.aggregate', module: 'jetty-all'
-      exclude group: 'org.eclipse.jetty.orbit', module: 'javax.servlet'
-      exclude group: 'org.apache.parquet', module: 'parquet-hadoop-bundle'
-      exclude group: 'com.tdunning', module: 'json'
-      exclude group: 'javax.transaction', module: 'transaction-api'
-      exclude group: 'com.zaxxer', module: 'HikariCP'
-    }
-  }
-}
-
-project(':iceberg-flink-runtime') {
-  apply plugin: 'com.github.johnrengelman.shadow'
-
-  tasks.jar.dependsOn tasks.shadowJar
-
-  configurations {
-    implementation {
-      exclude group: 'org.apache.flink'
-      // included in Flink
-      exclude group: 'org.slf4j'
-      exclude group: 'org.apache.commons'
-      exclude group: 'commons-pool'
-      exclude group: 'commons-codec'
-      exclude group: 'org.xerial.snappy'
-      exclude group: 'javax.xml.bind'
-      exclude group: 'javax.annotation'
-    }
-  }
-
-  dependencies {
-    implementation project(':iceberg-flink')
-    implementation project(':iceberg-aws')
-    implementation(project(':iceberg-nessie')) {
-      exclude group: 'com.google.code.findbugs', module: 'jsr305'
-    }
-  }
-
-  shadowJar {
-    configurations = [project.configurations.runtimeClasspath]
-
-    zip64 true
-
-    // include the LICENSE and NOTICE files for the shaded Jar
-    from(projectDir) {
-      include 'LICENSE'
-      include 'NOTICE'
-    }
-
-    // Relocate dependencies to avoid conflicts
-    relocate 'org.apache.avro', 'org.apache.iceberg.shaded.org.apache.avro'
-    relocate 'org.apache.parquet', 'org.apache.iceberg.shaded.org.apache.parquet'
-    relocate 'com.google', 'org.apache.iceberg.shaded.com.google'
-    relocate 'com.fasterxml', 'org.apache.iceberg.shaded.com.fasterxml'
-    relocate 'com.github.benmanes', 'org.apache.iceberg.shaded.com.github.benmanes'
-    relocate 'org.checkerframework', 'org.apache.iceberg.shaded.org.checkerframework'
-    relocate 'shaded.parquet', 'org.apache.iceberg.shaded.org.apache.parquet.shaded'
-    relocate 'org.apache.orc', 'org.apache.iceberg.shaded.org.apache.orc'
-    relocate 'io.airlift', 'org.apache.iceberg.shaded.io.airlift'
-    relocate 'org.threeten.extra', 'org.apache.iceberg.shaded.org.threeten.extra'
-
-    classifier null
-  }
-
-  jar {
-    enabled = false
-  }
-}
-
 project(':iceberg-hive-metastore') {
   dependencies {
     implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')