GCP: Add bundle jar for GCP-related dependencies (apache#8231)

Jonnathan7 · Aug 8, 2023 · 5150549 · 5150549
1 parent 00f771d
commit 5150549
Show file tree

Hide file tree

Showing 13 changed files with 635 additions and 3 deletions.
diff --git a/build.gradle b/build.gradle
@@ -608,8 +608,8 @@ project(':iceberg-gcp') {
     implementation project(':iceberg-common')
     implementation project(':iceberg-core')
 
-    implementation platform(libs.google.libraries.bom)
-    implementation "com.google.cloud:google-cloud-storage"
+    compileOnly platform(libs.google.libraries.bom)
+    compileOnly "com.google.cloud:google-cloud-storage"
 
     testImplementation "com.google.cloud:google-cloud-nio"
 

diff --git a/flink/v1.15/build.gradle b/flink/v1.15/build.gradle
@@ -162,6 +162,7 @@ project(":iceberg-flink:iceberg-flink-runtime-${flinkMajorVersion}") {
       exclude group: 'org.apache.httpcomponents', module: 'httpclient'
       exclude group: 'commons-logging', module: 'commons-logging'
     }
+    implementation project(':iceberg-gcp')
     implementation(project(':iceberg-nessie')) {
       exclude group: 'com.google.code.findbugs', module: 'jsr305'
     }

diff --git a/flink/v1.16/build.gradle b/flink/v1.16/build.gradle
@@ -162,6 +162,7 @@ project(":iceberg-flink:iceberg-flink-runtime-${flinkMajorVersion}") {
       exclude group: 'org.apache.httpcomponents', module: 'httpclient'
       exclude group: 'commons-logging', module: 'commons-logging'
     }
+    implementation project(':iceberg-gcp')
     implementation(project(':iceberg-nessie')) {
       exclude group: 'com.google.code.findbugs', module: 'jsr305'
     }

diff --git a/flink/v1.17/build.gradle b/flink/v1.17/build.gradle
@@ -162,6 +162,7 @@ project(":iceberg-flink:iceberg-flink-runtime-${flinkMajorVersion}") {
       exclude group: 'org.apache.httpcomponents', module: 'httpclient'
       exclude group: 'commons-logging', module: 'commons-logging'
     }
+    implementation project(':iceberg-gcp')
     implementation(project(':iceberg-nessie')) {
       exclude group: 'com.google.code.findbugs', module: 'jsr305'
     }

diff --git a/gcp-bundle/LICENSE b/gcp-bundle/LICENSE
diff --git a/gcp-bundle/NOTICE b/gcp-bundle/NOTICE
@@ -0,0 +1,6 @@
+
+Apache Iceberg
+Copyright 2017-2023 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
diff --git a/gcp-bundle/build.gradle b/gcp-bundle/build.gradle
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+project(":iceberg-gcp-bundle") {
+
+  apply plugin: 'com.github.johnrengelman.shadow'
+
+  tasks.jar.dependsOn tasks.shadowJar
+
+  dependencies {
+    implementation platform(libs.google.libraries.bom)
+    implementation "com.google.cloud:google-cloud-storage"
+  }
+
+  shadowJar {
+    archiveClassifier.set(null)
+    zip64 true
+
+    // include the LICENSE and NOTICE files for the shaded Jar
+    from(projectDir) {
+      include 'LICENSE'
+      include 'NOTICE'
+    }
+
+    // relocate GCP-specific versions
+    relocate 'com.fasterxml.jackson', 'org.apache.iceberg.gcp.shaded.com.fasterxml.jackson'
+    relocate 'com.google.common', 'org.apache.iceberg.gcp.shaded.com.google.common'
+    relocate 'com.google.errorprone', 'org.apache.iceberg.gcp.shaded.com.google.errorprone'
+    relocate 'com.google.gson', 'org.apache.iceberg.gcp.shaded.com.google.gson'
+    relocate 'com.google.protobuf', 'org.apache.iceberg.gcp.shaded.com.google.protobuf'
+    relocate 'org.apache.http', 'org.apache.iceberg.gcp.shaded.org.apache.http'
+    relocate 'io.netty', 'org.apache.iceberg.gcp.shaded.io.netty'
+  }
+
+  jar {
+    enabled = false
+  }
+}
diff --git a/hive-runtime/build.gradle b/hive-runtime/build.gradle
@@ -50,6 +50,7 @@ project(':iceberg-hive-runtime') {
       exclude group: 'org.apache.httpcomponents', module: 'httpclient'
       exclude group: 'commons-logging', module: 'commons-logging'
     }
+    implementation project(':iceberg-gcp')
   }
 
   shadowJar {

diff --git a/settings.gradle b/settings.gradle
@@ -33,6 +33,7 @@ include 'pig'
 include 'hive-metastore'
 include 'nessie'
 include 'gcp'
+include 'gcp-bundle'
 include 'dell'
 include 'snowflake'
 include 'delta-lake'
@@ -52,6 +53,7 @@ project(':pig').name = 'iceberg-pig'
 project(':hive-metastore').name = 'iceberg-hive-metastore'
 project(':nessie').name = 'iceberg-nessie'
 project(':gcp').name = 'iceberg-gcp'
+project(':gcp-bundle').name = 'iceberg-gcp-bundle'
 project(':dell').name = 'iceberg-dell'
 project(':snowflake').name = 'iceberg-snowflake'
 project(':delta-lake').name = 'iceberg-delta-lake'

diff --git a/spark/v3.1/build.gradle b/spark/v3.1/build.gradle
@@ -220,6 +220,7 @@ project(':iceberg-spark:iceberg-spark-runtime-3.1_2.12') {
       exclude group: 'org.apache.httpcomponents', module: 'httpclient'
       exclude group: 'commons-logging', module: 'commons-logging'
     }
+    implementation project(':iceberg-gcp')
     implementation project(':iceberg-hive-metastore')
     implementation(project(':iceberg-nessie')) {
       exclude group: 'com.google.code.findbugs', module: 'jsr305'

diff --git a/spark/v3.2/build.gradle b/spark/v3.2/build.gradle
@@ -215,6 +215,7 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio
       exclude group: 'org.apache.httpcomponents', module: 'httpclient'
       exclude group: 'commons-logging', module: 'commons-logging'
     }
+    implementation project(':iceberg-gcp')
     implementation project(':iceberg-hive-metastore')
     implementation(project(':iceberg-nessie')) {
       exclude group: 'com.google.code.findbugs', module: 'jsr305'

diff --git a/spark/v3.3/build.gradle b/spark/v3.3/build.gradle
@@ -221,6 +221,7 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio
       exclude group: 'org.apache.httpcomponents', module: 'httpclient'
       exclude group: 'commons-logging', module: 'commons-logging'
     }
+    implementation project(':iceberg-gcp')
     implementation project(':iceberg-hive-metastore')
     implementation(project(':iceberg-nessie')) {
       exclude group: 'com.google.code.findbugs', module: 'jsr305'

diff --git a/spark/v3.4/build.gradle b/spark/v3.4/build.gradle
@@ -220,6 +220,7 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio
       exclude group: 'org.apache.httpcomponents', module: 'httpclient'
       exclude group: 'commons-logging', module: 'commons-logging'
     }
+    implementation project(':iceberg-gcp')
     implementation project(':iceberg-hive-metastore')
     implementation(project(':iceberg-nessie')) {
       exclude group: 'com.google.code.findbugs', module: 'jsr305'
@@ -255,7 +256,8 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio
     }
 
     // Relocate dependencies to avoid conflicts
-    relocate 'com.google', 'org.apache.iceberg.shaded.com.google'
+    relocate 'com.google.errorprone', 'org.apache.iceberg.shaded.com.google.errorprone'
+    relocate 'com.google.flatbuffers', 'org.apache.iceberg.shaded.com.google.flatbuffers'
     relocate 'com.fasterxml', 'org.apache.iceberg.shaded.com.fasterxml'
     relocate 'com.github.benmanes', 'org.apache.iceberg.shaded.com.github.benmanes'
     relocate 'org.checkerframework', 'org.apache.iceberg.shaded.org.checkerframework'