Arrow: Define explicit netty-buffer dependency (apache#3420)

yqf1991 · Nov 1, 2021 · 62b4612 · 62b4612
1 parent ba7ccbb
commit 62b4612
Show file tree

Hide file tree

Showing 3 changed files with 10 additions and 0 deletions.
diff --git a/build.gradle b/build.gradle
@@ -489,8 +489,11 @@ project(':iceberg-arrow') {
     implementation("org.apache.arrow:arrow-memory-netty") {
       exclude group: 'com.google.code.findbugs', module: 'jsr305'
       exclude group: 'io.netty', module: 'netty-common'
+      exclude group: 'io.netty', module: 'netty-buffer'
     }
 
+    runtimeOnly("io.netty:netty-buffer")
+
     implementation("org.apache.parquet:parquet-avro") {
       exclude group: 'org.apache.avro', module: 'avro'
       // already shaded by Parquet

diff --git a/spark/v3.2/build.gradle b/spark/v3.2/build.gradle
@@ -64,6 +64,8 @@ project(':iceberg-spark:iceberg-spark-3.2') {
     compileOnly("org.apache.spark:spark-hive_2.12:${sparkVersion}") {
       exclude group: 'org.apache.avro', module: 'avro'
       exclude group: 'org.apache.arrow'
+      // to make sure io.netty.buffer only comes from project(':iceberg-arrow')
+      exclude group: 'io.netty', module: 'netty-buffer'
     }
 
     implementation("org.apache.orc:orc-core::nohive") {
@@ -82,6 +84,8 @@ project(':iceberg-spark:iceberg-spark-3.2') {
 
     testImplementation("org.apache.hadoop:hadoop-minicluster") {
       exclude group: 'org.apache.avro', module: 'avro'
+      // to make sure io.netty.buffer only comes from project(':iceberg-arrow')
+      exclude group: 'io.netty', module: 'netty-buffer'
     }
     testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts')
     testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
@@ -132,6 +136,8 @@ project(":iceberg-spark:iceberg-spark-3.2-extensions") {
     compileOnly("org.apache.spark:spark-hive_2.12:${sparkVersion}") {
       exclude group: 'org.apache.avro', module: 'avro'
       exclude group: 'org.apache.arrow'
+      // to make sure io.netty.buffer only comes from project(':iceberg-arrow')
+      exclude group: 'io.netty', module: 'netty-buffer'
     }
 
     testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts')

diff --git a/versions.props b/versions.props
@@ -16,6 +16,7 @@ com.google.guava:guava = 28.0-jre
 com.github.ben-manes.caffeine:caffeine = 2.8.4
 org.apache.arrow:arrow-vector = 5.0.0
 org.apache.arrow:arrow-memory-netty = 5.0.0
+io.netty:netty-buffer = 4.1.63.Final
 com.github.stephenc.findbugs:findbugs-annotations = 1.3.9-1
 com.aliyun.oss:aliyun-sdk-oss = 3.10.2
 javax.xml.bind:jaxb-api = 2.3.1