From 57477c3f1b7b28c938eebad73b9c9ab3efb924af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduard=20Tudenh=C3=B6fner?= Date: Fri, 17 Sep 2021 20:53:10 +0200 Subject: [PATCH] Build: Upgrade to Gradle 7.x (#2826) * Build: Upgrade to Gradle 7 * Build: Properly exclude stuff from shadedJar content This change is required after upgrading to ShadowJar plugin 7.0.0 due to https://github.com/johnrengelman/shadow/issues/505 not excluding files the same way it used to. * Fix ErrorProne / errors due to newer ErrorProne version * Fix checkstyle violations due to newer Checkstyle version --- .baseline/checkstyle/checkstyle.xml | 3 +- ...ectorizedParquetDefinitionLevelReader.java | 32 +- baseline.gradle | 15 +- build.gradle | 549 +++++++++++------- .../org/apache/iceberg/BaseTableScan.java | 4 +- .../org/apache/iceberg/BaseTransaction.java | 1 + .../java/org/apache/iceberg/FindFiles.java | 4 +- .../iceberg/IncrementalDataTableScan.java | 6 +- .../apache/iceberg/hadoop/HadoopCatalog.java | 10 +- .../org/apache/iceberg/TestTableMetadata.java | 24 +- deploy.gradle | 1 - gradle/wrapper/gradle-wrapper.properties | 21 +- gradlew | 290 +++++---- .../org/apache/iceberg/hive/HiveCatalog.java | 2 +- .../vector/CompatibilityHiveVectorUtils.java | 2 +- .../apache/iceberg/mr/hive/TestTables.java | 4 +- .../apache/iceberg/spark/SparkTableUtil.java | 8 +- spark3-runtime/LICENSE | 61 ++ versions.props | 1 + 19 files changed, 644 insertions(+), 394 deletions(-) diff --git a/.baseline/checkstyle/checkstyle.xml b/.baseline/checkstyle/checkstyle.xml index 54be02b39616..fc9f831e10c9 100644 --- a/.baseline/checkstyle/checkstyle.xml +++ b/.baseline/checkstyle/checkstyle.xml @@ -392,7 +392,6 @@ - @@ -420,7 +419,7 @@ - + diff --git a/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/parquet/VectorizedParquetDefinitionLevelReader.java b/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/parquet/VectorizedParquetDefinitionLevelReader.java index 8acfd9a7556f..eecf6e1fa59a 100644 --- a/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/parquet/VectorizedParquetDefinitionLevelReader.java +++ b/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/parquet/VectorizedParquetDefinitionLevelReader.java @@ -305,7 +305,7 @@ public void nextDictEncodedBatch( protected abstract void nextVal( FieldVector vector, int idx, ValuesAsBytesReader valuesReader, int typeWidth, byte[] byteArray); protected abstract void nextDictEncodedVal( - FieldVector vector, int startOffset, VectorizedDictionaryEncodedParquetValuesReader reader, + FieldVector vector, int idx, VectorizedDictionaryEncodedParquetValuesReader reader, int numValuesToRead, Dictionary dict, NullabilityHolder nullabilityHolder, int typeWidth, Mode mode); } @@ -320,10 +320,10 @@ protected void nextVal( @Override protected void nextDictEncodedVal( FieldVector vector, int idx, VectorizedDictionaryEncodedParquetValuesReader reader, - int numValues, Dictionary dict, NullabilityHolder nullabilityHolder, int typeWidth, Mode mode) { + int numValuesToRead, Dictionary dict, NullabilityHolder nullabilityHolder, int typeWidth, Mode mode) { if (Mode.RLE.equals(mode)) { reader.timestampMillisDictEncodedReader().nextBatch(vector, - idx, numValues, dict, nullabilityHolder, typeWidth); + idx, numValuesToRead, dict, nullabilityHolder, typeWidth); } else if (Mode.PACKED.equals(mode)) { vector.getDataBuffer().setLong((long) idx * typeWidth, dict.decodeToLong(reader.readInteger()) * 1000); } @@ -342,10 +342,10 @@ protected void nextVal( @Override protected void nextDictEncodedVal( FieldVector vector, int idx, VectorizedDictionaryEncodedParquetValuesReader reader, - int numValues, Dictionary dict, NullabilityHolder nullabilityHolder, int typeWidth, Mode mode) { + int numValuesToRead, Dictionary dict, NullabilityHolder nullabilityHolder, int typeWidth, Mode mode) { if (Mode.RLE.equals(mode)) { reader.fixedWidthBinaryDictEncodedReader() - .nextBatch(vector, idx, numValues, dict, nullabilityHolder, typeWidth); + .nextBatch(vector, idx, numValuesToRead, dict, nullabilityHolder, typeWidth); } else if (Mode.PACKED.equals(mode)) { ByteBuffer buffer = dict.decodeToBinary(reader.readInteger()).toByteBuffer(); vector.getDataBuffer().setBytes( @@ -366,10 +366,10 @@ protected void nextVal( @Override protected void nextDictEncodedVal( FieldVector vector, int idx, VectorizedDictionaryEncodedParquetValuesReader reader, - int numValues, Dictionary dict, NullabilityHolder nullabilityHolder, int typeWidth, Mode mode) { + int numValuesToRead, Dictionary dict, NullabilityHolder nullabilityHolder, int typeWidth, Mode mode) { if (Mode.RLE.equals(mode)) { reader.fixedLengthDecimalDictEncodedReader() - .nextBatch(vector, idx, numValues, dict, nullabilityHolder, typeWidth); + .nextBatch(vector, idx, numValuesToRead, dict, nullabilityHolder, typeWidth); } else if (Mode.PACKED.equals(mode)) { ByteBuffer decimalBytes = dict.decodeToBinary(reader.readInteger()).toByteBuffer(); byte[] vectorBytes = new byte[typeWidth]; @@ -390,10 +390,10 @@ protected void nextVal( @Override protected void nextDictEncodedVal( FieldVector vector, int idx, VectorizedDictionaryEncodedParquetValuesReader reader, - int numValues, Dictionary dict, NullabilityHolder nullabilityHolder, int typeWidth, Mode mode) { + int numValuesToRead, Dictionary dict, NullabilityHolder nullabilityHolder, int typeWidth, Mode mode) { if (Mode.RLE.equals(mode)) { reader.fixedSizeBinaryDictEncodedReader().nextBatch(vector, idx, - numValues, dict, nullabilityHolder, typeWidth); + numValuesToRead, dict, nullabilityHolder, typeWidth); } else if (Mode.PACKED.equals(mode)) { byte[] bytes = dict.decodeToBinary(reader.readInteger()).getBytes(); byte[] vectorBytes = new byte[typeWidth]; @@ -427,10 +427,10 @@ protected void nextVal( @Override protected void nextDictEncodedVal( FieldVector vector, int idx, VectorizedDictionaryEncodedParquetValuesReader reader, - int numValues, Dictionary dict, NullabilityHolder nullabilityHolder, int typeWidth, Mode mode) { + int numValuesToRead, Dictionary dict, NullabilityHolder nullabilityHolder, int typeWidth, Mode mode) { if (Mode.RLE.equals(mode)) { reader.varWidthBinaryDictEncodedReader().nextBatch(vector, idx, - numValues, dict, nullabilityHolder, typeWidth); + numValuesToRead, dict, nullabilityHolder, typeWidth); } else if (Mode.PACKED.equals(mode)) { ((BaseVariableWidthVector) vector).setSafe( idx, dict.decodeToBinary(reader.readInteger()).getBytesUnsafe()); @@ -448,10 +448,10 @@ protected void nextVal( @Override protected void nextDictEncodedVal( FieldVector vector, int idx, VectorizedDictionaryEncodedParquetValuesReader reader, - int numValues, Dictionary dict, NullabilityHolder nullabilityHolder, int typeWidth, Mode mode) { + int numValuesToRead, Dictionary dict, NullabilityHolder nullabilityHolder, int typeWidth, Mode mode) { if (Mode.RLE.equals(mode)) { reader.intBackedDecimalDictEncodedReader() - .nextBatch(vector, idx, numValues, dict, nullabilityHolder, typeWidth); + .nextBatch(vector, idx, numValuesToRead, dict, nullabilityHolder, typeWidth); } else if (Mode.PACKED.equals(mode)) { ((DecimalVector) vector).set(idx, dict.decodeToInt(reader.readInteger())); } @@ -468,10 +468,10 @@ protected void nextVal( @Override protected void nextDictEncodedVal( FieldVector vector, int idx, VectorizedDictionaryEncodedParquetValuesReader reader, - int numValues, Dictionary dict, NullabilityHolder nullabilityHolder, int typeWidth, Mode mode) { + int numValuesToRead, Dictionary dict, NullabilityHolder nullabilityHolder, int typeWidth, Mode mode) { if (Mode.RLE.equals(mode)) { reader.longBackedDecimalDictEncodedReader() - .nextBatch(vector, idx, numValues, dict, nullabilityHolder, typeWidth); + .nextBatch(vector, idx, numValuesToRead, dict, nullabilityHolder, typeWidth); } else if (Mode.PACKED.equals(mode)) { ((DecimalVector) vector).set( idx, dict.decodeToLong(reader.readInteger())); @@ -489,7 +489,7 @@ protected void nextVal( @Override protected void nextDictEncodedVal( FieldVector vector, int idx, VectorizedDictionaryEncodedParquetValuesReader reader, - int numValues, Dictionary dict, NullabilityHolder nullabilityHolder, int typeWidth, Mode mode) { + int numValuesToRead, Dictionary dict, NullabilityHolder nullabilityHolder, int typeWidth, Mode mode) { throw new UnsupportedOperationException(); } } diff --git a/baseline.gradle b/baseline.gradle index 39fcb7bd6342..25f7faa1406d 100644 --- a/baseline.gradle +++ b/baseline.gradle @@ -36,7 +36,7 @@ subprojects { apply plugin: 'com.palantir.baseline-checkstyle' apply plugin: 'com.palantir.baseline-error-prone' } - apply plugin: 'com.palantir.baseline-scalastyle' + apply plugin: 'com.github.alisiikh.scalastyle' apply plugin: 'com.palantir.baseline-class-uniqueness' apply plugin: 'com.palantir.baseline-reproducibility' apply plugin: 'com.palantir.baseline-exact-dependencies' @@ -47,7 +47,7 @@ subprojects { // So we apply Spotless manually to get a similar effect to baseline-format, but change the // import order. - pluginManager.withPlugin('com.diffplug.gradle.spotless') { + pluginManager.withPlugin('com.diffplug.spotless') { spotless { java { target 'src/main/java/**/*.java', 'src/test/java/**/*.java', 'src/jmh/java/**/*.java' @@ -82,4 +82,15 @@ subprojects { ) } } + + pluginManager.withPlugin('com.github.alisiikh.scalastyle') { + scalastyle { + config = file("${rootDir}/project/scalastyle_config.xml") + inputEncoding = 'UTF-8' + outputEncoding = 'UTF-8' + failOnWarning = false + verbose = false + quiet = false + } + } } diff --git a/build.gradle b/build.gradle index 851d78810d2a..cecf8b7e0bed 100644 --- a/build.gradle +++ b/build.gradle @@ -24,12 +24,13 @@ buildscript { gradlePluginPortal() } dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:5.0.0' - classpath 'com.palantir.baseline:gradle-baseline-java:3.36.2' + classpath 'gradle.plugin.com.github.jengelman.gradle.plugins:shadow:7.0.0' + classpath 'com.palantir.baseline:gradle-baseline-java:4.0.0' classpath 'com.palantir.gradle.gitversion:gradle-git-version:0.12.3' - classpath 'com.diffplug.spotless:spotless-plugin-gradle:5.14.0' - classpath 'gradle.plugin.org.inferred:gradle-processors:2.1.0' - classpath 'me.champeau.gradle:jmh-gradle-plugin:0.4.8' + classpath 'com.diffplug.spotless:spotless-plugin-gradle:5.14.1' + classpath 'gradle.plugin.org.inferred:gradle-processors:3.3.0' + classpath 'me.champeau.gradle:jmh-gradle-plugin:0.5.3' + classpath "com.github.alisiikh:gradle-scalastyle-plugin:3.4.1" } } @@ -71,10 +72,10 @@ allprojects { subprojects { apply plugin: 'nebula.dependency-recommender' - apply plugin: 'java' + apply plugin: 'java-library' configurations { - testCompile.extendsFrom compileOnly + testImplementation.extendsFrom compileOnly compileClasspath { // do not exclude Guava so the bundle project can reference classes. @@ -122,15 +123,15 @@ subprojects { targetCompatibility = '1.8' dependencies { - compile 'org.slf4j:slf4j-api' - compile 'com.github.stephenc.findbugs:findbugs-annotations' + implementation 'org.slf4j:slf4j-api' + implementation 'com.github.stephenc.findbugs:findbugs-annotations' - testCompile 'org.junit.vintage:junit-vintage-engine' - testCompile 'org.junit.jupiter:junit-jupiter-engine' - testCompile 'org.junit.jupiter:junit-jupiter' - testCompile 'org.slf4j:slf4j-simple' - testCompile 'org.mockito:mockito-core' - testCompile 'org.assertj:assertj-core' + testImplementation 'org.junit.vintage:junit-vintage-engine' + testImplementation 'org.junit.jupiter:junit-jupiter-engine' + testImplementation 'org.junit.jupiter:junit-jupiter' + testImplementation 'org.slf4j:slf4j-simple' + testImplementation 'org.mockito:mockito-core' + testImplementation 'org.assertj:assertj-core' } test { @@ -174,7 +175,7 @@ project(':iceberg-bundled-guava') { shadowJar { classifier null - configurations = [project.configurations.compileOnly] + configurations = [project.configurations.compileClasspath] zip64 true // include the LICENSE and NOTICE files for the shaded Jar @@ -201,44 +202,46 @@ project(':iceberg-bundled-guava') { project(':iceberg-api') { dependencies { - compile project(path: ':iceberg-bundled-guava', configuration: 'shadow') - compileOnly "com.google.errorprone:error_prone_annotations:2.3.3" - testCompile "org.apache.avro:avro" + implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow') + compileOnly "com.google.errorprone:error_prone_annotations" + testImplementation "org.apache.avro:avro" } } project(':iceberg-common') { dependencies { - compile project(path: ':iceberg-bundled-guava', configuration: 'shadow') + implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow') } } project(':iceberg-core') { dependencies { - compile project(':iceberg-api') - compile project(':iceberg-common') + api project(':iceberg-api') + implementation project(':iceberg-common') + implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow') - compile("org.apache.avro:avro") { + implementation("org.apache.avro:avro") { exclude group: 'org.tukaani' // xz compression is not supported } - compile "com.fasterxml.jackson.core:jackson-databind" - compile "com.fasterxml.jackson.core:jackson-core" - compile "com.github.ben-manes.caffeine:caffeine" + implementation "com.fasterxml.jackson.core:jackson-databind" + implementation "com.fasterxml.jackson.core:jackson-core" + implementation "com.github.ben-manes.caffeine:caffeine" compileOnly("org.apache.hadoop:hadoop-client") { exclude group: 'org.apache.avro', module: 'avro' exclude group: 'org.slf4j', module: 'slf4j-log4j12' } - testCompile "org.xerial:sqlite-jdbc" - testCompile project(path: ':iceberg-api', configuration: 'testArtifacts') + testImplementation "org.xerial:sqlite-jdbc" + testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') } } project(':iceberg-data') { dependencies { - compile project(':iceberg-api') - compile project(':iceberg-core') + implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow') + api project(':iceberg-api') + implementation project(':iceberg-core') compileOnly project(':iceberg-parquet') compileOnly project(':iceberg-orc') compileOnly("org.apache.hadoop:hadoop-common") { @@ -247,13 +250,30 @@ project(':iceberg-data') { exclude group: 'org.slf4j', module: 'slf4j-log4j12' } - testCompile("org.apache.hadoop:hadoop-client") { + implementation("org.apache.orc:orc-core::nohive") { + exclude group: 'org.apache.hadoop' + exclude group: 'commons-lang' + // These artifacts are shaded and included in the orc-core fat jar + exclude group: 'com.google.protobuf', module: 'protobuf-java' + exclude group: 'org.apache.hive', module: 'hive-storage-api' + } + + implementation("org.apache.parquet:parquet-avro") { + exclude group: 'org.apache.avro', module: 'avro' + // already shaded by Parquet + exclude group: 'it.unimi.dsi' + exclude group: 'org.codehaus.jackson' + } + + compileOnly "org.apache.avro:avro" + + testImplementation("org.apache.hadoop:hadoop-client") { exclude group: 'org.apache.avro', module: 'avro' exclude group: 'org.slf4j', module: 'slf4j-log4j12' } - testCompile project(path: ':iceberg-api', configuration: 'testArtifacts') - testCompile project(path: ':iceberg-core', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts') } test { @@ -264,8 +284,10 @@ project(':iceberg-data') { project(':iceberg-aws') { dependencies { - compile project(':iceberg-api') - compile project(':iceberg-core') + implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow') + api project(':iceberg-api') + implementation project(':iceberg-common') + implementation project(':iceberg-core') compileOnly 'software.amazon.awssdk:url-connection-client' compileOnly 'software.amazon.awssdk:s3' @@ -281,9 +303,9 @@ project(':iceberg-aws') { exclude group: 'com.google.code.gson', module: 'gson' } - testCompile 'software.amazon.awssdk:iam' - testCompile project(path: ':iceberg-api', configuration: 'testArtifacts') - testCompile("com.adobe.testing:s3mock-junit4") { + testImplementation 'software.amazon.awssdk:iam' + testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') + testImplementation("com.adobe.testing:s3mock-junit4") { exclude module: "spring-boot-starter-logging" exclude module: "logback-classic" exclude group: 'junit' @@ -301,7 +323,7 @@ project(':iceberg-aws') { configurations { integrationImplementation.extendsFrom testImplementation - integrationRuntime.extendsFrom testRuntime + integrationRuntime.extendsFrom testRuntimeOnly } task integrationTest(type: Test) { @@ -312,13 +334,14 @@ project(':iceberg-aws') { project(':iceberg-flink') { dependencies { - compile project(':iceberg-api') - compile project(':iceberg-common') - compile project(':iceberg-core') - compile project(':iceberg-data') - compile project(':iceberg-orc') - compile project(':iceberg-parquet') - compile project(':iceberg-hive-metastore') + implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow') + api project(':iceberg-api') + implementation project(':iceberg-common') + implementation project(':iceberg-core') + api project(':iceberg-data') + implementation project(':iceberg-orc') + implementation project(':iceberg-parquet') + implementation project(':iceberg-hive-metastore') compileOnly "org.apache.flink:flink-streaming-java_2.12" compileOnly "org.apache.flink:flink-streaming-java_2.12::tests" @@ -331,27 +354,44 @@ project(':iceberg-flink') { exclude group: 'org.apache.avro', module: 'avro' } - testCompile "org.apache.flink:flink-core" - testCompile "org.apache.flink:flink-runtime_2.12" - testCompile "org.apache.flink:flink-table-planner-blink_2.12" - testCompile ("org.apache.flink:flink-test-utils-junit") { + implementation("org.apache.parquet:parquet-avro") { + exclude group: 'org.apache.avro', module: 'avro' + // already shaded by Parquet + exclude group: 'it.unimi.dsi' + exclude group: 'org.codehaus.jackson' + } + + compileOnly "org.apache.avro:avro" + + implementation("org.apache.orc:orc-core::nohive") { + exclude group: 'org.apache.hadoop' + exclude group: 'commons-lang' + // These artifacts are shaded and included in the orc-core fat jar + exclude group: 'com.google.protobuf', module: 'protobuf-java' + exclude group: 'org.apache.hive', module: 'hive-storage-api' + } + + testImplementation "org.apache.flink:flink-core" + testImplementation "org.apache.flink:flink-runtime_2.12" + testImplementation "org.apache.flink:flink-table-planner-blink_2.12" + testImplementation ("org.apache.flink:flink-test-utils-junit") { exclude group: 'junit' } - testCompile("org.apache.flink:flink-test-utils_2.12") { + testImplementation("org.apache.flink:flink-test-utils_2.12") { exclude group: "org.apache.curator", module: 'curator-test' exclude group: 'junit' } - testCompile project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') - testCompile project(path: ':iceberg-api', configuration: 'testArtifacts') - testCompile project(path: ':iceberg-core', configuration: 'testArtifacts') - testCompile project(path: ':iceberg-data', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-data', configuration: 'testArtifacts') // By default, hive-exec is a fat/uber jar and it exports a guava library // that's really old. We use the core classifier to be able to override our guava // version. Luckily, hive-exec seems to work okay so far with this version of guava // See: https://github.com/apache/hive/blob/master/ql/pom.xml#L911 for more context. - testCompile("org.apache.hive:hive-exec::core") { + testImplementation("org.apache.hive:hive-exec::core") { exclude group: 'org.apache.avro', module: 'avro' exclude group: 'org.slf4j', module: 'slf4j-log4j12' exclude group: 'org.pentaho' // missing dependency @@ -363,7 +403,7 @@ project(':iceberg-flink') { exclude group: 'com.google.code.findbugs', module: 'jsr305' } - testCompile("org.apache.hive:hive-metastore") { + testImplementation("org.apache.hive:hive-metastore") { exclude group: 'org.apache.avro', module: 'avro' exclude group: 'org.slf4j', module: 'slf4j-log4j12' exclude group: 'org.pentaho' // missing dependency @@ -387,7 +427,7 @@ project(':iceberg-flink-runtime') { tasks.jar.dependsOn tasks.shadowJar configurations { - compile { + implementation { exclude group: 'org.apache.flink' // included in Flink exclude group: 'org.slf4j' @@ -401,15 +441,15 @@ project(':iceberg-flink-runtime') { } dependencies { - compile project(':iceberg-flink') - compile project(':iceberg-aws') - compile(project(':iceberg-nessie')) { + implementation project(':iceberg-flink') + implementation project(':iceberg-aws') + implementation(project(':iceberg-nessie')) { exclude group: 'com.google.code.findbugs', module: 'jsr305' } } shadowJar { - configurations = [project.configurations.compile] + configurations = [project.configurations.runtimeClasspath] zip64 true @@ -441,7 +481,12 @@ project(':iceberg-flink-runtime') { project(':iceberg-hive-metastore') { dependencies { - compile project(':iceberg-core') + implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow') + implementation project(':iceberg-core') + api project(':iceberg-api') + implementation project(':iceberg-common') + + implementation "com.github.ben-manes.caffeine:caffeine" compileOnly "org.apache.avro:avro" @@ -465,7 +510,7 @@ project(':iceberg-hive-metastore') { // that's really old. We use the core classifier to be able to override our guava // version. Luckily, hive-exec seems to work okay so far with this version of guava // See: https://github.com/apache/hive/blob/master/ql/pom.xml#L911 for more context. - testCompile("org.apache.hive:hive-exec::core") { + testImplementation("org.apache.hive:hive-exec::core") { exclude group: 'org.apache.avro', module: 'avro' exclude group: 'org.slf4j', module: 'slf4j-log4j12' exclude group: 'org.pentaho' // missing dependency @@ -477,7 +522,7 @@ project(':iceberg-hive-metastore') { exclude group: 'com.google.code.findbugs', module: 'jsr305' } - testCompile("org.apache.hive:hive-metastore") { + testImplementation("org.apache.hive:hive-metastore") { exclude group: 'org.apache.avro', module: 'avro' exclude group: 'org.slf4j', module: 'slf4j-log4j12' exclude group: 'org.pentaho' // missing dependency @@ -498,24 +543,26 @@ project(':iceberg-hive-metastore') { exclude group: 'org.slf4j', module: 'slf4j-log4j12' } - testCompile project(path: ':iceberg-api', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') } } project(':iceberg-mr') { configurations { - testCompile { + testImplementation { exclude group: 'org.apache.parquet', module: 'parquet-hadoop-bundle' } } dependencies { - compile project(':iceberg-api') - compile project(':iceberg-core') - compile project(':iceberg-data') - compile project(':iceberg-hive-metastore') - compile project(':iceberg-orc') - compile project(':iceberg-parquet') + implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow') + api project(':iceberg-api') + implementation project(':iceberg-common') + implementation project(':iceberg-core') + api project(':iceberg-data') + implementation project(':iceberg-hive-metastore') + implementation project(':iceberg-orc') + implementation project(':iceberg-parquet') compileOnly("org.apache.hadoop:hadoop-client") { exclude group: 'org.apache.avro', module: 'avro' @@ -535,23 +582,25 @@ project(':iceberg-mr') { compileOnly("org.apache.hive:hive-metastore") compileOnly("org.apache.hive:hive-serde") - testCompile "org.apache.calcite:calcite-core" - testCompile "org.apache.calcite:calcite-druid" + implementation "com.github.ben-manes.caffeine:caffeine" + + testImplementation "org.apache.calcite:calcite-core" + testImplementation "org.apache.calcite:calcite-druid" - testCompile project(path: ':iceberg-data', configuration: 'testArtifacts') - testCompile project(path: ':iceberg-api', configuration: 'testArtifacts') - testCompile project(path: ':iceberg-core', configuration: 'testArtifacts') - testCompile project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-data', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') - testCompile("org.apache.avro:avro:1.9.2") - testCompile("org.apache.calcite:calcite-core") - testCompile("com.esotericsoftware:kryo-shaded:4.0.2") - testCompile("com.fasterxml.jackson.core:jackson-annotations:2.6.5") - testCompile("org.apache.hive:hive-service") { + testImplementation("org.apache.avro:avro:1.9.2") + testImplementation("org.apache.calcite:calcite-core") + testImplementation("com.esotericsoftware:kryo-shaded:4.0.2") + testImplementation("com.fasterxml.jackson.core:jackson-annotations:2.6.5") + testImplementation("org.apache.hive:hive-service") { exclude group: 'org.apache.hive', module: 'hive-exec' } - testCompile("org.apache.tez:tez-dag") - testCompile("org.apache.tez:tez-mapreduce") + testImplementation("org.apache.tez:tez-dag") + testImplementation("org.apache.tez:tez-mapreduce") } test { @@ -569,16 +618,22 @@ if (jdkVersion == '8') { tasks.jar.dependsOn tasks.shadowJar + configurations { + implementation { + exclude group: 'com.github.luben' + } + } + dependencies { - compile project(':iceberg-data') - compile project(':iceberg-orc') + implementation project(':iceberg-data') + implementation project(':iceberg-orc') testCompileOnly project(path: ':iceberg-data', configuration: 'testArtifacts') testCompileOnly project(path: ':iceberg-orc', configuration: 'testArtifacts') } shadowJar { - configurations = [project.configurations.compile, project.configurations.compileOnly, project.configurations.testCompileOnly] + configurations = [project.configurations.compileClasspath, project.configurations.runtimeClasspath] zip64 true @@ -593,18 +648,24 @@ if (jdkVersion == '8') { // We really only need Iceberg and Orc classes, but with relocated references for storage-api classes (see above) // Unfortunately the include list feature of this shader plugin doesn't work as expected - exclude 'com/**/*' - exclude 'edu/**/*' - exclude 'io/**' - exclude 'javax/**' - exclude 'org/apache/avro/**/*' - exclude 'org/apache/commons/**/*' - exclude 'org/checkerframework/**/*' - exclude 'org/codehaus/**/*' - exclude 'org/intellij/**/*' - exclude 'org/jetbrains/**/*' - exclude 'org/slf4j/**/*' - exclude 'org/threeten/**/*' + dependencies { + exclude 'com/**/*' + exclude 'edu/**/*' + exclude 'io/**' + exclude 'javax/**' + exclude 'org/apache/avro/**/*' + exclude 'org/apache/commons/**/*' + exclude 'org/checkerframework/**/*' + exclude 'org/codehaus/**/*' + exclude 'org/intellij/**/*' + exclude 'org/jetbrains/**/*' + exclude 'org/slf4j/**/*' + exclude 'org/threeten/**/*' + exclude 'org/xerial/**/*' + exclude 'org/apache/parquet/**/*' + exclude 'org/apache/yetus/**/*' + exclude 'shaded/parquet/**/*' + } classifier null } @@ -633,13 +694,15 @@ if (jdkVersion == '8') { } dependencies { + compileOnly project(path: ':iceberg-bundled-guava', configuration: 'shadow') compileOnly project(':iceberg-api') compileOnly project(':iceberg-core') + compileOnly project(':iceberg-common') compileOnly project(':iceberg-hive-metastore') compileOnly project(':iceberg-parquet') compileOnly project(':iceberg-hive3-orc-bundle') compileOnly project(':iceberg-mr') - + compileOnly project(':iceberg-data') compileOnly("org.apache.hadoop:hadoop-client:3.1.0") { exclude group: 'org.apache.avro', module: 'avro' @@ -656,6 +719,15 @@ if (jdkVersion == '8') { exclude group: 'org.pentaho' // missing dependency exclude group: 'org.slf4j', module: 'slf4j-log4j12' } + + compileOnly("org.apache.orc:orc-core::nohive") { + exclude group: 'org.apache.hadoop' + exclude group: 'commons-lang' + // These artifacts are shaded and included in the orc-core fat jar + exclude group: 'com.google.protobuf', module: 'protobuf-java' + exclude group: 'org.apache.hive', module: 'hive-storage-api' + } + compileOnly("org.apache.hive:hive-metastore:3.1.2") { exclude group: 'org.apache.orc' } @@ -663,20 +735,21 @@ if (jdkVersion == '8') { exclude group: 'org.apache.orc' } - testCompile project(path: ':iceberg-api', configuration: 'testArtifacts') - testCompile project(path: ':iceberg-core', configuration: 'testArtifacts') - testCompile project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-data', configuration: 'testArtifacts') - testCompile("org.apache.avro:avro:1.9.2") - testCompile("org.apache.calcite:calcite-core") - testCompile("com.esotericsoftware:kryo-shaded:4.0.2") - testCompile("com.fasterxml.jackson.core:jackson-annotations:2.6.5") - testCompile("org.apache.hive:hive-service:3.1.2") { + testImplementation("org.apache.avro:avro:1.9.2") + testImplementation("org.apache.calcite:calcite-core") + testImplementation("com.esotericsoftware:kryo-shaded:4.0.2") + testImplementation("com.fasterxml.jackson.core:jackson-annotations:2.6.5") + testImplementation("org.apache.hive:hive-service:3.1.2") { exclude group: 'org.apache.hive', module: 'hive-exec' exclude group: 'org.apache.orc' } - testCompile("org.apache.tez:tez-dag:0.10.1") - testCompile("org.apache.tez:tez-mapreduce:0.10.1") + testImplementation("org.apache.tez:tez-dag:0.10.1") + testImplementation("org.apache.tez:tez-mapreduce:0.10.1") } test { @@ -692,7 +765,7 @@ project(':iceberg-hive-runtime') { tasks.jar.dependsOn tasks.shadowJar configurations { - compile { + implementation { exclude group: 'com.github.stephenc.findbugs' exclude group: 'commons-pool' exclude group: 'javax.annotation' @@ -704,18 +777,18 @@ project(':iceberg-hive-runtime') { } dependencies { - compile project(':iceberg-mr') + implementation project(':iceberg-mr') if (jdkVersion == '8') { - compile project(':iceberg-hive3') + implementation project(':iceberg-hive3') } - compile(project(':iceberg-nessie')) { + implementation(project(':iceberg-nessie')) { exclude group: 'com.google.code.findbugs', module: 'jsr305' } - compile project(':iceberg-aws') + implementation project(':iceberg-aws') } shadowJar { - configurations = [project.configurations.compile] + configurations = [project.configurations.runtimeClasspath] zip64 true @@ -747,10 +820,14 @@ project(':iceberg-hive-runtime') { project(':iceberg-orc') { dependencies { - compile project(':iceberg-api') - compile project(':iceberg-core') + implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow') + api project(':iceberg-api') + implementation project(':iceberg-core') + implementation("org.apache.avro:avro") { + exclude group: 'org.tukaani' // xz compression is not supported + } - compile("org.apache.orc:orc-core::nohive") { + implementation("org.apache.orc:orc-core::nohive") { exclude group: 'org.apache.hadoop' exclude group: 'commons-lang' // These artifacts are shaded and included in the orc-core fat jar @@ -767,16 +844,18 @@ project(':iceberg-orc') { exclude group: 'org.apache.avro', module: 'avro' } - testCompile project(path: ':iceberg-api', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') } } project(':iceberg-parquet') { dependencies { - compile project(':iceberg-api') - compile project(':iceberg-core') + implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow') + api project(':iceberg-api') + implementation project(':iceberg-core') + implementation project(':iceberg-common') - compile("org.apache.parquet:parquet-avro") { + implementation("org.apache.parquet:parquet-avro") { exclude group: 'org.apache.avro', module: 'avro' // already shaded by Parquet exclude group: 'it.unimi.dsi' @@ -788,32 +867,41 @@ project(':iceberg-parquet') { exclude group: 'org.apache.avro', module: 'avro' } - testCompile project(path: ':iceberg-core', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts') } } project(':iceberg-arrow') { dependencies { - compile project(':iceberg-api') - compile project(':iceberg-parquet') + implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow') + api project(':iceberg-api') + implementation project(':iceberg-core') + implementation project(':iceberg-parquet') - compile("org.apache.arrow:arrow-vector") { + implementation("org.apache.arrow:arrow-vector") { exclude group: 'io.netty', module: 'netty-buffer' exclude group: 'io.netty', module: 'netty-common' exclude group: 'com.google.code.findbugs', module: 'jsr305' } - compile("org.apache.arrow:arrow-memory-netty") { + implementation("org.apache.arrow:arrow-memory-netty") { exclude group: 'com.google.code.findbugs', module: 'jsr305' exclude group: 'io.netty', module: 'netty-common' } - testCompile project(path: ':iceberg-core', configuration: 'testArtifacts') + implementation("org.apache.parquet:parquet-avro") { + exclude group: 'org.apache.avro', module: 'avro' + // already shaded by Parquet + exclude group: 'it.unimi.dsi' + exclude group: 'org.codehaus.jackson' + } + + testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts') // To run ArrowReaderTest test cases, :netty-common is needed. // We import :netty-common through :arrow-memory-netty // so that the same version as used by the :arrow-memory-netty module is picked. - testCompile("org.apache.arrow:arrow-memory-netty") - testCompile("org.apache.hadoop:hadoop-common") - testCompile("org.apache.hadoop:hadoop-mapreduce-client-core") + testImplementation("org.apache.arrow:arrow-memory-netty") + testImplementation("org.apache.hadoop:hadoop-common") + testImplementation("org.apache.hadoop:hadoop-mapreduce-client-core") } } @@ -830,27 +918,43 @@ project(':iceberg-spark') { } dependencies { - compile project(':iceberg-api') - compile project(':iceberg-common') - compile project(':iceberg-core') - compile project(':iceberg-data') - compile project(':iceberg-orc') - compile project(':iceberg-parquet') - compile project(':iceberg-arrow') - compile project(':iceberg-hive-metastore') - + implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow') + api project(':iceberg-api') + implementation project(':iceberg-common') + implementation project(':iceberg-core') + api project(':iceberg-data') + implementation project(':iceberg-orc') + implementation project(':iceberg-parquet') + implementation project(':iceberg-arrow') + implementation project(':iceberg-hive-metastore') + + compileOnly "com.google.errorprone:error_prone_annotations" compileOnly "org.apache.avro:avro" compileOnly("org.apache.spark:spark-hive_2.11") { exclude group: 'org.apache.avro', module: 'avro' } - testCompile("org.apache.hadoop:hadoop-minicluster") { + implementation("org.apache.orc:orc-core::nohive") { + exclude group: 'org.apache.hadoop' + exclude group: 'commons-lang' + // These artifacts are shaded and included in the orc-core fat jar + exclude group: 'com.google.protobuf', module: 'protobuf-java' + exclude group: 'org.apache.hive', module: 'hive-storage-api' + } + + implementation("org.apache.arrow:arrow-vector") { + exclude group: 'io.netty', module: 'netty-buffer' + exclude group: 'io.netty', module: 'netty-common' + exclude group: 'com.google.code.findbugs', module: 'jsr305' + } + + testImplementation("org.apache.hadoop:hadoop-minicluster") { exclude group: 'org.apache.avro', module: 'avro' } - testCompile project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') - testCompile project(path: ':iceberg-api', configuration: 'testArtifacts') - testCompile project(path: ':iceberg-core', configuration: 'testArtifacts') - testCompile project(path: ':iceberg-data', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-data', configuration: 'testArtifacts') } test { @@ -881,29 +985,31 @@ if (jdkVersion == '8') { } dependencies { - compile project(':iceberg-api') - compile project(':iceberg-common') - compile project(':iceberg-core') - compile project(':iceberg-data') - compile project(':iceberg-orc') - compile project(':iceberg-parquet') - compile project(':iceberg-arrow') - compile project(':iceberg-hive-metastore') - compile project(':iceberg-spark') + implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow') + api project(':iceberg-api') + implementation project(':iceberg-common') + implementation project(':iceberg-core') + implementation project(':iceberg-data') + implementation project(':iceberg-orc') + implementation project(':iceberg-parquet') + implementation project(':iceberg-arrow') + implementation project(':iceberg-hive-metastore') + implementation project(':iceberg-spark') + implementation "com.github.ben-manes.caffeine:caffeine" compileOnly "org.apache.avro:avro" compileOnly("org.apache.spark:spark-hive_2.11") { exclude group: 'org.apache.avro', module: 'avro' } - testCompile project(path: ':iceberg-spark', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-spark', configuration: 'testArtifacts') - testCompile("org.apache.hadoop:hadoop-minicluster") { + testImplementation("org.apache.hadoop:hadoop-minicluster") { exclude group: 'org.apache.avro', module: 'avro' } - testCompile project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') - testCompile project(path: ':iceberg-api', configuration: 'testArtifacts') - testCompile project(path: ':iceberg-data', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-data', configuration: 'testArtifacts') } test { @@ -926,7 +1032,7 @@ if (jdkVersion == '8') { tasks.jar.dependsOn tasks.shadowJar configurations { - compile { + implementation { exclude group: 'org.apache.spark' // included in Spark exclude group: 'org.slf4j' @@ -940,15 +1046,15 @@ if (jdkVersion == '8') { } dependencies { - compile project(':iceberg-spark2') - compile project(':iceberg-aws') - compile(project(':iceberg-nessie')) { + implementation project(':iceberg-spark2') + implementation project(':iceberg-aws') + implementation(project(':iceberg-nessie')) { exclude group: 'com.google.code.findbugs', module: 'jsr305' } } shadowJar { - configurations = [project.configurations.compile] + configurations = [project.configurations.runtimeClasspath] zip64 true @@ -1004,15 +1110,16 @@ project(':iceberg-spark3') { } dependencies { - compile project(':iceberg-api') - compile project(':iceberg-common') - compile project(':iceberg-core') - compile project(':iceberg-data') - compile project(':iceberg-orc') - compile project(':iceberg-parquet') - compile project(':iceberg-arrow') - compile project(':iceberg-hive-metastore') - compile project(':iceberg-spark') + implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow') + api project(':iceberg-api') + implementation project(':iceberg-common') + implementation project(':iceberg-core') + implementation project(':iceberg-data') + implementation project(':iceberg-orc') + implementation project(':iceberg-parquet') + implementation project(':iceberg-arrow') + implementation project(':iceberg-hive-metastore') + implementation project(':iceberg-spark') compileOnly "org.apache.avro:avro" compileOnly("org.apache.spark:spark-hive_2.12:${project.ext.Spark30Version}") { @@ -1020,15 +1127,21 @@ project(':iceberg-spark3') { exclude group: 'org.apache.arrow' } - testCompile project(path: ':iceberg-spark', configuration: 'testArtifacts') + implementation("org.apache.arrow:arrow-vector") { + exclude group: 'io.netty', module: 'netty-buffer' + exclude group: 'io.netty', module: 'netty-common' + exclude group: 'com.google.code.findbugs', module: 'jsr305' + } + + testImplementation project(path: ':iceberg-spark', configuration: 'testArtifacts') - testCompile("org.apache.hadoop:hadoop-minicluster") { + testImplementation("org.apache.hadoop:hadoop-minicluster") { exclude group: 'org.apache.avro', module: 'avro' } - testCompile project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') - testCompile project(path: ':iceberg-api', configuration: 'testArtifacts') - testCompile project(path: ':iceberg-data', configuration: 'testArtifacts') - testCompile "org.xerial:sqlite-jdbc" + testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-data', configuration: 'testArtifacts') + testImplementation "org.xerial:sqlite-jdbc" spark31Implementation("org.apache.spark:spark-hive_2.12:${project.ext.Spark31Version}") { exclude group: 'org.apache.avro', module: 'avro' @@ -1060,7 +1173,7 @@ project(':iceberg-spark3') { } project(":iceberg-spark3-extensions") { - apply plugin: 'java' + apply plugin: 'java-library' apply plugin: 'scala' apply plugin: 'antlr' @@ -1082,27 +1195,35 @@ project(":iceberg-spark3-extensions") { /* The Gradle Antlr plugin erroneously adds both antlr-build and runtime dependencies to the runtime path. This bug https://github.com/gradle/gradle/issues/820 exists because older versions of Antlr do not have separate - runtime and compile dependencies and they do not want to break backwards compatibility. So to only end up with + runtime and implementation dependencies and they do not want to break backwards compatibility. So to only end up with the runtime dependency on the runtime classpath we remove the dependencies added by the plugin here. Then add the runtime dependency back to only the runtime configuration manually. */ - compile { + implementation { extendsFrom = extendsFrom.findAll { it != configurations.antlr } } } dependencies { compileOnly "org.scala-lang:scala-library" + compileOnly project(path: ':iceberg-bundled-guava', configuration: 'shadow') + compileOnly project(':iceberg-api') + compileOnly project(':iceberg-core') + compileOnly project(':iceberg-common') + compileOnly project(':iceberg-spark') compileOnly project(':iceberg-spark3') + compileOnly project(':iceberg-hive-metastore') compileOnly("org.apache.spark:spark-hive_2.12:${project.ext.Spark30Version}") { exclude group: 'org.apache.avro', module: 'avro' exclude group: 'org.apache.arrow' } - testCompile project(path: ':iceberg-api', configuration: 'testArtifacts') - testCompile project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') - testCompile project(path: ':iceberg-spark', configuration: 'testArtifacts') - testCompile project(path: ':iceberg-spark3', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') + + testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-spark', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-spark3', configuration: 'testArtifacts') spark31Implementation("org.apache.spark:spark-hive_2.12:${project.ext.Spark31Version}") { exclude group: 'org.apache.avro', module: 'avro' @@ -1111,7 +1232,7 @@ project(":iceberg-spark3-extensions") { // Required because we remove antlr plugin dependencies from the compile configuration, see note above // We shade this in Spark3 Runtime to avoid issues with Spark's Antlr Runtime - runtime "org.antlr:antlr4-runtime:4.7.1" + runtimeOnly "org.antlr:antlr4-runtime:4.7.1" antlr "org.antlr:antlr4:4.7.1" } @@ -1149,7 +1270,7 @@ project(':iceberg-spark3-runtime') { } configurations { - compile { + implementation { exclude group: 'org.apache.spark' // included in Spark exclude group: 'org.slf4j' @@ -1160,16 +1281,22 @@ project(':iceberg-spark3-runtime') { exclude group: 'javax.xml.bind' exclude group: 'javax.annotation' exclude group: 'com.github.luben' + exclude group: 'com.ibm.icu' + exclude group: 'org.glassfish' + exclude group: 'org.abego.treelayout' + exclude group: 'org.antlr', module: 'ST4' + exclude group: 'org.antlr', module: 'antlr4' } spark31Implementation.extendsFrom integrationImplementation spark31CompileOnly.extendsFrom integrationCompileOnly } dependencies { - compile project(':iceberg-spark3') - compile project(':iceberg-spark3-extensions') - compile project(':iceberg-aws') - compile(project(':iceberg-nessie')) { + api project(':iceberg-api') + implementation project(':iceberg-spark3') + implementation project(':iceberg-spark3-extensions') + implementation project(':iceberg-aws') + implementation(project(':iceberg-nessie')) { exclude group: 'com.google.code.findbugs', module: 'jsr305' } @@ -1184,12 +1311,13 @@ project(':iceberg-spark3-runtime') { // Not allowed on our classpath, only the runtime jar is allowed integrationCompileOnly project(':iceberg-spark3-extensions') integrationCompileOnly project(':iceberg-spark3') + integrationCompileOnly project(':iceberg-api') spark31Implementation "org.apache.spark:spark-hive_2.12:${project.ext.Spark31Version}" } shadowJar { - configurations = [project.configurations.compile] + configurations = [project.configurations.runtimeClasspath] zip64 true @@ -1217,7 +1345,7 @@ project(':iceberg-spark3-runtime') { relocate 'com.carrotsearch', 'org.apache.iceberg.shaded.com.carrotsearch' relocate 'org.threeten.extra', 'org.apache.iceberg.shaded.org.threeten.extra' // relocate Antlr runtime and related deps to shade Iceberg specific version - relocate 'org.antlr.v4.runtime', 'org.apache.iceberg.shaded.org.antlr.v4.runtime' + relocate 'org.antlr.v4', 'org.apache.iceberg.shaded.org.antlr.v4' classifier null } @@ -1249,10 +1377,18 @@ project(':iceberg-spark3-runtime') { project(':iceberg-pig') { dependencies { - compile project(':iceberg-api') - compile project(':iceberg-common') - compile project(':iceberg-core') - compile project(':iceberg-parquet') + implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow') + api project(':iceberg-api') + implementation project(':iceberg-common') + implementation project(':iceberg-core') + implementation project(':iceberg-parquet') + + implementation("org.apache.parquet:parquet-avro") { + exclude group: 'org.apache.avro', module: 'avro' + // already shaded by Parquet + exclude group: 'it.unimi.dsi' + exclude group: 'org.codehaus.jackson' + } compileOnly("org.apache.pig:pig") { exclude group: "junit", module: "junit" @@ -1262,7 +1398,7 @@ project(':iceberg-pig') { exclude group: 'org.apache.avro', module: 'avro' } - testCompile("org.apache.hadoop:hadoop-minicluster") { + testImplementation("org.apache.hadoop:hadoop-minicluster") { exclude group: 'org.apache.avro', module: 'avro' } } @@ -1274,9 +1410,10 @@ project(':iceberg-nessie') { } dependencies { - compile project(':iceberg-core') - compile project(path: ':iceberg-bundled-guava', configuration: 'shadow') - compile "org.projectnessie:nessie-client" + api project(':iceberg-api') + implementation project(':iceberg-core') + implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow') + implementation "org.projectnessie:nessie-client" testImplementation "org.projectnessie:nessie-jaxrs-testextension" // Need to "pull in" el-api explicitly :( @@ -1284,7 +1421,7 @@ project(':iceberg-nessie') { compileOnly "org.apache.hadoop:hadoop-common" - testCompile project(path: ':iceberg-api', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') } } diff --git a/core/src/main/java/org/apache/iceberg/BaseTableScan.java b/core/src/main/java/org/apache/iceberg/BaseTableScan.java index 356d909f6bba..52ec9eaed0a4 100644 --- a/core/src/main/java/org/apache/iceberg/BaseTableScan.java +++ b/core/src/main/java/org/apache/iceberg/BaseTableScan.java @@ -21,7 +21,7 @@ import java.time.Instant; import java.time.LocalDateTime; -import java.time.ZoneId; +import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.util.Collection; import java.util.Collections; @@ -306,6 +306,6 @@ private Schema lazyColumnProjection() { } private static String formatTimestampMillis(long millis) { - return DATE_FORMAT.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(millis), ZoneId.systemDefault())); + return DATE_FORMAT.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(millis), ZoneOffset.UTC)); } } diff --git a/core/src/main/java/org/apache/iceberg/BaseTransaction.java b/core/src/main/java/org/apache/iceberg/BaseTransaction.java index 8c5d498b92e2..3d0b31e49901 100644 --- a/core/src/main/java/org/apache/iceberg/BaseTransaction.java +++ b/core/src/main/java/org/apache/iceberg/BaseTransaction.java @@ -454,6 +454,7 @@ public TableMetadata refresh() { } @Override + @SuppressWarnings("ConsistentOverrides") public void commit(TableMetadata underlyingBase, TableMetadata metadata) { if (underlyingBase != current) { // trigger a refresh and retry diff --git a/core/src/main/java/org/apache/iceberg/FindFiles.java b/core/src/main/java/org/apache/iceberg/FindFiles.java index 1f466b501fe0..17d0753c36c0 100644 --- a/core/src/main/java/org/apache/iceberg/FindFiles.java +++ b/core/src/main/java/org/apache/iceberg/FindFiles.java @@ -21,7 +21,7 @@ import java.time.Instant; import java.time.LocalDateTime; -import java.time.ZoneId; +import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.util.Arrays; import java.util.List; @@ -109,7 +109,7 @@ public Builder asOfTime(long timestampMillis) { // case, there is no valid snapshot to read. Preconditions.checkArgument(lastSnapshotId != null, "Cannot find a snapshot older than %s", - DATE_FORMAT.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(timestampMillis), ZoneId.systemDefault()))); + DATE_FORMAT.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(timestampMillis), ZoneOffset.UTC))); return inSnapshot(lastSnapshotId); } diff --git a/core/src/main/java/org/apache/iceberg/IncrementalDataTableScan.java b/core/src/main/java/org/apache/iceberg/IncrementalDataTableScan.java index a02366b6b66e..98f430c7c7f0 100644 --- a/core/src/main/java/org/apache/iceberg/IncrementalDataTableScan.java +++ b/core/src/main/java/org/apache/iceberg/IncrementalDataTableScan.java @@ -54,10 +54,10 @@ public TableScan useSnapshot(long scanSnapshotId) { } @Override - public TableScan appendsBetween(long newFromSnapshotId, long newToSnapshotId) { - validateSnapshotIdsRefinement(newFromSnapshotId, newToSnapshotId); + public TableScan appendsBetween(long fromSnapshotId, long toSnapshotId) { + validateSnapshotIdsRefinement(fromSnapshotId, toSnapshotId); return new IncrementalDataTableScan(tableOps(), table(), schema(), - context().fromSnapshotId(newFromSnapshotId).toSnapshotId(newToSnapshotId)); + context().fromSnapshotId(fromSnapshotId).toSnapshotId(toSnapshotId)); } @Override diff --git a/core/src/main/java/org/apache/iceberg/hadoop/HadoopCatalog.java b/core/src/main/java/org/apache/iceberg/hadoop/HadoopCatalog.java index 7aa43491e003..4bcc832397a0 100644 --- a/core/src/main/java/org/apache/iceberg/hadoop/HadoopCatalog.java +++ b/core/src/main/java/org/apache/iceberg/hadoop/HadoopCatalog.java @@ -90,7 +90,7 @@ public class HadoopCatalog extends BaseMetastoreCatalog implements Closeable, Su private FileIO fileIO; private boolean suppressPermissionError = false; - public HadoopCatalog(){ + public HadoopCatalog() { } @Override @@ -111,11 +111,10 @@ public void initialize(String name, Map properties) { /** * The constructor of the HadoopCatalog. It uses the passed location as its warehouse directory. * - * @deprecated please use the no-arg constructor, setConf and initialize to construct the catalog. Will be removed in - * v0.13.0 - * * @param conf The Hadoop configuration * @param warehouseLocation The location used as warehouse directory + * @deprecated please use the no-arg constructor, setConf and initialize to construct the catalog. Will be removed in + * v0.13.0 */ @Deprecated public HadoopCatalog(Configuration conf, String warehouseLocation) { @@ -128,10 +127,9 @@ public HadoopCatalog(Configuration conf, String warehouseLocation) { * from the passed Hadoop configuration as its default file system, and use the default directory * iceberg/warehouse as the warehouse directory. * + * @param conf The Hadoop configuration * @deprecated please use the no-arg constructor, setConf and initialize to construct the catalog. Will be removed in * v0.13.0 - * - * @param conf The Hadoop configuration */ @Deprecated public HadoopCatalog(Configuration conf) { diff --git a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java index a3327ce318c1..06e9b1beec34 100644 --- a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java +++ b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java @@ -90,11 +90,11 @@ public void testJsonConversion() throws Exception { long previousSnapshotId = System.currentTimeMillis() - new Random(1234).nextInt(3600); Snapshot previousSnapshot = new BaseSnapshot( ops.io(), previousSnapshotId, null, previousSnapshotId, null, null, null, ImmutableList.of( - new GenericManifestFile(localInput("file:/tmp/manfiest.1.avro"), SPEC_5.specId()))); + new GenericManifestFile(localInput("file:/tmp/manfiest.1.avro"), SPEC_5.specId()))); long currentSnapshotId = System.currentTimeMillis(); Snapshot currentSnapshot = new BaseSnapshot( ops.io(), currentSnapshotId, previousSnapshotId, currentSnapshotId, null, null, 7, ImmutableList.of( - new GenericManifestFile(localInput("file:/tmp/manfiest.2.avro"), SPEC_5.specId()))); + new GenericManifestFile(localInput("file:/tmp/manfiest.2.avro"), SPEC_5.specId()))); List snapshotLog = ImmutableList.builder() .add(new SnapshotLogEntry(previousSnapshot.timestampMillis(), previousSnapshot.snapshotId())) @@ -172,11 +172,11 @@ public void testBackwardCompat() throws Exception { long previousSnapshotId = System.currentTimeMillis() - new Random(1234).nextInt(3600); Snapshot previousSnapshot = new BaseSnapshot( ops.io(), previousSnapshotId, null, previousSnapshotId, null, null, null, ImmutableList.of( - new GenericManifestFile(localInput("file:/tmp/manfiest.1.avro"), spec.specId()))); + new GenericManifestFile(localInput("file:/tmp/manfiest.1.avro"), spec.specId()))); long currentSnapshotId = System.currentTimeMillis(); Snapshot currentSnapshot = new BaseSnapshot( ops.io(), currentSnapshotId, previousSnapshotId, currentSnapshotId, null, null, null, ImmutableList.of( - new GenericManifestFile(localInput("file:/tmp/manfiest.2.avro"), spec.specId()))); + new GenericManifestFile(localInput("file:/tmp/manfiest.2.avro"), spec.specId()))); TableMetadata expected = new TableMetadata(null, 1, null, TEST_LOCATION, 0, System.currentTimeMillis(), 3, TableMetadata.INITIAL_SCHEMA_ID, @@ -288,11 +288,11 @@ public void testJsonWithPreviousMetadataLog() throws Exception { long previousSnapshotId = System.currentTimeMillis() - new Random(1234).nextInt(3600); Snapshot previousSnapshot = new BaseSnapshot( ops.io(), previousSnapshotId, null, previousSnapshotId, null, null, null, ImmutableList.of( - new GenericManifestFile(localInput("file:/tmp/manfiest.1.avro"), SPEC_5.specId()))); + new GenericManifestFile(localInput("file:/tmp/manfiest.1.avro"), SPEC_5.specId()))); long currentSnapshotId = System.currentTimeMillis(); Snapshot currentSnapshot = new BaseSnapshot( ops.io(), currentSnapshotId, previousSnapshotId, currentSnapshotId, null, null, null, ImmutableList.of( - new GenericManifestFile(localInput("file:/tmp/manfiest.2.avro"), SPEC_5.specId()))); + new GenericManifestFile(localInput("file:/tmp/manfiest.2.avro"), SPEC_5.specId()))); List reversedSnapshotLog = Lists.newArrayList(); long currentTimestamp = System.currentTimeMillis(); @@ -319,11 +319,11 @@ public void testAddPreviousMetadataRemoveNone() { long previousSnapshotId = System.currentTimeMillis() - new Random(1234).nextInt(3600); Snapshot previousSnapshot = new BaseSnapshot( ops.io(), previousSnapshotId, null, previousSnapshotId, null, null, null, ImmutableList.of( - new GenericManifestFile(localInput("file:/tmp/manfiest.1.avro"), SPEC_5.specId()))); + new GenericManifestFile(localInput("file:/tmp/manfiest.1.avro"), SPEC_5.specId()))); long currentSnapshotId = System.currentTimeMillis(); Snapshot currentSnapshot = new BaseSnapshot( ops.io(), currentSnapshotId, previousSnapshotId, currentSnapshotId, null, null, null, ImmutableList.of( - new GenericManifestFile(localInput("file:/tmp/manfiest.2.avro"), SPEC_5.specId()))); + new GenericManifestFile(localInput("file:/tmp/manfiest.2.avro"), SPEC_5.specId()))); List reversedSnapshotLog = Lists.newArrayList(); long currentTimestamp = System.currentTimeMillis(); @@ -359,11 +359,11 @@ public void testAddPreviousMetadataRemoveOne() { long previousSnapshotId = System.currentTimeMillis() - new Random(1234).nextInt(3600); Snapshot previousSnapshot = new BaseSnapshot( ops.io(), previousSnapshotId, null, previousSnapshotId, null, null, null, ImmutableList.of( - new GenericManifestFile(localInput("file:/tmp/manfiest.1.avro"), SPEC_5.specId()))); + new GenericManifestFile(localInput("file:/tmp/manfiest.1.avro"), SPEC_5.specId()))); long currentSnapshotId = System.currentTimeMillis(); Snapshot currentSnapshot = new BaseSnapshot( ops.io(), currentSnapshotId, previousSnapshotId, currentSnapshotId, null, null, null, ImmutableList.of( - new GenericManifestFile(localInput("file:/tmp/manfiest.2.avro"), SPEC_5.specId()))); + new GenericManifestFile(localInput("file:/tmp/manfiest.2.avro"), SPEC_5.specId()))); List reversedSnapshotLog = Lists.newArrayList(); long currentTimestamp = System.currentTimeMillis(); @@ -411,11 +411,11 @@ public void testAddPreviousMetadataRemoveMultiple() { long previousSnapshotId = System.currentTimeMillis() - new Random(1234).nextInt(3600); Snapshot previousSnapshot = new BaseSnapshot( ops.io(), previousSnapshotId, null, previousSnapshotId, null, null, null, ImmutableList.of( - new GenericManifestFile(localInput("file:/tmp/manfiest.1.avro"), SPEC_5.specId()))); + new GenericManifestFile(localInput("file:/tmp/manfiest.1.avro"), SPEC_5.specId()))); long currentSnapshotId = System.currentTimeMillis(); Snapshot currentSnapshot = new BaseSnapshot( ops.io(), currentSnapshotId, previousSnapshotId, currentSnapshotId, null, null, null, ImmutableList.of( - new GenericManifestFile(localInput("file:/tmp/manfiest.2.avro"), SPEC_5.specId()))); + new GenericManifestFile(localInput("file:/tmp/manfiest.2.avro"), SPEC_5.specId()))); List reversedSnapshotLog = Lists.newArrayList(); long currentTimestamp = System.currentTimeMillis(); diff --git a/deploy.gradle b/deploy.gradle index a2a51043c25b..e2af9e4bf2a4 100644 --- a/deploy.gradle +++ b/deploy.gradle @@ -22,7 +22,6 @@ if (project.hasProperty('release') && jdkVersion != '8') { } subprojects { - apply plugin: 'maven' // make pom files for deployment apply plugin: 'maven-publish' apply plugin: 'signing' diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 1277f7dac0f6..ffed3a254e91 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,24 +1,5 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-7.2-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-5.4.1-bin.zip diff --git a/gradlew b/gradlew index e718f9c62b82..bdfdde933d96 100755 --- a/gradlew +++ b/gradlew @@ -1,82 +1,133 @@ -#!/usr/bin/env sh +#!/bin/sh + +# +# Copyright © 2015-2021 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ############################################################################## -## -## Gradle start up script for UN*X -## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# ############################################################################## # Attempt to set APP_HOME + # Resolve links: $0 may be a link -PRG="$0" -# Need this for relative symlinks. -while [ -h "$PRG" ] ; do - ls=`ls -ld "$PRG"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG=`dirname "$PRG"`"/$link" - fi +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac done -SAVED="`pwd`" -cd "`dirname \"$PRG\"`/" >/dev/null -APP_HOME="`pwd -P`" -cd "$SAVED" >/dev/null + +APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit + +if [ ! -e $APP_HOME/gradle/wrapper/gradle-wrapper.jar ]; then + curl -o $APP_HOME/gradle/wrapper/gradle-wrapper.jar https://raw.githubusercontent.com/gradle/gradle/v7.2.0/gradle/wrapper/gradle-wrapper.jar +fi APP_NAME="Gradle" -APP_BASE_NAME=`basename "$0"` +APP_BASE_NAME=${0##*/} # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS="" +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' # Use the maximum available, or set MAX_FD != -1 to use that value. -MAX_FD="maximum" +MAX_FD=maximum warn () { echo "$*" -} +} >&2 die () { echo echo "$*" echo exit 1 -} +} >&2 # OS specific support (must be 'true' or 'false'). cygwin=false msys=false darwin=false nonstop=false -case "`uname`" in - CYGWIN* ) - cygwin=true - ;; - Darwin* ) - darwin=true - ;; - MINGW* ) - msys=true - ;; - NONSTOP* ) - nonstop=true - ;; +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; esac -if [ ! -e $APP_HOME/gradle/wrapper/gradle-wrapper.jar ]; then - curl -o $APP_HOME/gradle/wrapper/gradle-wrapper.jar https://raw.githubusercontent.com/gradle/gradle/v5.4.1/gradle/wrapper/gradle-wrapper.jar -fi - CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + # Determine the Java command to use to start the JVM. if [ -n "$JAVA_HOME" ] ; then if [ -x "$JAVA_HOME/jre/sh/java" ] ; then # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" + JAVACMD=$JAVA_HOME/jre/sh/java else - JAVACMD="$JAVA_HOME/bin/java" + JAVACMD=$JAVA_HOME/bin/java fi if [ ! -x "$JAVACMD" ] ; then die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME @@ -85,7 +136,7 @@ Please set the JAVA_HOME variable in your environment to match the location of your Java installation." fi else - JAVACMD="java" + JAVACMD=java which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the @@ -93,84 +144,95 @@ location of your Java installation." fi # Increase the maximum file descriptors if we can. -if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then - MAX_FD_LIMIT=`ulimit -H -n` - if [ $? -eq 0 ] ; then - if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then - MAX_FD="$MAX_FD_LIMIT" - fi - ulimit -n $MAX_FD - if [ $? -ne 0 ] ; then - warn "Could not set maximum file descriptor limit: $MAX_FD" - fi - else - warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" - fi +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac fi -# For Darwin, add options to specify how the application appears in the dock -if $darwin; then - GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" -fi +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) -# For Cygwin, switch paths to Windows format before running java -if $cygwin ; then - APP_HOME=`cygpath --path --mixed "$APP_HOME"` - CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` - JAVACMD=`cygpath --unix "$JAVACMD"` - - # We build the pattern for arguments to be converted via cygpath - ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` - SEP="" - for dir in $ROOTDIRSRAW ; do - ROOTDIRS="$ROOTDIRS$SEP$dir" - SEP="|" - done - OURCYGPATTERN="(^($ROOTDIRS))" - # Add a user-defined pattern to the cygpath arguments - if [ "$GRADLE_CYGPATTERN" != "" ] ; then - OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" - fi # Now convert the arguments - kludge to limit ourselves to /bin/sh - i=0 - for arg in "$@" ; do - CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` - CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option - - if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition - eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` - else - eval `echo args$i`="\"$arg\"" + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) fi - i=$((i+1)) + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg done - case $i in - (0) set -- ;; - (1) set -- "$args0" ;; - (2) set -- "$args0" "$args1" ;; - (3) set -- "$args0" "$args1" "$args2" ;; - (4) set -- "$args0" "$args1" "$args2" "$args3" ;; - (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; - (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; - (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; - (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; - (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; - esac fi -# Escape application args -save () { - for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done - echo " " -} -APP_ARGS=$(save "$@") - -# Collect all arguments for the java command, following the shell quoting and substitution rules -eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" - -# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong -if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then - cd "$(dirname "$0")" -fi +# Collect all arguments for the java command; +# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of +# shell script including quotes and variable substitutions, so put them in +# double quotes to make sure that they get re-expanded; and +# * put everything else in single quotes, so that it's not re-expanded. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' exec "$JAVACMD" "$@" diff --git a/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveCatalog.java b/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveCatalog.java index 3ae695973799..4911c696e47b 100644 --- a/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveCatalog.java +++ b/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveCatalog.java @@ -72,9 +72,9 @@ public HiveCatalog() { /** * Hive Catalog constructor. * + * @param conf Hadoop Configuration * @deprecated please use the no-arg constructor, setConf and initialize to construct the catalog. Will be removed in * v0.13.0 - * @param conf Hadoop Configuration */ @Deprecated public HiveCatalog(Configuration conf) { diff --git a/hive3/src/main/java/org/apache/iceberg/mr/hive/vector/CompatibilityHiveVectorUtils.java b/hive3/src/main/java/org/apache/iceberg/mr/hive/vector/CompatibilityHiveVectorUtils.java index d53fdada44c1..e24ea3503ba9 100644 --- a/hive3/src/main/java/org/apache/iceberg/mr/hive/vector/CompatibilityHiveVectorUtils.java +++ b/hive3/src/main/java/org/apache/iceberg/mr/hive/vector/CompatibilityHiveVectorUtils.java @@ -59,7 +59,7 @@ private CompatibilityHiveVectorUtils() { * Returns serialized mapwork instance from a job conf - ported from Hive source code LlapHiveUtils#findMapWork * * @param job JobConf instance - * @return + * @return a serialized {@link MapWork} based on the given job conf */ public static MapWork findMapWork(JobConf job) { String inputName = job.get(Utilities.INPUT_NAME, null); diff --git a/mr/src/test/java/org/apache/iceberg/mr/hive/TestTables.java b/mr/src/test/java/org/apache/iceberg/mr/hive/TestTables.java index aed7c9a5f10b..775ccf163db7 100644 --- a/mr/src/test/java/org/apache/iceberg/mr/hive/TestTables.java +++ b/mr/src/test/java/org/apache/iceberg/mr/hive/TestTables.java @@ -114,8 +114,8 @@ public String catalogName() { /** * The table properties string needed for the CREATE TABLE ... commands, - * like "TBLPROPERTIES('iceberg.catalog'='mycatalog') - * @return + * like {@code TBLPROPERTIES('iceberg.catalog'='mycatalog')} + * @return the tables properties string, such as {@code TBLPROPERTIES('iceberg.catalog'='mycatalog')} */ public String propertiesForCreateTableSQL(Map tableProperties) { Map properties = new HashMap<>(tableProperties); diff --git a/spark/src/main/java/org/apache/iceberg/spark/SparkTableUtil.java b/spark/src/main/java/org/apache/iceberg/spark/SparkTableUtil.java index fcfb973081c9..58c83118c76e 100644 --- a/spark/src/main/java/org/apache/iceberg/spark/SparkTableUtil.java +++ b/spark/src/main/java/org/apache/iceberg/spark/SparkTableUtil.java @@ -268,13 +268,13 @@ public static List getPartitionsByFilter(SparkSession spark, Tab * * For Parquet and ORC partitions, this will read metrics from the file footer. For Avro partitions, * metrics are set to null. - * @deprecated use {@link TableMigrationUtil#listPartition(Map, String, String, PartitionSpec, Configuration, - * MetricsConfig, NameMapping)} * * @param partition a partition * @param conf a serializable Hadoop conf * @param metricsConfig a metrics conf * @return a List of DataFile + * @deprecated use {@link TableMigrationUtil#listPartition(Map, String, String, PartitionSpec, Configuration, + * MetricsConfig, NameMapping)} */ @Deprecated public static List listPartition(SparkPartition partition, PartitionSpec spec, @@ -287,14 +287,14 @@ public static List listPartition(SparkPartition partition, PartitionSp * * For Parquet and ORC partitions, this will read metrics from the file footer. For Avro partitions, * metrics are set to null. - * @deprecated use {@link TableMigrationUtil#listPartition(Map, String, String, PartitionSpec, Configuration, - * MetricsConfig, NameMapping)} * * @param partition a partition * @param conf a serializable Hadoop conf * @param metricsConfig a metrics conf * @param mapping a name mapping * @return a List of DataFile + * @deprecated use {@link TableMigrationUtil#listPartition(Map, String, String, PartitionSpec, Configuration, + * MetricsConfig, NameMapping)} */ @Deprecated public static List listPartition(SparkPartition partition, PartitionSpec spec, diff --git a/spark3-runtime/LICENSE b/spark3-runtime/LICENSE index dab3b906188a..76648b691826 100644 --- a/spark3-runtime/LICENSE +++ b/spark3-runtime/LICENSE @@ -619,3 +619,64 @@ This binary includes code from Apache Commons. Copyright: 2020 The Apache Software Foundation Home page: https://commons.apache.org/ License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This binary artifact contains code from ANTLR. + +Copyright: 2012-2017 The ANTLR Project +Home page: https://github.com/antlr/antlr4 +License: https://github.com/antlr/antlr4/blob/master/LICENSE.txt (BSD 3-clause) + +License text: + +| [The "BSD 3-clause license"] +| Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. +| +| Redistribution and use in source and binary forms, with or without +| modification, are permitted provided that the following conditions +| are met: +| 1. Redistributions of source code must retain the above copyright +| notice, this list of conditions and the following disclaimer. +| 2. Redistributions in binary form must reproduce the above copyright +| notice, this list of conditions and the following disclaimer in the +| documentation and/or other materials provided with the distribution. +| 3. Neither the name of the copyright holder nor the names of its contributors +| may be used to endorse or promote products derived from this software +| without specific prior written permission. + +| THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +| IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +| OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +| IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +| INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +| NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +| DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +| THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +| (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +| THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +| ===== +| +| MIT License for codepointat.js from https://git.io/codepointat +| MIT License for fromcodepoint.js from https://git.io/vDW1m +| +| Copyright Mathias Bynens +| +| Permission is hereby granted, free of charge, to any person obtaining +| a copy of this software and associated documentation files (the +| "Software"), to deal in the Software without restriction, including +| without limitation the rights to use, copy, modify, merge, publish, +| distribute, sublicense, and/or sell copies of the Software, and to +| permit persons to whom the Software is furnished to do so, subject to +| the following conditions: +| +| The above copyright notice and this permission notice shall be +| included in all copies or substantial portions of the Software. +| +| THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +| EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +| MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +| NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +| LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +| OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +| WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/versions.props b/versions.props index 2a98bf444552..6bf81cd1d6b8 100644 --- a/versions.props +++ b/versions.props @@ -11,6 +11,7 @@ org.apache.spark:spark-hive_2.11 = 2.4.8 org.apache.spark:spark-avro_2.11 = 2.4.8 org.apache.pig:pig = 0.14.0 com.fasterxml.jackson.*:* = 2.11.4 +com.google.errorprone:error_prone_annotations = 2.3.3 com.google.guava:guava = 28.0-jre com.github.ben-manes.caffeine:caffeine = 2.8.4 org.apache.arrow:arrow-vector = 2.0.0