diff --git a/api/src/test/java/org/apache/iceberg/AssertHelpers.java b/api/src/test/java/org/apache/iceberg/AssertHelpers.java index e4eded3a968b..1874d1b75072 100644 --- a/api/src/test/java/org/apache/iceberg/AssertHelpers.java +++ b/api/src/test/java/org/apache/iceberg/AssertHelpers.java @@ -20,6 +20,8 @@ package org.apache.iceberg; import java.util.concurrent.Callable; +import org.apache.avro.AvroRuntimeException; +import org.apache.avro.generic.GenericRecord; import org.assertj.core.api.AbstractThrowableAssert; import org.assertj.core.api.Assertions; @@ -110,4 +112,16 @@ public static void assertThrowsCause(String message, .isInstanceOf(expected) .hasMessageContaining(containedInMessage); } + + /** + * A convenience method to check if an Avro field is empty. + * @param record The record to read from + * @param field The name of the field + */ + public static void assertEmptyAvroField(GenericRecord record, String field) { + AssertHelpers.assertThrows( + "Not a valid schema field: " + field, + AvroRuntimeException.class, + () -> record.get(field)); + } } diff --git a/core/src/test/java/org/apache/iceberg/TestManifestListVersions.java b/core/src/test/java/org/apache/iceberg/TestManifestListVersions.java index 6c145734ce4e..390ba0de12ae 100644 --- a/core/src/test/java/org/apache/iceberg/TestManifestListVersions.java +++ b/core/src/test/java/org/apache/iceberg/TestManifestListVersions.java @@ -136,9 +136,9 @@ public void testV1ForwardCompatibility() throws IOException { Assert.assertEquals("Added rows count", ADDED_ROWS, (long) generic.get("added_rows_count")); Assert.assertEquals("Existing rows count", EXISTING_ROWS, (long) generic.get("existing_rows_count")); Assert.assertEquals("Deleted rows count", DELETED_ROWS, (long) generic.get("deleted_rows_count")); - Assert.assertNull("Content", generic.get(ManifestFile.MANIFEST_CONTENT.name())); - Assert.assertNull("Sequence number", generic.get(ManifestFile.SEQUENCE_NUMBER.name())); - Assert.assertNull("Min sequence number", generic.get(ManifestFile.MIN_SEQUENCE_NUMBER.name())); + AssertHelpers.assertEmptyAvroField(generic, ManifestFile.MANIFEST_CONTENT.name()); + AssertHelpers.assertEmptyAvroField(generic, ManifestFile.SEQUENCE_NUMBER.name()); + AssertHelpers.assertEmptyAvroField(generic, ManifestFile.MIN_SEQUENCE_NUMBER.name()); } @Test @@ -158,9 +158,9 @@ public void testV2ForwardCompatibility() throws IOException { Assert.assertEquals("Added rows count", ADDED_ROWS, (long) generic.get("added_rows_count")); Assert.assertEquals("Existing rows count", EXISTING_ROWS, (long) generic.get("existing_rows_count")); Assert.assertEquals("Deleted rows count", DELETED_ROWS, (long) generic.get("deleted_rows_count")); - Assert.assertNull("Content", generic.get(ManifestFile.MANIFEST_CONTENT.name())); - Assert.assertNull("Sequence number", generic.get(ManifestFile.SEQUENCE_NUMBER.name())); - Assert.assertNull("Min sequence number", generic.get(ManifestFile.MIN_SEQUENCE_NUMBER.name())); + AssertHelpers.assertEmptyAvroField(generic, ManifestFile.MANIFEST_CONTENT.name()); + AssertHelpers.assertEmptyAvroField(generic, ManifestFile.SEQUENCE_NUMBER.name()); + AssertHelpers.assertEmptyAvroField(generic, ManifestFile.MIN_SEQUENCE_NUMBER.name()); } @Test diff --git a/core/src/test/java/org/apache/iceberg/avro/TestReadProjection.java b/core/src/test/java/org/apache/iceberg/avro/TestReadProjection.java index 9243a809ff1e..e71034483bdc 100644 --- a/core/src/test/java/org/apache/iceberg/avro/TestReadProjection.java +++ b/core/src/test/java/org/apache/iceberg/avro/TestReadProjection.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.Map; import org.apache.avro.generic.GenericData.Record; +import org.apache.iceberg.AssertHelpers; import org.apache.iceberg.Schema; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; @@ -148,7 +149,7 @@ public void testBasicProjection() throws Exception { ); Record projected = writeAndRead("basic_projection_id", writeSchema, idOnly, record); - Assert.assertNull("Should not project data", projected.get("data")); + AssertHelpers.assertEmptyAvroField(projected, "data"); Assert.assertEquals("Should contain the correct id value", 34L, (long) projected.get("id")); Schema dataOnly = new Schema( @@ -157,10 +158,10 @@ public void testBasicProjection() throws Exception { projected = writeAndRead("basic_projection_data", writeSchema, dataOnly, record); - Assert.assertNull("Should not project id", projected.get("id")); + AssertHelpers.assertEmptyAvroField(projected, "id"); int cmp = Comparators.charSequences() .compare("test", (CharSequence) projected.get("data")); - Assert.assertTrue("Should contain the correct data value", cmp == 0); + Assert.assertEquals("Should contain the correct data value", 0, cmp); } @Test @@ -184,7 +185,7 @@ public void testRename() throws Exception { Assert.assertEquals("Should contain the correct id value", 34L, (long) projected.get("id")); int cmp = Comparators.charSequences() .compare("test", (CharSequence) projected.get("renamed")); - Assert.assertTrue("Should contain the correct data/renamed value", cmp == 0); + Assert.assertEquals("Should contain the correct data/renamed value", 0, cmp); } @Test @@ -210,9 +211,8 @@ public void testNestedStructProjection() throws Exception { ); Record projected = writeAndRead("id_only", writeSchema, idOnly, record); - Record projectedLocation = (Record) projected.get("location"); + AssertHelpers.assertEmptyAvroField(projected, "location"); Assert.assertEquals("Should contain the correct id value", 34L, (long) projected.get("id")); - Assert.assertNull("Should not project location", projectedLocation); Schema latOnly = new Schema( Types.NestedField.optional(3, "location", Types.StructType.of( @@ -221,10 +221,10 @@ public void testNestedStructProjection() throws Exception { ); projected = writeAndRead("latitude_only", writeSchema, latOnly, record); - projectedLocation = (Record) projected.get("location"); - Assert.assertNull("Should not project id", projected.get("id")); + Record projectedLocation = (Record) projected.get("location"); + AssertHelpers.assertEmptyAvroField(projected, "id"); Assert.assertNotNull("Should project location", projected.get("location")); - Assert.assertNull("Should not project longitude", projectedLocation.get("long")); + AssertHelpers.assertEmptyAvroField(projectedLocation, "long"); Assert.assertEquals("Should project latitude", 52.995143f, (float) projectedLocation.get("lat"), 0.000001f); @@ -236,16 +236,16 @@ public void testNestedStructProjection() throws Exception { projected = writeAndRead("longitude_only", writeSchema, longOnly, record); projectedLocation = (Record) projected.get("location"); - Assert.assertNull("Should not project id", projected.get("id")); + AssertHelpers.assertEmptyAvroField(projected, "id"); Assert.assertNotNull("Should project location", projected.get("location")); - Assert.assertNull("Should not project latitutde", projectedLocation.get("lat")); + AssertHelpers.assertEmptyAvroField(projectedLocation, "lat"); Assert.assertEquals("Should project longitude", -1.539054f, (float) projectedLocation.get("long"), 0.000001f); Schema locationOnly = writeSchema.select("location"); projected = writeAndRead("location_only", writeSchema, locationOnly, record); projectedLocation = (Record) projected.get("location"); - Assert.assertNull("Should not project id", projected.get("id")); + AssertHelpers.assertEmptyAvroField(projected, "id"); Assert.assertNotNull("Should project location", projected.get("location")); Assert.assertEquals("Should project latitude", 52.995143f, (float) projectedLocation.get("lat"), 0.000001f); @@ -273,23 +273,23 @@ public void testMapProjection() throws IOException { Record projected = writeAndRead("id_only", writeSchema, idOnly, record); Assert.assertEquals("Should contain the correct id value", 34L, (long) projected.get("id")); - Assert.assertNull("Should not project properties map", projected.get("properties")); + AssertHelpers.assertEmptyAvroField(projected, "properties"); Schema keyOnly = writeSchema.select("properties.key"); projected = writeAndRead("key_only", writeSchema, keyOnly, record); - Assert.assertNull("Should not project id", projected.get("id")); + AssertHelpers.assertEmptyAvroField(projected, "id"); Assert.assertEquals("Should project entire map", properties, toStringMap((Map) projected.get("properties"))); Schema valueOnly = writeSchema.select("properties.value"); projected = writeAndRead("value_only", writeSchema, valueOnly, record); - Assert.assertNull("Should not project id", projected.get("id")); + AssertHelpers.assertEmptyAvroField(projected, "id"); Assert.assertEquals("Should project entire map", properties, toStringMap((Map) projected.get("properties"))); Schema mapOnly = writeSchema.select("properties"); projected = writeAndRead("map_only", writeSchema, mapOnly, record); - Assert.assertNull("Should not project id", projected.get("id")); + AssertHelpers.assertEmptyAvroField(projected, "id"); Assert.assertEquals("Should project entire map", properties, toStringMap((Map) projected.get("properties"))); } @@ -337,16 +337,16 @@ public void testMapOfStructsProjection() throws IOException { Record projected = writeAndRead("id_only", writeSchema, idOnly, record); Assert.assertEquals("Should contain the correct id value", 34L, (long) projected.get("id")); - Assert.assertNull("Should not project locations map", projected.get("locations")); + AssertHelpers.assertEmptyAvroField(projected, "locations"); projected = writeAndRead("all_locations", writeSchema, writeSchema.select("locations"), record); - Assert.assertNull("Should not project id", projected.get("id")); + AssertHelpers.assertEmptyAvroField(projected, "id"); Assert.assertEquals("Should project locations map", record.get("locations"), toStringMap((Map) projected.get("locations"))); projected = writeAndRead("lat_only", writeSchema, writeSchema.select("locations.lat"), record); - Assert.assertNull("Should not project id", projected.get("id")); + AssertHelpers.assertEmptyAvroField(projected, "id"); Map locations = toStringMap((Map) projected.get("locations")); Assert.assertNotNull("Should project locations map", locations); Assert.assertEquals("Should contain L1 and L2", @@ -355,28 +355,28 @@ public void testMapOfStructsProjection() throws IOException { Assert.assertNotNull("L1 should not be null", projectedL1); Assert.assertEquals("L1 should contain lat", 53.992811f, (float) projectedL1.get("lat"), 0.000001); - Assert.assertNull("L1 should not contain long", projectedL1.get("long")); + AssertHelpers.assertEmptyAvroField(projectedL1, "long"); Record projectedL2 = (Record) locations.get("L2"); Assert.assertNotNull("L2 should not be null", projectedL2); Assert.assertEquals("L2 should contain lat", 52.995143f, (float) projectedL2.get("lat"), 0.000001); - Assert.assertNull("L2 should not contain long", projectedL2.get("long")); + AssertHelpers.assertEmptyAvroField(projectedL2, "y"); projected = writeAndRead("long_only", writeSchema, writeSchema.select("locations.long"), record); - Assert.assertNull("Should not project id", projected.get("id")); + AssertHelpers.assertEmptyAvroField(projected, "id"); locations = toStringMap((Map) projected.get("locations")); Assert.assertNotNull("Should project locations map", locations); Assert.assertEquals("Should contain L1 and L2", Sets.newHashSet("L1", "L2"), locations.keySet()); projectedL1 = (Record) locations.get("L1"); Assert.assertNotNull("L1 should not be null", projectedL1); - Assert.assertNull("L1 should not contain lat", projectedL1.get("lat")); + AssertHelpers.assertEmptyAvroField(projectedL1, "lat"); Assert.assertEquals("L1 should contain long", -1.542616f, (float) projectedL1.get("long"), 0.000001); projectedL2 = (Record) locations.get("L2"); Assert.assertNotNull("L2 should not be null", projectedL2); - Assert.assertNull("L2 should not contain lat", projectedL2.get("lat")); + AssertHelpers.assertEmptyAvroField(projectedL2, "lat"); Assert.assertEquals("L2 should contain long", -1.539054f, (float) projectedL2.get("long"), 0.000001); @@ -390,7 +390,7 @@ public void testMapOfStructsProjection() throws IOException { ); projected = writeAndRead("latitude_renamed", writeSchema, latitiudeRenamed, record); - Assert.assertNull("Should not project id", projected.get("id")); + AssertHelpers.assertEmptyAvroField(projected, "id"); locations = toStringMap((Map) projected.get("locations")); Assert.assertNotNull("Should project locations map", locations); Assert.assertEquals("Should contain L1 and L2", @@ -399,14 +399,14 @@ public void testMapOfStructsProjection() throws IOException { Assert.assertNotNull("L1 should not be null", projectedL1); Assert.assertEquals("L1 should contain latitude", 53.992811f, (float) projectedL1.get("latitude"), 0.000001); - Assert.assertNull("L1 should not contain lat", projectedL1.get("lat")); - Assert.assertNull("L1 should not contain long", projectedL1.get("long")); + AssertHelpers.assertEmptyAvroField(projectedL1, "lat"); + AssertHelpers.assertEmptyAvroField(projectedL1, "long"); projectedL2 = (Record) locations.get("L2"); Assert.assertNotNull("L2 should not be null", projectedL2); Assert.assertEquals("L2 should contain latitude", 52.995143f, (float) projectedL2.get("latitude"), 0.000001); - Assert.assertNull("L2 should not contain lat", projectedL2.get("lat")); - Assert.assertNull("L2 should not contain long", projectedL2.get("long")); + AssertHelpers.assertEmptyAvroField(projectedL2, "lat"); + AssertHelpers.assertEmptyAvroField(projectedL2, "long"); } @Test @@ -429,16 +429,16 @@ public void testListProjection() throws IOException { Record projected = writeAndRead("id_only", writeSchema, idOnly, record); Assert.assertEquals("Should contain the correct id value", 34L, (long) projected.get("id")); - Assert.assertNull("Should not project values list", projected.get("values")); + AssertHelpers.assertEmptyAvroField(projected, "values"); Schema elementOnly = writeSchema.select("values.element"); projected = writeAndRead("element_only", writeSchema, elementOnly, record); - Assert.assertNull("Should not project id", projected.get("id")); + AssertHelpers.assertEmptyAvroField(projected, "id"); Assert.assertEquals("Should project entire list", values, projected.get("values")); Schema listOnly = writeSchema.select("values"); projected = writeAndRead("list_only", writeSchema, listOnly, record); - Assert.assertNull("Should not project id", projected.get("id")); + AssertHelpers.assertEmptyAvroField(projected, "id"); Assert.assertEquals("Should project entire list", values, projected.get("values")); } @@ -473,35 +473,35 @@ public void testListOfStructsProjection() throws IOException { Record projected = writeAndRead("id_only", writeSchema, idOnly, record); Assert.assertEquals("Should contain the correct id value", 34L, (long) projected.get("id")); - Assert.assertNull("Should not project points list", projected.get("points")); + AssertHelpers.assertEmptyAvroField(projected, "points"); projected = writeAndRead("all_points", writeSchema, writeSchema.select("points"), record); - Assert.assertNull("Should not project id", projected.get("id")); + AssertHelpers.assertEmptyAvroField(projected, "id"); Assert.assertEquals("Should project points list", record.get("points"), projected.get("points")); projected = writeAndRead("x_only", writeSchema, writeSchema.select("points.x"), record); - Assert.assertNull("Should not project id", projected.get("id")); + AssertHelpers.assertEmptyAvroField(projected, "id"); Assert.assertNotNull("Should project points list", projected.get("points")); List points = (List) projected.get("points"); Assert.assertEquals("Should read 2 points", 2, points.size()); Record projectedP1 = points.get(0); Assert.assertEquals("Should project x", 1, (int) projectedP1.get("x")); - Assert.assertNull("Should not project y", projectedP1.get("y")); + AssertHelpers.assertEmptyAvroField(projectedP1, "y"); Record projectedP2 = points.get(1); Assert.assertEquals("Should project x", 3, (int) projectedP2.get("x")); - Assert.assertNull("Should not project y", projectedP2.get("y")); + AssertHelpers.assertEmptyAvroField(projectedP2, "y"); projected = writeAndRead("y_only", writeSchema, writeSchema.select("points.y"), record); - Assert.assertNull("Should not project id", projected.get("id")); + AssertHelpers.assertEmptyAvroField(projected, "id"); Assert.assertNotNull("Should project points list", projected.get("points")); points = (List) projected.get("points"); Assert.assertEquals("Should read 2 points", 2, points.size()); projectedP1 = points.get(0); - Assert.assertNull("Should not project x", projectedP1.get("x")); + AssertHelpers.assertEmptyAvroField(projectedP1, "x"); Assert.assertEquals("Should project y", 2, (int) projectedP1.get("y")); projectedP2 = points.get(1); - Assert.assertNull("Should not project x", projectedP2.get("x")); + AssertHelpers.assertEmptyAvroField(projectedP2, "x"); Assert.assertEquals("Should project null y", null, projectedP2.get("y")); Schema yRenamed = new Schema( @@ -513,17 +513,17 @@ public void testListOfStructsProjection() throws IOException { ); projected = writeAndRead("y_renamed", writeSchema, yRenamed, record); - Assert.assertNull("Should not project id", projected.get("id")); + AssertHelpers.assertEmptyAvroField(projected, "id"); Assert.assertNotNull("Should project points list", projected.get("points")); points = (List) projected.get("points"); Assert.assertEquals("Should read 2 points", 2, points.size()); projectedP1 = points.get(0); - Assert.assertNull("Should not project x", projectedP1.get("x")); - Assert.assertNull("Should not project y", projectedP1.get("y")); + AssertHelpers.assertEmptyAvroField(projectedP1, "x"); + AssertHelpers.assertEmptyAvroField(projectedP1, "y"); Assert.assertEquals("Should project z", 2, (int) projectedP1.get("z")); projectedP2 = points.get(1); - Assert.assertNull("Should not project x", projectedP2.get("x")); - Assert.assertNull("Should not project y", projectedP2.get("y")); - Assert.assertEquals("Should project null z", null, projectedP2.get("z")); + AssertHelpers.assertEmptyAvroField(projectedP2, "x"); + AssertHelpers.assertEmptyAvroField(projectedP2, "y"); + Assert.assertNull("Should project null z", projectedP2.get("z")); } } diff --git a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetAvro.java b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetAvro.java index ed6a6ca6dfcc..faa2957645e7 100644 --- a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetAvro.java +++ b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetAvro.java @@ -20,8 +20,10 @@ package org.apache.iceberg.parquet; import java.math.BigDecimal; +import java.math.BigInteger; import java.util.List; import java.util.Map; +import java.util.WeakHashMap; import org.apache.avro.Conversion; import org.apache.avro.Conversions; import org.apache.avro.LogicalType; @@ -30,6 +32,7 @@ import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericFixed; import org.apache.avro.specific.SpecificData; +import org.apache.commons.math3.util.Pair; import org.apache.iceberg.avro.AvroSchemaVisitor; import org.apache.iceberg.avro.UUIDConversion; import org.apache.iceberg.relocated.com.google.common.base.Objects; @@ -49,8 +52,8 @@ static Schema parquetAvroSchema(Schema avroSchema) { static class ParquetDecimal extends LogicalType { private static final String NAME = "parquet-decimal"; - private int precision; - private int scale; + private final int precision; + private final int scale; ParquetDecimal(int precision, int scale) { super(NAME); @@ -155,12 +158,10 @@ public Long toLong(BigDecimal value, org.apache.avro.Schema schema, LogicalType } private static class FixedDecimalConversion extends Conversions.DecimalConversion { - private final LogicalType[] decimalsByScale = new LogicalType[39]; + private final WeakHashMap, LogicalType> decimalsByScale; private FixedDecimalConversion() { - for (int i = 0; i < decimalsByScale.length; i += 1) { - decimalsByScale[i] = LogicalTypes.decimal(i, i); - } + this.decimalsByScale = new WeakHashMap<>(); } @Override @@ -170,12 +171,16 @@ public String getLogicalTypeName() { @Override public BigDecimal fromFixed(GenericFixed value, Schema schema, LogicalType type) { - return super.fromFixed(value, schema, decimalsByScale[((ParquetDecimal) type).scale()]); + ParquetDecimal dec = (ParquetDecimal) type; + return new BigDecimal(new BigInteger(value.bytes()), dec.scale()); } @Override public GenericFixed toFixed(BigDecimal value, Schema schema, LogicalType type) { - return super.toFixed(value, schema, decimalsByScale[((ParquetDecimal) type).scale()]); + ParquetDecimal dec = (ParquetDecimal) type; + Pair key = new Pair<>(dec.precision(), dec.scale()); + return super.toFixed(value, schema, + decimalsByScale.computeIfAbsent(key, k -> LogicalTypes.decimal(k.getFirst(), k.getSecond()))); } } diff --git a/parquet/src/test/java/org/apache/iceberg/TestHelpers.java b/parquet/src/test/java/org/apache/iceberg/TestHelpers.java index 9b7b8decfef0..13fdb10d7c07 100644 --- a/parquet/src/test/java/org/apache/iceberg/TestHelpers.java +++ b/parquet/src/test/java/org/apache/iceberg/TestHelpers.java @@ -20,6 +20,8 @@ package org.apache.iceberg; import java.util.concurrent.Callable; +import org.apache.avro.AvroRuntimeException; +import org.apache.avro.generic.GenericRecord; import org.junit.Assert; public class TestHelpers { @@ -85,4 +87,17 @@ private static void handleException(String message, throw e; } } + + /** + * A convenience method to assert if an Avro field is empty + * @param record The record to read from + * @param field The name of the field + */ + public static void assertEmptyAvroField(GenericRecord record, String field) { + TestHelpers.assertThrows( + "Not a valid schema field: " + field, + AvroRuntimeException.class, + "Not a valid schema field: " + field, + () -> record.get(field)); + } } diff --git a/parquet/src/test/java/org/apache/iceberg/avro/TestReadProjection.java b/parquet/src/test/java/org/apache/iceberg/avro/TestReadProjection.java index 9243a809ff1e..685d4ee88f19 100644 --- a/parquet/src/test/java/org/apache/iceberg/avro/TestReadProjection.java +++ b/parquet/src/test/java/org/apache/iceberg/avro/TestReadProjection.java @@ -24,6 +24,7 @@ import java.util.Map; import org.apache.avro.generic.GenericData.Record; import org.apache.iceberg.Schema; +import org.apache.iceberg.TestHelpers; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.Maps; @@ -148,7 +149,7 @@ public void testBasicProjection() throws Exception { ); Record projected = writeAndRead("basic_projection_id", writeSchema, idOnly, record); - Assert.assertNull("Should not project data", projected.get("data")); + TestHelpers.assertEmptyAvroField(projected, "data"); Assert.assertEquals("Should contain the correct id value", 34L, (long) projected.get("id")); Schema dataOnly = new Schema( @@ -157,10 +158,10 @@ public void testBasicProjection() throws Exception { projected = writeAndRead("basic_projection_data", writeSchema, dataOnly, record); - Assert.assertNull("Should not project id", projected.get("id")); + TestHelpers.assertEmptyAvroField(projected, "id"); int cmp = Comparators.charSequences() .compare("test", (CharSequence) projected.get("data")); - Assert.assertTrue("Should contain the correct data value", cmp == 0); + Assert.assertEquals("Should contain the correct data value", 0, cmp); } @Test @@ -210,9 +211,8 @@ public void testNestedStructProjection() throws Exception { ); Record projected = writeAndRead("id_only", writeSchema, idOnly, record); - Record projectedLocation = (Record) projected.get("location"); + TestHelpers.assertEmptyAvroField(projected, "location"); Assert.assertEquals("Should contain the correct id value", 34L, (long) projected.get("id")); - Assert.assertNull("Should not project location", projectedLocation); Schema latOnly = new Schema( Types.NestedField.optional(3, "location", Types.StructType.of( @@ -221,10 +221,10 @@ public void testNestedStructProjection() throws Exception { ); projected = writeAndRead("latitude_only", writeSchema, latOnly, record); - projectedLocation = (Record) projected.get("location"); - Assert.assertNull("Should not project id", projected.get("id")); + Record projectedLocation = (Record) projected.get("location"); + TestHelpers.assertEmptyAvroField(projected, "id"); Assert.assertNotNull("Should project location", projected.get("location")); - Assert.assertNull("Should not project longitude", projectedLocation.get("long")); + TestHelpers.assertEmptyAvroField(projectedLocation, "long"); Assert.assertEquals("Should project latitude", 52.995143f, (float) projectedLocation.get("lat"), 0.000001f); @@ -236,16 +236,16 @@ public void testNestedStructProjection() throws Exception { projected = writeAndRead("longitude_only", writeSchema, longOnly, record); projectedLocation = (Record) projected.get("location"); - Assert.assertNull("Should not project id", projected.get("id")); + TestHelpers.assertEmptyAvroField(projected, "id"); Assert.assertNotNull("Should project location", projected.get("location")); - Assert.assertNull("Should not project latitutde", projectedLocation.get("lat")); + TestHelpers.assertEmptyAvroField(projectedLocation, "lat"); Assert.assertEquals("Should project longitude", -1.539054f, (float) projectedLocation.get("long"), 0.000001f); Schema locationOnly = writeSchema.select("location"); projected = writeAndRead("location_only", writeSchema, locationOnly, record); projectedLocation = (Record) projected.get("location"); - Assert.assertNull("Should not project id", projected.get("id")); + TestHelpers.assertEmptyAvroField(projected, "id"); Assert.assertNotNull("Should project location", projected.get("location")); Assert.assertEquals("Should project latitude", 52.995143f, (float) projectedLocation.get("lat"), 0.000001f); @@ -273,23 +273,23 @@ public void testMapProjection() throws IOException { Record projected = writeAndRead("id_only", writeSchema, idOnly, record); Assert.assertEquals("Should contain the correct id value", 34L, (long) projected.get("id")); - Assert.assertNull("Should not project properties map", projected.get("properties")); + TestHelpers.assertEmptyAvroField(projected, "properties"); Schema keyOnly = writeSchema.select("properties.key"); projected = writeAndRead("key_only", writeSchema, keyOnly, record); - Assert.assertNull("Should not project id", projected.get("id")); + TestHelpers.assertEmptyAvroField(projected, "id"); Assert.assertEquals("Should project entire map", properties, toStringMap((Map) projected.get("properties"))); Schema valueOnly = writeSchema.select("properties.value"); projected = writeAndRead("value_only", writeSchema, valueOnly, record); - Assert.assertNull("Should not project id", projected.get("id")); + TestHelpers.assertEmptyAvroField(projected, "id"); Assert.assertEquals("Should project entire map", properties, toStringMap((Map) projected.get("properties"))); Schema mapOnly = writeSchema.select("properties"); projected = writeAndRead("map_only", writeSchema, mapOnly, record); - Assert.assertNull("Should not project id", projected.get("id")); + TestHelpers.assertEmptyAvroField(projected, "id"); Assert.assertEquals("Should project entire map", properties, toStringMap((Map) projected.get("properties"))); } @@ -337,16 +337,16 @@ public void testMapOfStructsProjection() throws IOException { Record projected = writeAndRead("id_only", writeSchema, idOnly, record); Assert.assertEquals("Should contain the correct id value", 34L, (long) projected.get("id")); - Assert.assertNull("Should not project locations map", projected.get("locations")); + TestHelpers.assertEmptyAvroField(projected, "locations"); projected = writeAndRead("all_locations", writeSchema, writeSchema.select("locations"), record); - Assert.assertNull("Should not project id", projected.get("id")); + TestHelpers.assertEmptyAvroField(projected, "id"); Assert.assertEquals("Should project locations map", record.get("locations"), toStringMap((Map) projected.get("locations"))); projected = writeAndRead("lat_only", writeSchema, writeSchema.select("locations.lat"), record); - Assert.assertNull("Should not project id", projected.get("id")); + TestHelpers.assertEmptyAvroField(projected, "id"); Map locations = toStringMap((Map) projected.get("locations")); Assert.assertNotNull("Should project locations map", locations); Assert.assertEquals("Should contain L1 and L2", @@ -355,28 +355,28 @@ public void testMapOfStructsProjection() throws IOException { Assert.assertNotNull("L1 should not be null", projectedL1); Assert.assertEquals("L1 should contain lat", 53.992811f, (float) projectedL1.get("lat"), 0.000001); - Assert.assertNull("L1 should not contain long", projectedL1.get("long")); + TestHelpers.assertEmptyAvroField(projectedL1, "long"); Record projectedL2 = (Record) locations.get("L2"); Assert.assertNotNull("L2 should not be null", projectedL2); Assert.assertEquals("L2 should contain lat", 52.995143f, (float) projectedL2.get("lat"), 0.000001); - Assert.assertNull("L2 should not contain long", projectedL2.get("long")); + TestHelpers.assertEmptyAvroField(projectedL2, "long"); projected = writeAndRead("long_only", writeSchema, writeSchema.select("locations.long"), record); - Assert.assertNull("Should not project id", projected.get("id")); + TestHelpers.assertEmptyAvroField(projected, "id"); locations = toStringMap((Map) projected.get("locations")); Assert.assertNotNull("Should project locations map", locations); Assert.assertEquals("Should contain L1 and L2", Sets.newHashSet("L1", "L2"), locations.keySet()); projectedL1 = (Record) locations.get("L1"); Assert.assertNotNull("L1 should not be null", projectedL1); - Assert.assertNull("L1 should not contain lat", projectedL1.get("lat")); + TestHelpers.assertEmptyAvroField(projectedL1, "lat"); Assert.assertEquals("L1 should contain long", -1.542616f, (float) projectedL1.get("long"), 0.000001); projectedL2 = (Record) locations.get("L2"); Assert.assertNotNull("L2 should not be null", projectedL2); - Assert.assertNull("L2 should not contain lat", projectedL2.get("lat")); + TestHelpers.assertEmptyAvroField(projectedL2, "lat"); Assert.assertEquals("L2 should contain long", -1.539054f, (float) projectedL2.get("long"), 0.000001); @@ -390,7 +390,7 @@ public void testMapOfStructsProjection() throws IOException { ); projected = writeAndRead("latitude_renamed", writeSchema, latitiudeRenamed, record); - Assert.assertNull("Should not project id", projected.get("id")); + TestHelpers.assertEmptyAvroField(projected, "id"); locations = toStringMap((Map) projected.get("locations")); Assert.assertNotNull("Should project locations map", locations); Assert.assertEquals("Should contain L1 and L2", @@ -399,14 +399,14 @@ public void testMapOfStructsProjection() throws IOException { Assert.assertNotNull("L1 should not be null", projectedL1); Assert.assertEquals("L1 should contain latitude", 53.992811f, (float) projectedL1.get("latitude"), 0.000001); - Assert.assertNull("L1 should not contain lat", projectedL1.get("lat")); - Assert.assertNull("L1 should not contain long", projectedL1.get("long")); + TestHelpers.assertEmptyAvroField(projectedL1, "lat"); + TestHelpers.assertEmptyAvroField(projectedL1, "long"); projectedL2 = (Record) locations.get("L2"); Assert.assertNotNull("L2 should not be null", projectedL2); Assert.assertEquals("L2 should contain latitude", 52.995143f, (float) projectedL2.get("latitude"), 0.000001); - Assert.assertNull("L2 should not contain lat", projectedL2.get("lat")); - Assert.assertNull("L2 should not contain long", projectedL2.get("long")); + TestHelpers.assertEmptyAvroField(projectedL2, "lat"); + TestHelpers.assertEmptyAvroField(projectedL2, "long"); } @Test @@ -429,16 +429,16 @@ public void testListProjection() throws IOException { Record projected = writeAndRead("id_only", writeSchema, idOnly, record); Assert.assertEquals("Should contain the correct id value", 34L, (long) projected.get("id")); - Assert.assertNull("Should not project values list", projected.get("values")); + TestHelpers.assertEmptyAvroField(projected, "values"); Schema elementOnly = writeSchema.select("values.element"); projected = writeAndRead("element_only", writeSchema, elementOnly, record); - Assert.assertNull("Should not project id", projected.get("id")); + TestHelpers.assertEmptyAvroField(projected, "id"); Assert.assertEquals("Should project entire list", values, projected.get("values")); Schema listOnly = writeSchema.select("values"); projected = writeAndRead("list_only", writeSchema, listOnly, record); - Assert.assertNull("Should not project id", projected.get("id")); + TestHelpers.assertEmptyAvroField(projected, "id"); Assert.assertEquals("Should project entire list", values, projected.get("values")); } @@ -473,36 +473,36 @@ public void testListOfStructsProjection() throws IOException { Record projected = writeAndRead("id_only", writeSchema, idOnly, record); Assert.assertEquals("Should contain the correct id value", 34L, (long) projected.get("id")); - Assert.assertNull("Should not project points list", projected.get("points")); + TestHelpers.assertEmptyAvroField(projected, "points"); projected = writeAndRead("all_points", writeSchema, writeSchema.select("points"), record); - Assert.assertNull("Should not project id", projected.get("id")); + TestHelpers.assertEmptyAvroField(projected, "id"); Assert.assertEquals("Should project points list", record.get("points"), projected.get("points")); projected = writeAndRead("x_only", writeSchema, writeSchema.select("points.x"), record); - Assert.assertNull("Should not project id", projected.get("id")); + TestHelpers.assertEmptyAvroField(projected, "id"); Assert.assertNotNull("Should project points list", projected.get("points")); List points = (List) projected.get("points"); Assert.assertEquals("Should read 2 points", 2, points.size()); Record projectedP1 = points.get(0); Assert.assertEquals("Should project x", 1, (int) projectedP1.get("x")); - Assert.assertNull("Should not project y", projectedP1.get("y")); + TestHelpers.assertEmptyAvroField(projectedP1, "y"); Record projectedP2 = points.get(1); Assert.assertEquals("Should project x", 3, (int) projectedP2.get("x")); - Assert.assertNull("Should not project y", projectedP2.get("y")); + TestHelpers.assertEmptyAvroField(projectedP2, "y"); projected = writeAndRead("y_only", writeSchema, writeSchema.select("points.y"), record); - Assert.assertNull("Should not project id", projected.get("id")); + TestHelpers.assertEmptyAvroField(projected, "id"); Assert.assertNotNull("Should project points list", projected.get("points")); points = (List) projected.get("points"); Assert.assertEquals("Should read 2 points", 2, points.size()); projectedP1 = points.get(0); - Assert.assertNull("Should not project x", projectedP1.get("x")); + TestHelpers.assertEmptyAvroField(projectedP1, "x"); Assert.assertEquals("Should project y", 2, (int) projectedP1.get("y")); projectedP2 = points.get(1); - Assert.assertNull("Should not project x", projectedP2.get("x")); - Assert.assertEquals("Should project null y", null, projectedP2.get("y")); + TestHelpers.assertEmptyAvroField(projectedP2, "x"); + Assert.assertNull("Should project null y", projectedP2.get("y")); Schema yRenamed = new Schema( Types.NestedField.optional(22, "points", @@ -513,17 +513,17 @@ public void testListOfStructsProjection() throws IOException { ); projected = writeAndRead("y_renamed", writeSchema, yRenamed, record); - Assert.assertNull("Should not project id", projected.get("id")); + TestHelpers.assertEmptyAvroField(projected, "id"); Assert.assertNotNull("Should project points list", projected.get("points")); points = (List) projected.get("points"); Assert.assertEquals("Should read 2 points", 2, points.size()); projectedP1 = points.get(0); - Assert.assertNull("Should not project x", projectedP1.get("x")); - Assert.assertNull("Should not project y", projectedP1.get("y")); + TestHelpers.assertEmptyAvroField(projectedP1, "x"); + TestHelpers.assertEmptyAvroField(projectedP1, "y"); Assert.assertEquals("Should project z", 2, (int) projectedP1.get("z")); projectedP2 = points.get(1); - Assert.assertNull("Should not project x", projectedP2.get("x")); - Assert.assertNull("Should not project y", projectedP2.get("y")); - Assert.assertEquals("Should project null z", null, projectedP2.get("z")); + TestHelpers.assertEmptyAvroField(projectedP2, "x"); + TestHelpers.assertEmptyAvroField(projectedP2, "y"); + Assert.assertNull("Should project null z", projectedP2.get("z")); } } diff --git a/versions.props b/versions.props index d2bafd8489dc..e889b0f5d5c8 100644 --- a/versions.props +++ b/versions.props @@ -1,5 +1,5 @@ org.slf4j:* = 1.7.25 -org.apache.avro:avro = 1.9.2 +org.apache.avro:avro = 1.10.1 org.apache.calcite:* = 1.10.0 org.apache.flink:* = 1.12.1 org.apache.hadoop:* = 2.7.3