Skip to content

Commit

Permalink
Core: Use Avro 1.10.1 (apache#1648)
Browse files Browse the repository at this point in the history 
Co-authored-by: Fokko Driesprong <[email protected]>
  • Loading branch information
@Fokko
Fokko and Fokko Driesprong authored Jul 13, 2021
1 parent 1a903f6 commit b3fb81a
Show file tree
Hide file tree
Showing 7 changed files with 141 additions and 107 deletions.
14 changes: 14 additions & 0 deletions api/src/test/java/org/apache/iceberg/AssertHelpers.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
package org.apache.iceberg;

import java.util.concurrent.Callable;
import org.apache.avro.AvroRuntimeException;
import org.apache.avro.generic.GenericRecord;
import org.assertj.core.api.AbstractThrowableAssert;
import org.assertj.core.api.Assertions;

Expand Down Expand Up @@ -110,4 +112,16 @@ public static void assertThrowsCause(String message,
.isInstanceOf(expected)
.hasMessageContaining(containedInMessage);
}

/**
* A convenience method to check if an Avro field is empty.
* @param record The record to read from
* @param field The name of the field
*/
public static void assertEmptyAvroField(GenericRecord record, String field) {
AssertHelpers.assertThrows(
"Not a valid schema field: " + field,
AvroRuntimeException.class,
() -> record.get(field));
}
}
12 changes: 6 additions & 6 deletions core/src/test/java/org/apache/iceberg/TestManifestListVersions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,9 +136,9 @@ public void testV1ForwardCompatibility() throws IOException {
Assert.assertEquals("Added rows count", ADDED_ROWS, (long) generic.get("added_rows_count"));
Assert.assertEquals("Existing rows count", EXISTING_ROWS, (long) generic.get("existing_rows_count"));
Assert.assertEquals("Deleted rows count", DELETED_ROWS, (long) generic.get("deleted_rows_count"));
Assert.assertNull("Content", generic.get(ManifestFile.MANIFEST_CONTENT.name()));
Assert.assertNull("Sequence number", generic.get(ManifestFile.SEQUENCE_NUMBER.name()));
Assert.assertNull("Min sequence number", generic.get(ManifestFile.MIN_SEQUENCE_NUMBER.name()));
AssertHelpers.assertEmptyAvroField(generic, ManifestFile.MANIFEST_CONTENT.name());
AssertHelpers.assertEmptyAvroField(generic, ManifestFile.SEQUENCE_NUMBER.name());
AssertHelpers.assertEmptyAvroField(generic, ManifestFile.MIN_SEQUENCE_NUMBER.name());
}

@Test
Expand All @@ -158,9 +158,9 @@ public void testV2ForwardCompatibility() throws IOException {
Assert.assertEquals("Added rows count", ADDED_ROWS, (long) generic.get("added_rows_count"));
Assert.assertEquals("Existing rows count", EXISTING_ROWS, (long) generic.get("existing_rows_count"));
Assert.assertEquals("Deleted rows count", DELETED_ROWS, (long) generic.get("deleted_rows_count"));
Assert.assertNull("Content", generic.get(ManifestFile.MANIFEST_CONTENT.name()));
Assert.assertNull("Sequence number", generic.get(ManifestFile.SEQUENCE_NUMBER.name()));
Assert.assertNull("Min sequence number", generic.get(ManifestFile.MIN_SEQUENCE_NUMBER.name()));
AssertHelpers.assertEmptyAvroField(generic, ManifestFile.MANIFEST_CONTENT.name());
AssertHelpers.assertEmptyAvroField(generic, ManifestFile.SEQUENCE_NUMBER.name());
AssertHelpers.assertEmptyAvroField(generic, ManifestFile.MIN_SEQUENCE_NUMBER.name());
}

@Test
Expand Down
92 changes: 46 additions & 46 deletions core/src/test/java/org/apache/iceberg/avro/TestReadProjection.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import java.util.List;
import java.util.Map;
import org.apache.avro.generic.GenericData.Record;
import org.apache.iceberg.AssertHelpers;
import org.apache.iceberg.Schema;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
Expand Down Expand Up @@ -148,7 +149,7 @@ public void testBasicProjection() throws Exception {
);

Record projected = writeAndRead("basic_projection_id", writeSchema, idOnly, record);
Assert.assertNull("Should not project data", projected.get("data"));
AssertHelpers.assertEmptyAvroField(projected, "data");
Assert.assertEquals("Should contain the correct id value", 34L, (long) projected.get("id"));

Schema dataOnly = new Schema(
Expand All @@ -157,10 +158,10 @@ public void testBasicProjection() throws Exception {

projected = writeAndRead("basic_projection_data", writeSchema, dataOnly, record);

Assert.assertNull("Should not project id", projected.get("id"));
AssertHelpers.assertEmptyAvroField(projected, "id");
int cmp = Comparators.charSequences()
.compare("test", (CharSequence) projected.get("data"));
Assert.assertTrue("Should contain the correct data value", cmp == 0);
Assert.assertEquals("Should contain the correct data value", 0, cmp);
}

@Test
Expand All @@ -184,7 +185,7 @@ public void testRename() throws Exception {
Assert.assertEquals("Should contain the correct id value", 34L, (long) projected.get("id"));
int cmp = Comparators.charSequences()
.compare("test", (CharSequence) projected.get("renamed"));
Assert.assertTrue("Should contain the correct data/renamed value", cmp == 0);
Assert.assertEquals("Should contain the correct data/renamed value", 0, cmp);
}

@Test
Expand All @@ -210,9 +211,8 @@ public void testNestedStructProjection() throws Exception {
);

Record projected = writeAndRead("id_only", writeSchema, idOnly, record);
Record projectedLocation = (Record) projected.get("location");
AssertHelpers.assertEmptyAvroField(projected, "location");
Assert.assertEquals("Should contain the correct id value", 34L, (long) projected.get("id"));
Assert.assertNull("Should not project location", projectedLocation);

Schema latOnly = new Schema(
Types.NestedField.optional(3, "location", Types.StructType.of(
Expand All @@ -221,10 +221,10 @@ public void testNestedStructProjection() throws Exception {
);

projected = writeAndRead("latitude_only", writeSchema, latOnly, record);
projectedLocation = (Record) projected.get("location");
Assert.assertNull("Should not project id", projected.get("id"));
Record projectedLocation = (Record) projected.get("location");
AssertHelpers.assertEmptyAvroField(projected, "id");
Assert.assertNotNull("Should project location", projected.get("location"));
Assert.assertNull("Should not project longitude", projectedLocation.get("long"));
AssertHelpers.assertEmptyAvroField(projectedLocation, "long");
Assert.assertEquals("Should project latitude",
52.995143f, (float) projectedLocation.get("lat"), 0.000001f);

Expand All @@ -236,16 +236,16 @@ public void testNestedStructProjection() throws Exception {

projected = writeAndRead("longitude_only", writeSchema, longOnly, record);
projectedLocation = (Record) projected.get("location");
Assert.assertNull("Should not project id", projected.get("id"));
AssertHelpers.assertEmptyAvroField(projected, "id");
Assert.assertNotNull("Should project location", projected.get("location"));
Assert.assertNull("Should not project latitutde", projectedLocation.get("lat"));
AssertHelpers.assertEmptyAvroField(projectedLocation, "lat");
Assert.assertEquals("Should project longitude",
-1.539054f, (float) projectedLocation.get("long"), 0.000001f);

Schema locationOnly = writeSchema.select("location");
projected = writeAndRead("location_only", writeSchema, locationOnly, record);
projectedLocation = (Record) projected.get("location");
Assert.assertNull("Should not project id", projected.get("id"));
AssertHelpers.assertEmptyAvroField(projected, "id");
Assert.assertNotNull("Should project location", projected.get("location"));
Assert.assertEquals("Should project latitude",
52.995143f, (float) projectedLocation.get("lat"), 0.000001f);
Expand Down Expand Up @@ -273,23 +273,23 @@ public void testMapProjection() throws IOException {

Record projected = writeAndRead("id_only", writeSchema, idOnly, record);
Assert.assertEquals("Should contain the correct id value", 34L, (long) projected.get("id"));
Assert.assertNull("Should not project properties map", projected.get("properties"));
AssertHelpers.assertEmptyAvroField(projected, "properties");

Schema keyOnly = writeSchema.select("properties.key");
projected = writeAndRead("key_only", writeSchema, keyOnly, record);
Assert.assertNull("Should not project id", projected.get("id"));
AssertHelpers.assertEmptyAvroField(projected, "id");
Assert.assertEquals("Should project entire map",
properties, toStringMap((Map) projected.get("properties")));

Schema valueOnly = writeSchema.select("properties.value");
projected = writeAndRead("value_only", writeSchema, valueOnly, record);
Assert.assertNull("Should not project id", projected.get("id"));
AssertHelpers.assertEmptyAvroField(projected, "id");
Assert.assertEquals("Should project entire map",
properties, toStringMap((Map) projected.get("properties")));

Schema mapOnly = writeSchema.select("properties");
projected = writeAndRead("map_only", writeSchema, mapOnly, record);
Assert.assertNull("Should not project id", projected.get("id"));
AssertHelpers.assertEmptyAvroField(projected, "id");
Assert.assertEquals("Should project entire map",
properties, toStringMap((Map) projected.get("properties")));
}
Expand Down Expand Up @@ -337,16 +337,16 @@ public void testMapOfStructsProjection() throws IOException {

Record projected = writeAndRead("id_only", writeSchema, idOnly, record);
Assert.assertEquals("Should contain the correct id value", 34L, (long) projected.get("id"));
Assert.assertNull("Should not project locations map", projected.get("locations"));
AssertHelpers.assertEmptyAvroField(projected, "locations");

projected = writeAndRead("all_locations", writeSchema, writeSchema.select("locations"), record);
Assert.assertNull("Should not project id", projected.get("id"));
AssertHelpers.assertEmptyAvroField(projected, "id");
Assert.assertEquals("Should project locations map",
record.get("locations"), toStringMap((Map) projected.get("locations")));

projected = writeAndRead("lat_only",
writeSchema, writeSchema.select("locations.lat"), record);
Assert.assertNull("Should not project id", projected.get("id"));
AssertHelpers.assertEmptyAvroField(projected, "id");
Map<String, ?> locations = toStringMap((Map) projected.get("locations"));
Assert.assertNotNull("Should project locations map", locations);
Assert.assertEquals("Should contain L1 and L2",
Expand All @@ -355,28 +355,28 @@ public void testMapOfStructsProjection() throws IOException {
Assert.assertNotNull("L1 should not be null", projectedL1);
Assert.assertEquals("L1 should contain lat",
53.992811f, (float) projectedL1.get("lat"), 0.000001);
Assert.assertNull("L1 should not contain long", projectedL1.get("long"));
AssertHelpers.assertEmptyAvroField(projectedL1, "long");
Record projectedL2 = (Record) locations.get("L2");
Assert.assertNotNull("L2 should not be null", projectedL2);
Assert.assertEquals("L2 should contain lat",
52.995143f, (float) projectedL2.get("lat"), 0.000001);
Assert.assertNull("L2 should not contain long", projectedL2.get("long"));
AssertHelpers.assertEmptyAvroField(projectedL2, "y");

projected = writeAndRead("long_only",
writeSchema, writeSchema.select("locations.long"), record);
Assert.assertNull("Should not project id", projected.get("id"));
AssertHelpers.assertEmptyAvroField(projected, "id");
locations = toStringMap((Map) projected.get("locations"));
Assert.assertNotNull("Should project locations map", locations);
Assert.assertEquals("Should contain L1 and L2",
Sets.newHashSet("L1", "L2"), locations.keySet());
projectedL1 = (Record) locations.get("L1");
Assert.assertNotNull("L1 should not be null", projectedL1);
Assert.assertNull("L1 should not contain lat", projectedL1.get("lat"));
AssertHelpers.assertEmptyAvroField(projectedL1, "lat");
Assert.assertEquals("L1 should contain long",
-1.542616f, (float) projectedL1.get("long"), 0.000001);
projectedL2 = (Record) locations.get("L2");
Assert.assertNotNull("L2 should not be null", projectedL2);
Assert.assertNull("L2 should not contain lat", projectedL2.get("lat"));
AssertHelpers.assertEmptyAvroField(projectedL2, "lat");
Assert.assertEquals("L2 should contain long",
-1.539054f, (float) projectedL2.get("long"), 0.000001);

Expand All @@ -390,7 +390,7 @@ public void testMapOfStructsProjection() throws IOException {
);

projected = writeAndRead("latitude_renamed", writeSchema, latitiudeRenamed, record);
Assert.assertNull("Should not project id", projected.get("id"));
AssertHelpers.assertEmptyAvroField(projected, "id");
locations = toStringMap((Map) projected.get("locations"));
Assert.assertNotNull("Should project locations map", locations);
Assert.assertEquals("Should contain L1 and L2",
Expand All @@ -399,14 +399,14 @@ public void testMapOfStructsProjection() throws IOException {
Assert.assertNotNull("L1 should not be null", projectedL1);
Assert.assertEquals("L1 should contain latitude",
53.992811f, (float) projectedL1.get("latitude"), 0.000001);
Assert.assertNull("L1 should not contain lat", projectedL1.get("lat"));
Assert.assertNull("L1 should not contain long", projectedL1.get("long"));
AssertHelpers.assertEmptyAvroField(projectedL1, "lat");
AssertHelpers.assertEmptyAvroField(projectedL1, "long");
projectedL2 = (Record) locations.get("L2");
Assert.assertNotNull("L2 should not be null", projectedL2);
Assert.assertEquals("L2 should contain latitude",
52.995143f, (float) projectedL2.get("latitude"), 0.000001);
Assert.assertNull("L2 should not contain lat", projectedL2.get("lat"));
Assert.assertNull("L2 should not contain long", projectedL2.get("long"));
AssertHelpers.assertEmptyAvroField(projectedL2, "lat");
AssertHelpers.assertEmptyAvroField(projectedL2, "long");
}

@Test
Expand All @@ -429,16 +429,16 @@ public void testListProjection() throws IOException {

Record projected = writeAndRead("id_only", writeSchema, idOnly, record);
Assert.assertEquals("Should contain the correct id value", 34L, (long) projected.get("id"));
Assert.assertNull("Should not project values list", projected.get("values"));
AssertHelpers.assertEmptyAvroField(projected, "values");

Schema elementOnly = writeSchema.select("values.element");
projected = writeAndRead("element_only", writeSchema, elementOnly, record);
Assert.assertNull("Should not project id", projected.get("id"));
AssertHelpers.assertEmptyAvroField(projected, "id");
Assert.assertEquals("Should project entire list", values, projected.get("values"));

Schema listOnly = writeSchema.select("values");
projected = writeAndRead("list_only", writeSchema, listOnly, record);
Assert.assertNull("Should not project id", projected.get("id"));
AssertHelpers.assertEmptyAvroField(projected, "id");
Assert.assertEquals("Should project entire list", values, projected.get("values"));
}

Expand Down Expand Up @@ -473,35 +473,35 @@ public void testListOfStructsProjection() throws IOException {

Record projected = writeAndRead("id_only", writeSchema, idOnly, record);
Assert.assertEquals("Should contain the correct id value", 34L, (long) projected.get("id"));
Assert.assertNull("Should not project points list", projected.get("points"));
AssertHelpers.assertEmptyAvroField(projected, "points");

projected = writeAndRead("all_points", writeSchema, writeSchema.select("points"), record);
Assert.assertNull("Should not project id", projected.get("id"));
AssertHelpers.assertEmptyAvroField(projected, "id");
Assert.assertEquals("Should project points list",
record.get("points"), projected.get("points"));

projected = writeAndRead("x_only", writeSchema, writeSchema.select("points.x"), record);
Assert.assertNull("Should not project id", projected.get("id"));
AssertHelpers.assertEmptyAvroField(projected, "id");
Assert.assertNotNull("Should project points list", projected.get("points"));
List<Record> points = (List<Record>) projected.get("points");
Assert.assertEquals("Should read 2 points", 2, points.size());
Record projectedP1 = points.get(0);
Assert.assertEquals("Should project x", 1, (int) projectedP1.get("x"));
Assert.assertNull("Should not project y", projectedP1.get("y"));
AssertHelpers.assertEmptyAvroField(projectedP1, "y");
Record projectedP2 = points.get(1);
Assert.assertEquals("Should project x", 3, (int) projectedP2.get("x"));
Assert.assertNull("Should not project y", projectedP2.get("y"));
AssertHelpers.assertEmptyAvroField(projectedP2, "y");

projected = writeAndRead("y_only", writeSchema, writeSchema.select("points.y"), record);
Assert.assertNull("Should not project id", projected.get("id"));
AssertHelpers.assertEmptyAvroField(projected, "id");
Assert.assertNotNull("Should project points list", projected.get("points"));
points = (List<Record>) projected.get("points");
Assert.assertEquals("Should read 2 points", 2, points.size());
projectedP1 = points.get(0);
Assert.assertNull("Should not project x", projectedP1.get("x"));
AssertHelpers.assertEmptyAvroField(projectedP1, "x");
Assert.assertEquals("Should project y", 2, (int) projectedP1.get("y"));
projectedP2 = points.get(1);
Assert.assertNull("Should not project x", projectedP2.get("x"));
AssertHelpers.assertEmptyAvroField(projectedP2, "x");
Assert.assertEquals("Should project null y", null, projectedP2.get("y"));

Schema yRenamed = new Schema(
Expand All @@ -513,17 +513,17 @@ public void testListOfStructsProjection() throws IOException {
);

projected = writeAndRead("y_renamed", writeSchema, yRenamed, record);
Assert.assertNull("Should not project id", projected.get("id"));
AssertHelpers.assertEmptyAvroField(projected, "id");
Assert.assertNotNull("Should project points list", projected.get("points"));
points = (List<Record>) projected.get("points");
Assert.assertEquals("Should read 2 points", 2, points.size());
projectedP1 = points.get(0);
Assert.assertNull("Should not project x", projectedP1.get("x"));
Assert.assertNull("Should not project y", projectedP1.get("y"));
AssertHelpers.assertEmptyAvroField(projectedP1, "x");
AssertHelpers.assertEmptyAvroField(projectedP1, "y");
Assert.assertEquals("Should project z", 2, (int) projectedP1.get("z"));
projectedP2 = points.get(1);
Assert.assertNull("Should not project x", projectedP2.get("x"));
Assert.assertNull("Should not project y", projectedP2.get("y"));
Assert.assertEquals("Should project null z", null, projectedP2.get("z"));
AssertHelpers.assertEmptyAvroField(projectedP2, "x");
AssertHelpers.assertEmptyAvroField(projectedP2, "y");
Assert.assertNull("Should project null z", projectedP2.get("z"));
}
}
Loading

0 comments on commit b3fb81a

Please sign in to comment.