Core: Add sort_order_id to SCAN_COLUMNS to address null sort order ID…

… after planned data files (apache#8873)
regadas · Oct 20, 2023 · 0cc74f1 · 0cc74f1
1 parent 7a60b00
commit 0cc74f1
Show file tree

Hide file tree

Showing 2 changed files with 32 additions and 1 deletion.
diff --git a/core/src/main/java/org/apache/iceberg/BaseScan.java b/core/src/main/java/org/apache/iceberg/BaseScan.java
@@ -48,7 +48,8 @@ abstract class BaseScan<ThisT, T extends ScanTask, G extends ScanTaskGroup<T>>
           "record_count",
           "partition",
           "key_metadata",
-          "split_offsets");
+          "split_offsets",
+          "sort_order_id");
 
   private static final List<String> STATS_COLUMNS =
       ImmutableList.of(

diff --git a/core/src/test/java/org/apache/iceberg/ScanTestBase.java b/core/src/test/java/org/apache/iceberg/ScanTestBase.java
@@ -224,4 +224,34 @@ public void testReAddingPartitionField() throws Exception {
       }
     }
   }
+
+  @Test
+  public void testDataFileSorted() throws Exception {
+    Schema schema =
+        new Schema(
+            required(1, "a", Types.IntegerType.get()), required(2, "b", Types.StringType.get()));
+    File dir = temp.newFolder();
+    dir.delete();
+    this.table =
+        TestTables.create(
+            dir, "test_data_file_sorted", schema, PartitionSpec.unpartitioned(), formatVersion);
+    table
+        .newFastAppend()
+        .appendFile(
+            DataFiles.builder(PartitionSpec.unpartitioned())
+                .withPath("/path/to/data/a.parquet")
+                .withFileSizeInBytes(10)
+                .withRecordCount(1)
+                .withSortOrder(
+                    SortOrder.builderFor(table.schema()).asc("a", NullOrder.NULLS_FIRST).build())
+                .build())
+        .commit();
+
+    TableScan scan = table.newScan();
+    try (CloseableIterable<FileScanTask> tasks = scan.planFiles()) {
+      for (FileScanTask fileScanTask : tasks) {
+        Assertions.assertThat(fileScanTask.file().sortOrderId()).isEqualTo(1);
+      }
+    }
+  }
 }