From 71e8ba719f12741949fa83d53ad6fce0ec491cf1 Mon Sep 17 00:00:00 2001 From: Robin Linacre Date: Mon, 25 Nov 2024 17:10:09 +0000 Subject: [PATCH] Use parquet rather than checkpoint to break lineage The reason for this is that checkpointing can cause the schema to be lost if there are zero rows. Whereas parquet preserves the schema --- tests/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/helpers.py b/tests/helpers.py index bd18eecf9f..487d33f86f 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -74,7 +74,7 @@ def db_api_args(self): return { "spark_session": self.spark, "num_partitions_on_repartition": 1, - "break_lineage_method": "checkpoint", + "break_lineage_method": "parquet", } def convert_frame(self, df):