Fix append without duplicates (#83)

* fix: append without duplicates * chore: update readme
MrPowers · Feb 15, 2023 · e5b3f18 · e5b3f18
1 parent 0f8771c
commit e5b3f18
Show file tree

Hide file tree

Showing 3 changed files with 5 additions and 1 deletion.
diff --git a/README.md b/README.md
@@ -311,6 +311,7 @@ Here is data to be appended:
 +----+----+----+
 |   2|   R|   T| # duplicate col1
 |   8|   A|   B|
+|   8|   C|   D| # duplicate col1
 |  10|   X|   Y|
 +----+----+----+
 ```

diff --git a/mack/__init__.py b/mack/__init__.py
@@ -405,9 +405,11 @@ def append_without_duplicates(
 
     condition_columns = " AND ".join(condition_columns)
 
+    deduplicated_append_df = append_df.drop_duplicates(p_keys)
+
     # Insert records without duplicates
     delta_table.alias("old").merge(
-        append_df.alias("new"), condition_columns
+        deduplicated_append_df.alias("new"), condition_columns
     ).whenNotMatchedInsertAll().execute()
 
 

diff --git a/tests/test_public_interface.py b/tests/test_public_interface.py
@@ -468,6 +468,7 @@ def test_append_without_duplicates_single_column(tmp_path):
         [
             (2, "R", "T"),  # duplicate
             (8, "A", "B"),
+            (8, "B", "C"),  # duplicate
             (10, "X", "Y"),
         ],
         ["col1", "col2", "col3"],