ARROW-16204: [C++][Dataset] Default error existing_data_behaviour for…

… writing dataset ignores a single file Closes apache#12898 from jorisvandenbossche/ARROW-16204 Authored-by: Joris Van den Bossche <[email protected]> Signed-off-by: Joris Van den Bossche <[email protected]>
davidhao3300 · Apr 22, 2022 · 912e2bb · 912e2bb
1 parent 16638a4
commit 912e2bb
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 3 deletions.
diff --git a/cpp/src/arrow/dataset/dataset_writer.cc b/cpp/src/arrow/dataset/dataset_writer.cc
@@ -423,7 +423,7 @@ Status EnsureDestinationValid(const FileSystemDatasetWriteOptions& options) {
       // If the path doesn't exist then continue
       return Status::OK();
     }
-    if (maybe_files->size() > 1) {
+    if (maybe_files->size() > 0) {
       return Status::Invalid(
           "Could not write to ", options.base_dir,
           " as the directory is not empty and existing_data_behavior is to error");

diff --git a/cpp/src/arrow/dataset/dataset_writer_test.cc b/cpp/src/arrow/dataset/dataset_writer_test.cc
@@ -453,6 +453,18 @@ TEST_F(DatasetWriterTestFixture, ErrOnExistingData) {
   ASSERT_RAISES(Invalid, DatasetWriter::Make(write_options_));
   AssertEmptyFiles(
       {"testdir/chunk-0.arrow", "testdir/chunk-5.arrow", "testdir/blah.txt"});
+
+  // only a single file in the target directory
+  fs::TimePoint mock_now2 = std::chrono::system_clock::now();
+  ASSERT_OK_AND_ASSIGN(
+      std::shared_ptr<fs::FileSystem> fs2,
+      MockFileSystem::Make(
+          mock_now2, {::arrow::fs::Dir("testdir"), fs::File("testdir/part-0.arrow")}));
+  filesystem_ = std::dynamic_pointer_cast<MockFileSystem>(fs2);
+  write_options_.filesystem = filesystem_;
+  write_options_.base_dir = "testdir";
+  ASSERT_RAISES(Invalid, DatasetWriter::Make(write_options_));
+  AssertEmptyFiles({"testdir/part-0.arrow"});
 }
 
 }  // namespace internal

diff --git a/docs/source/python/dataset.rst b/docs/source/python/dataset.rst
@@ -570,7 +570,7 @@ dataset to be partitioned.  For example:
     part = ds.partitioning(
         pa.schema([("c", pa.int16())]), flavor="hive"
     )
-    ds.write_dataset(table, "sample_dataset", format="parquet", partitioning=part)
+    ds.write_dataset(table, "partitioned_dataset", format="parquet", partitioning=part)
 
 This will create two files.  Half our data will be in the dataset_root/c=1 directory and
 the other half will be in the dataset_root/c=2 directory.
@@ -701,7 +701,7 @@ into memory:
     new_part = ds.partitioning(
         pa.schema([("c", pa.int16())]), flavor=None
     )
-    input_dataset = ds.dataset("sample_dataset", partitioning=old_part)
+    input_dataset = ds.dataset("partitioned_dataset", partitioning=old_part)
     # A scanner can act as an iterator of record batches but you could also receive
     # data from the network (e.g. via flight), from your own scanning, or from any
     # other method that yields record batches.  In addition, you can pass a dataset