fix fs rm bug at the end

playgroundai · Sep 14, 2023 · 728d6ee · 728d6ee
1 parent 268328b
commit 728d6ee
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 2 deletions.
diff --git a/examples/test_coco_mds/download_mds.sh b/examples/test_coco_mds/download_mds.sh
@@ -9,7 +9,7 @@ img2dataset \
     --url_col "URL" \
     --caption_col "TEXT" \
     --output_format mosaicstreaming \
-    --output_folder gs://pai-datasets-private/test-img2dataset/coco/ \
+    --output_folder gs://pai-datasets-private/test-img2dataset/coco \
     --temp_download_folder ./tmp_coco_mds \
     --processes_count 4 \
     --thread_count 8 \

diff --git a/img2dataset/main.py b/img2dataset/main.py
@@ -133,7 +133,11 @@ def make_path_absolute(path):
 
     if temp_download_folder is None:
         temp_download_folder = output_folder + "/_tmp"
+
+    temp_download_folder = make_path_absolute(temp_download_folder)
+
     fs, tmp_dir = fsspec.core.url_to_fs(temp_download_folder)
+
     if not fs.exists(tmp_dir):
         fs.mkdir(tmp_dir)
 
@@ -268,7 +272,7 @@ def signal_handler(signal_arg, frame):  # pylint: disable=unused-argument
     logger_process.join()
 
     if not hasattr(fs, "s3"):
-        fs.rm(tmp_dir, recursive=True)
+        fs.rmdir(tmp_dir)
 
 
 def main():