fix: don't rewrite clean file if exists!

WAcry · Mar 26, 2023 · dfee696 · dfee696
1 parent eac7734
commit dfee696
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/clean.py b/clean.py
@@ -6,8 +6,10 @@
 import pandas as pd
 
 
-prompt_generation_dir = "prompts-reponses"
+prompt_generation_dir = "raw_data_sanity_cleaned_delobotomized"
 for file in glob.glob(os.path.join(prompt_generation_dir, "*.jsonl")):
+    if "clean" in file:
+        continue
     data = []
     print(file)
     with open(file) as f: