From dfee6963a1839366b3b5307b72f866c6e5529c3e Mon Sep 17 00:00:00 2001
From: Zach Nussbaum <zanussbaum@gmail.com>
Date: Sun, 26 Mar 2023 18:15:22 +0000
Subject: [PATCH] fix: don't rewrite clean file if exists!

---
 clean.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/clean.py b/clean.py
index 9cf8bf57640a..324b99fd7089 100644
--- a/clean.py
+++ b/clean.py
@@ -6,8 +6,10 @@
 import pandas as pd
 
 
-prompt_generation_dir = "prompts-reponses"
+prompt_generation_dir = "raw_data_sanity_cleaned_delobotomized"
 for file in glob.glob(os.path.join(prompt_generation_dir, "*.jsonl")):
+    if "clean" in file:
+        continue
     data = []
     print(file)
     with open(file) as f: