remove redundant cleanup param

dbanys · Apr 18, 2021 · e1740ef · e1740ef
1 parent ad6e3b2
commit e1740ef
Showing 1 changed file with 0 additions and 10 deletions.
diff --git a/aitextgen/aitextgen.py b/aitextgen/aitextgen.py
@@ -472,7 +472,6 @@ def generate_to_file(
         destination_path: str = None,
         sample_delim: str = "=" * 20 + "\n",
         seed: int = None,
-        cleanup: bool = True,
         **kwargs,
     ) -> None:
         """
@@ -516,15 +515,6 @@ def generate_to_file(
         for _ in range(n // batch_size):
             gen_texts = self.generate(n=batch_size, return_as_list=True, **kwargs)
 
-            # Remove empty texts and strip out extra newlines/extra spaces
-            if cleanup:
-                texts_to_clean = gen_texts
-                gen_texts = []
-                for text in texts_to_clean:
-                    clean_text = text.strip().strip("\n")
-                    if clean_text and len(clean_text) >= 2:
-                        gen_texts.append(clean_text)
-
             for gen_text in gen_texts:
                 f.write("{}\n{}".format(gen_text, sample_delim))
             pbar.update(batch_size)