Do not upload temporary directory to artifacts (#841) (#849)

mozilla · Sep 16, 2024 · d7235e0 · d7235e0
1 parent 34b9c1d
commit d7235e0
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 2 deletions.
diff --git a/pipeline/alignments/align.py b/pipeline/alignments/align.py
@@ -56,7 +56,7 @@ def run(
     src = os.environ["SRC"]
     trg = os.environ["TRG"]
 
-    tmp_dir = os.path.join(os.path.dirname(output_path), "tmp")
+    tmp_dir = os.path.join(os.path.dirname(output_path), "tmp_aln")
     os.makedirs(tmp_dir, exist_ok=True)
 
     corpus_src = decompress(corpus_src)
@@ -100,6 +100,9 @@ def run(
             remapped_aln += ".zst"
         shutil.move(remapped_aln, output_path)
 
+    # Remove temporary directory to prevent uploading to artifacts
+    shutil.rmtree(tmp_dir)
+
 
 def decompress(file_path: str):
     if file_path.endswith(".zst"):

diff --git a/pipeline/alignments/generate-shortlist.sh b/pipeline/alignments/generate-shortlist.sh
@@ -27,7 +27,7 @@ fi
 cd "$(dirname "${0}")"
 
 mkdir -p "${output_dir}"
-dir="${output_dir}/tmp"
+dir="${output_dir}/tmp_shortlist"
 mkdir -p "${dir}"
 
 corpus_src="${corpus_prefix}.${SRC}.zst"