Add support to delete output after azkaban.jobtype.examples.java.Word…

…Count (azkaban#3289)
dfsyke · May 11, 2023 · 90f7367 · 90f7367
1 parent 355a4c2
commit 90f7367
Showing 1 changed file with 12 additions and 3 deletions.
diff --git a/az-hadoop-jobtype-plugin/src/main/java/azkaban/jobtype/examples/java/WordCount.java b/az-hadoop-jobtype-plugin/src/main/java/azkaban/jobtype/examples/java/WordCount.java
@@ -46,14 +46,17 @@ public class WordCount extends AbstractHadoopJob {
 
   private final String inputPath;
   private final String outputPath;
-  private boolean forceOutputOverrite;
+  private final boolean forceOutputOverwrite;
+  private final boolean outputDelete;
 
   public WordCount(String name, Props props) {
     super(name, props);
     this.inputPath = props.getString("input.path");
     this.outputPath = props.getString("output.path");
-    this.forceOutputOverrite =
+    this.forceOutputOverwrite =
         props.getBoolean("force.output.overwrite", false);
+    this.outputDelete =
+        props.getBoolean("output.delete", false);
   }
 
   public static class Map extends MapReduceBase implements
@@ -121,13 +124,19 @@ public void run() throws Exception {
     FileInputFormat.addInputPath(jobconf, new Path(inputPath));
     FileOutputFormat.setOutputPath(jobconf, new Path(outputPath));
 
-    if (forceOutputOverrite) {
+    if (forceOutputOverwrite) {
       FileSystem fs =
           FileOutputFormat.getOutputPath(jobconf).getFileSystem(jobconf);
       fs.delete(FileOutputFormat.getOutputPath(jobconf), true);
     }
 
     super.run();
+
+    if (outputDelete) {
+      FileSystem fs =
+          FileOutputFormat.getOutputPath(jobconf).getFileSystem(jobconf);
+      fs.delete(FileOutputFormat.getOutputPath(jobconf), true);
+    }
   }
 
 }