Dr. Elephant should check for finished directory before listing

meteorchenwu · Nov 17, 2017 · 35d06d9 · 35d06d9
1 parent e756226
commit 35d06d9
Showing 1 changed file with 12 additions and 9 deletions.
diff --git a/app/com/linkedin/drelephant/mapreduce/fetchers/MapReduceFSFetcherHadoop2.java b/app/com/linkedin/drelephant/mapreduce/fetchers/MapReduceFSFetcherHadoop2.java
@@ -153,14 +153,17 @@ private DataFiles getHistoryFiles(AnalyticJob job) throws IOException {
 
     // Search files in done dir
     String jobHistoryDirPath = getHistoryDir(job);
-    RemoteIterator<LocatedFileStatus> it = _fs.listFiles(new Path(jobHistoryDirPath), false);
-    while (it.hasNext() && (jobConfPath == null || jobHistPath == null)) {
-      String name = it.next().getPath().getName();
-      if (name.contains(jobId)) {
-        if (name.endsWith("_conf.xml")) {
-          jobConfPath = jobHistoryDirPath + name;
-        } else if (name.endsWith(".jhist")) {
-          jobHistPath = jobHistoryDirPath + name;
+
+    if (_fs.exists(new Path(jobHistoryDirPath))) {
+      RemoteIterator<LocatedFileStatus> it = _fs.listFiles(new Path(jobHistoryDirPath), false);
+      while (it.hasNext() && (jobConfPath == null || jobHistPath == null)) {
+        String name = it.next().getPath().getName();
+        if (name.contains(jobId)) {
+          if (name.endsWith("_conf.xml")) {
+            jobConfPath = jobHistoryDirPath + name;
+          } else if (name.endsWith(".jhist")) {
+            jobHistPath = jobHistoryDirPath + name;
+          }
         }
       }
     }
@@ -178,7 +181,7 @@ private DataFiles getHistoryFiles(AnalyticJob job) throws IOException {
     }
     if (jobHistPath == null) {
       try {
-        it = _fs.listFiles(new Path(intermediateDirPath), false);
+        RemoteIterator<LocatedFileStatus> it = _fs.listFiles(new Path(intermediateDirPath), false);
         while (it.hasNext()) {
           String name = it.next().getPath().getName();
           if (name.contains(jobId) && name.endsWith(".jhist")) {