Merge pull request yasserg#65 from MadEgg/shutdown-empty

Allow crawler to keep running with an empty queue

Author: yasserg

src/main/java/edu/uci/ics/crawler4j/crawler/CrawlConfig.java

@@ -123,6 +123,11 @@ public class CrawlConfig {
    */
   private boolean onlineTldListUpdate = false;
 
+  /**
+   * Should the crawler stop running when the queue is empty?
+   */
+  private boolean shutdownOnEmptyQueue = true;
+  
   /**
    * If crawler should run behind a proxy, this parameter can be used for
    * specifying the proxy host.
@@ -378,6 +383,17 @@ public boolean isFollowRedirects() {
   public void setFollowRedirects(boolean followRedirects) {
     this.followRedirects = followRedirects;
   }
+  
+  public boolean isShutdownOnEmptyQueue() {
+      return shutdownOnEmptyQueue;
+  }
+  
+  /**
+   * Should the crawler stop running when the queue is empty?
+   */
+  public void setShutdownOnEmptyQueue(boolean shutdown) {
+      shutdownOnEmptyQueue = shutdown;
+  }
 
   public boolean isOnlineTldListUpdate() {
     return onlineTldListUpdate;

src/main/java/edu/uci/ics/crawler4j/crawler/CrawlController.java

@@ -36,6 +36,13 @@
 import edu.uci.ics.crawler4j.url.WebURL;
 import edu.uci.ics.crawler4j.util.IO;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+
 /**
  * The controller that manages a crawling session. This class creates the
  * crawler threads and monitors their progress.
@@ -254,8 +261,11 @@ public void run() {
                     someoneIsWorking = true;
                   }
                 }
-                if (!someoneIsWorking) {
-                  // Make sure again that none of the threads are alive.
+                boolean shut_on_empty = config.isShutdownOnEmptyQueue();
+                if (!someoneIsWorking && shut_on_empty) {
+                  // Make sure again that none of the threads
+                  // are
+                  // alive.
                   logger.info("It looks like no thread is working, waiting for 10 seconds to make sure...");
                   sleep(10);
 
@@ -511,4 +521,4 @@ public void shutdown() {
     pageFetcher.shutDown();
     frontier.finish();
   }
-}
+}