Merge pull request yasserg#46 from yasserg/pr/42

Added default request header configuration

Author: yasserg

src/main/java/edu/uci/ics/crawler4j/crawler/CrawlConfig.java

@@ -18,8 +18,13 @@
 package edu.uci.ics.crawler4j.crawler;
 
 import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
 import java.util.List;
 
+import org.apache.http.Header;
+import org.apache.http.message.BasicHeader;
+
 import edu.uci.ics.crawler4j.crawler.authentication.AuthInfo;
 
 public class CrawlConfig {
@@ -54,6 +59,11 @@ public class CrawlConfig {
    */
   private String userAgentString = "crawler4j (http://code.google.com/p/crawler4j/)";
 
+  /**
+   * Default request header values.
+   */
+  private Collection<BasicHeader> defaultHeaders = new HashSet<BasicHeader>();
+
   /**
    * Politeness delay in milliseconds (delay between sending two requests to
    * the same host).
@@ -229,6 +239,24 @@ public void setUserAgentString(String userAgentString) {
     this.userAgentString = userAgentString;
   }
 
+  /**
+   * Return a copy of the default header collection.
+   */
+  public Collection<BasicHeader> getDefaultHeaders() {
+    return new HashSet<>(defaultHeaders);
+  }
+
+  /**
+   * Set the default header collection (creating copies of the provided headers).
+   */
+  public void setDefaultHeaders(Collection<? extends Header> defaultHeaders) {
+    Collection<BasicHeader> copiedHeaders = new HashSet<>();
+    for (Header header : defaultHeaders) {
+      copiedHeaders.add(new BasicHeader(header.getName(), header.getValue()));
+    }
+    this.defaultHeaders = copiedHeaders;
+  }
+
   public int getPolitenessDelay() {
     return politenessDelay;
   }

src/main/java/edu/uci/ics/crawler4j/fetcher/PageFetcher.java

@@ -119,6 +119,7 @@ public boolean isTrusted(final X509Certificate[] chain, String authType) {
     clientBuilder.setDefaultRequestConfig(requestConfig);
     clientBuilder.setConnectionManager(connectionManager);
     clientBuilder.setUserAgent(config.getUserAgentString());
+    clientBuilder.setDefaultHeaders(config.getDefaultHeaders());
 
     if (config.getProxyHost() != null) {
       if (config.getProxyUsername() != null) {