Merge pull request yasserg#74 from albert0815/albert0815-close-entity

Fix for issue yasserg#52 - Close entity when page is to big

src/main/java/edu/uci/ics/crawler4j/fetcher/PageFetcher.java

@@ -31,7 +31,7 @@
 import edu.uci.ics.crawler4j.crawler.authentication.NtAuthInfo;
 import org.apache.http.Header;
 import org.apache.http.HttpHost;
-import org.apache.http.HttpResponse;
+import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.HttpStatus;
 import org.apache.http.NameValuePair;
 import org.apache.http.auth.AuthScope;
@@ -234,7 +234,7 @@ public PageFetchResult fetchPage(WebURL webUrl)
         lastFetchTime = (new Date()).getTime();
       }
 
-      HttpResponse response = httpClient.execute(request);
+      CloseableHttpResponse response = httpClient.execute(request);
       fetchResult.setEntity(response.getEntity());
       fetchResult.setResponseHeaders(response.getAllHeaders());
 
@@ -274,6 +274,8 @@ public PageFetchResult fetchPage(WebURL webUrl)
             }
           }
           if (size > config.getMaxDownloadSize()) {
+            //fix issue #52 - consume entity
+            response.close();
             throw new PageBiggerThanMaxSizeException(size);
           }
         }
@@ -307,4 +309,4 @@ protected HttpUriRequest newHttpUriRequest(String url) {
     return new HttpGet(url);
   }
 
-}
+}