Merge pull request yasserg#418 from djp3/patch-1

Update the maximum size of a robots.txt file
lrrp · Nov 1, 2019 · d3c1157 · d3c1157
2 parents 4fcddc8 + 42407db
commit d3c1157
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/crawler4j/src/main/java/edu/uci/ics/crawler4j/robotstxt/RobotstxtServer.java b/crawler4j/src/main/java/edu/uci/ics/crawler4j/robotstxt/RobotstxtServer.java
@@ -135,8 +135,8 @@ private HostDirectives fetchDirectives(URL url) throws IOException, InterruptedE
             if (fetchResult.getStatusCode() == HttpStatus.SC_OK) {
                 Page page = new Page(robotsTxtUrl);
                 // Most recent answer on robots.txt max size is
-                // https://goo.gl/OqpKbP
-                fetchResult.fetchContent(page, 10_000 * 1024);
+                // https://developers.google.com/search/reference/robots_txt
+                fetchResult.fetchContent(page, 500 * 1024);
                 if (Util.hasPlainTextContent(page.getContentType())) {
                     String content;
                     if (page.getContentCharset() == null) {