Converted crawler user agent string to lowercase to address bug with …

…non-lowercase values
zytops · Apr 2, 2016 · 9076c81 · 9076c81
1 parent bb6bb78
commit 9076c81
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/main/java/edu/uci/ics/crawler4j/robotstxt/RobotstxtParser.java b/src/main/java/edu/uci/ics/crawler4j/robotstxt/RobotstxtParser.java
@@ -59,7 +59,7 @@ public static HostDirectives parse(String content, String myUserAgent) {
 
       if (line.matches(PATTERNS_USERAGENT)) {
         String ua = line.substring(PATTERNS_USERAGENT_LENGTH).trim().toLowerCase();
-        inMatchingUserAgent = "*".equals(ua) || ua.contains(myUserAgent);
+        inMatchingUserAgent = "*".equals(ua) || ua.contains(myUserAgent.toLowerCase());
       } else if (line.matches(PATTERNS_DISALLOW)) {
         if (!inMatchingUserAgent) {
           continue;