Merge pull request yasserg#272 from yasserg/pr/270

Fix href characters encoding issue
shiguang1120 · Dec 20, 2017 · 312408a · 312408a
2 parents 8f16a21 + b7f3582
commit 312408a
Show file tree

Hide file tree

Showing 4 changed files with 31 additions and 22 deletions.
diff --git a/crawler4j/checkstyle.xml b/crawler4j/checkstyle.xml
@@ -6,7 +6,7 @@
 
     <module name="TreeWalker">
         <module name="LineLength">
-            <property name="max" value="100"/>
+            <property name="max" value="120"/>
             <property name="ignorePattern" value="^(package .*;)|(import .*;)|(\{@link .*)$"/>
         </module>
         <module name="FileContentsHolder"/>

diff --git a/crawler4j/src/main/java/edu/uci/ics/crawler4j/parser/Parser.java b/crawler4j/src/main/java/edu/uci/ics/crawler4j/parser/Parser.java
@@ -20,6 +20,8 @@
 import java.io.ByteArrayInputStream;
 import java.io.InputStream;
 import java.io.UnsupportedEncodingException;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 import java.util.HashSet;
 import java.util.Set;
 
@@ -132,7 +134,10 @@ public void parse(Page page, String contextURL)
                 String hrefLoweredCase = href.trim().toLowerCase();
                 if (!hrefLoweredCase.contains("javascript:") &&
                     !hrefLoweredCase.contains("mailto:") && !hrefLoweredCase.contains("@")) {
-                    String url = URLCanonicalizer.getCanonicalURL(href, contextURL);
+                    // Prefer page's content charset to encode href url
+                    Charset hrefCharset = ((page.getContentCharset() == null) || page.getContentCharset().isEmpty()) ?
+                                          StandardCharsets.UTF_8 : Charset.forName(page.getContentCharset());
+                    String url = URLCanonicalizer.getCanonicalURL(href, contextURL, hrefCharset);
                     if (url != null) {
                         WebURL webURL = new WebURL();
                         webURL.setURL(url);

diff --git a/crawler4j/src/main/java/edu/uci/ics/crawler4j/url/URLCanonicalizer.java b/crawler4j/src/main/java/edu/uci/ics/crawler4j/url/URLCanonicalizer.java
@@ -23,6 +23,8 @@
 import java.net.URL;
 import java.net.URLDecoder;
 import java.net.URLEncoder;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.Objects;
@@ -40,6 +42,10 @@ public static String getCanonicalURL(String url) {
     }
 
     public static String getCanonicalURL(String href, String context) {
+        return getCanonicalURL(href, context, StandardCharsets.UTF_8);
+    }
+
+    public static String getCanonicalURL(String href, String context, Charset charset) {
 
         try {
             URL canonicalURL =
@@ -75,7 +81,7 @@ public static String getCanonicalURL(String href, String context) {
             Map<String, String> params = createParameterMap(canonicalURL.getQuery());
             final String queryString;
             if ((params != null) && !params.isEmpty()) {
-                String canonicalParams = canonicalize(params);
+                String canonicalParams = canonicalize(params, charset);
                 queryString = (canonicalParams.isEmpty() ? "" : ("?" + canonicalParams));
             } else {
                 queryString = "";
@@ -143,9 +149,11 @@ private static Map<String, String> createParameterMap(String queryString) {
      *
      * @param paramsMap
      *            Parameter map whose name-value pairs are in order of insertion.
+     * @param charset
+     *            Charset of html page
      * @return Canonical form of query string.
      */
-    private static String canonicalize(Map<String, String> paramsMap) {
+    private static String canonicalize(Map<String, String> paramsMap, Charset charset) {
         if ((paramsMap == null) || paramsMap.isEmpty()) {
             return "";
         }
@@ -159,36 +167,27 @@ private static String canonicalize(Map<String, String> paramsMap) {
             if (sb.length() > 0) {
                 sb.append('&');
             }
-            sb.append(percentEncodeRfc3986(pair.getKey()));
+            sb.append(percentEncodeRfc3986(pair.getKey(), charset));
             if (!pair.getValue().isEmpty()) {
                 sb.append('=');
-                sb.append(percentEncodeRfc3986(pair.getValue()));
+                sb.append(percentEncodeRfc3986(pair.getValue(), charset));
             }
         }
         return sb.toString();
     }
 
-    /**
-     * Percent-encode values according the RFC 3986. The built-in Java
-     * URLEncoder does not encode according to the RFC, so we make the extra
-     * replacements.
-     *
-     * @param string
-     *            Decoded string.
-     * @return Encoded string per RFC 3986.
-     */
-    private static String percentEncodeRfc3986(String string) {
+    private static String normalizePath(final String path) {
+        return path.replace("%7E", "~").replace(" ", "%20");
+    }
+
+    private static String percentEncodeRfc3986(String string, Charset charset) {
         try {
             string = string.replace("+", "%2B");
             string = URLDecoder.decode(string, "UTF-8");
-            string = URLEncoder.encode(string, "UTF-8");
+            string = URLEncoder.encode(string, charset.name());
             return string.replace("+", "%20").replace("*", "%2A").replace("%7E", "~");
         } catch (Exception e) {
             return string;
         }
     }
-
-    private static String normalizePath(final String path) {
-        return path.replace("%7E", "~").replace(" ", "%20");
-    }
 }
diff --git a/crawler4j/src/test/java/edu/uci/ics/crawler4j/tests/URLCanonicalizerTest.java b/crawler4j/src/test/java/edu/uci/ics/crawler4j/tests/URLCanonicalizerTest.java
@@ -2,6 +2,8 @@
 
 import static org.junit.Assert.assertEquals;
 
+import java.nio.charset.Charset;
+
 import org.junit.Test;
 
 import edu.uci.ics.crawler4j.url.URLCanonicalizer;
@@ -79,6 +81,9 @@ public void testCanonizalier() {
 
         assertEquals("http://foo.bar/mydir/myfile?page=2",
                      URLCanonicalizer.getCanonicalURL("?page=2", "http://foo.bar/mydir/myfile"));
-
+        // test href with charset
+        assertEquals("http://www.example.com/3.asp?DengJh=%BA%E91700718",
+                URLCanonicalizer.getCanonicalURL("3.asp?DengJh=洪1700718", "http://www.example.com",
+                        Charset.forName("gb2312")));
     }
 }