fix encoding issues

src/wfuzz/externals/reqresp/Request.py

@@ -19,6 +19,8 @@
 from .exceptions import ReqRespException
 from .Response import Response
 
+from wfuzz.utils import python2_3_convert_to_unicode
+
 from .TextParser import TextParser
 
 
@@ -295,7 +297,7 @@ def to_pycurl_object(c, req):
             if PYCURL_PATH_AS_IS:
                 c.setopt(pycurl.PATH_AS_IS, 1)
 
-            c.setopt(pycurl.URL, req.completeUrl)
+            c.setopt(pycurl.URL, python2_3_convert_to_unicode(req.completeUrl))
 
             if req.getConnTimeout():
                 c.setopt(pycurl.CONNECTTIMEOUT, req.getConnTimeout())
@@ -311,11 +313,11 @@ def to_pycurl_object(c, req):
                     c.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_NTLM)
                 elif authMethod == "digest":
                     c.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_DIGEST)
-                c.setopt(pycurl.USERPWD, userpass)
+                c.setopt(pycurl.USERPWD, python2_3_convert_to_unicode(userpass))
             else:
                 c.unsetopt(pycurl.USERPWD)
 
-            c.setopt(pycurl.HTTPHEADER, req.getHeaders())
+            c.setopt(pycurl.HTTPHEADER, python2_3_convert_to_unicode(req.getHeaders()))
 
             curl_options = {
                 "GET": pycurl.HTTPGET,
@@ -334,13 +336,13 @@ def to_pycurl_object(c, req):
                 c.setopt(pycurl.CUSTOMREQUEST, req.method)
 
             if req.postdata:
-                c.setopt(pycurl.POSTFIELDS, req.postdata)
+                c.setopt(pycurl.POSTFIELDS, python2_3_convert_to_unicode(req.postdata))
 
             c.setopt(pycurl.FOLLOWLOCATION, 1 if req.followLocation else 0)
 
             proxy = req.getProxy()
             if proxy is not None:
-                c.setopt(pycurl.PROXY, proxy)
+                c.setopt(pycurl.PROXY, python2_3_convert_to_unicode(proxy))
                 if req.proxytype == "SOCKS5":
                     c.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_SOCKS5)
                 elif req.proxytype == "SOCKS4":

src/wfuzz/externals/reqresp/Response.py

@@ -5,6 +5,8 @@
 
 from .TextParser import TextParser
 
+from wfuzz.utils import python2_3_convert_from_unicode
+
 
 class Response:
         def __init__(self, protocol="", code="", message=""):
@@ -92,7 +94,8 @@ def parseResponse(self, rawheader, rawbody=None, type="curl"):
                 self._headers = []
 
                 tp = TextParser()
-                tp.setSource("string", rawheader.decode('utf-8', errors='replace'))
+                rawheader = python2_3_convert_from_unicode(rawheader.decode("utf-8", errors='replace'))
+                tp.setSource("string", rawheader)
 
                 tp.readUntil("(HTTP\S*) ([0-9]+)")
                 while True:
@@ -169,4 +172,6 @@ def parseResponse(self, rawheader, rawbody=None, type="curl"):
                         rawbody = deflated_data
                         self.delHeader("Content-Encoding")
 
-                self.__content = rawbody.decode('utf-8', errors='replace')
+                # TODO: Try to get encoding from content
+                self.__content = python2_3_convert_from_unicode(rawbody.decode("unicode_escape", errors='replace'))
+                # self.__content = python2_3_convert_from_unicode(rawbody.decode("utf-8", errors='replace'))

src/wfuzz/fuzzobjects.py

@@ -20,6 +20,8 @@
 from .facade import Facade
 from .mixins import FuzzRequestUrlMixing, FuzzRequestSoupMixing
 
+from .utils import python2_3_convert_to_unicode
+
 auth_header = namedtuple("auth_header", "method credentials")
 
 
@@ -787,7 +789,7 @@ def update(self, exception=None):
 
         if self.history and self.history.content:
             m = hashlib.md5()
-            m.update(self.history.content.encode('utf-8'))
+            m.update(python2_3_convert_to_unicode(self.history.content))
             self.md5 = m.hexdigest()
 
             self.chars = len(self.history.content)

src/wfuzz/options.py

@@ -4,16 +4,17 @@
 from .fuzzobjects import FuzzResult, FuzzStats
 from .filter import FuzzResFilter
 from .core import requestGenerator
-from .utils import json_minify
+from .utils import (
+    json_minify,
+    python2_3_convert_from_unicode
+)
 
 from .core import Fuzzer
 from .myhttp import HttpPool
 
 from .externals.reqresp.cache import HttpCache
 
 from collections import defaultdict
-import sys
-import six
 
 # python 2 and 3
 try:
@@ -142,17 +143,6 @@ def validate(self):
         except TypeError:
             return "Bad options: Filter must be specified in the form of [int, ... , int]."
 
-    # pycurl does not like unicode strings
-    def _convert_from_unicode(self, text):
-        if isinstance(text, dict):
-            return {self._convert_from_unicode(key): self._convert_from_unicode(value) for key, value in list(text.items())}
-        elif isinstance(text, list):
-            return [self._convert_from_unicode(element) for element in text]
-        elif isinstance(text, six.string_types):
-            return text.encode('utf-8')
-        else:
-            return text
-
     def export_to_file(self, filename):
         try:
             with open(filename, 'w') as f:
@@ -175,11 +165,7 @@ def import_json(self, data):
                 for section in js['wfuzz_recipe'].keys():
                     for k, v in js['wfuzz_recipe'].items():
                         if k not in self.keys_not_to_dump:
-                            # python 2 and 3 hack
-                            if sys.version_info >= (3, 0):
-                                self.data[k] = v
-                            else:
-                                self.data[k] = self._convert_from_unicode(v)
+                            self.data[k] = python2_3_convert_from_unicode(v)
             else:
                 raise FuzzExceptBadRecipe("Unsupported recipe version.")
         except KeyError:

src/wfuzz/utils.py

@@ -1,5 +1,7 @@
 import re
 import os
+import sys
+import six
 
 
 def json_minify(string, strip_space=True):
@@ -103,3 +105,28 @@ def find_file_in_paths(name, path):
             return os.path.join(root, name)
 
     return None
+
+
+def python2_3_convert_from_unicode(text):
+    if sys.version_info >= (3, 0):
+        return text
+    else:
+        return convert_to_unicode(text)
+
+
+def python2_3_convert_to_unicode(text):
+    if sys.version_info >= (3, 0):
+        return convert_to_unicode(text)
+    else:
+        return text
+
+
+def convert_to_unicode(text):
+    if isinstance(text, dict):
+        return {convert_to_unicode(key): convert_to_unicode(value) for key, value in list(text.items())}
+    elif isinstance(text, list):
+        return [convert_to_unicode(element) for element in text]
+    elif isinstance(text, six.string_types):
+        return text.encode("utf-8", errors='ignore')
+    else:
+        return text

tests/test_acceptance.py

@@ -1,3 +1,6 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
 import os
 import unittest
 import tempfile
@@ -36,9 +39,21 @@
 # script args
 
 testing_tests = [
+    # not working due to content being decode as unicode not utf-8
+    # ("test_encode_cookie2_utf8_return", "%s/anything" % HTTPBIN_URL, [["は国"]], dict(cookie=["test=FUZZ"], filter="content~'test=\\\\u00e3\\\\u0081\\\\u00af\\\\u00e5\\\\u009b\\\\u00bd'"), [(200, '/anything')], None),
+    # ("test_encode_header_utf8_return", "%s/headers" % HTTPBIN_URL, [["は国"]], dict(headers=[("myheader", "FUZZ")], filter="content~'Myheader' and content~'\\\\u00e3\\\\u0081\\\\u00af\\\\u00e5\\\\u009b\\\\u00bd'"), [(200, '/headers')], None),
 ]
 
 basic_tests = [
+    # encoding tests
+    ("test_encode_path", "%s/FUZZ" % HTTPBIN_URL, [["は国"]], dict(), [(404, '/は国')], None),
+    ("test_encode_basic_auth", "%s/basic-auth/FUZZ/FUZZ" % HTTPBIN_URL, [["は国"]], dict(auth=("basic", "FUZZ:FUZZ")), [(200, '/basic-auth/は国/は国')], None),
+    ("test_encode_postdata", "%s/anything" % HTTPBIN_URL, [["は国"]], dict(postdata="a=FUZZ", filter="content~'は国'"), [(200, '/anything')], None),
+    ("test_encode_url_filter", "%s/FUZZ" % HTTPBIN_URL, [["は国"]], dict(filter="url~'は国'"), [(404, '/は国')], None),
+    ("test_encode_var", "%s/anything?var=FUZZ" % HTTPBIN_URL, [["は国"]], dict(filter="content~'\"は国\"'"), [(200, '/anything')], None),
+    ("test_encode_redirect", "%s/redirect-to?url=FUZZ" % HTTPBIN_URL, [["は国"]], dict(filter="headers.response.Location='%C3%A3%C2%81%C2%AF%C3%A5%C2%9B%C2%BD'"), [(302, '/redirect-to')], None),
+    ("test_encode_cookie", "%s/cookies" % HTTPBIN_URL, [["は国"]], dict(cookie=["cookie1=FUZZ"], follow=True, filter="content~FUZZ"), [(200, '/cookies')], None),
+
     # postdata tests
     # pycurl does not allow it ("test_get_postdata", "%s/FUZZ?var=1&var2=2" % HTTPBIN_URL, [["anything"]], dict(postdata='a=1', filter="content~'\"form\":{\"a\":\"1\"}'"), [(200, '/anything')], None),
     ("test_allmethods_postdata", "%s/FUZZ?var=1&var2=2" % HTTPBIN_URL, [["anything"], ['PUT', 'POST', 'DELETE'], ['333888']], dict(method='FUZ2Z', postdata='a=FUZ3Z', filter="content~FUZ2Z and content~'\"form\":{\"a\":\"' and content~FUZ3Z"), [(200, '/anything')] * 3, None),
@@ -309,6 +324,11 @@ def duplicate_tests_diff_params(test_list, group, next_extra_params, previous_ex
 
     """
     for test_name, url, payloads, params, expected_res, exception_str in test_list:
+
+        # mitmproxy does not go well with encodings. temporary bypass encoding checks with proxy
+        if group == "_proxy_" and "encode" in test_name:
+            continue
+
         next_extra = dict(list(params.items()) + list(next_extra_params.items()))
         new_test = "%s_%s" % (test_name, group)