try to detect encoding when loading file

setup.py

@@ -22,7 +22,8 @@
     'pyparsing>=2.2.0',
     'future>=0.16.0',
     'six>=1.10.0',
-    'configparser>=3.5.0'
+    'configparser>=3.5.0',
+    'chardet',
 ]
 
 setup(

src/wfuzz/plugins/payloads/file.py

@@ -1,6 +1,7 @@
 from wfuzz.externals.moduleman.plugin import moduleman_plugin
 from wfuzz.exception import FuzzExceptBadFile
 from wfuzz.plugin_api.base import BasePayload
+from wfuzz.utils import open_file_detect_encoding
 
 
 @moduleman_plugin
@@ -25,7 +26,7 @@ def __init__(self, params):
         BasePayload.__init__(self, params)
 
         try:
-            self.f = open(self.find_file(self.params["fn"]), "r")
+            self.f = open_file_detect_encoding(self.find_file(self.params["fn"]))
         except IOError as e:
             raise FuzzExceptBadFile("Error opening file. %s" % str(e))
 

src/wfuzz/utils.py

@@ -2,6 +2,7 @@
 import os
 import sys
 import six
+from chardet.universaldetector import UniversalDetector
 
 
 def json_minify(string, strip_space=True):
@@ -130,3 +131,24 @@ def convert_to_unicode(text):
         return text.encode("utf-8", errors='ignore')
     else:
         return text
+
+
+def open_file_detect_encoding(file_path):
+    def detect_encoding(file_path):
+        detector = UniversalDetector()
+        detector.reset()
+
+        with open(file_path, mode='rb') as file_to_detect:
+            for line in file_to_detect:
+                detector.feed(line)
+                if detector.done:
+                    break
+        detector.close()
+
+        print(detector.result)
+        return detector.result
+
+    if sys.version_info >= (3, 0):
+        return open(file_path, "r", encoding=detect_encoding(file_path).get('encoding', 'utf-8'))
+    else:
+        return open(file_path, "r")