Merge pull request #36 from plowsec/fix_py3_ida77

Port to python 3.10 and IDA 7.7

exporters/base_support.py

@@ -189,7 +189,7 @@ def json_loads(line):
 def _pickle_method(method):
   func_name = method.im_func.__name__
   obj = method.im_self
-  cls = method.im_class
+  cls = method.__class__
   return _unpickle_method, (func_name, obj, cls)
 
 def _unpickle_method(func_name, obj, cls):

hooks/example1.py

@@ -21,12 +21,12 @@ def get_export_range(self):
     segname = ".libopenssl.so.RO"
     for s in Segments():
       if SegName(s) == segname:
-        start_ea, end_ea = SegStart(s), SegEnd(s)
+        start_ea, end_ea = get_segm_start(s), get_segm_end(s)
         self.pigaios.log("Limiting the export to 0x%08x -> 0x%08x" % (start_ea, end_ea))
         return start_ea, end_ea
 
     # We didn't find the segment, export the whole database
-    return MinEA(), MaxEA()
+    return inf_get_min_ea(), inf_get_max_ea()
 
   def before_export_function(self, f, name):
     """

ml/pigaios_ml.py

@@ -51,7 +51,7 @@ def warn(*args, **kwargs):
 from sklearn import naive_bayes
 from sklearn import linear_model
 from sklearn import neural_network
-from sklearn.externals import joblib
+import joblib
 from sklearn.model_selection import cross_val_score
 from sklearn.utils.validation import check_is_fitted
 
@@ -112,7 +112,7 @@ class CPigaiosMultiClassifier(object):
   def __init__(self, random_state=None):
     self.clfs = {}
     for classifier, name, args in ML_CLASSIFIERS:
-      has_seed = 'random_state' in dir(classifier.__init__.im_class())
+      has_seed = 'random_state' in dir(classifier.__class__)
       if has_seed:
         self.clfs[name] = classifier(random_state=random_state)
         for arg_name, arg_value in args:
@@ -178,7 +178,7 @@ def load_data(self, dataset="dataset.csv"):
       next(reader, None)
       for row in reader:
         is_match = row[2]
-        x_values.append(map(float, row[3:]))
+        x_values.append(list(map(float, row[3:])))
         y_values.append([float(is_match)])
 
     return np.array(x_values), np.array(y_values)
@@ -193,7 +193,7 @@ def predict(self):
     zeros_bad = 0
     total_matches = 0
     for i in range(0, len(X)):
-      tmp = X[i]
+      tmp = np.asarray(X[i])
       ret = self.clf.predict(tmp.reshape(1, -1))
       ret = round(ret)
       if ret == y[i]:

sourceimp_core.py

@@ -49,8 +49,9 @@
 
   from ml.pigaios_ml import CPigaiosClassifier, CPigaiosMultiClassifier
   has_ml = True
-except ImportError:
+except ImportError as ie:
   has_ml = False
+  log("Don't have ml: %s" %(str(ie.args)))
 
 try:
   long        # Python 2
@@ -136,7 +137,8 @@ def seems_false_positive(src_name, bin_name):
 
 #-------------------------------------------------------------------------------
 def json_loads(line):
-  return json.loads(line.decode("utf-8","ignore"))
+  data = line.decode("utf-8", "ignore") if hasattr(line, "decode") else line
+  return json.loads(data)
 
 #-------------------------------------------------------------------------------
 PROFILING = os.getenv("DIAPHORA_PROFILE") is not None
@@ -251,7 +253,7 @@ def get_compare_functions_data(self, src_id, bin_id, heur):
         raise Exception("Unknow data type for field %s" % field)
 
     tmp = []
-    header = ret.keys()
+    header = list(ret.keys())
     header.sort()
 
     for key in ML_FIELDS_ORDER:
@@ -282,7 +284,7 @@ def compare_functions(self, src_id, bin_id, heuristic):
           self.ml_classifier = CPigaiosClassifier()
           self.ml_model = self.ml_classifier.load_model()
 
-        line = map(float, line)
+        line = list(map(float, line))
         ml = self.ml_model.predict_proba(np.array(line).reshape(1, -1))
 
     fields = COMPARE_FIELDS
@@ -594,7 +596,6 @@ def find_initial_rows(self):
         if score >= 0.3 or ml == 1.0:
           self.add_match(match_id, func_ea, match_name, heur_name.capitalize(),
                          score, reasons, ml, qr)
-
         if score < min_score and score > 0.0:
           min_score = score
         if score > max_score:

sourceimp_ida.py

@@ -36,9 +36,9 @@
 from pygments.formatters import HtmlFormatter
 from pygments.lexers import NasmLexer, CppLexer
 
-from idaapi import (Choose2, PluginForm, Form, init_hexrays_plugin, load_plugin,
+from idaapi import (Choose, PluginForm, Form, init_hexrays_plugin, load_plugin,
                     get_func, decompile, tag_remove, show_wait_box, info,
-                    hide_wait_box, replace_wait_box, askyn_c, reg_read_string,
+                    hide_wait_box, replace_wait_box, ask_yn, reg_read_string,
                     reg_write_string)
 
 import sourceimp_core
@@ -62,9 +62,9 @@
 LITTLE_ORANGE = 0x026AFD
 
 #-------------------------------------------------------------------------------
-def log(msg):
-  Message("[%s] %s\n" % (time.asctime(), msg))
-  replace_wait_box(msg)
+def log(msg_raw):
+  msg("[%s] %s\n" % (time.asctime(), msg_raw))
+  replace_wait_box(msg_raw)
 
 #-------------------------------------------------------------------------------
 def indent_source(src):
@@ -259,7 +259,7 @@ def _trunc(self, s, changed, max_col=120):
     return res
 
 #-------------------------------------------------------------------------------
-class CDiffChooser(Choose2):
+class CDiffChooser(Choose):
   def __init__(self, differ, title, matches, importer_obj):
     self.importer = importer_obj
     self.differ = differ
@@ -268,7 +268,7 @@ def __init__(self, differ, title, matches, importer_obj):
       self.columns.append(["FP?", 6])
       self.columns.append(["Reasons", 40])
 
-    Choose2.__init__(self, title, columns, Choose2.CH_MULTI)
+    Choose.__init__(self, title, columns, Choose.CH_MULTI)
     self.n = 0
     self.icon = -1
     self.selcount = 0
@@ -278,7 +278,7 @@ def __init__(self, differ, title, matches, importer_obj):
 
     for i, match in enumerate(matches):
       ea, name, heuristic, score, reason, ml, qr = matches[match]
-      bin_func_name = GetFunctionName(long(ea))
+      bin_func_name = get_func_name(long(ea))
       line = ["%03d" % i, "%05d" % match, name, "0x%08x" % long(ea), bin_func_name, str(score), str(ml), str((score + ml)/2), str(qr), heuristic, reason]
       if _DEBUG:
         maybe_false_positive = int(seems_false_positive(name, bin_func_name))
@@ -368,22 +368,22 @@ def OnCommand(self, n, cmd_id):
         cdiffer.Show(src, title)
       cur.close()
     elif cmd_id == self.cmd_import_all:
-      if askyn_c(0, "HIDECANCEL\nDo you really want to import all matched functions as well as struct, union, enum and typedef definitions?") == 1:
+      if ask_yn(0, "HIDECANCEL\nDo you really want to import all matched functions as well as struct, union, enum and typedef definitions?") == 1:
         import_items = []
         for item in self.items:
           src_id, src_name, bin_ea = int(item[1]), item[2], int(item[3], 16)
           import_items.append([src_id, src_name, bin_ea])
 
         self.importer.import_items(import_items)
     elif cmd_id == self.cmd_import_selected:
-      if len(self.selected_items) == 1 or askyn_c(1, "HIDECANCEL\nDo you really want to import the selected functions?") == 1:
+      if len(self.selected_items) == 1 or ask_yn(1, "HIDECANCEL\nDo you really want to import the selected functions?") == 1:
         import_items = []
         for index in self.selected_items:
           item = self.items[index]
           src_id, src_name, bin_ea = int(item[1]), item[2], int(item[3], 16)
           import_items.append([src_id, src_name, bin_ea])
 
-        import_definitions = askyn_c(0, "HIDECANCEL\nDo you also want to import all struct, union, enum and typedef definitions?") == 1
+        import_definitions = ask_yn(0, "HIDECANCEL\nDo you also want to import all struct, union, enum and typedef definitions?") == 1
         self.importer.import_items(import_items, import_definitions = import_definitions)
     elif cmd_id == self.cmd_diff_c:
       html_diff = CHtmlDiff()
@@ -421,7 +421,7 @@ class CIDABinaryToSourceImporter(CBinaryToSourceImporter):
   def __init__(self, project_script):
     self.hooks = None
     self.project_script = project_script
-    CBinaryToSourceImporter.__init__(self, GetIdbPath())
+    CBinaryToSourceImporter.__init__(self, get_idb_path())
 
     show_wait_box("Finding matches...")
     self.src_db = None
@@ -473,7 +473,7 @@ def different_versions(self):
           msg  = "HIDECANCEL\nDatabase version (%s) is different to current version (%s).\n"
           msg += "Do you want to re-create the database?"
           msg += "\n\nNOTE: Selecting 'NO' will try to use the non updated database."
-          ret = askyn_c(0, msg % (version, VERSION_VALUE)) == 1
+          ret = ask_yn(0, msg % (version, VERSION_VALUE)) == 1
         elif status != "done":
           ret = True
         else:
@@ -567,7 +567,7 @@ def decompile(self, ea):
     return "\n".join(self.pseudo[ea])
 
   def get_function_name(self, ea):
-    return GetFunctionName(ea)
+    return get_func_name(ea)
 
   def import_src(self, src_db):
     self.load_hooks()
@@ -607,7 +607,7 @@ def import_items(self, import_items, import_definitions=True):
       cur.close()
 
     for src_id, src_name, bin_ea in import_items:
-      bin_name = GetFunctionName(bin_ea)
+      bin_name = get_func_name(bin_ea)
       if is_ida_func(bin_name):
         MakeName(bin_ea, src_name)
         proto = self.get_source_field_name(src_id, "prototype")