使用redis HyperLogLog进行计数

BaseMessage.py

@@ -101,6 +101,8 @@ def WebLogicScan(self):
                         bug = BugList(oldurl=self.domain, bugurl=bugurl, bugname=bugname,
                                       buggrade=redispool.hget('bugtype', bugname),
                                       payload=bugurl, bugdetail=bugdetail)
+                        redispool.pfadd(redispool.hget('bugtype', bugname), bugurl)
+                        redispool.pfadd(bugname, bugurl)
                         db.session.add(bug)
                 db.session.commit()
         except Exception as e:

POCScan/POCScan.py

@@ -13,6 +13,8 @@ def POCScanConsole(attackurl,url):
                 if rep.status_code!=404 and poc.expression in rep.text:
                     bug = BugList(oldurl=attackurl, bugurl=url, bugname=poc.name,buggrade=redispool.hget('bugtype', poc.name), payload=url+poc,
                                   bugdetail=rep.text)
+                    redispool.pfadd(redispool.hget('bugtype', poc.name), url)
+                    redispool.pfadd(poc.name, url)
                     db.session.add(bug)
             except Exception as e:
                 print(e)

POCScan/selfpocscan.py

@@ -22,6 +22,9 @@ def informationpoc_check(oldurl,informationurl):
             if vulnerable:
                 bug = BugList(oldurl=oldurl, bugurl=bugurl, bugname=bugname, buggrade=redispool.hget('bugtype', bugname),
                               payload=payload, bugdetail=bugdetail)
+                # 使用 HyperLogLog  进行 漏洞四等级 计数
+                redispool.pfadd(redispool.hget('bugtype', bugname),bugurl)
+                redispool.pfadd(bugname,bugurl)
                 db.session.add(bug)
         db.session.commit()
 
@@ -39,6 +42,8 @@ def cmspoc_check(oldurl,cmsurl):
             if vulnerable:
                 bug = BugList(oldurl=oldurl, bugurl=bugurl, bugname=bugname, buggrade=redispool.hget('bugtype', bugname),
                               payload=payload, bugdetail=bugdetail)
+                redispool.pfadd(redispool.hget('bugtype', bugname),bugurl)
+                redispool.pfadd(bugname,bugurl)
                 db.session.add(bug)
         db.session.commit()
 
@@ -57,6 +62,8 @@ def industrial_check(oldurl,industrialurl):
             if vulnerable:
                 bug = BugList(oldurl=oldurl, bugurl=bugurl, bugname=bugname, buggrade=redispool.hget('bugtype', bugname),
                               payload=payload, bugdetail=bugdetail)
+                redispool.pfadd(redispool.hget('bugtype', bugname),bugurl)
+                redispool.pfadd(bugname,bugurl)
                 db.session.add(bug)
         db.session.commit()
 
@@ -74,6 +81,8 @@ def hardware_check(oldurl,hardwareurl):
             if vulnerable:
                 bug = BugList(oldurl=oldurl, bugurl=bugurl, bugname=bugname, buggrade=redispool.hget('bugtype', bugname),
                               payload=payload, bugdetail=bugdetail)
+                redispool.pfadd(redispool.hget('bugtype', bugname),bugurl)
+                redispool.pfadd(bugname,bugurl)
                 db.session.add(bug)
         db.session.commit()
 

SZheConsole.py

@@ -35,6 +35,8 @@ def BugScanConsole(attackurl):
                     vulnerable, payload,bugdetail=getattr(Bug, value)()
                     if vulnerable:
                             bug = BugList(oldurl=attackurl,bugurl=url,bugname=value,buggrade=redispool.hget('bugtype', value),payload=payload,bugdetail=bugdetail)
+                            redispool.pfadd(redispool.hget('bugtype', value), url)
+                            redispool.pfadd(value, url)
                             db.session.add(bug)
                 db.session.commit()
             Bug.POCScan()

get_message.py

@@ -248,6 +248,8 @@ def SenFileScan(domain,url):
                     if not core.is_similar_page(rep404,rep.text,radio=0.85):
                         print(url)
                         bug = BugList(oldurl=domain, bugurl=url, bugname="SenDir",buggrade=redispool.hget('bugtype', "SenDir"),payload=url, bugdetail=rep.text)
+                        redispool.pfadd(redispool.hget('bugtype', "SenDir"), url)
+                        redispool.pfadd("SenDir", url)
                         db.session.add(bug)
                 except Exception as e:
                     # print(e)

index.py

@@ -13,21 +13,24 @@
 
 
 def GetBit():
-    #获取数据库中各个漏洞的数量
-    seriouscount = BugList.query.filter(BugList.buggrade=='serious').count()
-    highcount = BugList.query.filter(BugList.buggrade=='high').count()
-    mediumcount = BugList.query.filter(BugList.buggrade=='medium').count()
-    lowcount = BugList.query.filter(BugList.buggrade=='low').count()
+    '''
+    操作redis HyperLogLog进行计数
+    :return:
+    '''
+    seriouscount = redispool.pfcount('serious')
+    highcount =  redispool.pfcount('high')
+    mediumcount =  redispool.pfcount('medium')
+    lowcount =  redispool.pfcount('low')
     allcount=seriouscount+highcount+mediumcount+lowcount
-    sqlcount=BugList.query.filter(BugList.bugname=='SQLBugScan').count()
-    comincount=BugList.query.filter(BugList.bugname=='ComInScan').count()
-    weblogiccount=BugList.query.filter(BugList.bugname=='WebLogicScan').count()
-    fileincount=BugList.query.filter(BugList.bugname=='FileIncludeScan').count()
-    sendircount=BugList.query.filter(BugList.bugname=='SenDir').count()
-    robotscount=BugList.query.filter(BugList.bugname=='robots文件发现').count()
-    phpinfocount=BugList.query.filter(BugList.bugname=='phpstudy探针').count()
-    gitcount=BugList.query.filter(BugList.bugname=='git源码泄露扫描').count()
-    phpstudycount=BugList.query.filter(BugList.bugname=='phpstudy phpmyadmin默认密码漏洞').count()
+    sqlcount= redispool.pfcount('SQLBugScan')
+    comincount= redispool.pfcount('ComInScan')
+    weblogiccount= redispool.pfcount('WebLogicScan')
+    fileincount= redispool.pfcount('FileIncludeScan')
+    sendircount= redispool.pfcount('SenDir')
+    robotscount= redispool.pfcount('robots文件发现')
+    phpinfocount= redispool.pfcount('phpstudy探针')
+    gitcount= redispool.pfcount('git源码泄露扫描')
+    phpstudycount= redispool.pfcount('phpstudy phpmyadmin默认密码漏洞')
     otherpoc=allcount-sqlcount-comincount-weblogiccount-fileincount-sendircount-robotscount-phpinfocount-gitcount
     bugbit={
         'seriouscount':seriouscount,
@@ -36,18 +39,32 @@ def GetBit():
         'lowcount':lowcount,
         'allcount': allcount,
     }
-    bugtype={
-        'SQLBugScan':sqlcount,
-        'ComInScan':comincount,
-        'WebLogicScan':weblogiccount,
-        'FileIncludeScan':fileincount,
-        'SenDir':sendircount,
-        'robots文件发现':robotscount,
-        'phpstudy探针':phpinfocount,
-        'git源码泄露扫描':gitcount,
-        'phpstudy phpmyadmin默认密码漏洞':phpstudycount,
-        'POC扫描漏洞':otherpoc
-    }
+    if allcount==0:
+        bugtype = {
+            'SQLBugScan': 0,
+            'ComInScan': 0,
+            'WebLogicScan': 0,
+            'FileIncludeScan': 0,
+            'SenDir': 0,
+            'robots文件发现': 0,
+            'phpstudy探针': 0,
+            'git源码泄露扫描': 0,
+            'phpstudy phpmyadmin默认密码漏洞': 0,
+            'POC扫描漏洞': 0
+        }
+    else:
+        bugtype={
+            'SQLBugScan':sqlcount/allcount*100,
+            'ComInScan':comincount/allcount*100,
+            'WebLogicScan':weblogiccount/allcount*100,
+            'FileIncludeScan':fileincount/allcount*100,
+            'SenDir':sendircount/allcount*100,
+            'robots文件发现':robotscount/allcount*100,
+            'phpstudy探针':phpinfocount/allcount*100,
+            'git源码泄露扫描':gitcount/allcount*100,
+            'phpstudy phpmyadmin默认密码漏洞':phpstudycount/allcount*100,
+            'POC扫描漏洞':otherpoc/allcount*100
+        }
     return bugbit,bugtype
 
 def save_log(ip, email):
@@ -86,7 +103,6 @@ def index(page=None):
     if not page:
         page = 1
     per_page = 10
-    print(bugbit['allcount'])
     paginate = BaseInfo.query.order_by(BaseInfo.date.desc()).paginate(page, per_page, error_out=False)
     infos = paginate.items
     return render_template('homeOne.html', paginate=paginate, infos=infos,bugbit=bugbit,bugtype=bugtype)
@@ -95,9 +111,9 @@ def index(page=None):
 @app.route('/POCmanage',methods=['GET','POST'])
 # @login_required
 def POCmanage():
-    bugbit = GetBit()
+    bugbit,bugtype=GetBit()
     if request.method == 'GET':
-        return render_template('pocmanage.html',bugbit=bugbit)
+        return render_template('pocmanage.html',bugbit=bugbit,bugtype=bugtype)
     else:
         pocname=request.form.get('pocname')
         rule=request.form.get('rule')
@@ -107,14 +123,14 @@ def POCmanage():
         poc = POC(name=pocname, rule=rule, expression=expression)
         db.session.add(poc)
         db.session.commit()
-        return render_template('pocmanage.html',bugbit=bugbit)
+        return render_template('pocmanage.html',bugbit=bugbit,bugtype=bugtype)
 
 
 @app.route('/setting')
 # @login_required
 def setting():
-    bugbit = GetBit()
-    return render_template('setting.html',bugbit=bugbit)
+    bugbit,bugtype=GetBit()
+    return render_template('setting.html',bugbit=bugbit,bugtype=bugtype)
 
 
 @app.route('/editinfo',methods=['GET','POST'])
@@ -162,6 +178,7 @@ def editinfo():
 @app.route('/domaindetail')
 # @login_required
 def domaindetail(id=None):
+    bugbit, bugtype = GetBit()
     if not id:
         baseinfo = BaseInfo.query.order_by(BaseInfo.id.desc()).first()
     else:
@@ -171,32 +188,33 @@ def domaindetail(id=None):
     else:
         deepinfo=DomainInfo.query.filter(DomainInfo.baseinfoid == baseinfo.id).first()
     buglist=BugList.query.filter(BugList.oldurl == baseinfo.url).all()
-    return render_template('domain-detail.html',baseinfo=baseinfo,deepinfo=deepinfo,buglist=buglist)
+    return render_template('domain-detail.html',baseinfo=baseinfo,deepinfo=deepinfo,buglist=buglist,bugbit=bugbit,bugtype=bugtype)
 
 
 @app.route('/buglist/<int:page>',methods=['GET'])
 @app.route('/buglist')
 # @login_required
 def buglist(page=None):
-    bugbit = GetBit()
+    bugbit,bugtype=GetBit()
     if not page:
         page = 1
     per_page = 10
     paginate = BugList.query.order_by(BugList.id.desc()).paginate(page, per_page, error_out=False)
     bugs = paginate.items
-    return render_template('bug-list.html', paginate=paginate, bugs=bugs,bugbit=bugbit)
+    return render_template('bug-list.html', paginate=paginate, bugs=bugs,bugbit=bugbit,bugtype=bugtype)
 
 
 @app.route('/bugdetail/<int:id>',methods=['GET'])
 @app.route('/bugdetail')
 # @login_required
 def bugdetail(id=None):
+    bugbit, bugtype = GetBit()
     if not id:
         buginfo = BugList.query.order_by(BugList.id.desc()).first()
     else:
         buginfo = BugList.query.filter(BugList.id == id).first()
     oldurlinfo=BaseInfo.query.filter(BaseInfo.url==buginfo.oldurl).first()
-    return render_template('bug-details.html',buginfo=buginfo,oldurlinfo=oldurlinfo)
+    return render_template('bug-details.html',buginfo=buginfo,oldurlinfo=oldurlinfo,bugbit=bugbit,bugtype=bugtype)
 
 
 @app.route('/user', methods=['GET', 'POST'])
@@ -216,13 +234,13 @@ def user():
 @app.route('/test_console', methods=['GET', 'POST'])
 # @login_required
 def console():
-    bugbit = GetBit()
+    bugbit,bugtype=GetBit()
     if request.method == 'GET':
-        return render_template('console.html',bugbit=bugbit)
+        return render_template('console.html',bugbit=bugbit,bugtype=bugtype)
     else:
         urls = request.form.get('urls')
         executor.submit(SZheConsole, urls)
-        return render_template('console.html',bugbit=bugbit)
+        return render_template('console.html',bugbit=bugbit,bugtype=bugtype)
 
 
 @app.route('/login/', methods=['GET', 'POST'])
@@ -306,13 +324,13 @@ def about():
 @app.route('/log_detail/<int:page>', methods=['GET'])
 # @login_required
 def log_detail(page=None):
-    bugbit = GetBit()
+    bugbit,bugtype=GetBit()
     if not page:
         page = 1
     per_page = 38
     paginate = Log.query.order_by(Log.date.desc()).paginate(page, per_page, error_out=False)
     logs = paginate.items
-    return render_template('log_detail.html', paginate=paginate, logs=logs,bugbit=bugbit)
+    return render_template('log_detail.html', paginate=paginate, logs=logs,bugbit=bugbit,bugtype=bugtype)
 
 
 @app.errorhandler(404)

templates/base.html

@@ -68,7 +68,7 @@ <h2>漏洞类型统计</h2>
                 <div class="skill">
                     <div class="skill-name skill-one">{{name}}</div>
                     <div class="skill-bar">
-                        <div class="skill-per" per="18">
+                        <div class="skill-per" per="{{num}}">
                         </div>
                     </div>
                 </div>
@@ -146,19 +146,19 @@ <h2>漏洞类型统计</h2>
                     },
                     data: [
                         {
-                            value: 35,
+                            value: {{bugbit.lowcount}},
                             name: "低危",
                         },
                         {
-                            value: 66,
+                            value: {{bugbit.mediumcount}},
                             name: "中危",
                         },
                         {
-                            value: 98,
+                            value: {{bugbit.highcount}},
                             name: "高危",
                         },
                         {
-                            value: 55,
+                            value: {{bugbit.seriouscount}},
                             name: "严重",
                         },
                     ],