Site updated: 2019-11-07 12:47:57

2018/04/13/爬虫漏洞监控/index.html

@@ -92,11 +92,10 @@
 <meta property="og:image" content="https://i.imgur.com/vMyTInd.png">
 <meta property="og:image" content="https://i.imgur.com/ZD0jmCp.png">
 <meta property="og:image" content="https://i.imgur.com/0aJzerh.png">
-<meta property="og:image" content="https://i.imgur.com/imTIPIJ.png">
 <meta property="og:image" content="https://i.imgur.com/OiNOmVq.png">
 <meta property="og:image" content="https://i.imgur.com/pauOYGz.png">
 <meta property="og:image" content="https://i.imgur.com/ONkk7pW.png">
-<meta property="og:updated_time" content="2019-11-07T03:52:36.924Z">
+<meta property="og:updated_time" content="2019-11-07T04:13:21.509Z">
 <meta name="twitter:card" content="summary">
 <meta name="twitter:title" content="Python爬虫最新漏洞监控实现定时邮件推送和可视化">
 <meta name="twitter:description" content="0x00前言最近由于实验室有需求，要对漏洞进行及时监控并进行分析，于是就参考浮萍牛之前作品，并根据网上不同的源，进行漏洞监控脚本的书写，小试牛刀，涉及的过程有，先对漏洞源进行爬虫，然后进行关键词提取，进行数据库存储，并进行可视化，实现邮件定时推送等等。 0x01代码结构  首先对收集的源进行爬虫，这里暂时只提取了四个源(CVE,exploit_db,hacker_news,theat_book)">
@@ -386,7 +385,7 @@ <h1 id="0x02效果如下"><a href="#0x02效果如下" class="headerlink" title="
 <p><img src="https://i.imgur.com/ZD0jmCp.png" alt></p>
 <p><img src="https://i.imgur.com/0aJzerh.png" alt></p>
 <h1 id="0x03实现过程"><a href="#0x03实现过程" class="headerlink" title="0x03实现过程"></a>0x03实现过程</h1><h2 id="爬虫提取"><a href="#爬虫提取" class="headerlink" title="爬虫提取"></a>爬虫提取</h2><p>首先是对几个源进行爬虫，这里只用了BeautifulSoup来进行html标签的提取等等</p>
-<p><img src="https://i.imgur.com/imTIPIJ.png" alt></p>
+<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br></pre></td><td class="code"><pre><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">get_exploit_db_info</span><span class="params">()</span>:</span></span><br><span class="line">    headers = &#123;</span><br><span class="line">        <span class="string">'User-Agent'</span>:<span class="string">'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:56.0) Gecko/20100101 Firefox/56.0'</span>,</span><br><span class="line">        <span class="string">'Accept'</span>:<span class="string">'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'</span>,</span><br><span class="line">        <span class="string">'Accept-Language'</span>:<span class="string">'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3'</span>,</span><br><span class="line">        <span class="string">'Cookie'</span>:<span class="string">'_ga=GA1.3.151879025.1520582790; _gid=GA1.3.1728776549.1523330453; PHPSESSID=niirk5p8e6glodfoqphqteud73; _gat=1'</span></span><br><span class="line">    &#125;</span><br><span class="line">    url = <span class="string">'https://www.exploit-db.com/'</span></span><br><span class="line">    res = requests.get(url=url,headers=headers,timeout=<span class="number">60</span>)</span><br><span class="line">    <span class="comment">#print(res.text)</span></span><br><span class="line">    soup = bs(res.text,<span class="string">'html.parser'</span>)</span><br><span class="line">    trs = soup.find_all(<span class="string">'tr'</span>,&#123;<span class="string">'class'</span>:<span class="string">'featured'</span>&#125;)</span><br><span class="line">    total_exploit_db =len(trs)</span><br><span class="line">    <span class="comment">#print(trs)</span></span><br><span class="line">    select_msg = <span class="string">''</span></span><br><span class="line">    keywords = config.keywords</span><br><span class="line">    wordlist = []</span><br><span class="line">    <span class="keyword">try</span>:</span><br><span class="line">        <span class="keyword">for</span> tr <span class="keyword">in</span> trs:</span><br><span class="line">            description = tr.find_all(<span class="string">'td'</span>)[<span class="number">4</span>].find(<span class="string">'a'</span>).string</span><br><span class="line">            <span class="comment">#print(description)</span></span><br><span class="line">            date = tr.find_all(<span class="string">'td'</span>)[<span class="number">0</span>].string</span><br><span class="line">            platform = tr.find_all(<span class="string">'td'</span>)[<span class="number">5</span>].find(<span class="string">'a'</span>).string</span><br><span class="line">            <span class="comment">#print(platform) </span></span><br><span class="line">            site = tr.find_all(<span class="string">'td'</span>)[<span class="number">4</span>].find(<span class="string">'a'</span>)[<span class="string">'href'</span>]</span><br><span class="line">            <span class="keyword">for</span> k <span class="keyword">in</span> keywords:</span><br><span class="line">                <span class="keyword">if</span> k <span class="keyword">in</span> description:</span><br><span class="line">                    keyword = k</span><br><span class="line">                    wordlist.append(keyword)</span><br><span class="line">                    data = &#123;<span class="string">'date'</span>:date,<span class="string">'description'</span>:description,<span class="string">'platform'</span>:platform&#125;</span><br><span class="line">                    storage_to_db.storage_data(data)</span><br><span class="line">                    select_msg += <span class="string">'&lt;p&gt;&lt;b&gt;发布日期：'</span>+date+<span class="string">'&lt;/p&gt;&lt;/b&gt;'</span>+<span class="string">'&lt;br&gt;&lt;b&gt;漏洞描述：&lt;/b&gt;'</span>+<span class="string">'&lt;a href ="'</span>+site\</span><br><span class="line">            +<span class="string">'"&gt;'</span>+description+ <span class="string">'&lt;/a&gt;&lt;/br&gt;'</span></span><br><span class="line">    <span class="keyword">except</span> Exception <span class="keyword">as</span> e:</span><br><span class="line">        <span class="keyword">pass</span></span><br><span class="line">            </span><br><span class="line">    <span class="keyword">if</span> len(trs) == <span class="number">0</span>:</span><br><span class="line">        msg = nowtime + <span class="string">'&lt;p&gt;今日Exploit_db风和日丽，没有大事发生。&lt;/p&gt;'</span></span><br><span class="line">        dataframe = pd.DataFrame(&#123;<span class="string">'exploit_db告警总数'</span>:[<span class="number">0</span>],<span class="string">'exploit_db检索数'</span>:[<span class="number">0</span>]&#125;)</span><br><span class="line">        dataframe.to_csv(os.path.join(config.output_path, <span class="string">'exploit_db_data.csv'</span>),encoding=<span class="string">"gb2312"</span>)</span><br><span class="line">        <span class="keyword">return</span> msg</span><br><span class="line">    <span class="keyword">else</span>:</span><br><span class="line">        msg_header = <span class="string">'&lt;p&gt;今日Exploit_db一共&lt;font size="3" color="red"&gt;'</span> + str(len(trs))+<span class="string">'&lt;/font&gt;条。'</span>      </span><br><span class="line">        <span class="keyword">if</span> len(wordlist) == <span class="number">0</span>:</span><br><span class="line">            key_msg = <span class="string">'根据设置的关键字，未匹配到关注的Exploit_db信息。&lt;/p&gt;'</span></span><br><span class="line">            msg = nowtime + msg_header + key_msg</span><br><span class="line">            dataframe = pd.DataFrame(&#123;<span class="string">'exploit_db告警总数'</span>:[len(trs)],<span class="string">'exploit_db检索数'</span>:[<span class="number">0</span>]&#125;)</span><br><span class="line">            dataframe.to_csv(os.path.join(config.output_path, <span class="string">'exploit_db_data.csv'</span>),encoding=<span class="string">"gb2312"</span>)</span><br><span class="line">            <span class="keyword">return</span> msg</span><br><span class="line">        <span class="keyword">else</span>:</span><br><span class="line">            key_msg = <span class="string">'&lt;/p&gt;根据设置的关键字，关注的Exploit_db信息一共&lt;font size="3" color="red"&gt;'</span> + str(len(wordlist))+<span class="string">'&lt;/font&gt;个。具体如下：&lt;br&gt;&lt;br&gt;'</span></span><br><span class="line">            msg = nowtime + key_msg + select_msg</span><br><span class="line">            dataframe = pd.DataFrame(&#123;<span class="string">'exploit_db告警总数'</span>:[len(trs)],<span class="string">'exploit_db检索数'</span>:[len(wordlist)]&#125;)</span><br><span class="line">            dataframe.to_csv(os.path.join(config.output_path, <span class="string">'exploit_db_data.csv'</span>),encoding=<span class="string">"gb2312"</span>)</span><br><span class="line">            <span class="keyword">return</span> msg</span><br></pre></td></tr></table></figure>
 <h2 id="数据库存储"><a href="#数据库存储" class="headerlink" title="数据库存储"></a>数据库存储</h2><p>其实脚本里对存储的数据并没有进行调用，不存储亦可，方便后期统计，起初用过mysql存储，但是对于每个源来说，爬取的字段数并不一致，用mysql存储的话就需要用不同的表，相对麻烦。于是参照一些大牛的套路，就用MongoDB进行存储，用了之后感觉挺好的，同时还搭配了一款MongoDB数据库查询软件，GUI棒棒哒</p>
 <p><img src="https://i.imgur.com/OiNOmVq.png" alt></p>
 <p>这里参考浮萍牛的MongoDB配置</p>

2018/07/10/Python白盒审计工具整理/index.html

@@ -87,6 +87,7 @@
 <meta property="og:site_name" content="Poochi - Royal Never Give Up">
 <meta property="og:description" content="0x01前言好久没更博了突然才发现，最近也是经历了一些工作上的变动，从乙方到甲方，包括工作方式和工作内容也或多或少也在逐渐偏移。甲方对于基础安全的建设需要做的也还有很多，以及SDL的落地还差哪些，最近在考虑Python源码的白盒审计，于是找了一些开源的工具和脚本进行测试。 0x02工具为了便于对Python源码白盒审计，这里找了网上开源的一些Python脚本工具，对代码进行简单匹配查询，快速定位危">
 <meta property="og:locale" content="zh-Hans">
+<meta property="og:image" content="http://yoursite.com/2018/07/10/images/1573101979861.jpg">
 <meta property="og:image" content="https://ws1.sinaimg.cn/large/81a93ae7ly1ft54wnjc17j21aw04imy2.jpg">
 <meta property="og:image" content="https://ws1.sinaimg.cn/large/81a93ae7ly1ft53b1o8ohj20lc04h751.jpg">
 <meta property="og:image" content="https://ws1.sinaimg.cn/large/81a93ae7ly1ft53c1tkbxj214q0bdjto.jpg">
@@ -105,11 +106,11 @@
 <meta property="og:image" content="https://ws1.sinaimg.cn/large/81a93ae7ly1ft54j1bol1j20u90h0who.jpg">
 <meta property="og:image" content="https://ws1.sinaimg.cn/large/81a93ae7ly1ft54jfmy01j20jd05qwf2.jpg">
 <meta property="og:image" content="https://ws1.sinaimg.cn/large/81a93ae7ly1ft54jtdctij20j504vq3q.jpg">
-<meta property="og:updated_time" content="2018-07-10T14:29:24.000Z">
+<meta property="og:updated_time" content="2019-11-07T04:47:28.312Z">
 <meta name="twitter:card" content="summary">
 <meta name="twitter:title" content="Python白盒审计工具整理">
 <meta name="twitter:description" content="0x01前言好久没更博了突然才发现，最近也是经历了一些工作上的变动，从乙方到甲方，包括工作方式和工作内容也或多或少也在逐渐偏移。甲方对于基础安全的建设需要做的也还有很多，以及SDL的落地还差哪些，最近在考虑Python源码的白盒审计，于是找了一些开源的工具和脚本进行测试。 0x02工具为了便于对Python源码白盒审计，这里找了网上开源的一些Python脚本工具，对代码进行简单匹配查询，快速定位危">
-<meta name="twitter:image" content="https://ws1.sinaimg.cn/large/81a93ae7ly1ft54wnjc17j21aw04imy2.jpg">
+<meta name="twitter:image" content="http://yoursite.com/2018/07/10/images/1573101979861.jpg">
 
 
 
@@ -383,7 +384,7 @@ <h1 class="post-title" itemprop="name headline">Python白盒审计工具整理</
 
         <h1 id="0x01前言"><a href="#0x01前言" class="headerlink" title="0x01前言"></a>0x01前言</h1><p>好久没更博了突然才发现，最近也是经历了一些工作上的变动，从乙方到甲方，包括工作方式和工作内容也或多或少也在逐渐偏移。甲方对于基础安全的建设需要做的也还有很多，以及SDL的落地还差哪些，最近在考虑Python源码的白盒审计，于是找了一些开源的工具和脚本进行测试。</p>
 <h1 id="0x02工具"><a href="#0x02工具" class="headerlink" title="0x02工具"></a>0x02工具</h1><p>为了便于对Python源码白盒审计，这里找了网上开源的一些Python脚本工具，对代码进行简单匹配查询，快速定位危险函数和参数调用，易于发现一些常见的Web漏洞等。</p>
-<p>这里对以下脚本工具进行测试。</p>
+<p>这里对以下脚本工具进行测试。<br><img src="../images/1573101979861.jpg" alt></p>
 <p><img src="https://ws1.sinaimg.cn/large/81a93ae7ly1ft54wnjc17j21aw04imy2.jpg" alt></p>
 <h2 id="bandit"><a href="#bandit" class="headerlink" title="bandit"></a>bandit</h2><p>项目地址：<a href="https://github.com/PyCQA/bandit" target="_blank" rel="noopener">https://github.com/PyCQA/bandit</a></p>
 <p><img src="https://ws1.sinaimg.cn/large/81a93ae7ly1ft53b1o8ohj20lc04h751.jpg" alt></p>

sitemap.xml

@@ -2,16 +2,23 @@
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
 
   <url>
-    <loc>http://yoursite.com/2018/05/06/%E7%88%AC%E5%8F%96%E7%94%B5%E5%BD%B1%E3%80%8A%E5%90%8E%E6%9D%A5%E7%9A%84%E6%88%91%E4%BB%AC%E3%80%8B%E8%B1%86%E7%93%A3%E5%BD%B1%E8%AF%84%E8%BF%9B%E8%A1%8C%E6%96%87%E6%9C%AC%E6%95%B0%E6%8D%AE%E6%9C%BA%E5%99%A8%E5%AD%A6%E4%B9%A0%E6%A3%80%E6%B5%8B%E6%81%B6%E6%84%8F%E8%AF%84%E8%AE%BA/</loc>
+    <loc>http://yoursite.com/2018/07/10/Python%E7%99%BD%E7%9B%92%E5%AE%A1%E8%AE%A1%E5%B7%A5%E5%85%B7%E6%95%B4%E7%90%86/</loc>
 
-    <lastmod>2019-11-07T04:03:35.205Z</lastmod>
+    <lastmod>2019-11-07T04:47:28.312Z</lastmod>
 
   </url>
 
   <url>
     <loc>http://yoursite.com/2018/04/13/%E7%88%AC%E8%99%AB%E6%BC%8F%E6%B4%9E%E7%9B%91%E6%8E%A7/</loc>
 
-    <lastmod>2019-11-07T03:52:36.924Z</lastmod>
+    <lastmod>2019-11-07T04:13:21.509Z</lastmod>
+
+  </url>
+
+  <url>
+    <loc>http://yoursite.com/2018/05/06/%E7%88%AC%E5%8F%96%E7%94%B5%E5%BD%B1%E3%80%8A%E5%90%8E%E6%9D%A5%E7%9A%84%E6%88%91%E4%BB%AC%E3%80%8B%E8%B1%86%E7%93%A3%E5%BD%B1%E8%AF%84%E8%BF%9B%E8%A1%8C%E6%96%87%E6%9C%AC%E6%95%B0%E6%8D%AE%E6%9C%BA%E5%99%A8%E5%AD%A6%E4%B9%A0%E6%A3%80%E6%B5%8B%E6%81%B6%E6%84%8F%E8%AF%84%E8%AE%BA/</loc>
+
+    <lastmod>2019-11-07T04:03:35.205Z</lastmod>
 
   </url>
 
@@ -57,13 +64,6 @@
 
   </url>
 
-  <url>
-    <loc>http://yoursite.com/2018/07/10/Python%E7%99%BD%E7%9B%92%E5%AE%A1%E8%AE%A1%E5%B7%A5%E5%85%B7%E6%95%B4%E7%90%86/</loc>
-
-    <lastmod>2018-07-10T14:29:24.000Z</lastmod>
-
-  </url>
-
   <url>
     <loc>http://yoursite.com/2018/03/12/hello-world/</loc>