update image download ignored

vinsonzou · Jan 7, 2016 · c93b3f7 · c93b3f7
1 parent 9789145
commit c93b3f7
Show file tree

Hide file tree

Showing 14 changed files with 12 additions and 4 deletions.
diff --git a/scrapy/wooyun/wooyun/__init__.pyc b/scrapy/wooyun/wooyun/__init__.pyc
diff --git a/scrapy/wooyun/wooyun/items.pyc b/scrapy/wooyun/wooyun/items.pyc
diff --git a/scrapy/wooyun/wooyun/pipelines.pyc b/scrapy/wooyun/wooyun/pipelines.pyc
diff --git a/scrapy/wooyun/wooyun/settings.py b/scrapy/wooyun/wooyun/settings.py
@@ -33,6 +33,10 @@
 MONGODB_PORT = 27017
 MONGODB_DB = 'wooyun'
 MONGODB_COLLECTION = 'wooyun_list'
+#image ignored downoading
+IMAGE_DOWLOAD_IGNORED = {
+    'www.quip.com'
+}
 # Crawl responsibly by identifying yourself (and your website) on the user-agent
 #USER_AGENT = 'wooyun (+http://www.yourdomain.com)'
 

diff --git a/scrapy/wooyun/wooyun/settings.pyc b/scrapy/wooyun/wooyun/settings.pyc
diff --git a/scrapy/wooyun/wooyun/spiders/WooyunSpider.py b/scrapy/wooyun/wooyun/spiders/WooyunSpider.py
@@ -77,16 +77,20 @@ def parse_detail(self,response):
             #同时，在piplines.py存放时，作相应的反向处理
             image_urls = response.xpath("//img[contains(@src, '/upload/')]/@src").extract()
             for u in image_urls:
-                if u.startswith('https://'):
-                    continue
-                #skip www.quip.com, can'nt be downloaded
-                if 'www.quip.com' in u:
+                if self.__check_ingnored_image(u):
                     continue
                 if u.startswith('/'):
                     u = 'http://www.wooyun.org' + u
                 item['image_urls'].append(u)
         return item
 
+    def __check_ingnored_image(self,image_url):
+        for ignored_url in settings['IMAGE_DOWLOAD_IGNORED']:
+            if ignored_url in image_url:
+                return True
+
+        return False
+
     def __search_mongodb(self,wooyun_id):
         #
         wooyun_id_exsist = True if self.collection.find({'wooyun_id':wooyun_id}).count()>0 else False

diff --git a/scrapy/wooyun/wooyun/spiders/WooyunSpider.pyc b/scrapy/wooyun/wooyun/spiders/WooyunSpider.pyc
diff --git a/scrapy/wooyun/wooyun/spiders/__init__.pyc b/scrapy/wooyun/wooyun/spiders/__init__.pyc
diff --git a/scrapy/wooyun_drops/wooyun_drops/__init__.pyc b/scrapy/wooyun_drops/wooyun_drops/__init__.pyc
diff --git a/scrapy/wooyun_drops/wooyun_drops/items.pyc b/scrapy/wooyun_drops/wooyun_drops/items.pyc
diff --git a/scrapy/wooyun_drops/wooyun_drops/pipelines.pyc b/scrapy/wooyun_drops/wooyun_drops/pipelines.pyc
diff --git a/scrapy/wooyun_drops/wooyun_drops/settings.pyc b/scrapy/wooyun_drops/wooyun_drops/settings.pyc
diff --git a/scrapy/wooyun_drops/wooyun_drops/spiders/WooyunSpider.pyc b/scrapy/wooyun_drops/wooyun_drops/spiders/WooyunSpider.pyc
diff --git a/scrapy/wooyun_drops/wooyun_drops/spiders/__init__.pyc b/scrapy/wooyun_drops/wooyun_drops/spiders/__init__.pyc