From c93b3f7655965f709c7ae239fe7d2c60cbe9371f Mon Sep 17 00:00:00 2001
From: hanc00l <hancool@163.com>
Date: Thu, 7 Jan 2016 17:12:28 +0800
Subject: [PATCH] update image download ignored

---
 scrapy/wooyun/wooyun/__init__.pyc                | Bin 148 -> 0 bytes
 scrapy/wooyun/wooyun/items.pyc                   | Bin 644 -> 0 bytes
 scrapy/wooyun/wooyun/pipelines.pyc               | Bin 4047 -> 0 bytes
 scrapy/wooyun/wooyun/settings.py                 |   4 ++++
 scrapy/wooyun/wooyun/settings.pyc                | Bin 769 -> 0 bytes
 scrapy/wooyun/wooyun/spiders/WooyunSpider.py     |  12 ++++++++----
 scrapy/wooyun/wooyun/spiders/WooyunSpider.pyc    | Bin 4393 -> 0 bytes
 scrapy/wooyun/wooyun/spiders/__init__.pyc        | Bin 156 -> 0 bytes
 scrapy/wooyun_drops/wooyun_drops/__init__.pyc    | Bin 160 -> 0 bytes
 scrapy/wooyun_drops/wooyun_drops/items.pyc       | Bin 630 -> 0 bytes
 scrapy/wooyun_drops/wooyun_drops/pipelines.pyc   | Bin 5491 -> 0 bytes
 scrapy/wooyun_drops/wooyun_drops/settings.pyc    | Bin 810 -> 0 bytes
 .../wooyun_drops/spiders/WooyunSpider.pyc        | Bin 4240 -> 0 bytes
 .../wooyun_drops/spiders/__init__.pyc            | Bin 168 -> 0 bytes
 14 files changed, 12 insertions(+), 4 deletions(-)
 delete mode 100644 scrapy/wooyun/wooyun/__init__.pyc
 delete mode 100644 scrapy/wooyun/wooyun/items.pyc
 delete mode 100644 scrapy/wooyun/wooyun/pipelines.pyc
 delete mode 100644 scrapy/wooyun/wooyun/settings.pyc
 delete mode 100644 scrapy/wooyun/wooyun/spiders/WooyunSpider.pyc
 delete mode 100644 scrapy/wooyun/wooyun/spiders/__init__.pyc
 delete mode 100644 scrapy/wooyun_drops/wooyun_drops/__init__.pyc
 delete mode 100644 scrapy/wooyun_drops/wooyun_drops/items.pyc
 delete mode 100644 scrapy/wooyun_drops/wooyun_drops/pipelines.pyc
 delete mode 100644 scrapy/wooyun_drops/wooyun_drops/settings.pyc
 delete mode 100644 scrapy/wooyun_drops/wooyun_drops/spiders/WooyunSpider.pyc
 delete mode 100644 scrapy/wooyun_drops/wooyun_drops/spiders/__init__.pyc

diff --git a/scrapy/wooyun/wooyun/__init__.pyc b/scrapy/wooyun/wooyun/__init__.pyc
deleted file mode 100644
index c4e556cde05f2416a59d3dc7a3c8c990f4d20b41..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 148
zcmZSn%*&<v?q_H+0~9a<X$K%KW&si@3=F{<AQ3+eAi;n}6x#yD^h1kNi;DF#67!Pt
z^K<me^Ybf9^WqCilX5bX^^22>5(_G!f-ovRJ~J<~BtBlRpt6JmXo5{{eoARhsvXFl
IVjyM!0IFdl0RR91

diff --git a/scrapy/wooyun/wooyun/items.pyc b/scrapy/wooyun/wooyun/items.pyc
deleted file mode 100644
index 3f2ed18bcd03e7535ac93de795bc1b8a5ac0ec3f..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 644
zcmb`EzfQw25XR3*8~P_*c#5!)9U%k^NGuGgDnu45O-$>`iKF-;lo{T*J8*VW<Od*$
zKmTl>KVN*|akY3jes4+`pOE@{YUYZhqK|<B$O!@v00#!|45|>m06qcU^1w!{lL<x3
zA9`MJLHMfm&^OO0>OW&f(~r~)l8{r*Km>3ItxZ1hc<OQF@rB1TkLMmQJYISnL$*%n
zTu4?VI3wY_DQdyF%a`+7SG}^sh}KNm743ji!Y3(I<u+6W3Y4`#YjGZM-HOHqqC_Qd
zPHP|T=eew$SJC5Mcg|tk?>G*vKzmzztd%=k*2PZnzEj4<yP-KHOJU&1+z_Q3BRZ4r
zi>B0CrQ<Pezg4nKPbrU+=T1@q%ym0t)@Vu1T#`g#%)B><Sz@c6S#hb^y16a?n}xfB
P+tWqg*%Fw5Bo2Q8eM63F

diff --git a/scrapy/wooyun/wooyun/pipelines.pyc b/scrapy/wooyun/wooyun/pipelines.pyc
deleted file mode 100644
index 5dcbc8e523c707996dfb2f952077bcb729c742f8..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 4047
zcmb_fTXWmS6<&bUg|d`5u`P>D-9V}}<}@~0ySde)WLjP7$&4bokYcAQGZ+e3k)S~U
z!7eLn#(GlU`vcma(_ho+TYo_NeFq>Vcj7mR!olv@%em}#&RNv{v$XL1;ENZbs{U$t
z{|=A&2S|i}M<q%fP1}w-a)j@y#8o3#9l6S>=GD|uZL+4Wj_Q;4j5?Z;c1<O-YBZ;g
z=E^m7mCUPALmf5HpHX^2onZ4>^;e}Xl={=r4HeBzxbsu)B9%I5HK1VpU;OR095n4b
z74BJW<B`7P7;SBw_RS?{yU6lytRC6-vEbV*?PuGY&*NMtajK_o;EtPk%tH_j2Z+(o
z3+6&P8!*+C2JEa!V@CN0tvVKjT+WA`Ej;F95UjJHu4?jCQ_T}sEx%DJkiE;5dxlz#
zs8w|Qw9{EPPuC;+9(dpG?>*b!-V6?Q{1-dEUA)42zV9ElTrhlb#0O}FSqf`xoTY)W
z1sqV?PkoF|PFi>?A?9neI^P*xn9iV=hFO+$PP6Q6oCf*$IEllK35#BSRxu~9ymFXn
z=V#*iAc)i01_6Z|>Y|*GFFwSSc;N*auzlz#m_2mWg*1YibZg4Ola6XO>601Z%?J-3
z6W*+{b80xR-Z*O5m@Hl(vJP65v1b=RCyv_Qv>#sSZtd^x?ra@?v%lB6K{e(;@-y+Y
zB}0&~mBc!={wybugk@jcN=u?+5fNrdQhHamNwR()F1AEuU)x=Dbpd0PN_!)1sT!t~
zCq%KbwaMd17uS0bv?I$k&RjOXjjmx>DCdrI8^7s^&>%Sx9v9C-4&?$9#hD!5EA?GM
z)6Rf|NoMq=4L&!&UhN5Kky5GuGxR4=q7SJBA5nZ3I#y6}2!(Tis76cyLFhVFM=<8l
zLJZbb48S4U5PNt&$GqW;g4I!7y{fC?d!9@R9L}lpx_VP*<CUw5zgPGK^l--584E=Q
zwp~y|WGSGv+;*A#w6ET{YPhJ1e@#{{{hw$kGTjOZ@AC^}QEFLkH~02-E3!*DB<QCB
zA_-E<L|8&U9`*V<7#E2Vh)LH7EUS=?qZkfU<{rqD^>p2My(rR=XEToh5go4@qW3<M
zr{O8QHC4ZkjeR)InQK?gFMfJB&8$~VX)$|AG=y1|EF!IQ>F`+dEcco8WXegLMkU;`
zacYHiSd6u>{?JRH5JNmpk*{o!rCMf0acKQp+=B3k^zpba_LmCCin7TSEza>J3~6JG
zN+4JkH|3eJ0SxQen;4s#9q88vMqfYN^CAniF#)yUnV1z2<-YGMIZN(m&Z4vIeB?Hq
zPe51PmZT-da}Y4G1c8_q1fwh(C+PYd^Y4({CHWP|rOwj2hIH@q(C8jXEwfIe)|hS7
zY!jT9QaBEJf6<58ZWi{Ech^<IO24D7qKY!vh7?4bY1S4V^G^^)7Eaoc$h^fsIbiIe
zZXk6bTFq}Hc-VJk&a+7jN}ZOt#Tr0mILnz}Fn1vK5OosT)!v!xjpByv(5yt<MWo+F
zZU9&is>l$CU0hKjOlqhIrRWCstNm?7og*Va!VPX2EvhEPzJ!8*{GJN8m{s?D`T#i$
zhCKl}9Oy!OvGG!%)SF`Op&funfOt`cG3y;`&%njes?@FB{jJ~c1_y`x{*EEE+XFjF
zBrE_64O@U?sn8n5J>pe}v&Q!1(U%e=x+t=u$`}1KE6R}KCM_mpk&*B$ago5Nze*+}
zXAf#h)cN<(^FJc_IR^=};GIwif(hs;qr9<5sfD(ok)2t>B$Q7J*e$S^fb}0x1`Cf*
z8D+%w>_DPHEWO@~jReqRNScOa-SwQCP8|Vy2XxuJ@7#5zs7|0NF#R`{yp2h3+AlG=
zA%H{qLA?U_fL~xtGCFPx2d@QsfLe3=hftln#5{o3L;(ZX0Hxx5%EHAO_(PeS%-Trd
znOUM4sH_$)1D$LxKuD$7MJd)%8$Ui%FRKNpWQ!9`FDF!kODcNrCkE#MXQuBx|1(g(
zO?Ukg#11APjw4nWp0;<KMQ4P#9_99IRRX|g;*dKM>lTyOz}g&F-_z66b{Pfjtmqr&
z_l<be_e>Ff_2&m`Kdc*4!%&*?AFP`P0-+qRFht`9uJy0HL7`8+>aKM{V>-s3CA!;2
z(@^x432W0~8QtY5ZohGEMQC@8ni;5OP$1&4TdCKjeig4XG@ap(V_lqeA3g52Kkc@k
zblami?Y4(T+;Vlpwdv$XkK3QNpKv1P$(N1)1*BgIF9|!=KFBVtIgRZ=?w*CtlU}GT
z6DDGpI5gv9-@{w(#Fv31V{o9<0U_5=4CK?~?U3G~3Jg-^c+LvK<OBDXbI*C~-osBm
zkNi(C=szT3iSoY$xzy^qRP!HU+h606yA0~oZArU6OP9Y%f~U$`zhz543rNZ@o;JRr
zPA<QwDyDuN>YU$I#;40Ag=WV?d<{rl6rtB|izM)kvd(`^xrU!Mi?|@%#8bX+&MkZY
E1MOXQCIA2c

diff --git a/scrapy/wooyun/wooyun/settings.py b/scrapy/wooyun/wooyun/settings.py
index 7616419..63f66f1 100644
--- a/scrapy/wooyun/wooyun/settings.py
+++ b/scrapy/wooyun/wooyun/settings.py
@@ -33,6 +33,10 @@
 MONGODB_PORT = 27017
 MONGODB_DB = 'wooyun'
 MONGODB_COLLECTION = 'wooyun_list'
+#image ignored downoading
+IMAGE_DOWLOAD_IGNORED = {
+    'www.quip.com'
+}
 # Crawl responsibly by identifying yourself (and your website) on the user-agent
 #USER_AGENT = 'wooyun (+http://www.yourdomain.com)'
 
diff --git a/scrapy/wooyun/wooyun/settings.pyc b/scrapy/wooyun/wooyun/settings.pyc
deleted file mode 100644
index a1312989dcecf868ecedf8b0e4abe61e6cb1bac2..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 769
zcmZuu&5oNe5FSE95|UI^d+Dj_sfv1l+n(B0NpM$4{0Ycz<qKjl%erL&t({hRguYoH
zq3_Wb=s-xam5T8<GoQcljBWMLQvZE9zRqFzRJ5OPimwO^fFdeTDnOOap(Efbun{m7
zF4!|*V>BAD30%;gr^ro#j(fy~8PEw76QEOII?x%gIZ(adc@As=bOCG$@#;C2lN!R0
zrrGam(ZvVX5G`lLqu>fs;#~*%W7h7CmbXR4>p~d(AuG{1y$J`JWA`B5A`JVS^<Gn#
zjlDi8e|5_D|5M)d-Nf1VB5B+v&#GgN9Haa$t3)AQqh}Z<{&!U-zL`R1lIP~_t`w%6
z;bPS#-Zw(>pBzs)KNU(VE~LL&A|a;;>mW&e%OkQoeH=PAiBd1Hw=Rk03#5E<^AFqW
zoP>C3=!C>|{6QC5S0weUFR4vFTU$4guQ1sSHkO;lNf40}-__ZMww0WzuDd5vxxm=*
zaKreJ+t>60|0=N8X-uMP67`xR6$Vivbxb-E_FBGt5^e(5C7Z+v{1vir2JsO?b1U$v
z&HJp*o2D`kFPFBv+X`>=@a_Y7G@_7_*QGGpeSe1^9vZf*ip#EV@diN~%@j>ll?e`A
M)f60~esrGl7ho8@I{*Lx

diff --git a/scrapy/wooyun/wooyun/spiders/WooyunSpider.py b/scrapy/wooyun/wooyun/spiders/WooyunSpider.py
index 455a4e0..1f74540 100644
--- a/scrapy/wooyun/wooyun/spiders/WooyunSpider.py
+++ b/scrapy/wooyun/wooyun/spiders/WooyunSpider.py
@@ -77,16 +77,20 @@ def parse_detail(self,response):
             #同时，在piplines.py存放时，作相应的反向处理
             image_urls = response.xpath("//img[contains(@src, '/upload/')]/@src").extract()
             for u in image_urls:
-                if u.startswith('https://'):
-                    continue
-                #skip www.quip.com, can'nt be downloaded
-                if 'www.quip.com' in u:
+                if self.__check_ingnored_image(u):
                     continue
                 if u.startswith('/'):
                     u = 'http://www.wooyun.org' + u
                 item['image_urls'].append(u)
         return item
 
+    def __check_ingnored_image(self,image_url):
+        for ignored_url in settings['IMAGE_DOWLOAD_IGNORED']:
+            if ignored_url in image_url:
+                return True
+
+        return False
+        
     def __search_mongodb(self,wooyun_id):
         #
         wooyun_id_exsist = True if self.collection.find({'wooyun_id':wooyun_id}).count()>0 else False
diff --git a/scrapy/wooyun/wooyun/spiders/WooyunSpider.pyc b/scrapy/wooyun/wooyun/spiders/WooyunSpider.pyc
deleted file mode 100644
index 2d938ae347ae916398c77c2625812a7e40804c16..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 4393
zcmcIn-%}jN5$?S^4(<>FSwE4KM3#+pk}TG;WGAJ<DiB~>RR~lI5;la|T5gwPVQ=@e
zGZ&x&PfAtZQ<azeE&f09XXL@puH^fA?+(eSO8j6rW_G5hXQrq7>z<zSk7M<p)_?!H
zDZRfkzOyJMLKWiQkwl~|vLWKIUy{U;4M)e&F3Cno+hs}0(yquxrQoYbG9>M)Y*f)7
zlBgtm!?J^URr!m^TaoWKM!4E)V;DQd|Hfa#LCcPC<G@BXZb!=nsSeV5mK}A|=Qe7e
zvYE)*IBgkwZoshGiNh!#9H)YTXDDVL6^>L0PLUmH;T&a(z(z%)AqmSe$xo&D49O&=
zE%OZFFvZ>&mT*KHkiDjjnuK+2)Fm9%#;AC!4N7FIs18(a!y^>U=`3%V+vx6E+j-);
zr&M>l+cIt%9r&H@b`m#TI{~(rbKflbFXq1U7Z$&sd$qJ?FM_zVGC#NEudc0liv#+3
z&|ke=m|Gi=nmj&*;CBVZ{2i4QId<f&BgY~>41xxXQ=(1wOG<=RKrFvctWZ_9P!c~-
z(oRXPaFMFmVc8qu+`Vdl<(hVDIt7}XBx7A^b){8%v{AyRK*6%5pf>MDW*m)nmbS8R
z8#<aYPo_f41HM>U{$^$2nZLT|y<YUx@O{$D6>rT_&waweGyB0i-T9TJrN#NR=PS#F
zbqMu1wF<aS(2D$aaEOD6B-uff+fi*bgTyyB%Of>rw?kxlB@EdMsG3=tMok-MDVTCV
z5qc2m9I52G!V8{ZK8Yi2L`yV_CRYfzRoZ5jB*hvHGN=knl<assu15rT+)C*Iprgy|
zAF$Z~U9>XK(Q;pzC^znIkT$a{=>e;cCDY7<&QXup{}}Za;}*_U_|A0aNLAwdaT;6S
zrv!$wNNL=;>|8G0Kz+$+@(3(++IbX1DTx65171bASXF0TZjjNtj7k;UOfnM{v}>gC
z+<6q3EIqRm6}rc^;SFqND326ppv4HgCW`qRDmVwu01tqR;3Pmj8Vn65>K*VDU>}~6
zKM`A|?_l_(ENKDw65EI0(q5PYCqjMoZAlik9?1!4Lvmb^D<}A1b<{&T9UfF=fIP0M
z3P%JpUOR8rWnG<v(l47y)UMmvoNXq7F<;#wXzy&fHafJ8yM}Q7V|YKNJz^9w)a`%8
z6i#xd!bS~)YK+59V0SGQl1ByR#`;5fkhUU)bT?1-GB^tbDUhUOFM0!CwTk0TkekR)
zVq-O;dumw*^E@)0EJduQN7>B6qbP(?ltpQ%Jxj4u1vv+ySP^V`W_X~VtHi%W!w}=*
z+;qmB@lwsX?$pszfRk+ij=!SwGjIUrK*ixcVX&~(r0t8i`3`C-YSJ<h0o0i9wW}}y
zaG+5_>836W`<HuYf^zl%C@cqes`ub<Q{LR{A%a3J8+w%M+U+bna&7L~&<)(#-8|Y+
zb-FRF+MwFK-=R{I0W5jYv}$_ONn(2%UDgnI?Qg*7zZ(yH-$X&)-1U3mSZ976Bqs81
zbLEd%eaz)~=M^XjBOAm?gW6LZc%N`wV{4r58}B~5L|}3G`EhtwhW4hg%Uu*h=>(x`
z%(>`%+_x|B%rcK+#-T1;j)hHWXuTX61@v-+NhkNzpBWEu{lVsN=b0X-UVQI!4r}DV
zG;)p%IBO)o&j|}-%~5Pn;r`@5Co@MMDl9NE_Wl%(pO=~S^ny5!6OQWy5K>3JE8+_7
z;c!C23q%gNKN#$OQI|p@b4mB?9qKYWKre6y!!T@N0Hj_AlAU&NbadWo><h097QXuH
zE__X9T7@Piv4wg;K3eK2`?~I>W<gLrTxC#Q*!or&SBGQ|ImHcf<o|Tu{NSWmrXZ?!
z<v%(h{B20Cf}3Md&m=T|t{03sWxb%UHs}t^h5&Nidz-Uy_|<J>X$y+)+golJ|7r7!
zEqC|P=EJSN@5Nvi?*S_HKko~)^a6#a#x{w10b5=4Se-)2<`(!|F4d3k2S?b-w@01R
zDBmrE>0dwmL!-O~&_0W~r_=wZjE^Z}ab0L654v_Y%MIiG%u9yvTZ?&~<xdSgwQF}C
z{6X2`R>}m3A!paN6HPn*k2au_-nDccvo>)rZT<Q2$%CTt`DsIy@W2C){E}tA>@RC6
zbN`j#XC0(h%>3du?hYUKhB1sdt{b;on@G|SjZ)K?HF@*XTX)=UC&_})y>oZVCCTuT
zVlr-!X=+IAXXXb7)2G>agIjvMrZkv((`>n@icpI5ucInrFjlpNQ8No8?-uz`#M1?M
z)u^n-2Dvo{F)m+5prC_8gqi}r6Edmu7;hfEpF`_C<!s8(J+pxM2&+FtF+89+HRm3F
zpAD5uA39~{e0j1o<vi9>cWM=e({ZPYGUi+>u1q47V$Y)(Vt<em0<>^nBvxO-H3#u;
zzasg4W>TqMkxa7qD&#WD>O1OB#a(h0L{;)moQA6RX4Xxu_ZjHk=csgAp+9b|52(Q%
z<41>v=?MnTBx|N=e|}r8gJ!4>DLpI|?+(Jv_tT&q`Mx5-$JH8k6L!hT5zF^aknklE
z`eD{4eos-O2-jDL$86SP4&!~winpCd3wku+=}EQq>pl`ZTBwF%E}^Od1hrDFeBPO;
z*UGig+DPpa%`pq;^UbQ6f+h()ry{V8dX%PyMP4}#cikyoSagfxO`?09>1i~U&J=cd
S%Ct#e>=Vv8{3e`<@_zuIzQZK|

diff --git a/scrapy/wooyun/wooyun/spiders/__init__.pyc b/scrapy/wooyun/wooyun/spiders/__init__.pyc
deleted file mode 100644
index 4a58b4a60143256a3486d815f3b1deeda8493fe0..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 156
zcmZSn%*&<v?q_H+0~9a<X$K%KW&si@3=F{<AQ3+eAi;n}6uSb&^h1kNi;DF#67!Pt
z^K<me^Ybf9^WqCilX5bX^^22>5(_G!f-tJMATtG|B|bhgFS8^*Uaz3Cgac@jO>TZl
OX-=vg$i8AAW&i;D5+*VL

diff --git a/scrapy/wooyun_drops/wooyun_drops/__init__.pyc b/scrapy/wooyun_drops/wooyun_drops/__init__.pyc
deleted file mode 100644
index e94e697bdc112d100967502dd03ca0a14df6d764..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 160
zcmZSn%*&<v?q_H+0~9a<X$K%KW&si@3=F{<AQ3+eAi;n}6ng^2^h1kNi;DF#67!Pt
z^K<me^Ybf9^WqCilX5bX^^22>5(_F}f+<D$1;wbG`1s7c%#!$cy@JXT4xni^x%nxj
NIjMFaJBxvs0RX!SC>sC(

diff --git a/scrapy/wooyun_drops/wooyun_drops/items.pyc b/scrapy/wooyun_drops/wooyun_drops/items.pyc
deleted file mode 100644
index ee5f357a7c91be159e584a5c074a2ac28d4bc629..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 630
zcmcJM!A^uQ5Qe9);L2j+W8g(zz!>9Ujh8*JQ6rZIp)TDNN@$7UtZ(cC=yX;$^#Lr*
z&o|RyI+=!_XUW|rdaU7Rh4{TeGZ!cgz6=yVi4cGQ*fV%#(1h>-@DA`?1}<ZpEihYt
z;o8^)@k#4>>hG=Sj)fV9Z_!MPf|W`JB7lA9%9IO_BadT`Pdr|FeCl!H@d}D<hG#*R
zqSz%0=Y7=)&Z$SvJKapmxiRjUdaQ=oMu=-ER6}H?Rf4vw$+mi=DotgDm7O4`MrWtT
z*}B0^qq1%it?d+@A-ifPIED=sA=ez5!ZaW<Yoz?ah|%P2)z@09{Eg<vhv`MhIzP1b
z8*D~Bn7@w1L7U5AE?m(SnmI>_!;~GLAZ3|rv2a37y4#^2{;waJ(A8ng*X{yLfRcva
DMpKNw

diff --git a/scrapy/wooyun_drops/wooyun_drops/pipelines.pyc b/scrapy/wooyun_drops/wooyun_drops/pipelines.pyc
deleted file mode 100644
index 2737e810baffc5661514eed2ada17eca731db227..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 5491
zcmcIo-EUk+6`y-|y|$A$cG7&A6uKcFTH3^mP~&_M5RILdAlDApCMlNCWq0q`zMI{>
zw|8c7EX7i#v{flZL4v9httxmxR05?uKwE{6KJc%^PWsFP4?Mu{cV^dXS1AOk_Il2I
zo-=38oUggLe+=z-;Y{IbO?CcqsK13`-T{d4@2Q4T^BZkX&3iKLtA?+dzMA)y*IAcS
z^SLfgUd`vb?LIZ%C+(bS6jZZc&G$Q;ylM=n=AfD%M88kz9clr@3+iR1ZYlNs`JF22
z?~V>^jP4>+v!y{W82=l8(<Kj0J3xjjR@=C#dm~0$8@CosZ)7q{(#uvi?Jy9YPg;w~
z<i#s-svB`jZ-{{uFQS;!02%_2LxV3I7rV28qx#aoIdjs$A@kBGs9?6#2h1Aju;fuD
zP|OnmAYcJ~HOeZdMi+c_;HFZcY%*Ko;%P(nnq)7ko-db=oAbvbyBlNA&rClzGkGzb
zy%b!%6xdyxhOW#6b0r@Gfk<QzTD7EwW7#-qg~n!(gcAg}FuJi&LahXt@1Rxwp3#{p
zFI8H#Bx#hFlVqjc3e$GA5!cG5mQ~V~?qCElnysC*BSkZouE=$SVcd#s7?Ke~zLZyE
z#X?0BiCjeka;SR>BB=Z7mNdegbaTo=yq+2zrF{BitWU-uJsB$~+pp>a>ZYgagI(ku
zRJYj@_n5Xj0J}n)ylD4s>Q2l|O<kIpyF4>p+DSJ00n!zby5)eV$3!F6EgKZLfS~63
zB4VB-s>&9%q|tDa7BY=waS`ISRMSOmr_j|I4x?17($to$VTrpfZY!is8b><YzDwbm
zl2k(%WTSoP8rqHW_IvyA+qfMbPHu-+$9s_rx`l-zSWZtn-iOe%qU(*s=-zn(65qa)
zQ0;5nv2&C!px;$7%9b3l$RQ4b-;RoTRFPFESB|;{%|h4fbVT<&o>t~*nN&s?dQM&7
za9-8>RE!}QYF>?^jAKE)pdc9duj_f0B~+po)OJ9v=GD!-GQSu30d-J8Pe2VQvMC7e
zRCRc>LYLqHj<*)oO<&cARQ9hf=I&2z3Klj<!9$MUf)6Htj-1#6`E%qB2^BXrxMe3C
zW41)=a5fS*D~md8XAL8|C|x6VT!UL(OtKZxS}cfxqs&o&j)$N#C}QILaZ{{Bk&cQs
zDbkFh>JdYoKL$T+iVsK7H)3N?wo^LqBj(tDThvNyv9qv5|0woUOFHf*(mIt66(CJg
z`z6$&*l^FqhhejlhTRhd8&nqJR^;?JX}7Epp3B-=hMui7z&|Awx8NXcn6$L4h~k<J
z9wQ^vwMbXniy}M6qhPouF(e`cV(S!1j2I+FA}%6Lj13`yiY2*CVl|<@VIZLGHKjC5
zYTB5POnngy7}}HahrJ<h$Uo?x_6pts@324UJq38k7e7HMvrM2ExCIr%?ZPsAH4cNJ
zalj^^+~ZKuXaLoqigutvRNsYaNJYC*4XbDb)gBe?MfI49_NnnzpLz-_xL#1%57|I>
zbXN3rKQ@|f)BM-}a(n%c@2<V}>iWAs6RD?_6w%Agz5derJMUxe-re8HP1!hgGix9I
zX6=;^*MITBCvU$hw`LLP+|<2~etG}RHy-@y<p+1(7o?V6Hf=iaWnEpn^W(Kw-V+Nz
z&u#M2Tc5mid;Qn1-TT{5<gVdatv1XzWd8nd|Gf74dmThZj%HGg8`|N%^{e}LKkm%Y
z1uw^$Zr}>q)X4QW|FHhkJ8O4;zG(rKqgpXq7Tml0#)FUFUVH6>P1LJ#tCFoaJ>k7L
zRil6IYI`W5M_gyeqG=4<r&&x_w%Xg*Fk;An@}BS>_x5<ZyaB%?HbEhT!eJPSG=Z5!
z?S>A+fO7$FJ~&8ll7N|1k9BE%&N)m#DUP5Rnts6>%ncS$_S;c(uSsq+TltZmOQw=q
zrSaK0<WVfw|0{<?#>nkz1k@ECc>=}!6@UQ&%^}(W&43vZEbh9m%qel{?897ql|3Ww
z9_uA+NuU7^)ES`*m>6L6L$E;9kyvDELKR_)F@rNMfPf?xA4C*`gQ_EPArTXMcIND?
z`IeWveMqf>bRWYHW93f4^R9uv@4g@=%ln=V4g(?x;}~ZF=9YA(i?J!T;2DdX{2jx{
zNxOs@`l4c#V1+GKb<sf7(Gf|@re-FdnF?p;W`av%r_d8^nUgF_xYEd8BPRWH%5A%F
z>a4g2on=YZiEWFmBy%o&$u=7$awwD=yh&ur-PvhyCk{S1fhED?1oV&Mj>FH?fiUbk
zRFZeuI92~qFdPdDa1>&#l5ygEH6ur5_9kv?IfHykoE19{zQPuBuHZ=k7gH*B$(cN5
zvwgX~*y9CkVGaQ(M2Wm#^hN-Oy<xz^{xR=hFJ>_KiDpwGP!MRg%ft90Vn@^xF#!4q
z#pfuiKKD+Vl`~IEu2(K26T)E$%3`F)&8g#Nj6#o@n!Cpi)xe~U*ouWYafvokddzq9
zHu^w{q^AyBFZT)=qc{|Np~O@s=;TV|^G`Ik8hdf0?~MI|STaNi`M|1OS}JKD0@W-;
zMky9(@i1CgGQu!y6{fpdU@TsR&F@sJ0}RNkE;oeLBh6*RMOF(hf;I^Ev+Y>9(BX(P
z_#f!5f(yX&Yh%@ov$B3jjZXgHq*{gha9)dONE3F$UkyuZgn8-)+KYcz*P!0BCB8eR
zgGs>P8vv4Q+NG|ih$Cj2yyjvxiB{-GbThRpM?~*C-XxC9(?DIaHpK(l4USd6I@V4r
zF`l(!wWL`#NVxE{t($WF#kS5?#+q?!tZvMiM@OtCiN({pl6G;=JtA(+nDV*P`swkr
zXU>f^E7%w#=KsOox;3#GnL1~35A@#2a4Lq(&9j)?N&;)X{HZnNhqL+6Wm|~>F(5zU
z`Q>xQ=DqPnLmXNPC*|DOvs}yU*ZJC`Xvs@D(`lnp(*d`XXL`@!$#uJ%3+qO{g>3M^
z0R~Y@HUux0nNV+*BaH1GW$6*wlz;bDgBJ6$&f*P{^tYX{-da+o^MSts+@7y1F0<Mk
zWahAyys(s4^bUD>WbE?uyY&h0gulmo(s$30^H?5yoj{VQZ?p9rK#yf<ecl-rqXc&~
zbaKe3_6_DsyqADiAyUh9fMzGuUS><a%n-N_P_Wz9Lhl1tXHeg$>6D+jOhDO7z)z}m
le2|eWF5t%7dp1$F3od2-CV4P>0d^t79YJyRNdJMS{sTS2Ku-Vw

diff --git a/scrapy/wooyun_drops/wooyun_drops/settings.pyc b/scrapy/wooyun_drops/wooyun_drops/settings.pyc
deleted file mode 100644
index 4ea1b8d49b932b686376d3921a70bab88e3b8502..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 810
zcmZ`%O^?$s5S=vJ^m{?Tg$qa2Q^ILa2$42s6)9h8(k}8P3URh;$ktKqg8CEqXZ#<;
zor&A-c9DpaH{<6wGqK0!@5%7b%3YOkd`z?-@snQ>7yv~yU~K?3T8DOko4`82Ot@kf
zz`AHGU_H2^ny1M1fp&onfa1asXb-9$&_1vc&;hV9&>^r1(2>qR26hSQ^cj{@V}u_<
z?01`-ZAByHtA7YvHk?&k$qihPZ(At0MYFdX-c&W;RMO`6#Tt#TO>(3;?&P39Pk$B_
zip^Shvj_dJR{sv+Kl&T}mud06%0wW_qCQ2c3k*`*U8TC{+!nR0<ZJY7+rjs?F61{y
zszUM7(OY!X09Wfa#hs9f|Ku2MA$2GmO(*E9KEf={^2iN|y2MGE_#Rp0VeBmflB(xO
zMdaoQt95=x!aVU468O<km$=s?58cnXM?SgBAX6_fIEd$Nkf&L^AP;<7XPJ0z_DGGM
zN_E5h5EG_{SY0!WqwCn4<tbUL$U<vQR1z;THN+$*!keiVXVZBc1Z1B1aWq919zpsz
zb(Ru)@7xueQi$3)wx-#Asqr)q7vV%6&i1@{j;xg8o3*r?y*|(P_k!)}>O*_!<QoL5
UGcYXEG<rCOre)yh=+Sz{U#*_akpKVy

diff --git a/scrapy/wooyun_drops/wooyun_drops/spiders/WooyunSpider.pyc b/scrapy/wooyun_drops/wooyun_drops/spiders/WooyunSpider.pyc
deleted file mode 100644
index 48b248cf5393320b281a70c1942eb49d953447fa..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 4240
zcmcgv-BKLK6+SZy3k&GalCX(Mg{oN5I*OQBRY}<@DJ2BhaY_Q!fJ8!~ni^(?Wngw@
znC>-D1-_B<2)Rz>5pt2rizJVbi(KacobNlm{Kzi*&cIGj_vzE!=jS`8&$Rw`W#Zp|
z?Y&B5_}9Yw2_7>*lj7$|A<`Gw6!AFrrSPQh%ck$fEh$>E*-~EgM`UwE`y*0}O1~|e
zZO1n%#hCQRWpf<kwqzr++mUTg(lPl;<c-K5HYX%)$y#?DC+A<|uj`>_IZR`l*}R`Q
z$`~kva(__SY{gL~K-s9Oqe1z(&HCr-OlECf_Kcl6V_2*6G;7ZE9l^v?JZ29~1~nlO
zTAzydJXOrs&WL2Aly8Qw5%(IE8L09V^luOK*c_8|oC-nBw4<Glq!ZeKdXw6jlyI#}
ziEJCqfy!;Dg6qy#P0zfC;f}TS!yrhTsy6fIbAi2%$rlTsEl1B6ejhC@f4cDU*}A<6
z;<MGog=f*)`f9j*M!yRB%NI)v>u02dO7m#GSMiv?ps^yyp1kqoSR{fK(D7y0r)&0n
zCBn%dw%#Vf+rC(58opqfZC|FjNn7lg?2dEoZu@lSjt)Dz1ooRD<Al;Cl-3^7Ch3Q@
zE`7aXDX47*nYo5ezbboGx&_PJF%R#gmI^*!UHNQv>1njK9KKo()!nD07pvjArFBmU
zOHb`P7lw<g&z>zWu7AF|;(Wpg$V;oZtK(i4_2WY*CQ?)fSz{-)m&8S6Y}I7y!a+@d
zhFocHph>E-%o3YdC72oj5Oxr$k5qEq;W>3!Eb<Ho(G!X5`sk)xDs55~h1;V`1~q}n
zitUgl2zg3KEtDPuFuKj&0lQt$MH}-Hz2K$E8WZfqWl~i|;D8j>gRLS@0+TdxeRN7x
zsJ%Tf8ZRc;P?);tnXix3JW-UFxs4)9W9SC)uX(q<+y47#Z+QvT;lnsy#AC=vG~frQ
zxs0-=;kwc#V>pdQjhz&g$sBElG}@raVpEluqMh>zL`Y|Gu%T??^x+KQnBXx`fk6X(
z5E%#^XoYA%hl%1uqXrR#0V1e*D7M9z!t_Z?N*4$|`-h*<j|d$E7ut8<__Fl+133Y0
zRE|ev`ec+=K@c5d4C9RnSYCPi8?vEMLZMd7Cuor_@-P1QBq?HJ9^XBv?-Rmh{6&8E
zbztha98S$Wp1aE#1G3EyZTG{$6|DZxpZ`AB{Zo4H!@2+d?H@S^lGAtQCuqJa8*!W9
zPO52JU3XZ=cE{3WP3F)_Xy{ZEmpvCU4tE+CVYZLRwwA!I<HlrBT^Sn<n!+gbLj?e@
zfhIF`RU)Y~2yJB%gsOysF)LFYX+)5G9>Lh^Qc%Mzp&HO$!+@Ceu6ft|j{hTX!U2fj
zIRg-F{0#sBQ$Wt)b3$e5^)m564uGjOgboqV67xHSf`SH+nrdcGF@gmbFfuFqvqPBB
zC*OvNVgVKh6m}k5zz1?ub9CS4w#YCQ2TyjIY+KzN&S4|<9iY?1iPb)=yj$g^OG`fn
zM|3e~M3KqjCfSLG={x)>IKmIO5wAk$_%yR|UUX?5^?LX-&UZL}3JJszZvV@1Xm|%_
z-NU1|vmmKmfj4KIADPk#%0<rq3t@r_)cMZhmQ;_yEpQ@I!=sBN`0hMOa4>S3F~~eB
zkGL42uUF>oC^tosG(*mD<>Ua9Ko0mjXZ#bKe4F3xJSj8eoI2yY@@>wmXFHz?G{hYQ
z`A&8uIZAbnYmuyg0>m717$gK~H~`oA0J01R*!Y|c1IP9T2@bMemq$IP-wj9&G<T%=
zr#CdrJLlpfF&;i<R$O}hKu%DF-IU|D?2SqDrI+41?{m)^kQSAOn$_F?lE018h2Ny?
zPDy%O(rG4EGQW#sebgVU!&btN=wn7t5NJY#JJHt!li=p>Rn3uxdk|>23{$Em&;ePG
zHT&tDA>=P+X@;K*wLZ{)9%(tx(|a!-*qy4;YyEo{vO*pntpd(z`!K{lqBTX{M_Sax
zY8Z1tPzQOx7d$adVhClXc!Ro1XAcez=JyAAJ)cy4!^}3Y+xI_Kf%0C-(uZNXWBWyy
zS4T*3Ln^D*QiV-j<W`eK_zO%p8VaIXMGAQ>XeHyctZSrj-E^x;kCc{XNtI?=1YXDo
zJU2(3uXjx9l22+QsvmyLb122&GBSP`;G_q5%p4l=I^Iq1zV}P-=U#yS6{P%*ce{1P
zd(V5{pYgRGYJramEaEZ0LPK=p=kclqz>zI}jR<jtH}b^Zh&1<DIg~9H2`nmzOK#(E
zsVM8xK_S7a2HehJNs^LkfC`i6t)qNa(x-8O%8KiqNr`R!co-IjS_4Mep~;QC3>)g#
z%r|d`-$E!uvBm$$cXc@d8bxK?&!R}t9N{WS2L*@Z<c!u3aluMGjnb+g=cQ3zW8=n^
z6k4T+vse)FmJXTTHP5Rr!+8wPwbtGbS44-Do1Z4;7Mix#_Bwv2b=A8*(P?$2I%A#b
zE-M`uz^w6b?-lu$i!v_JOH{vRE;?_X!C^EYwNi4m7UH<vcHb!6%2o0z!KLY|3G<Qj
T+HaUIG|^o5u6tAX$Gi1^dh55v

diff --git a/scrapy/wooyun_drops/wooyun_drops/spiders/__init__.pyc b/scrapy/wooyun_drops/wooyun_drops/spiders/__init__.pyc
deleted file mode 100644
index 8e64400843c3382a10633b6f9daa5102e25ccbde..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 168
zcmZSn%*&<v?q_H+0~9a<X$K%KW&si@3=F{<AQ3+eAi;n}6bAyu^h1kNi;DF#67!Pt
z^K<me^Ybf9^WqCilX5bX^^22>5(_F}f+<D$1;wbG;)2W+koNfa%)HE!_;|g7$`THs
TsW!R!DWy57b|AZpftUdRH2Nz{