整理代码

config.py

@@ -9,3 +9,5 @@
 
 free_ipproxy_database = 'ipproxy'
 free_ipproxy_table = 'free_ipproxy'
+
+data_port = '8000'

ipproxytool/spiders/proxy/basespider.py

@@ -2,8 +2,11 @@
 
 import sys
 
-from utils import *
-from config import *
+import datetime
+
+import config
+import utils
+
 from sqlhelper import SqlHelper
 from scrapy.spiders import Spider
 from scrapy.http import Request
@@ -31,9 +34,9 @@ def init(self):
             'download_timeout': self.timeout,
         }
 
-        make_dir(self.dir_log)
+        utils.make_dir(self.dir_log)
 
-        command = get_create_table_command(free_ipproxy_table)
+        command = utils.get_create_table_command(config.free_ipproxy_table)
         self.sql.execute(command)
 
     def start_requests(self):
@@ -55,7 +58,7 @@ def error_parse(self, failure):
         pass
 
     def add_proxy(self, proxy):
-        sql_insert_proxy(self.sql, free_ipproxy_table, proxy)
+        utils.sql_insert_proxy(self.sql, config.free_ipproxy_table, proxy)
 
     def write(self, data):
         with open('%s/%s.html' % (

ipproxytool/spiders/validator/assetstore.py

@@ -1,9 +1,12 @@
 #-*- coding: utf-8 -*-
+import json
+import time
+
+import config
+import utils
 
 from scrapy.http import Request
 from validator import Validator
-from config import *
-from utils import *
 
 
 class AssetStoreSpider(Validator):
@@ -60,13 +63,13 @@ def get_unity_version(self, response):
             'X-Unity-Session': '26c4202eb475d02864b40827dfff11a14657aa41',
         }
 
-        count = get_table_length(self.sql, self.name)
-        count_free = get_table_length(self.sql, free_ipproxy_table)
+        count = utils.get_table_length(self.sql, self.name)
+        count_free = utils.get_table_length(self.sql, config.free_ipproxy_table)
 
         for i in range(0, count + count_free):
-            table = self.name if (i < count) else free_ipproxy_table
+            table = self.name if (i < count) else config.free_ipproxy_table
 
-            proxy = get_proxy_info(self.sql, table, i)
+            proxy = utils.get_proxy_info(self.sql, table, i)
             if proxy == None:
                 continue
 

ipproxytool/spiders/validator/gather.py

@@ -1,8 +1,5 @@
 #-*- coding: utf-8 -*-
 
-from scrapy.http import Request
-from config import *
-from utils import *
 from validator import Validator
 
 

ipproxytool/spiders/validator/validator.py

@@ -1,10 +1,14 @@
 #-*- coding: utf-8 -*-
+import time
+
+import datetime
+
+import utils
 
 from scrapy import Request
 from scrapy.spiders import Spider
 from config import free_ipproxy_table
 from sqlhelper import SqlHelper
-from utils import *
 
 
 class Validator(Spider):
@@ -22,19 +26,19 @@ def __init__(self, name = None, **kwargs):
         self.success_mark = ''
 
     def init(self):
-        make_dir(self.dir_log)
+        utils.make_dir(self.dir_log)
 
-        command = get_create_table_command(self.name)
+        command = utils.get_create_table_command(self.name)
         self.sql.create_table(command)
 
     def start_requests(self):
-        count = get_table_length(self.sql, self.name)
-        count_free = get_table_length(self.sql, free_ipproxy_table)
+        count = utils.get_table_length(self.sql, self.name)
+        count_free = utils.get_table_length(self.sql, free_ipproxy_table)
 
         for i in range(0, count + count_free):
             table = self.name if (i < count) else free_ipproxy_table
 
-            proxy = get_proxy_info(self.sql, table, i)
+            proxy = utils.get_proxy_info(self.sql, table, i)
             if proxy == None:
                 continue
 
@@ -72,14 +76,14 @@ def success_parse(self, response):
 
             if table == self.name:
                 if speed > self.timeout:
-                    command = get_delete_data_command(table, id)
+                    command = utils.get_delete_data_command(table, id)
                     self.sql.execute(command)
                 else:
-                    command = get_update_data_command(table, id, speed)
+                    command = utils.get_update_data_command(table, id, speed)
                     self.sql.execute(command)
             else:
                 if speed < self.timeout:
-                    command = get_insert_data_command(self.name)
+                    command = utils.get_insert_data_command(self.name)
                     msg = (None, proxy.get('ip'), proxy.get('port'), proxy.get('country'), proxy.get('anonymity'),
                            proxy.get('https'), speed, proxy.get('source'), None)
 
@@ -93,7 +97,7 @@ def error_parse(self, failure):
         id = failure.request.meta.get('id')
 
         if table == self.name:
-            command = get_delete_data_command(table, id)
+            command = utils.get_delete_data_command(table, id)
             self.sql.execute(command)
         else:
             # TODO... 如果 ip 验证失败应该针对特定的错误类型，进行处理

runserver.py

@@ -1,12 +1,11 @@
 #-*- coding: utf-8 -*-
 
-import BaseHTTPServer
+import config
 
-from server.ipproxyserver import IpProxyServer
+from server import dataserver
 from utils import kill_ports
 
 if __name__ == '__main__':
-    kill_ports(['8000'])
+    kill_ports([config.data_port])
 
-    server = BaseHTTPServer.HTTPServer(('0.0.0.0', 8000), IpProxyServer)
-    server.serve_forever()
+    dataserver.start_api_server()

runspider.py

@@ -5,14 +5,15 @@
 import sys
 import scrapydo
 import time
+import utils
+import config
 
 from sqlhelper import SqlHelper
 from ipproxytool.spiders.proxy.xicidaili import XiCiDaiLiSpider
 from ipproxytool.spiders.proxy.sixsixip import SixSixIpSpider
 from ipproxytool.spiders.proxy.ip181 import IpOneEightOneSpider
 from ipproxytool.spiders.proxy.kuaidaili import KuaiDaiLiSpider
 from ipproxytool.spiders.proxy.gatherproxy import GatherproxySpider
-from config import free_ipproxy_table
 
 scrapydo.setup()
 
@@ -31,9 +32,9 @@
     sql = SqlHelper()
 
     while True:
-        print('*******************run spider start...*******************')
+        utils.log('*******************run spider start...*******************')
 
-        command = 'delete from {0} where `save_time` < now() - {1};'.format(free_ipproxy_table, 3600)
+        command = 'DELETE FROM {0} WHERE save_time < now() - {1}'.format(config.free_ipproxy_table, 1800)
         sql.execute(command)
 
         items = scrapydo.run_spider(XiCiDaiLiSpider)
@@ -42,5 +43,5 @@
         items = scrapydo.run_spider(KuaiDaiLiSpider)
         items = scrapydo.run_spider(GatherproxySpider)
 
-        print('*******************run spider waiting...*******************')
+        utils.log('*******************run spider waiting...*******************')
         time.sleep(300)

runvalidator.py

@@ -5,6 +5,7 @@
 import sys
 import time
 import scrapydo
+import utils
 
 from scrapy import cmdline
 from scrapy.crawler import CrawlerProcess
@@ -28,9 +29,9 @@
     )
 
     while True:
-        print('----------------validator start...-----------------------')
-        # items = scrapydo.run_spider(DoubanSpider)
-        items = scrapydo.run_spider(GatherSpider)
-        items = scrapydo.run_spider(AssetStoreSpider)
-        print('*************************validator waiting...*************************')
+        utils.log('----------------validator start...-----------------------')
+        items = scrapydo.run_spider(DoubanSpider)
+        # items = scrapydo.run_spider(GatherSpider)
+        # items = scrapydo.run_spider(AssetStoreSpider)
+        utils.log('*************************validator waiting...*************************')
         time.sleep(60)

server/dataserver.py

@@ -45,4 +45,8 @@ def GET(self):
         name = inputs.get('name')
         ip = inputs.get('ip')
         command = "DELETE FROM {0} WHERE ip=\'{1}\'".format(name, ip)
-        return sql.execute(command)
+        sql.execute(command)
+
+        command = "SELECT ip FROM {0} WHERE ip=\'{1}\'".format(name, ip)
+        res = sql.query_one(command)
+        return res is None

server/ipproxyserver.py

@@ -67,7 +67,7 @@ def do_GET(self):
 
             table_name = dict.get('name')
 
-            command = "select * from {0}".format(table_name)
+            command = "SELECT * FROM {0}".format(table_name)
 
             data = sql.query(command)
             # data = str(data)

singleton.py

@@ -1,17 +1,9 @@
 #-*- coding: utf-8 -*-
 
-import threading
-
 class Singleton(object):
-
-    lock = threading.Lock()
-
     def __new__(cls, *args, **kw):
         if not hasattr(cls, '_instance'):
-            Singleton.lock.acquire(True)
             orig = super(Singleton, cls)
             cls._instance = orig.__new__(cls, *args, **kw)
             cls.is_init = False
-            Singleton.lock.release()
         return cls._instance
-

sqlhelper.py

@@ -2,19 +2,19 @@
 
 import logging
 import mysql.connector
+import utils
+import config
 
 from singleton import Singleton
-from utils import log
-from config import *
 
 
 class SqlHelper(Singleton):
     def __init__(self):
-        self.database_name = free_ipproxy_database
+        self.database_name = config.free_ipproxy_database
         self.init()
 
     def init(self):
-        self.database = mysql.connector.connect(**database_config)
+        self.database = mysql.connector.connect(**config.database_config)
         self.cursor = self.database.cursor()
 
         self.create_database()
@@ -23,58 +23,58 @@ def init(self):
     def create_database(self):
         try:
             command = 'CREATE DATABASE IF NOT EXISTS %s DEFAULT CHARACTER SET \'utf8\' ' % self.database_name
-            log('sql helper create_database command:%s' % command)
+            utils.log('sql helper create_database command:%s' % command)
             self.cursor.execute(command)
         except Exception, e:
-            log('SqlHelper create_database exception:%s' % str(e), logging.WARNING)
+            utils.log('SqlHelper create_database exception:%s' % str(e), logging.WARNING)
 
     def create_table(self, command):
         try:
-            log('sql helper create_table command:%s' % command)
+            utils.log('sql helper create_table command:%s' % command)
             self.cursor.execute(command)
             self.database.commit()
         except Exception, e:
-            log('sql helper create_table exception:%s' % str(e), logging.WARNING)
+            utils.log('sql helper create_table exception:%s' % str(e), logging.WARNING)
 
     def insert_data(self, command, data):
         try:
-            log('insert_data command:%s, data:%s' % (command, data))
+            utils.log('insert_data command:%s, data:%s' % (command, data))
 
             self.cursor.execute(command, data)
             self.database.commit()
         except Exception, e:
-            log('sql helper insert_data exception msg:%s' % str(e), logging.WARNING)
+            utils.log('sql helper insert_data exception msg:%s' % str(e), logging.WARNING)
 
     def execute(self, command):
         try:
-            log('sql helper execute command:%s' % command)
+            utils.log('sql helper execute command:%s' % command)
             data = self.cursor.execute(command)
             self.database.commit()
             return data
         except Exception, e:
-            log('sql helper execute exception msg:%s' % str(e))
+            utils.log('sql helper execute exception msg:%s' % str(e))
             return None
 
     def query(self, command):
         try:
-            log('sql helper execute command:%s' % command)
+            utils.log('sql helper execute command:%s' % command)
 
             self.cursor.execute(command)
             data = self.cursor.fetchall()
 
             return data
         except Exception, e:
-            log('sql helper execute exception msg:%s' % str(e))
+            utils.log('sql helper execute exception msg:%s' % str(e))
             return None
 
     def query_one(self, command):
         try:
-            log('sql helper execute command:%s' % command)
+            utils.log('sql helper execute command:%s' % command)
 
             self.cursor.execute(command)
             data = self.cursor.fetchone()
 
             return data
         except Exception, e:
-            log('sql helper execute exception msg:%s' % str(e))
+            utils.log('sql helper execute exception msg:%s' % str(e))
             return None

utils.py

@@ -106,7 +106,7 @@ def get_table_length(sql, table_name):
 
 # 通过指定 id 得到代理信息
 def get_proxy_info(sql, table_name, id):
-    command = ('select * from {0} limit {1},1;'.format(table_name, id))
+    command = ('SELECT * FROM {0} limit {1},1;'.format(table_name, id))
     result = sql.query_one(command)
     if result != None:
         data = {