diff --git a/Schedule/ProxyCheck.py b/Schedule/ProxyCheck.py index 5f0c13d40..10d62aa98 100644 --- a/Schedule/ProxyCheck.py +++ b/Schedule/ProxyCheck.py @@ -13,7 +13,7 @@ __author__ = 'J_hao' import sys -from time import sleep +import threading from threading import Thread sys.path.append('../') @@ -26,32 +26,36 @@ class ProxyCheck(ProxyManager, Thread): - def __init__(self): + def __init__(self, queue, item_dict): ProxyManager.__init__(self) Thread.__init__(self) self.log = LogHandler('proxy_check') + self.queue = queue + self.item_dict = item_dict def run(self): self.db.changeTable(self.useful_proxy_queue) - while True: - for proxy, count in self.db.getAll().items(): - if validUsefulProxy(proxy): - # 验证通过计数器减1 - if count and int(count) > 0: - self.db.put(proxy, num=int(count) - 1) - else: - pass - self.log.info('ProxyCheck: {} validation pass'.format(proxy)) + while self.queue.qsize(): + proxy = self.queue.get() + count = self.item_dict[proxy] + if validUsefulProxy(proxy): + # 验证通过计数器减1 + if count and int(count) > 0: + self.db.put(proxy, num=int(count) - 1) else: - self.log.info('ProxyCheck: {} validation fail'.format(proxy)) - if count and int(count) > FAIL_COUNT: - self.log.info('ProxyCheck: {} fail too many, delete!'.format(proxy)) - self.db.delete(proxy) - else: - self.db.put(proxy, num=int(count) + 1) - sleep(60 * 5) + pass + print('ProxyCheck: {} validation pass'.format(proxy)) + else: + print('ProxyCheck: {} validation fail'.format(proxy)) + if count and int(count) > FAIL_COUNT: + print('ProxyCheck: {} fail too many, delete!'.format(proxy)) + self.db.delete(proxy) + else: + self.db.put(proxy, num=int(count) + 1) + self.queue.task_done() if __name__ == '__main__': - p = ProxyCheck() - p.run() + # p = ProxyCheck() + # p.run() + pass diff --git a/Schedule/ProxyValidSchedule.py b/Schedule/ProxyValidSchedule.py index 8345a83a4..f5a605aad 100644 --- a/Schedule/ProxyValidSchedule.py +++ b/Schedule/ProxyValidSchedule.py @@ -17,13 +17,17 @@ sys.path.append('../') from Schedule.ProxyCheck import ProxyCheck +from Manager.ProxyManager import ProxyManager +from queue import Queue +import time -class ProxyValidSchedule(object): +class ProxyValidSchedule(ProxyManager, object): def __init__(self): - pass + ProxyManager.__init__(self) + self.queue = Queue() - def __validProxy(self, threads=5): + def __validProxy(self, threads=10): """ 验证useful_proxy代理 :param threads: 线程数 @@ -31,7 +35,7 @@ def __validProxy(self, threads=5): """ thread_list = list() for index in range(threads): - thread_list.append(ProxyCheck()) + thread_list.append(ProxyCheck(self.queue, self.item_dict)) for thread in thread_list: thread.daemon = True @@ -41,7 +45,20 @@ def __validProxy(self, threads=5): thread.join() def main(self): - self.__validProxy() + self.put_queue() + while True: + if self.queue.qsize(): + self.__validProxy() + else: + print('Time sleep 5 minutes.') + time.sleep(60 * 1) + self.put_queue() + + def put_queue(self): + self.db.changeTable(self.useful_proxy_queue) + self.item_dict = self.db.getAll() + for item in self.item_dict: + self.queue.put(item) def run(): @@ -51,4 +68,4 @@ def run(): if __name__ == '__main__': p = ProxyValidSchedule() - p.main() + p.main() \ No newline at end of file