添加 mongodb 数据库支持

config.py

@@ -1,10 +1,7 @@
 # coding=utf-8
 
-db_type = 'mysql'
-# database_type = 'redis'
-# database_type = 'sqlite'
-
 DB_config = {
+    # 'db_type': 'mongodb',
     'db_type': 'mysql',
 
     'mysql': {
@@ -20,6 +17,12 @@
         'password': '123456',
         'db': 1,
     },
+    'mongodb':{
+        'host': 'localhost',
+        'port': 27017,
+        'username': '',
+        'password': '',
+    }
 }
 
 database = 'ipproxy'

proxy.py

@@ -1,4 +1,4 @@
-#-*- coding: utf-8 -*-
+# -*- coding: utf-8 -*-
 
 
 class Proxy(object):
@@ -27,7 +27,35 @@ def __str__(self):
 
         return str(data)
 
-    def set_value(self, ip, port, country, anonymity, source = 'unkonw', https = 'no', speed = -1, vali_count = 0):
+    def __dict__(self):
+        data = {
+            'ip': self.ip,
+            'port': self.port,
+            'country': self.country,
+            'anonymity': self.anonymity,
+            'https': self.https,
+            'speed': self.speed,
+            'source': self.source,
+            'vali_count': self.vali_count,
+        }
+
+        return data
+
+    def get_dict(self):
+        data = {
+            'ip': self.ip,
+            'port': self.port,
+            'country': self.country,
+            'anonymity': self.anonymity,
+            'https': self.https,
+            'speed': self.speed,
+            'source': self.source,
+            'vali_count': self.vali_count,
+        }
+
+        return data
+
+    def set_value(self, ip, port, country, anonymity, source='unkonw', https='no', speed=-1, vali_count=0):
         self.ip = ip
         self.port = port
         self.country = country

server/dataserver.py

@@ -1,4 +1,4 @@
-#-*- coding: utf-8 -*-
+# -*- coding: utf-8 -*-
 
 import json
 import logging
@@ -37,13 +37,13 @@ def GET(self):
 
             proxy = Proxy()
             proxy.set_value(
-                    ip = inputs.get('ip'),
-                    port = inputs.get('port'),
-                    country = inputs.get('country', None),
-                    anonymity = inputs.get('anonymity', None),
-                    https = inputs.get('https', 'no'),
-                    speed = inputs.get('speed', -1),
-                    source = inputs.get('source', name),
+                ip=inputs.get('ip'),
+                port=inputs.get('port'),
+                country=inputs.get('country', None),
+                anonymity=inputs.get('anonymity', None),
+                https=inputs.get('https', 'no'),
+                speed=inputs.get('speed', -1),
+                source=inputs.get('source', name),
             )
 
             sql.insert_proxy(name, proxy)
@@ -59,8 +59,8 @@ def GET(self):
             sql = SqlManager()
             inputs = web.input()
             name = inputs.get('name')
-            anonymity = inputs.get('anonymity', '%')
-            https = inputs.get('https', '%')
+            anonymity = inputs.get('anonymity', '')
+            https = inputs.get('https', '')
             order = inputs.get('order', 'speed')
             sort = inputs.get('sort', 'asc')
             count = inputs.get('count', 100)
@@ -74,11 +74,12 @@ def GET(self):
             }
             result = sql.select_proxy(name, **kwargs)
             data = [{
-                'id': item[0], 'ip': item[1], 'port': item[2], 'anonymity': item[4], 'https': item[5],
-                'speed': item[6], 'save_time': str(item[8])
+                'ip': item.get('ip'), 'port': item.get('port'),
+                'anonymity': item.get('anonymity'), 'https': item.get('https'),
+                'speed': item.get('speed'), 'save_time': item.get('save_time', '')
             } for item in result]
 
-            data = json.dumps(data, indent = 4)
+            data = json.dumps(data, indent=4)
             return data
         except Exception, e:
             logging.exception('select exception msg:%s' % e)

sql/__init__.py

@@ -2,4 +2,5 @@
 
 import sql
 from mysql import MySql
+from mongodb import Mongodb
 from sql_manager import SqlManager

sql/mongodb.py

@@ -0,0 +1,91 @@
+# -*- coding: utf-8 -*-
+
+import logging
+import pymongo
+import config
+import json
+import datetime
+
+from proxy import Proxy
+from sql import Sql
+
+
+class Mongodb(Sql):
+    def __init__(self, **kwargs):
+        super(Mongodb, self).__init__(**kwargs)
+        self.client = pymongo.MongoClient(**kwargs)
+        self.db = self.client[config.database]
+
+    def init_database(self, database_name):
+        pass
+
+    def init_proxy_table(self, table_name):
+        pass
+
+    def insert_proxy(self, table_name, proxy):
+        data = proxy.get_dict()
+        data['save_time'] = str(datetime.datetime.now())
+        self.db[table_name].insert(data)
+
+    def select_proxy(self, table_name, **kwargs):
+        filter = {}
+        if kwargs.get('anonymity') != '':
+            filter['anonymity'] = kwargs.get('anonymity')
+        if kwargs.get('https') != '':
+            filter['https'] = kwargs.get('https')
+
+        data = [item for item in self.db[table_name].find(filter).limit(int(kwargs.get('count')))]
+        return data
+
+    def update_proxy(self, table_name, proxy):
+        self.db[table_name].update_one(
+            {'_id': proxy.id},
+            {'$set':
+                 {'https': proxy.https, 'speed': proxy.speed, 'vali_count': proxy.vali_count,
+                  'anonymity': proxy.anonymity, 'save_time': str(datetime.datetime.now())}})
+
+    def delete_proxy(self, table_name, proxy):
+        return self.del_proxy_with_id(table_name, proxy.id)
+
+    def delete_old(self, table_name, day):
+        start = datetime.datetime.now()
+        end = datetime.datetime.now()
+        pass
+
+    def get_proxy_count(self, table_name):
+        count = self.db[table_name].find().count()
+        logging.debug('count:%s' % count)
+        return count
+
+    def get_proxy_ids(self, table_name):
+        ids = self.db[table_name].distinct('_id')
+        logging.debug('ids:%s' % ids)
+        return ids
+
+    def get_proxy_with_id(self, table_name, id):
+        data = self.db[table_name].find_one({'_id': id})
+        logging.debug(data)
+        proxy = Proxy()
+        proxy.set_value(
+            ip=data.get('ip'),
+            port=data.get('port'),
+            country=data.get('country'),
+            anonymity=data.get('country'),
+            https=data.get('https'),
+            speed=data.get('speed'),
+            source=data.get('source'),
+            vali_count=data.get('vali_count')
+        )
+        proxy.id = data.get('_id')
+        return proxy
+
+    def del_proxy_with_id(self, table_name, id):
+        self.db[table_name].delete_one({'_id': id})
+        return True
+
+    def del_proxy_with_ip(self, table_name, ip):
+        self.db[table_name].delete_one({'ip': ip})
+        return True
+
+    def commit(self):
+        pass

sql/mysql.py

@@ -1,4 +1,4 @@
-#-*- coding: utf-8 -*-
+# -*- coding: utf-8 -*-
 
 import logging
 import utils
@@ -73,15 +73,24 @@ def insert_proxy(self, table_name, proxy):
             logging.exception('mysql insert_proxy exception msg:%s' % e)
 
     def select_proxy(self, table_name, **kwargs):
+        filter = {}
+        for k, v in kwargs.items():
+            if v != '':
+                filter[k] = v
+
         try:
             command = "SELECT * FROM {name} WHERE anonymity LIKE '{anonymity}' AND https LIKE '{https}' ORDER BY " \
                       "{order} {sort} limit {count}". \
-                format(name = table_name, anonymity = kwargs.get('anonymity', '%'),
-                       https = kwargs.get('https', '%'), order = kwargs.get('order', 'save_time'),
-                       sort = kwargs.get('sort', 'desc'), count = kwargs.get('count', 100))
+                format(name=table_name, anonymity=filter.get('anonymity', '%'),
+                       https=filter.get('https', '%'), order=filter.get('order', 'save_time'),
+                       sort=filter.get('sort', 'desc'), count=filter.get('count', 100))
 
             result = self.query(command)
-            return result
+            data = [{
+                'ip': item[1], 'port': item[2], 'anonymity': item[4], 'https': item[5],
+                'speed': item[6], 'save_time': str(item[8])
+            } for item in result]
+            return data
         except Exception, e:
             logging.exception('mysql select_proxy exception msg:%s' % e)
         return []
@@ -91,21 +100,21 @@ def update_proxy(self, table_name, proxy):
             command = "UPDATE {table_name} set https='{https}', speed={speed}, " \
                       "vali_count={vali_count}, anonymity = {anonymity},save_time={save_time} " \
                       "where id={id};".format(
-                    table_name = table_name, https = proxy.https,
-                    speed = proxy.speed, id = proxy.id, vali_count = proxy.vali_count, anonymity = proxy.anonymity,
-                    save_time = 'NOW()')
+                table_name=table_name, https=proxy.https,
+                speed=proxy.speed, id=proxy.id, vali_count=proxy.vali_count, anonymity=proxy.anonymity,
+                save_time='NOW()')
             logging.debug('mysql update_proxy command:%s' % command)
             self.cursor.execute(command)
         except Exception, e:
             logging.exception('mysql update_proxy exception msg:%s' % e)
 
     def delete_proxy(self, table_name, proxy):
-        self.del_proxy_with_id(table_name = table_name, id = proxy.id)
+        self.del_proxy_with_id(table_name=table_name, id=proxy.id)
 
     def delete_old(self, table_name, day):
         try:
             command = "DELETE FROM {table} where save_time < SUBDATE(NOW(), INTERVAL {day} DAY)".format(
-                    table = config.free_ipproxy_table, day = day)
+                table=config.free_ipproxy_table, day=day)
 
             self.cursor.execute(command)
             self.commit()
@@ -154,14 +163,14 @@ def get_proxy_with_id(self, table_name, id):
                 # }
                 proxy = Proxy()
                 proxy.set_value(
-                        ip = result[1],
-                        port = result[2],
-                        country = result[3],
-                        anonymity = result[4],
-                        https = result[5],
-                        speed = result[6],
-                        source = result[7],
-                        vali_count = result[9])
+                    ip=result[1],
+                    port=result[2],
+                    country=result[3],
+                    anonymity=result[4],
+                    https=result[5],
+                    speed=result[6],
+                    source=result[7],
+                    vali_count=result[9])
                 proxy.id = result[0]
                 proxy.save_time = result[8]
         except Exception, e:
@@ -201,7 +210,7 @@ def create_table(self, command):
         except Exception, e:
             logging.exception('mysql create_table exception:%s' % e)
 
-    def insert_data(self, command, data, commit = False):
+    def insert_data(self, command, data, commit=False):
         try:
             logging.debug('mysql insert_data command:%s, data:%s' % (command, data))
             x = self.cursor.execute(command, data)
@@ -214,7 +223,7 @@ def insert_data(self, command, data, commit = False):
     def commit(self):
         self.conn.commit()
 
-    def execute(self, command, commit = True):
+    def execute(self, command, commit=True):
         try:
             logging.debug('mysql execute command:%s' % command)
             data = self.cursor.execute(command)
@@ -225,7 +234,7 @@ def execute(self, command, commit = True):
             logging.exception('mysql execute exception msg:%s' % e)
             return None
 
-    def query(self, command, commit = False):
+    def query(self, command, commit=False):
         try:
             logging.debug('mysql execute command:%s' % command)
 
@@ -238,7 +247,7 @@ def query(self, command, commit = False):
             logging.exception('mysql execute exception msg:%s' % e)
             return None
 
-    def query_one(self, command, commit = False):
+    def query_one(self, command, commit=False):
         try:
             logging.debug('mysql execute command:%s' % command)
 

sql/sql_manager.py

@@ -4,7 +4,6 @@
 
 from sql import Sql
 
-
 class SqlManager(object):
     def __init__(self):
         db_type = config.DB_config.get('db_type', 'mysql')
@@ -17,6 +16,9 @@ def __init__(self):
             pass
         elif db_type == 'sqlite':
             pass
+        elif db_type == 'mongodb':
+            from mongodb import Mongodb
+            self.sql = Mongodb(**db_config)
         else:  # default mysql
             from mysql import MySql
             self.sql = MySql(**config.DB_config.get('db_type'))