diff --git a/buildout.cfg b/buildout.cfg index bad7463..2eec192 100644 --- a/buildout.cfg +++ b/buildout.cfg @@ -10,11 +10,12 @@ auto-checkout = find-links = http://op:x9W3jZ@dist.quintagroup.com/op/ [sources] -openprocurement_client = git https://github.com/openprocurement/openprocurement.client.python branch=use_requests +openprocurement_client = git https://github.com/openprocurement/openprocurement.client.python branch=upstream [test] recipe = zc.recipe.egg:scripts dependent-scripts = true eggs = openprocurement.bridge.basic [test] - + redis + lazydb diff --git a/openprocurement/bridge/basic/databridge.py b/openprocurement/bridge/basic/databridge.py index 0d14578..0a8bcc4 100644 --- a/openprocurement/bridge/basic/databridge.py +++ b/openprocurement/bridge/basic/databridge.py @@ -15,9 +15,9 @@ import gevent.pool from gevent import sleep, spawn from gevent.queue import PriorityQueue, Queue -from openprocurement_client.client import TendersClient as APIClient from openprocurement_client.exceptions import RequestFailed -from openprocurement_client.sync import ResourceFeeder +from openprocurement_client.resources.sync import ResourceFeeder +from openprocurement_client.resources.tenders import TendersClient as APIClient from pkg_resources import iter_entry_points from yaml import load diff --git a/openprocurement/bridge/basic/storages/redis_plugin.py b/openprocurement/bridge/basic/storages/redis_plugin.py index f26c2a7..cd49fe5 100644 --- a/openprocurement/bridge/basic/storages/redis_plugin.py +++ b/openprocurement/bridge/basic/storages/redis_plugin.py @@ -1,5 +1,9 @@ # -*- coding: utf-8 -*- -class Db(object): +import redis +from lazydb import Db + + +class DbProxy(object): """ Database proxy """ def __init__(self, config): @@ -10,23 +14,6 @@ def __init__(self, config): self._port = None self._host = None - if 'cache_host' in self.config['storage_config']: - import redis - self._backend = "redis" - self._host = self.config['storage_config'].get('cache_host') - self._port = self.config['storage_config'].get('cache_port') or 6379 - self._db_name = self.config['storage_config'].get('cache_db_name') or 0 - self.db = redis.StrictRedis(host=self._host, port=self._port, db=self._db_name) - self.set_value = self.db.set - self.has_value = self.db.exists - else: - from lazydb import Db - self._backend = "lazydb" - self._db_name = self.config['storage_config'].get('cache_db_name') or 'databridge_cache_db' - self.db = Db(self._db_name) - self.set_value = self.db.put - self.has_value = self.db.has - def get(self, key): return self.db.get(key) @@ -37,5 +24,35 @@ def has(self, key): return self.has_value(key) -def includeme(config): - return Db(config) +class DbRedis(DbProxy): + """ Database proxy for redis """ + + def __init__(self, config): + super(DbRedis, self).__init__(config) + self._backend = "redis" + self._host = self.config['storage_config'].get('cache_host') + self._port = self.config['storage_config'].get('cache_port') or 6379 + self._db_name = self.config['storage_config'].get('cache_db_name') or 0 + self.db = redis.StrictRedis(host=self._host, port=self._port, db=self._db_name) + self.set_value = self.db.set + self.has_value = self.db.exists + + +class DbLazy(DbProxy): + """ Database proxy for LazyDB """ + + def __init__(self, config): + super(DbLazy, self).__init__(config) + self._backend = "lazydb" + self._db_name = self.config['storage_config'].get('cache_db_name') or 'cache_db_name' + self.db = Db(self._db_name) + self.set_value = self.db.put + self.has_value = self.db.has + + +def redis_includeme(config): + return DbRedis(config) + + +def lazy_includeme(config): + return DbLazy(config) diff --git a/openprocurement/bridge/basic/tests/databridge.py b/openprocurement/bridge/basic/tests/databridge.py index 5af7ad1..25ad603 100644 --- a/openprocurement/bridge/basic/tests/databridge.py +++ b/openprocurement/bridge/basic/tests/databridge.py @@ -166,7 +166,7 @@ def test_create_api_client(self, mock_APIClient): del bridge - @patch('openprocurement_client.api_base_client.Session') + @patch('openprocurement_client.templates.Session') def test__get_average_request_duration(self, mocked_session): mocked_session.request.return_value = MockedResponse(200) bridge = BasicDataBridge(self.config) @@ -212,7 +212,7 @@ def test__calculate_st_dev(self): stdev = bridge._calculate_st_dev([]) self.assertEqual(stdev, 0) - @patch('openprocurement_client.api_base_client.Session') + @patch('openprocurement_client.templates.Session') def test__mark_bad_clients(self, mocked_session): mocked_session.request.return_value = MockedResponse(200) bridge = BasicDataBridge(self.config) @@ -243,7 +243,7 @@ def test__mark_bad_clients(self, mocked_session): to_destroy += 1 self.assertEqual(to_destroy, 3) - @patch('openprocurement_client.api_base_client.Session') + @patch('openprocurement_client.templates.Session') def test_perfomance_watcher(self, mocked_session): mocked_session.request.return_value = MockedResponse(200) bridge = BasicDataBridge(self.config) diff --git a/openprocurement/bridge/basic/tests/test_redis_storage.py b/openprocurement/bridge/basic/tests/test_redis_storage.py new file mode 100644 index 0000000..5e51bee --- /dev/null +++ b/openprocurement/bridge/basic/tests/test_redis_storage.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- +import unittest + +from mock import MagicMock, patch +from openprocurement.bridge.basic.storages.redis_plugin import redis_includeme, lazy_includeme + + +class TestDbs(unittest.TestCase): + + def setUp(self): + self.config = { + 'storage_config': { + 'cache_host': '127.0.0.1', + 'cache_port': '6379', + 'cache_db_name': '0' + } + } + with patch('openprocurement.bridge.basic.storages.redis_plugin.redis') as mocked_redis: + StrictRedis_mock = MagicMock() + StrictRedis_mock.configure_mock(**{'set': None, 'exists': None}) + mocked_redis.StrictRedis.return_value = StrictRedis_mock + + self.db = redis_includeme(self.config) + self.db.db = dict() + + def set_value(key, value): + self.db.db[key] = value + + self.db.set_value = set_value + self.db.has_value = lambda x: x in self.db.db + + @patch('openprocurement.bridge.basic.storages.redis_plugin.redis') + def test_redis_includeme(self, mocked_redis): + config = { + 'storage_config': { + 'cache_host': '127.0.0.1', + 'cache_port': '6379', + 'cache_db_name': '0' + } + } + StrictRedis_mock = MagicMock() + StrictRedis_mock.configure_mock(**{'set': None, 'exists': None}) + mocked_redis.StrictRedis.return_value = StrictRedis_mock + + db = redis_includeme(config) + + self.assertEqual(db._backend, 'redis') + self.assertEqual(db._db_name, config['storage_config']['cache_db_name']) + self.assertEqual(db._port, config['storage_config']['cache_port']) + self.assertEqual(db._host, config['storage_config']['cache_host']) + self.assertEqual(db._host, config['storage_config']['cache_host']) + self.assertEqual(db.set_value, None) + self.assertEqual(db.has_value, None) + + @patch('openprocurement.bridge.basic.storages.redis_plugin.Db') + def test_cache_host_in_config(self, mocked_db): + db = lazy_includeme(self.config) + + self.assertEqual(db._backend, 'lazydb') + self.assertEqual(db._db_name, self.config['storage_config']['cache_db_name']) + + def test_get(self): + self.assertEquals(self.db.get('test'), None) + self.db.set_value('test', 'test') + self.assertEquals(self.db.get('test'), 'test') + + def test_put(self): + self.db.put('test_put', 'test_put') + self.assertEquals(self.db.get('test_put'), 'test_put') + + def test_has(self): + self.assertEquals(self.db.has('test_has'), False) + self.db.set_value('test_has', 'test_has') + self.assertEquals(self.db.has('test_has'), True) \ No newline at end of file diff --git a/setup.py b/setup.py index f680020..7b4a422 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,8 @@ 'openprocurement.bridge.basic.storage_plugins': [ 'couchdb = openprocurement.bridge.basic.storages.couchdb_plugin:includeme', 'elasticsearch = openprocurement.bridge.basic.storages.elasticsearch_plugin:includeme', - 'redis = openprocurement.bridge.basic.storages.redis_plugin:includeme' + 'redis = openprocurement.bridge.basic.storages.redis_plugin:redis_includeme', + 'lazy = openprocurement.bridge.basic.storages.redis_plugin:lazy_includeme' ], 'openprocurement.bridge.basic.filter_plugins': [ 'basic_couchdb = openprocurement.bridge.basic.filters:BasicCouchDBFilter',