Add tests for filter greenlets

openprocurement/bridge/basic/tests/filters.py

@@ -0,0 +1,140 @@
+# -*- coding: utf-8 -*-
+import unittest
+from copy import deepcopy
+from datetime import datetime
+from uuid import uuid4
+
+from gevent.queue import PriorityQueue
+from mock import MagicMock, patch
+from munch import munchify
+
+from openprocurement.bridge.basic.filters import BasicCouchDBFilter, BasicElasticSearchFilter
+from openprocurement.bridge.basic.tests.base import TEST_CONFIG
+
+
+class TestBasicCouchDBFilter(unittest.TestCase):
+
+    config = deepcopy(TEST_CONFIG['main'])
+    config['storage_config']['bulk_query_limit'] = 1
+
+    def setUp(self):
+        self.old_date_modified = datetime.now().isoformat()
+        self.id_1 = uuid4().hex
+        self.date_modified_1 = datetime.now().isoformat()
+        self.id_2 = uuid4().hex
+        self.date_modified_2 = datetime.now().isoformat()
+        self.id_3 = uuid4().hex
+        self.date_modified_3 = datetime.now().isoformat()
+        self.queue = PriorityQueue()
+        self.input_queue = PriorityQueue()
+        self.db = MagicMock()
+        self.bulk = {
+            self.id_1: self.date_modified_1,
+            self.id_2: self.date_modified_2,
+            self.id_3: self.date_modified_3
+        }
+        self.priority_cache = {self.id_1: 1, self.id_2: 1, self.id_3: 1}
+        self.return_value = [
+            munchify({'id': self.id_1, 'key': self.date_modified_1}),
+            munchify({'id': self.id_2, 'key': self.old_date_modified}),
+            munchify({'id': self.id_3, 'key': self.old_date_modified})
+        ]
+        self.db.view.return_value = self.return_value
+
+    def test__check_bulk(self):
+        self.queue.put((1000, self.id_3))
+        couchdb_filter = BasicCouchDBFilter(self.config, self.input_queue, self.queue, self.db)
+        self.assertEqual(self.queue.qsize(), 1)
+
+        couchdb_filter._check_bulk(self.bulk, self.priority_cache)
+        self.assertEqual(self.queue.qsize(), 2)
+
+        self.db.view.side_effect = [Exception(), Exception(), Exception('test')]
+        self.bulk = {}
+        with self.assertRaises(Exception) as e:
+            couchdb_filter._check_bulk(self.bulk, self.priority_cache)
+        self.assertEqual(e.exception.message, 'test')
+
+    @patch('openprocurement.bridge.basic.filters.INFINITY')
+    def test__run(self, mocked_infinity):
+        couchdb_filter = BasicCouchDBFilter(self.config, self.input_queue, self.queue, self.db)
+        self.input_queue.put((1, {'id': self.id_1, 'dateModified': self.date_modified_1}))
+        self.input_queue.put((1, {'id': self.id_2, 'dateModified': self.date_modified_2}))
+        self.input_queue.put((1, {'id': self.id_3, 'dateModified': self.date_modified_3}))
+        mocked_infinity.__nonzero__.side_effect = [True] * 5 + [False]
+        self.assertEqual(self.queue.qsize(), 0)
+        self.assertEqual(self.input_queue.qsize(), 3)
+
+        couchdb_filter._run()
+        self.assertEqual(self.queue.qsize(), 2)
+        self.assertEqual(self.input_queue.qsize(), 0)
+
+
+class TestBasicElasticSearchFilter(unittest.TestCase):
+
+    config = deepcopy(TEST_CONFIG['main'])
+
+    def test__check_bulk(self):
+        input_queue = PriorityQueue()
+        queue = PriorityQueue()
+        old_date_modified = datetime.now().isoformat()
+        id_1 = uuid4().hex
+        date_modified_1 = datetime.now().isoformat()
+        id_2 = uuid4().hex
+        date_modified_2 = datetime.now().isoformat()
+        id_3 = uuid4().hex
+        date_modified_3 = datetime.now().isoformat()
+        db = MagicMock()
+        bulk = {
+            id_1: date_modified_1,
+            id_2: date_modified_2,
+            id_3: date_modified_3
+        }
+        priority_cache = {id_1: 1, id_2: 1, id_3: 1}
+        return_value = {
+            u'docs': [
+                {
+                    u'_type': u'Tender',
+                    u'_source': {
+                        u'dateModified': date_modified_1
+                    },
+                    u'_index': u'bridge_tenders',
+                    u'_version': 1,
+                    u'found': True,
+                    u'_id': id_1
+                },
+                {
+                    u'_type': u'Tender',
+                    u'_source': {
+                        u'dateModified': old_date_modified
+                    },
+                    u'_index': u'bridge_tenders',
+                    u'_version': 1,
+                    u'found': True,
+                    u'_id': id_2
+                },
+                {
+                    u'found': False,
+                    u'_type': u'Tender',
+                    u'_id': id_3,
+                    u'_index': u'bridge_tenders'
+                }
+            ]
+        }
+        db.mget.return_value = return_value
+        elastic_filter = BasicElasticSearchFilter(self.config, input_queue, queue, db)
+        self.assertEqual(queue.qsize(), 0)
+
+        elastic_filter._check_bulk(bulk, priority_cache)
+        self.assertEqual(queue.qsize(), 2)
+
+
+def suite():
+    suite = unittest.TestSuite()
+    suite.addTest(unittest.makeSuite(TestBasicCouchDBFilter))
+    suite.addTest(unittest.makeSuite(TestBasicElasticSearchFilter))
+    return suite
+
+
+if __name__ == '__main__':
+    unittest.main(defaultTest='suite')

openprocurement/bridge/basic/tests/test_elasticsearch_storage.py

@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 import unittest
 from copy import deepcopy
-from datetime import datetime
 
 from mock import patch
 
@@ -16,39 +15,6 @@ class TestElasticsearchStorage(unittest.TestCase):
     id_1 = '2bf7359509c2436d96f903c745d09ab5'
     id_2 = 'ffb2de965f02491bb44a9209cdc5c320'
 
-    # @patch('openprocurement.bridge.basic.storages.elasticsearch_plugin.Elasticsearch')
-    # def test_filter_bulk(self, mocked_elastic):
-    #     mocked_elastic().indices.get_settings.return_value = {}
-    #     db = ElasticsearchStorage(self.config)
-    #     bulk = {
-    #         self.id_1: '2017-10-10T12:22:19.781980+03:00',
-    #         self.id_2: datetime.now().isoformat()
-    #     }
-    #     rows = {
-    #         u'docs': [
-    #             {
-    #                 u'_type': u'Tender',
-    #                 u'_source': {
-    #                     u'dateModified': u'2017-10-10T12:22:19.781980+03:00'
-    #                 },
-    #                 u'_index': u'bridge_tenders',
-    #                 u'_version': 1,
-    #                 u'found': True,
-    #                 u'_id': u'2bf7359509c2436d96f903c745d09ab5'
-    #             },
-    #             {
-    #                 u'found': False,
-    #                 u'_type': u'Tender',
-    #                 u'_id': u'ffb2de965f02491bb44a9209cdc5c320',
-    #                 u'_index': u'bridge_tenders'
-    #             }
-    #         ]
-    #     }
-    #     db.db.mget.return_value = rows
-    #     resp_dict = db.filter_bulk(bulk)
-    #     self.assertEqual(resp_dict[self.id_1], bulk[self.id_1])
-    #     self.assertEqual(resp_dict[self.id_2], False)
-
     @patch('openprocurement.bridge.basic.storages.elasticsearch_plugin.Elasticsearch')
     def test_save_bulk(self, mocked_elastic):
         id_3 = '4dbb346095554f788b568c5c29e9ef23'