Merge pull request #10 from 5monkeys/feature/configurable-cache-pipe

Configurable cache pipe

cio/backends/__init__.py

@@ -29,15 +29,18 @@ def get_backend(backend):
     else:
         # Parse uri or package
         if '://' in backend:
+            config['BACKEND'] = backend
+
             scheme, _config = backend.split('://', 1)
             if scheme not in BACKENDS:
                 raise InvalidBackend('Invalid content-io backend scheme "%s"' % scheme)
-            package = backend = 'cio.backends.%s' % BACKENDS[scheme]
+            package = 'cio.backends.%s' % BACKENDS[scheme]
             class_name = 'Backend'
 
             # Parse config
             name, _, params = _config.partition('?')
-            config['NAME'] = name
+            if name:
+                config['NAME'] = name
             if params:
                 config.update(dict(param.split('=') for param in params.split('&')))
         elif '.' in backend:
@@ -77,6 +80,7 @@ def setup(self):
         # Validate backend
         if self._is_valid_backend(backend):
             self._backend = backend
+            self._update_backend_settings(backend.config)
         else:
             raise InvalidBackend('Invalid content-io %s backend "%s"' % (self._scope(), self._conf))
 
@@ -86,6 +90,9 @@ def _scope(self):
     def _get_backend_config(self):
         raise NotImplementedError  # pragma: no cover
 
+    def _update_backend_settings(self, config):
+        raise NotImplementedError  # pragma: no cover
+
     def _is_valid_backend(self, backend):
         raise NotImplementedError  # pragma: no cover
 
@@ -119,6 +126,9 @@ class CacheManager(BackendManager, CacheBackend):
     def _get_backend_config(self):
         return settings.CACHE
 
+    def _update_backend_settings(self, config):
+        settings.CACHE = config
+
     def get(self, uri):
         uri = self._clean_get_uri(uri)
         return self.backend.get(uri)
@@ -164,6 +174,9 @@ class StorageManager(BackendManager, StorageBackend):
     def _get_backend_config(self):
         return settings.STORAGE
 
+    def _update_backend_settings(self, config):
+        settings.STORAGE = config
+
     def get(self, uri):
         uri = self._clean_get_uri(uri)
         return self.backend.get(uri)

cio/conf/__init__.py

@@ -6,14 +6,48 @@
 from contextlib import contextmanager
 from types import ModuleType
 from . import default_settings
+from ..utils.thread import ThreadLocalObject
 
 logger = logging.getLogger(__name__)
 
 
+class LocalSettings(ThreadLocalObject):
+
+    def __init__(self, base):
+        super(LocalSettings, self).__init__()
+        self._base = base
+        self._local = {}
+
+    def __contains__(self, key):
+        return key in self._local
+
+    def get(self, var):
+        return self._local[var]
+
+    def set(self, **vars):
+        def deepupdate(original, update):
+            for key, value in original.iteritems():
+                if key not in update:
+                    update[key] = value
+                elif isinstance(value, dict):
+                    deepupdate(value, update[key])
+            return update
+
+        for setting, value in six.iteritems(vars):
+            if isinstance(value, dict):
+                base_value = self._base.get(setting)
+                if base_value and isinstance(base_value, dict):
+                    deepupdate(base_value, value)
+
+            self._local[setting] = value
+
+
 class Settings(dict):
 
     def __init__(self, conf=None, **settings):
+        super(Settings, self).__init__()
         self._listeners = set()
+        self._local = LocalSettings(self)
         self.configure(conf=conf, **settings)
 
     @contextmanager
@@ -34,25 +68,43 @@ def deepcopy(self):
             copy[key] = value
         return copy
 
-    def configure(self, conf=None, **settings):
+    def configure(self, conf=None, local=False, **settings):
         if isinstance(conf, ModuleType):
             conf = conf.__dict__
 
-        for setting, value in six.iteritems(conf or settings):
-            if setting.isupper():
-                self[setting] = value
+        if local:
+            self._local.set(**conf or settings)
+
+        else:
+            for setting, value in six.iteritems(conf or settings):
+                if setting.isupper():
+                    self[setting] = value
 
-        for callback in self._listeners:
-            try:
-                callback()
-            except Exception as e:
-                logger.warn('Failed to notify callback about new settings; %s', e)
+            for callback in self._listeners:
+                try:
+                    callback()
+                except Exception as e:
+                    logger.warn('Failed to notify callback about new settings; %s', e)
 
     def watch(self, callback):
         self._listeners.add(callback)
 
-    __getattr__ = dict.__getitem__
-    __setattr__ = dict.__setitem__
+    def __getitem__(self, key):
+        """
+        First try environment specific setting, then this config
+        """
+        if key in self._local:
+            return self._local.get(key)
+
+        return super(Settings, self).__getitem__(key)
+
+    __getattr__ = __getitem__
+
+    def __setattr__(self, name, value):
+        if name.isupper():
+            self[name] = value
+        else:
+            super(Settings, self).__setattr__(name, value)
 
 
 settings = Settings(default_settings)

cio/pipeline/pipes/cache.py

@@ -3,6 +3,7 @@
 
 import six
 from .base import BasePipe
+from ...conf import settings
 from ...backends import cache
 
 
@@ -34,7 +35,10 @@ def get_response(self, response):
                 # Empty node meta to be coherent with cached nodes
                 node.meta.clear()
 
-        if nodes:
+        pipe_config = settings.CACHE.get('PIPE', {})
+        cache_on_get = pipe_config.get('CACHE_ON_GET', True)
+
+        if nodes and cache_on_get:
             cache.set_many(nodes)
 
         return response

runtests.py

@@ -22,8 +22,8 @@ def main():
     # Configure setup
     from cio.conf import settings
     settings.configure(STORAGE={
-        'BACKEND': 'sqlite://',
-        'NAME': ':memory:',
+        'BACKEND': 'sqlite://:memory:?foo=bar',
+        'NAME': '__overridden__',
         'OPTIONS': {
             'check_same_thread': False
         }

tests/__init__.py

@@ -59,13 +59,13 @@ def assertCache(self, calls=-1, hits=-1, misses=-1, sets=-1):
         yield
 
         if calls >= 0:
-            assert cb.calls == calls
+            assert cb.calls == calls, '%s != %s' % (cb.calls, calls)
         if hits >= 0:
-            assert cb.hits == hits
+            assert cb.hits == hits, '%s != %s' % (cb.hits, hits)
         if misses >= 0:
-            assert cb.misses == misses
+            assert cb.misses == misses, '%s != %s' % (cb.misses, misses)
         if sets >= 0:
-            assert cb.sets == sets
+            assert cb.sets == sets, '%s != %s' % (cb.sets, sets)
 
 
     @contextmanager
@@ -78,15 +78,20 @@ def assertDB(self, calls=-1, selects=-1, inserts=-1, updates=-1, deletes=-1):
 
         count = lambda cmd: len([q for q in backend.queries if q['sql'].split(' ', 1)[0].upper().startswith(cmd)])
         if calls >= 0:
-            assert len(backend.queries) == calls
+            call_count = len(backend.queries)
+            assert call_count == calls, '%s != %s' % (call_count, calls)
         if selects >= 0:
-            assert count('SELECT') == selects
+            select_count = count('SELECT')
+            assert select_count == selects, '%s != %s' % (select_count, selects)
         if inserts >= 0:
-            assert count('INSERT') == inserts
+            insert_count = count('INSERT')
+            assert insert_count == inserts, '%s != %s' % (insert_count, inserts)
         if updates >= 0:
-            assert count('UPDATE') == updates
+            update_count = count('UPDATE')
+            assert update_count == updates, '%s != %s' % (update_count, updates)
         if deletes >= 0:
-            assert count('DELETE') == deletes
+            delete_count = count('DELETE')
+            assert delete_count == deletes, '%s != %s' % (delete_count, deletes)
 
         backend.stop_debug()
 

tests/test_api.py

@@ -1,6 +1,8 @@
 import cio
 import six
+import threading
 from cio.backends import cache
+from cio.conf import settings
 from cio.conf.exceptions import ImproperlyConfigured
 from cio.pipeline import pipeline
 from cio.backends import storage
@@ -46,6 +48,28 @@ def test_get_with_context(self):
         content = node.render(firstname=u'Jonas', lastname=u'Lundberg')
         self.assertEqual(content, u'{Welcome} Jonas Lundberg!')
 
+    def test_get_with_local_cache_pipe_settings(self):
+        def assert_local_thread():
+            settings.configure(local=True, CACHE={'PIPE': {'CACHE_ON_GET': False}})
+            self.assertIn('BACKEND', settings.CACHE, 'Cache settings should be merged')
+
+            # Test twice so that not the first get() caches the reponse in pipeline
+            with self.assertCache(calls=1, misses=1, hits=0, sets=0):
+                cio.get('local/settings', default=u'default', lazy=False)
+            with self.assertCache(calls=1, misses=1, hits=0, sets=0):
+                cio.get('local/settings', default=u'default', lazy=False)
+
+        thread = threading.Thread(target=assert_local_thread)
+        thread.start()
+        thread.join()
+
+        # Back on main thread, settings should not be affected
+        # Test twice to make sure first get chaches the reponse in pipeline
+        with self.assertCache(calls=2, misses=1, hits=0, sets=1):
+            cio.get('local/settings', default=u'default', lazy=False)
+        with self.assertCache(calls=1, misses=0, hits=1, sets=0):
+            cio.get('local/settings', default=u'default', lazy=False)
+
     def test_set(self):
         with self.assertRaises(URI.Invalid):
             cio.set('page/title', 'fail')

tests/test_app.py

@@ -22,6 +22,21 @@ def test_settings(self):
             self.assertEqual(settings.STORAGE, 'bogus.newstorage')
         self.assertEqual(settings.STORAGE, pre)
 
+        self.assertEqual(settings.STORAGE['NAME'], ':memory:', "Should've been overridden")
+
+        settings.STORAGE['PIPE'] = {'FOO': 'bar'}
+        def assert_local_thread_settings():
+            settings.configure(local=True, STORAGE={'PIPE': {'HAM': 'spam'}})
+            self.assertEqual(settings.STORAGE['PIPE']['FOO'], 'bar')
+            self.assertEqual(settings.STORAGE['PIPE']['HAM'], 'spam')
+
+        thread = threading.Thread(target=assert_local_thread_settings)
+        thread.start()
+        thread.join()
+
+        self.assertEqual(settings.STORAGE['PIPE']['FOO'], 'bar')
+        self.assertNotIn('HAM', settings.STORAGE['PIPE'])
+
     def test_environment(self):
         """
         'default': {