Merge pull request shanbay#23 from yandy/add-cache

add cache

requirements.txt

@@ -1,5 +1,7 @@
 grpcio>=1.4.0,<1.5.0
 orator
+mysqlclient
+redis
 celery
 python-consul
 pytest

sea/cli.py

@@ -102,6 +102,7 @@ class NewCmd(AbstractCommand):
         'consul': [],
         'orator': ['configs/development/orator.py.tmpl',
                    'configs/testing/orator.py.tmpl'],
+        'cache': [],
         'celery': ['configs/development/celery.py.tmpl',
                    'configs/testing/celery.py.tmpl',
                    'app/tasks.py.tmpl'],
@@ -116,6 +117,8 @@ def opt(self, subparsers):
             help='skip add git files and run git init')
         p.add_argument(
             '--skip-orator', action='store_true', help='skip orator')
+        p.add_argument(
+            '--skip-cache', action='store_true', help='skip cache')
         p.add_argument(
             '--skip-celery', action='store_true', help='skip celery')
         p.add_argument(

sea/contrib/extensions/cache/__init__.py

@@ -0,0 +1,75 @@
+import functools
+import logging
+
+from sea.extensions import AbstractExtension
+from . import backends
+
+
+logger = logging.getLogger(__name__)
+DEFAULT_KEY_TYPES = (str, int, float, bool)
+
+
+def _trans_key(v):
+    if isinstance(v, bytes):
+        return v.decode()
+    if v is None or isinstance(v, DEFAULT_KEY_TYPES):
+        return str(v)
+    else:
+        raise ValueError('only str, int, float, bool can be key')
+
+
+def default_key(f, *args, **kwargs):
+    keys = [_trans_key(v) for v in args]
+    keys += sorted(
+        ['{}={}'.format(k, _trans_key(v)) for k, v in kwargs.items()])
+    return '{}.{}.{}'.format(f.__module__, f.__name__, '.'.join(keys))
+
+
+class Cache(AbstractExtension):
+
+    PROTO_METHODS = ('get', 'get_many', 'set', 'set_many', 'delete',
+                     'delete_many', 'expire', 'expireat', 'clear')
+
+    def __init__(self):
+        self._backend = None
+
+    def init_app(self, app):
+        self.app = app
+        opts = app.config.get_namespace('CACHE_')
+        backend_cls = getattr(backends, opts.pop('backend'))
+        # default ttl: 60 * 60 * 48
+        self.default_ttl = opts.pop('default_ttl', 172800)
+        self._backend = backend_cls(**opts)
+
+    def cached(self, ttl=None, cache_key=default_key, unless=None):
+        if ttl is None:
+            ttl = self.default_ttl
+
+        def decorator(f):
+            def make_cache_key(*args, **kwargs):
+                if callable(cache_key):
+                    key = cache_key(f, *args, **kwargs)
+                else:
+                    key = cache_key
+                return '{}.{}'.format(self.app.name, key)
+
+            @functools.wraps(f)
+            def wrapper(*args, **kwargs):
+                if callable(unless) and unless(*args, **kwargs):
+                    return f(*args, **kwargs)
+                key = make_cache_key(*args, **kwargs)
+                rv = self._backend.get(key)
+                if rv is None:
+                    rv = f(*args, **kwargs)
+                    self._backend.set(key, rv, ttl=ttl)
+                return rv
+
+            wrapper.uncached = f
+
+            return wrapper
+        return decorator
+
+    def __getattr__(self, name):
+        if name in self.PROTO_METHODS:
+            return getattr(self._backend, name)
+        return super().__getattr__(name)

sea/contrib/extensions/cache/backends.py

@@ -0,0 +1,159 @@
+import abc
+import pickle
+import time
+
+
+class BaseBackend(metaclass=abc.ABCMeta):
+
+    @abc.abstractmethod
+    def get(self, key):
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def get_many(self, keys):
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def set(self, key, value, ttl=None):
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def set_many(self, mapping):
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def delete(self, key):
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def delete_many(self, keys):
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def expire(self, key, seconds):
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def expireat(self, key, timestamp):
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def clear(self):
+        raise NotImplementedError
+
+
+class Redis(BaseBackend):
+
+    def __init__(self, *args, **kwargs):
+        import redis
+        self._client = redis.StrictRedis(*args, **kwargs)
+
+    def get(self, key):
+        return self._client.get(key)
+
+    def get_many(self, keys):
+        return self._client.mget(keys)
+
+    def set(self, key, value, ttl=None):
+        return self._client.set(key, value, ex=ttl)
+
+    def set_many(self, mapping):
+        return self._client.mset(mapping)
+
+    def delete(self, key):
+        return self._client.delete(key)
+
+    def delete_many(self, keys):
+        return self._client.delete(keys)
+
+    def expire(self, key, seconds):
+        return self._client.expire(key, seconds)
+
+    def expireat(self, key, timestamp):
+        return self._client.expireat(key, int(timestamp))
+
+    def clear(self):
+        return self._client.flushdb()
+
+
+class Simple(BaseBackend):
+
+    def __init__(self, threshold=500, default_ttl=600):
+        self._cache = {}
+        self.threshold = threshold
+        self.default_ttl = default_ttl
+
+    def _ttl2expire(self, ttl):
+        if ttl is None:
+            ttl = self.default_ttl
+        now = int(time.time())
+        return now + ttl
+
+    def _expired(self, ts):
+        now = int(time.time())
+        return now > ts
+
+    def _prune(self):
+        toremove = []
+        for k, (exp, v) in self._cache.items():
+            if self._expired(exp):
+                toremove.append(k)
+        for k in toremove:
+            self._cache.pop(k, None)
+        return len(self._cache)
+
+    def get(self, key):
+        exp, v = self._cache.get(key, (None, None))
+        if exp is None:
+            return None
+        if self._expired(exp):
+            self._cache.pop(key)
+            return None
+        return pickle.loads(v)
+
+    def get_many(self, keys):
+        return [self.get(k) for k in keys]
+
+    def set(self, key, value, ttl=None):
+        if len(self._cache) >= self.threshold \
+                and self._prune() >= self.threshold:
+            return False
+        self._cache[key] = (
+            self._ttl2expire(ttl), pickle.dumps(
+                value, pickle.HIGHEST_PROTOCOL))
+        return True
+
+    def set_many(self, mapping):
+        for k, v in mapping.items():
+            self.set(k, v)
+        return True
+
+    def delete(self, key):
+        try:
+            self._cache.pop(key)
+            return 1
+        except KeyError:
+            pass
+        return 0
+
+    def delete_many(self, keys):
+        return sum([self.delete(k) for k in keys])
+
+    def expire(self, key, seconds):
+        try:
+            exp, v = self._cache[key]
+        except KeyError:
+            return 0
+        self._cache[key] = (self._ttl2expire(seconds), v)
+        return 1
+
+    def expireat(self, key, timestamp):
+        try:
+            exp, v = self._cache[key]
+        except KeyError:
+            return 0
+        self._cache[key] = (timestamp, v)
+        return 1
+
+    def clear(self):
+        return self._cache.clear()

sea/contrib/extensions/orator/__init__.py

@@ -8,7 +8,9 @@ def __init__(self):
         self._client = None
 
     def init_app(self, app):
-        self._client = orator.DatabaseManager(app.config.get('ORATOR'))
+        conf_m = app.config.get('ORATOR')
+        self._client = orator.DatabaseManager(conf_m.DATABASES)
+        orator.Model.set_connection_resolver(self._client)
 
     def __getattr__(self, name):
         return getattr(self._client, name)

sea/template/app/extensions.py.tmpl

@@ -4,6 +4,9 @@ from sea.contrib.extensions.consul import Consul
 {%- if not skip_orator -%}
 from sea.contrib.extensions.orator import Orator
 {% endif %}
+{%- if not skip_cache -%}
+from sea.contrib.extensions.cache import Cache
+{% endif %}
 {%- if not skip_celery -%}
 from sea.contrib.extensions.celery import Celery
 {% endif %}
@@ -14,6 +17,9 @@ consul = Consul()
 {%- if not skip_orator -%}
 db = Orator()
 {% endif %}
+{%- if not skip_cache -%}
+cache = Cache()
+{% endif %}
 {%- if not skip_celery -%}
 celeryapp = Celery()
 {% endif %}

sea/template/configs/default/__init__.py.tmpl

@@ -1,2 +1,12 @@
 {% if not skip_orator -%}from . import orator as ORATOR{%- endif %}
 {% if not skip_celery -%}from . import celery as CELERY{%- endif %}
+{% if not skip_cache -%}
+CACHE_BACKEND = 'Redis'
+CACHE_HOST = 'localhost'
+CACHE_DB = 0
+CACHE_PORT = 6379
+CACHE_PASSWORD = 'abcdef'
+{%- endif %}
+
+TESTING = False
+DEBUG = True

sea/template/configs/default/orator.py.tmpl

@@ -1,5 +1,6 @@
 DATABASES = {
-    'sqlite': {
+    'default': 'db',
+    'db': {
         'driver': 'mysql',
         'host': 'localhost',
         'database': {{ project }},

sea/template/configs/development/__init__.py.tmpl

@@ -1,3 +1 @@
 from configs.default import *
-
-TESTING = False

sea/template/configs/testing/__init__.py.tmpl

@@ -1,5 +1,3 @@
 from configs.default import *
-{% if not skip_orator -%}from . import orator as ORATOR{%- endif %}
-{% if not skip_celery -%}from . import celery as CELERY{%- endif %}
 
 TESTING = True

sea/template/requirements.txt.tmpl

@@ -1,5 +1,9 @@
 {%- if not skip_orator -%}
 orator
+mysqlclient
+{% endif -%}
+{%- if not skip_cache -%}
+redis
 {% endif -%}
 {%- if not skip_consul -%}
 python-consul

tests/test_cli.py

@@ -19,8 +19,9 @@ def test_cmd_server():
     with pytest.raises(ValueError):
         cli.main()
     sys.argv = 'sea -w ./tests/wd s -b 127.0.0.1'.split()
-    with mock.patch('sea.cli.Server', autospec=True):
-        assert isinstance(cli.main(), mock.Mock)
+    with mock.patch('sea.cli.Server', autospec=True) as mocked:
+        cli.main()
+        mocked.return_value.run.assert_called_with()
 
 
 def test_cmd_console():
@@ -29,12 +30,12 @@ def test_cmd_console():
         cli.main()
     sys.argv = 'sea -w ./tests/wd c'.split()
     mocked = mock.MagicMock()
-    mocked.embed = mock.MagicMock(return_value='Embed Called')
-    mocked.interact = mock.MagicMock(return_value='Interact Called')
     with mock.patch.dict('sys.modules', {'IPython': mocked, 'code': mocked}):
-        assert cli.main() == 'Embed Called'
+        cli.main()
+        assert mocked.embed.called
         mocked.embed.side_effect = ImportError
-        assert cli.main() == 'Interact Called'
+        cli.main()
+        assert mocked.interact.called
 
 
 def test_cmd_new():
@@ -65,6 +66,7 @@ class MyprojectServicer(myproject_pb2_grpc.MyprojectServicer, metaclass=Servicer
     assert os.path.exists('./tests/myproject/app/tasks.py')
 
     correct_code = """\
+    redis
     celery
     pytest
     """

tests/test_config.py

@@ -39,4 +39,4 @@ class App:
     a.config = Config('root', {'n_x': json.dumps({'foo': 'bar'})})
     assert 'foo' in a.x
     a.x = json.dumps({'a': 1})
-    assert 'a' in a.x
+    assert 'a' in a.config['n_x']