Skip to content

Commit

Permalink
WIP on the re-implementation of fact caching and various backends.
Browse files Browse the repository at this point in the history
  • Loading branch information
joshdrake authored and Michael DeHaan committed Aug 11, 2014
1 parent fb5a140 commit aa41904
Show file tree
Hide file tree
Showing 11 changed files with 466 additions and 48 deletions.
59 changes: 59 additions & 0 deletions lib/ansible/cache/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
# (c) 2014, Michael DeHaan <[email protected]>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.

from collections import MutableMapping

from ansible import utils
from ansible import constants as C
from ansible import errors


class FactCache(MutableMapping):

def __init__(self, *args, **kwargs):
self._plugin = utils.plugins.cache_loader.get(C.CACHE_PLUGIN)
if self._plugin is None:
return

def __getitem__(self, key):
if key not in self:
raise KeyError
return self._plugin.get(key)

def __setitem__(self, key, value):
self._plugin.set(key, value)

def __delitem__(self, key):
self._plugin.delete(key)

def __contains__(self, key):
return self._plugin.contains(key)

def __iter__(self):
return iter(self._plugin.keys())

def __len__(self):
return len(self._plugin.keys())

def copy(self):
"""
Return a primitive copy of the keys and values from the cache.
"""
return dict([(k, v) for (k, v) in self.iteritems()])

def keys(self):
return self._plugin.keys()
15 changes: 15 additions & 0 deletions lib/ansible/cache/base.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
class BaseCacheModule(object):
def get(self, key):
raise NotImplementedError("Subclasses of {} must implement the '{}' method".format(self.__class__.__name__, self.__name__))

def set(self, key, value):
raise NotImplementedError("Subclasses of {} must implement the '{}' method".format(self.__class__.__name__, self.__name__))

def keys(self):
raise NotImplementedError("Subclasses of {} must implement the '{}' method".format(self.__class__.__name__, self.__name__))

def contains(self, key):
raise NotImplementedError("Subclasses of {} must implement the '{}' method".format(self.__class__.__name__, self.__name__))

def delete(self, key):
raise NotImplementedError("Subclasses of {} must implement the '{}' method".format(self.__class__.__name__, self.__name__))
65 changes: 65 additions & 0 deletions lib/ansible/cache/file.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
# (c) 2014, Michael DeHaan <[email protected]>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
from __future__ import absolute_import

import collections
import json
import os
import shutil
import sys
import tempfile
from datetime import datetime

from ansible import constants as C
from ansible.cache.memory import CacheModule as MemoryCacheModule


class CacheModule(MemoryCacheModule):
def __init__(self):
super(CacheModule, self).__init__()
self._timeout = int(C.CACHE_PLUGIN_TIMEOUT)
self._filename = '/tmp/ansible_facts.json'

if os.access(self._filename, os.R_OK):
mtime = datetime.fromtimestamp(os.path.getmtime(self._filename))
if self._timeout == 0 or (datetime.now() - mtime).total_seconds() < self._timeout:
with open(self._filename, 'rb') as f:
# we could make assumptions about the MemoryCacheModule here if we wanted
# to be more efficient, but performance isn't the priority with this module
data = json.load(f)
if isinstance(data, collections.Mapping):
for k, v in data.items():
super(CacheModule, self).set(k, v)

def set(self, *args, **kwargs):
super(CacheModule, self).set(*args, **kwargs)
self.flush()

def delete(self, *args, **kwargs):
super(CacheModule, self).delete(*args, **kwargs)
self.flush()

def flush(self):
temp = tempfile.TemporaryFile('r+b')

try:
json.dump(self._cache, temp, separators=(',', ':'))
temp.seek(0)
with open(self._filename, 'w+b') as f:
shutil.copyfileobj(temp, f)
finally:
temp.close()
112 changes: 112 additions & 0 deletions lib/ansible/cache/memcached.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
# (c) 2014, Michael DeHaan <[email protected]>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
import collections
import sys
import time

from ansible import constants as C
from ansible.cache.base import BaseCacheModule

try:
import memcache
except ImportError:
print 'python-memcached is required for the memcached fact cache'
sys.exit(1)


class CacheModuleKeys(collections.MutableSet):
"""
A set subclass that keeps track of insertion time and persists
the set in memcached.
"""
PREFIX = 'ansible_cache_keys'

def __init__(self, cache, *args, **kwargs):
self._cache = cache
self._keyset = dict(*args, **kwargs)

def __contains__(self, key):
return key in self._keyset

def __iter__(self):
return iter(self._keyset)

def __len__(self):
return len(self._keyset)

def add(self, key):
self._keyset[key] = time.time()
self._cache.set(self.PREFIX, self._keyset)

def discard(self, key):
del self._keyset[key]
self._cache.set(self.PREFIX, self._keyset)

def remove_by_timerange(self, s_min, s_max):
for k in self._keyset.keys():
t = self._keyset[k]
if s_min < t < s_max:
del self._keyset[k]
self._cache.set(self.PREFIX, self._keyset)


class CacheModule(BaseCacheModule):

def __init__(self, *args, **kwargs):
if C.CACHE_PLUGIN_CONNECTION:
connection = C.CACHE_PLUGIN_CONNECTION.split(',')
else:
connection = ['127.0.0.1:11211']

self._timeout = C.CACHE_PLUGIN_TIMEOUT
self._prefix = C.CACHE_PLUGIN_PREFIX
self._cache = memcache.Client(connection, debug=0)
self._keys = CacheModuleKeys(self._cache, self._cache.get(CacheModuleKeys.PREFIX) or [])

def _make_key(self, key):
return "{}{}".format(self._prefix, key)

def _expire_keys(self):
if self._timeout > 0:
expiry_age = time.time() - self._timeout
self._keys.remove_by_timerange(0, expiry_age)

def get(self, key):
value = self._cache.get(self._make_key(key))
# guard against the key not being removed from the zset;
# this could happen in cases where the timeout value is changed
# between invocations
if value is None:
self.delete(key)
raise KeyError
return value

def set(self, key, value):
self._cache.set(self._make_key(key), value, time=self._timeout)
self._keys.add(key)

def keys(self):
self._expire_keys()
return list(iter(self._keys))

def contains(self, key):
self._expire_keys()
return key in self._keys

def delete(self, key):
self._cache.delete(self._make_key(key))
self._keys.discard(key)
37 changes: 37 additions & 0 deletions lib/ansible/cache/memory.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
# (c) 2014, Michael DeHaan <[email protected]>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.


class CacheModule(object):

def __init__(self, *args, **kwargs):
self._cache = {}

def get(self, key):
return self._cache.get(key)

def set(self, key, value):
self._cache[key] = value

def keys(self):
return self._cache.keys()

def contains(self, key):
return key in self._cache

def delete(self, key):
del self._cache[key]
Loading

0 comments on commit aa41904

Please sign in to comment.