Add cache_prefix setting (fixes #227)

CHANGELOG.rst

@@ -10,6 +10,7 @@ This document describes changes between each past release.
 **New features**
 
 - Default console log renderer now has colours (#671)
+- A ``cache_prefix`` setting was added for cache backends. (#680)
 
 
 3.0.0 (2016-02-26)

CONTRIBUTORS.rst

@@ -11,6 +11,7 @@ Contributors
 * FooBarQuaxx
 * Greeshma <greeshmabalabadra@gmail.com>
 * Hiromipaw <silvia@nopressure.co.uk>
+* Lavish Aggarwal <lucky.lavish@gmail.com>
 * Mathieu Leplatre <mathieu@mozilla.com>
 * Michiel de Jong <michiel@unhosted.org>
 * Nicolas Perriault <nperriault@mozilla.com>

cliquet/__init__.py

@@ -32,6 +32,7 @@
     'cache_backend': '',
     'cache_url': '',
     'cache_pool_size': 25,
+    'cache_prefix': '',
     'cors_origins': '*',
     'cors_max_age_seconds': 3600,
     'eos': None,

cliquet/cache/__init__.py

@@ -11,6 +11,7 @@
 class CacheBase(object):
 
     def __init__(self, *args, **kwargs):
+        self.prefix = kwargs['cache_prefix']
         pass
 
     def initialize_schema(self):

cliquet/cache/memory.py

@@ -25,30 +25,32 @@ def flush(self):
         self._store = {}
 
     def ttl(self, key):
-        ttl = self._ttl.get(key)
+        ttl = self._ttl.get(self.prefix + key)
         if ttl is not None:
             return (ttl - utils.msec_time()) / 1000.0
         return -1
 
     def expire(self, key, ttl):
-        self._ttl[key] = utils.msec_time() + int(ttl * 1000.0)
+        self._ttl[self.prefix + key] = utils.msec_time() + int(ttl * 1000.0)
 
     def set(self, key, value, ttl=None):
-        self._store[key] = value
         if ttl is not None:
             self.expire(key, ttl)
+        self._store[self.prefix + key] = value
 
     def get(self, key):
         current = utils.msec_time()
-        expired = [k for k, v in self._ttl.items() if current > v]
-        for key in expired:
-            self.delete(key)
-        return self._store.get(key)
+        expired = [k for k, v in self._ttl.items() if current >= v]
+        for expired_item_key in expired:
+            self.delete(expired_item_key[len(self.prefix):])
+        return self._store.get(self.prefix + key)
 
     def delete(self, key):
+        key = self.prefix + key
         self._ttl.pop(key, None)
         self._store.pop(key, None)
 
 
 def load_from_config(config):
-    return Cache()
+    settings = config.get_settings()
+    return Cache(cache_prefix=settings['cache_prefix'])

cliquet/cache/postgresql/__init__.py

@@ -90,7 +90,7 @@ def ttl(self, key):
            AND ttl IS NOT NULL;
         """
         with self.client.connect(readonly=True) as conn:
-            result = conn.execute(query, dict(key=key))
+            result = conn.execute(query, dict(key=self.prefix + key))
             if result.rowcount > 0:
                 return result.fetchone()['ttl']
         return -1
@@ -100,7 +100,7 @@ def expire(self, key, ttl):
         UPDATE cache SET ttl = sec2ttl(:ttl) WHERE key = :key;
         """
         with self.client.connect() as conn:
-            conn.execute(query, dict(ttl=ttl, key=key))
+            conn.execute(query, dict(ttl=ttl, key=self.prefix + key))
 
     def set(self, key, value, ttl=None):
         query = """
@@ -114,24 +114,26 @@ def set(self, key, value, ttl=None):
         """
         value = json.dumps(value)
         with self.client.connect() as conn:
-            conn.execute(query, dict(key=key, value=value, ttl=ttl))
+            conn.execute(query, dict(key=self.prefix + key,
+                                     value=value, ttl=ttl))
 
     def get(self, key):
         purge = "DELETE FROM cache WHERE ttl IS NOT NULL AND now() > ttl;"
         query = "SELECT value FROM cache WHERE key = :key;"
         with self.client.connect() as conn:
             conn.execute(purge)
-            result = conn.execute(query, dict(key=key))
+            result = conn.execute(query, dict(key=self.prefix + key))
             if result.rowcount > 0:
                 value = result.fetchone()['value']
                 return json.loads(value)
 
     def delete(self, key):
         query = "DELETE FROM cache WHERE key = :key"
         with self.client.connect() as conn:
-            conn.execute(query, dict(key=key))
+            conn.execute(query, dict(key=self.prefix + key))
 
 
 def load_from_config(config):
+    settings = config.get_settings()
     client = create_from_config(config, prefix='cache_')
-    return Cache(client=client)
+    return Cache(client=client, cache_prefix=settings['cache_prefix'])

cliquet/cache/redis.py

@@ -20,7 +20,13 @@ class Cache(CacheBase):
 
         cliquet.cache_pool_size = 50
 
+    If the database is used for multiple Kinto deployement cache, you
+    may want to add a prefix to every key to avoid collision::
+
+        cliquet.cache_prefix = stack1_
+
     :noindex:
+
     """
 
     def __init__(self, client, *args, **kwargs):
@@ -41,32 +47,33 @@ def flush(self):
 
     @wrap_redis_error
     def ttl(self, key):
-        return self._client.ttl(key)
+        return self._client.ttl(self.prefix + key)
 
     @wrap_redis_error
     def expire(self, key, ttl):
-        self._client.pexpire(key, int(ttl * 1000))
+        self._client.pexpire(self.prefix + key, int(ttl * 1000))
 
     @wrap_redis_error
     def set(self, key, value, ttl=None):
         value = json.dumps(value)
         if ttl:
-            self._client.psetex(key, int(ttl * 1000), value)
+            self._client.psetex(self.prefix + key, int(ttl * 1000), value)
         else:
-            self._client.set(key, value)
+            self._client.set(self.prefix + key, value)
 
     @wrap_redis_error
     def get(self, key):
-        value = self._client.get(key)
+        value = self._client.get(self.prefix + key)
         if value:
             value = value.decode('utf-8')
             return json.loads(value)
 
     @wrap_redis_error
     def delete(self, key):
-        self._client.delete(key)
+        self._client.delete(self.prefix + key)
 
 
 def load_from_config(config):
+    settings = config.get_settings()
     client = create_from_config(config, prefix='cache_')
-    return Cache(client)
+    return Cache(client, cache_prefix=settings['cache_prefix'])

cliquet/storage/postgresql/client.py

@@ -76,6 +76,7 @@ def create_from_config(config, prefix=''):
     # Custom Cliquet settings, unsupported by SQLAlchemy.
     settings.pop(prefix + 'backend', None)
     settings.pop(prefix + 'max_fetch_size', None)
+    settings.pop(prefix + 'prefix', None)
     transaction_per_request = settings.pop('transaction_per_request', False)
 
     url = settings[prefix + 'url']

cliquet/tests/test_cache.py

@@ -15,7 +15,7 @@
 
 class CacheBaseTest(unittest.TestCase):
     def setUp(self):
-        self.cache = CacheBase()
+        self.cache = CacheBase(cache_prefix='')
 
     def test_mandatory_overrides(self):
         calls = [
@@ -56,6 +56,16 @@ def tearDown(self):
         super(BaseTestCache, self).tearDown()
         self.cache.flush()
 
+    def get_backend_prefix(self, prefix):
+        settings_prefix = self.settings.copy()
+        settings_prefix['cache_prefix'] = prefix
+        config_prefix = self._get_config(settings=settings_prefix)
+
+        # initiating cache backend with prefix:
+        backend_prefix = self.backend.load_from_config(config_prefix)
+
+        return backend_prefix
+
     def test_backend_error_is_raised_anywhere(self):
         self.client_error_patcher.start()
         calls = [
@@ -143,9 +153,89 @@ def test_ttl_return_none_if_unknown(self):
         ttl = self.cache.ttl('unknown')
         self.assertTrue(ttl < 0)
 
+    def test_cache_prefix_is_set(self):
+        backend_prefix = self.get_backend_prefix(prefix='prefix_')
+
+        # Set the value
+        backend_prefix.set('key', 'foo')
+
+        # Validate that it was set with the prefix.
+        obtained = self.cache.get('prefix_key')
+        self.assertEqual(obtained, 'foo')
+
+    def test_cache_when_prefix_is_not_set(self):
+        backend_prefix = self.get_backend_prefix(prefix='')
+
+        # Set a value
+        backend_prefix.set('key', 'foo')
+
+        # Validate that it was set with no prefix
+        obtained = self.cache.get('key')
+        self.assertEqual(obtained, 'foo')
+
+    def test_prefix_value_use_to_get_data(self):
+        backend_prefix = self.get_backend_prefix(prefix='prefix_')
+
+        # Set the value with the prefix
+        self.cache.set('prefix_key', 'foo')
+
+        # Validate that the prefix was added
+        obtained = backend_prefix.get('key')
+        self.assertEqual(obtained, 'foo')
+
+    def test_prefix_value_use_to_delete_data(self):
+        backend_prefix = self.get_backend_prefix(prefix='prefix_')
+        # Set the value
+        self.cache.set('prefix_key', 'foo')
+
+        # Delete the value
+        backend_prefix.delete('key')
+
+        # Validate that the value was deleted
+        obtained = self.cache.get('prefix_key')
+        self.assertEqual(obtained, None)
+
+    def test_prefix_value_used_with_ttl(self):
+        backend_prefix = self.get_backend_prefix(prefix='prefix_')
+
+        self.cache.set('prefix_key', 'foo', 10)
+
+        # Validate that the ttl add the prefix to the key.
+        obtained = backend_prefix.ttl('key')
+        self.assertLessEqual(obtained, 10)
+        self.assertGreater(obtained, 9)
+
+    def test_prefix_value_used_with_expire(self):
+        backend_prefix = self.get_backend_prefix(prefix='prefix_')
+
+        self.cache.set('prefix_foobar', 'toto', 10)
+
+        # expiring the ttl of key
+        backend_prefix.expire('foobar', 0)
+
+        # Make sure the TTL was set accordingly.
+        ttl = self.cache.ttl('prefix_foobar')
+        self.assertLessEqual(ttl, 0)
+
+        # The record should have expired
+        retrieved = self.cache.get('prefix_foobar')
+        self.assertIsNone(retrieved)
+
 
 class MemoryCacheTest(BaseTestCache, unittest.TestCase):
     backend = memory_backend
+    settings = {
+        'cache_prefix': ''
+    }
+
+    def get_backend_prefix(self, prefix):
+        backend_prefix = BaseTestCache.get_backend_prefix(self, prefix)
+
+        # Share the store between both client for tests.
+        backend_prefix._ttl = self.cache._ttl
+        backend_prefix._store = self.cache._store
+
+        return backend_prefix
 
     def test_backend_error_is_raised_anywhere(self):
         pass
@@ -161,7 +251,8 @@ class RedisCacheTest(BaseTestCache, unittest.TestCase):
     backend = redis_backend
     settings = {
         'cache_url': '',
-        'cache_pool_size': 10
+        'cache_pool_size': 10,
+        'cache_prefix': ''
     }
 
     def setUp(self):
@@ -185,7 +276,8 @@ class PostgreSQLCacheTest(BaseTestCache, unittest.TestCase):
     backend = postgresql_backend
     settings = {
         'cache_pool_size': 10,
-        'cache_url': 'postgres://postgres:postgres@localhost:5432/testdb'
+        'cache_url': 'postgres://postgres:postgres@localhost:5432/testdb',
+        'cache_prefix': ''
     }
 
     def setUp(self):

cliquet_docs/reference/configuration.rst

@@ -298,6 +298,7 @@ Backend
 
     cliquet.cache_backend = cliquet.cache.redis
     cliquet.cache_url = redis://localhost:6379/0
+    cliquet.cache_prefix = stack1_
 
     # Control number of pooled connections
     # cliquet.storage_pool_size = 50
@@ -539,4 +540,4 @@ possible to change the ``initialization_sequence`` setting.
                                       cliquet.initialization.setup_backoff
                                       cliquet.initialization.setup_statsd
                                       cliquet.initialization.setup_listeners
-                                      cliquet.events.setup_transaction_hook
+                                      cliquet.events.setup_transaction_hook