Add support of redis clusters (#236)

* Add support of redis clusters (#1)

Add Redis cluster support as session management backend

beaker/cache.py

@@ -22,6 +22,7 @@
 import beaker.ext.google as google
 import beaker.ext.mongodb as mongodb
 import beaker.ext.redisnm as redisnm
+import beaker.ext.redisclusternm as redisclusternm
 from functools import wraps
 
 # Initialize the cache region dict
@@ -126,7 +127,8 @@ def _init(self):
  'ext:sqla': sqla.SqlaNamespaceManager,
  'ext:google': google.GoogleNamespaceManager,
  'ext:mongodb': mongodb.MongoNamespaceManager,
- 'ext:redis': redisnm.RedisNamespaceManager
+ 'ext:redis': redisnm.RedisNamespaceManager,
+ 'ext:rediscluster': redisclusternm.RedisClusterNamespaceManager,
 })
 
 

beaker/docs/modules/rediscluster.rst

@@ -0,0 +1,10 @@
+:mod:`beaker.ext.redisclusternm` -- Redis cluster NameSpace Manager and Synchronizer
+==============================================================================
+
+.. automodule:: beaker.ext.redisclusternm
+
+Module Contents
+---------------
+
+.. autoclass:: RedisClusterNamespaceManager
+.. autoclass:: RedisClusterSynchronizer

beaker/ext/redisclusternm.py

@@ -0,0 +1,87 @@
+import os
+import threading
+import time
+import pickle
+
+from beaker.container import NamespaceManager
+
+try:
+ import redis
+except ImportError:
+ redis = None
+
+from beaker.ext.redisnm import RedisNamespaceManager, RedisSynchronizer
+from beaker._compat import string_type
+
+
+class RedisClusterNamespaceManager(RedisNamespaceManager):
+ """Provides the :class:`.NamespaceManager` API over Redis cluster.
+
+ Provided ``urls`` can be both multiple redis connection strings separated by a comma or
+ an already existing RedisCluster instance.
+
+ Unlike a StrictRedis connection string, a RedisCluster one does not support
+ database indicators, it is zero by default.
+
+ Example: `redis://node-1:7001,redis://node-2:7002`
+
+ Additional options can be passed in kwargs (e.g. `username="redis", password="secure_password"`).
+
+ The data will be stored into redis keys, with their name
+ starting with ``beaker_cache:``.
+ """
+
+ def __init__(self, namespace, urls, timeout=None, **kwargs):
+ super(RedisNamespaceManager, self).__init__(namespace)
+ self.lock_dir = None # Redis uses redis itself for locking.
+ self.timeout = timeout
+ self.nodes = []
+ self.options = kwargs
+
+ if redis is None:
+ raise RuntimeError('redis is not available')
+
+ if isinstance(urls, string_type):
+ for url in urls.split(','):
+ url_options = redis.connection.parse_url(url)
+ if 'db' in url_options:
+ raise redis.exceptions.RedisClusterException(
+ "A ``db`` querystring option can only be 0 in cluster mode"
+ )
+ self.nodes.append(redis.cluster.ClusterNode(
+ host=url_options.get('host'),
+ port=url_options.get('port')
+ ))
+ self.client = RedisClusterNamespaceManager.clients.get(
+ urls, redis.cluster.RedisCluster, startup_nodes=self.nodes, **kwargs
+ )
+ else:
+ self.client = urls
+
+ def get_creation_lock(self, key):
+ return RedisClusterSynchronizer(self._format_key(key), self.client, self.nodes, **self.options)
+
+
+class RedisClusterSynchronizer(RedisSynchronizer):
+ """Synchronizer based on redis cluster.
+
+ Provided ``urls`` can be both multiple redis connection strings separated by a comma or
+ an already existing RedisCluster instance.
+
+ Unlike a StrictRedis connection string, a RedisCluster one does not support
+ database indicators, it is zero by default.
+
+ Example: ``redis://node-1:7001,redis://node-2:7002,
+
+ This Synchronizer only supports 1 reader or 1 writer at time, not concurrent readers.
+ """
+
+ def __init__(self, identifier, urls, nodes=None, **kwargs):
+ super(RedisSynchronizer, self).__init__()
+ self.identifier = 'beaker_lock:%s' % identifier
+ if isinstance(urls, string_type):
+ self.client = RedisClusterNamespaceManager.clients.get(
+ urls, redis.cluster.RedisCluster, startup_nodes=nodes, **kwargs
+ )
+ else:
+ self.client = urls

tests/test_managers/test_ext_rediscluster.py

@@ -0,0 +1,16 @@
+from beaker.cache import Cache
+from . import base
+
+
+class TestRedis(base.CacheManagerBaseTests):
+ CACHE_ARGS = {
+ 'type': 'ext:rediscluster',
+ 'url': 'redis://localhost:6379'
+ }
+
+ def test_client_reuse(self):
+ cache1 = Cache('test1', **self.CACHE_ARGS)
+ cli1 = cache1.namespace.client
+ cache2 = Cache('test2', **self.CACHE_ARGS)
+ cli2 = cache2.namespace.client
+ self.assertTrue(cli1 is cli2)