cache: add a simple RedisCache implementation (alongside local and uwsgi)

proxy_ip_resolver: add option to use RedisCache if redis_cache_key set in config
proxy_ip_resolver: add 'delete' option to delete ip from cache, closes #145

Author: ikreymer

pywb/framework/cache.py

@@ -5,6 +5,9 @@
     uwsgi_cache = False
 
 
+from redis import StrictRedis
+
+
 #=================================================================
 class UwsgiCache(object):  # pragma: no cover
     def __setitem__(self, item, value):
@@ -27,7 +30,31 @@ def __getitem__(self, item):
 
 
 #=================================================================
-def create_cache():
+class RedisCache(object):
+    def __init__(self, redis_url):
+        # must be of the form redis://host:port/db/key
+        redis_url, key = redis_url.rsplit('/', 1)
+        self.redis = StrictRedis.from_url(redis_url)
+        self.key = key
+
+    def __setitem__(self, item, value):
+        self.redis.hset(self.key, item, value)
+
+    def __getitem__(self, item):
+        return self.redis.hget(self.key, item)
+
+    def __contains__(self, item):
+        return self.redis.hexists(self.key, item)
+
+    def __delitem__(self, item):
+        self.redis.hdel(self.key, item)
+
+
+#=================================================================
+def create_cache(redis_url_key=None):
+    if redis_url_key:
+        return RedisCache(redis_url_key)
+
     if uwsgi_cache:  # pragma: no cover
         return UwsgiCache()
     else:

pywb/framework/proxy_resolvers.py

@@ -114,7 +114,7 @@ def read_basic_auth_coll(value):
 class IPCacheResolver(BaseCollResolver):
     def __init__(self, routes, config):
         super(IPCacheResolver, self).__init__(routes, config)
-        self.cache = create_cache()
+        self.cache = create_cache(config.get('redis_cache_key'))
         self.magic_name = config['magic_name']
 
     def _get_ip(self, env):
@@ -131,17 +131,22 @@ def _get_ip(self, env):
     def get_proxy_coll_ts(self, env):
         ip = env['REMOTE_ADDR']
         qs = env.get('pywb.proxy_query')
+
         if qs:
             res = urlparse.parse_qs(qs)
 
             if 'ip' in res:
                 ip = res['ip'][0]
 
-            if 'coll' in res:
-                self.cache[ip + ':c'] = res['coll'][0]
+            if 'delete' in res:
+                del self.cache[ip + ':c']
+                del self.cache[ip + ':t']
+            else:
+                if 'coll' in res:
+                    self.cache[ip + ':c'] = res['coll'][0]
 
-            if 'ts' in res:
-                self.cache[ip + ':t'] = res['ts'][0]
+                if 'ts' in res:
+                    self.cache[ip + ':t'] = res['ts'][0]
 
         coll = self.cache[ip + ':c']
         ts = self.cache[ip + ':t']

tests/test_proxy_http_ip.py

@@ -81,3 +81,14 @@ def test_proxy_ip_change_ts_for_ip(self):
         resp = self.get_url('http://www.iana.org/', '127.0.0.3')
         self._assert_basic_html(resp)
         assert '"20140127171238"' in resp.body
+
+    def test_proxy_ip_delete_ip(self):
+        resp = self.get_url('http://info.pywb.proxy/')
+        assert resp.json == {'ip': '127.0.0.1', 'coll': 'all', 'ts': '1996'}
+
+        resp = self.get_url('http://info.pywb.proxy/set?delete=true')
+        assert resp.json == {'ip': '127.0.0.1', 'coll': None, 'ts': None}
+
+        resp = self.get_url('http://info.pywb.proxy/')
+        assert resp.json == {'ip': '127.0.0.1', 'coll': None, 'ts': None}
+