replay: make range cache available by default in replay_views since its

inited on first use. remove
separate subclass. 'enable_ranges' can be set to false to disable range
cache altogether
improve tests

Author: ikreymer

pywb/utils/test/test_loaders.py

@@ -53,6 +53,8 @@
 
 >>> extract_client_cookie(dict(HTTP_COOKIE='a=b; c=d'), 'x')
 
+>>> extract_client_cookie(dict(HTTP_COOKIE='x'), 'x')
+
 >>> extract_client_cookie({}, 'y')
 """
 

pywb/webapp/cached_replay.py

@@ -120,11 +120,7 @@ def __init__(self, query_handler, config=None):
         resolving_loader = ResolvingLoader(paths=paths,
                                            record_loader=record_loader)
 
-        enable_cache = config.get('enable_cache')
-        if enable_cache:
-            self.replay = CachedReplayView(resolving_loader, config)
-        else:
-            self.replay = ReplayView(resolving_loader, config)
+        self.replay = ReplayView(resolving_loader, config)
 
         self.fallback_handler = None
         self.fallback_name = config.get('fallback')

pywb/webapp/handlers.py

@@ -24,21 +24,6 @@ def cleanup(self):
             shutil.rmtree(self.temp_dir, True)
             self.temp_dir = None
 
-    def __call__(self, wbrequest, digest, wbresponse_func):
-        result = wbrequest.extract_range()
-        if not result:
-            return None, None
-
-        # no longer needed -- handled at frontend rewrite
-        #if wbrequest.env.get('HTTP_X_IGNORE_RANGE_ARG'):
-        #    wbrequest.wb_url.url = result[0]
-        #    return None, None
-
-        return self.handle_range(wbrequest,
-                                 digest,
-                                 wbresponse_func,
-                                 *result)
-
     def handle_range(self, wbrequest, digest, wbresponse_func,
                      url, start, end, use_206):
 

pywb/webapp/rangecache.py

@@ -15,6 +15,8 @@
 from views import J2TemplateView, add_env_globals
 from views import J2HtmlCapturesView, HeadInsertView
 
+from rangecache import range_cache
+
 
 #=================================================================
 class CaptureException(WbException):
@@ -49,6 +51,8 @@ def __init__(self, content_loader, config):
         else:
             self.response_class = WbResponse
 
+        self.enable_range_cache = config.get('enable_ranges', True)
+
         self._reporter = config.get('reporter')
 
     def render_content(self, wbrequest, cdx_lines, cdx_loader):
@@ -77,10 +81,10 @@ def render_content(self, wbrequest, cdx_lines, cdx_loader):
 
                     first = False
 
-                response = self.replay_capture(wbrequest,
-                                               cdx,
-                                               cdx_loader,
-                                               failed_files)
+                response = self.cached_replay_capture(wbrequest,
+                                                      cdx,
+                                                      cdx_loader,
+                                                      failed_files)
 
             except (CaptureException, ArchiveLoadFailed) as ce:
                 import traceback
@@ -99,6 +103,33 @@ def render_content(self, wbrequest, cdx_lines, cdx_loader):
 
         raise last_e
 
+    def cached_replay_capture(self, wbrequest, cdx, cdx_loader, failed_files):
+        def get_capture():
+            return self.replay_capture(wbrequest,
+                                       cdx,
+                                       cdx_loader,
+                                       failed_files)
+
+        if not self.enable_range_cache:
+            return get_capture()
+
+        range_info = wbrequest.extract_range()
+
+        if not range_info:
+            return get_capture()
+
+        range_status, range_iter = (range_cache.
+            handle_range(wbrequest,
+                         cdx.get('digest'),
+                         get_capture,
+                         *range_info))
+
+        response = self.response_class(range_status,
+                                       range_iter,
+                                       wbrequest=wbrequest,
+                                       cdx=cdx)
+        return response
+
     def replay_capture(self, wbrequest, cdx, cdx_loader, failed_files):
         (status_headers, stream) = (self.content_loader.
                                     resolve_headers_and_payload(cdx,

pywb/webapp/replay_views.py

@@ -38,9 +38,9 @@ collections:
         index_paths: ./sample_archive/cdx/
         fallback: live
 
-    pywb-rangecache:
+    pywb-norange:
         index_paths: ./sample_archive/cdx/
-        enable_cache: true
+        enable_ranges: False
 
 # indicate if cdx files are sorted by SURT keys -- eg: com,example)/
 # SURT keys are recommended for future indices, but non-SURT cdxs

tests/test_config.yaml

@@ -101,15 +101,6 @@ def test_replay_content(self):
         assert 'new _WBWombat' in resp.body, resp.body
         assert '/pywb/20140127171238/http://www.iana.org/time-zones"' in resp.body
 
-    def test_replay_content_with_rangecache(self):
-        resp = self.testapp.get('/pywb-rangecache/20140127171238/http://www.iana.org/')
-        self._assert_basic_html(resp)
-
-        assert '"20140127171238"' in resp.body
-        assert 'wb.js' in resp.body
-        assert 'new _WBWombat' in resp.body, resp.body
-        assert '/pywb-rangecache/20140127171238/http://www.iana.org/time-zones"' in resp.body
-
     def test_replay_non_frame_content(self):
         resp = self.testapp.get('/pywb-nonframe/20140127171238/http://www.iana.org/')
         self._assert_basic_html(resp)
@@ -142,6 +133,11 @@ def test_replay_url_agnostic_revisit(self):
         assert 'wb.js' in resp.body
         assert '/pywb/20130729195151/http://www.iana.org/domains/example"' in resp.body
 
+    def test_video_info_not_found(self):
+        # not actually archived, but ensure video info path is tested
+        resp = self.testapp.get('/pywb/vi_/https://www.youtube.com/watch?v=DjFZyFWSt1M', status=404)
+        assert resp.status_int == 404
+
     def test_replay_cdx_mod(self):
         resp = self.testapp.get('/pywb/20140127171239cdx_/http://www.iana.org/_css/2013.1/print.css')
         self._assert_basic_text(resp)
@@ -177,7 +173,7 @@ def test_replay_identity_1(self):
 
     def test_replay_range_cache_content(self):
         headers = [('Range', 'bytes=0-200')]
-        resp = self.testapp.get('/pywb-rangecache/20140127171251id_/http://example.com', headers=headers)
+        resp = self.testapp.get('/pywb/20140127171251id_/http://example.com', headers=headers)
 
         assert resp.status_int == 206
         assert resp.headers['Accept-Ranges'] == 'bytes'
@@ -186,9 +182,22 @@ def test_replay_range_cache_content(self):
 
         assert 'wb.js' not in resp.body
 
+    def test_replay_content_ignore_range(self):
+        headers = [('Range', 'bytes=0-200')]
+        resp = self.testapp.get('/pywb-norange/20140127171251id_/http://example.com', headers=headers)
+
+        # range request ignored
+        assert resp.status_int == 200
+
+        # full response
+        assert resp.content_length == 1270, resp.content_length
+
+        # identity, no header insertion
+        assert 'wb.js' not in resp.body
+
     def test_replay_range_cache_content_bound_end(self):
         headers = [('Range', 'bytes=10-10000')]
-        resp = self.testapp.get('/pywb-rangecache/20140127171251id_/http://example.com', headers=headers)
+        resp = self.testapp.get('/pywb/20140127171251id_/http://example.com', headers=headers)
 
         assert resp.status_int == 206
         assert resp.headers['Accept-Ranges'] == 'bytes'
@@ -201,7 +210,7 @@ def test_replay_range_cache_content_bound_end(self):
     def test_replay_redir_no_cache(self):
         headers = [('Range', 'bytes=10-10000')]
         # Range ignored
-        resp = self.testapp.get('/pywb-rangecache/20140126200927/http://www.iana.org/domains/root/db/', headers=headers)
+        resp = self.testapp.get('/pywb/20140126200927/http://www.iana.org/domains/root/db/', headers=headers)
         assert resp.content_length == 0
         assert resp.status_int == 302