Record unbound Range requests only, don't convert bounded range requests to unbounded (#845)

* record fix: don't convert 0-N range requests to unbounded for recording, only record 0- (unbounded range requests)
* tests: attempt to fix tests, disable youtube-dl related tests
* bump to 2.7.4

Author: ikreymer

.github/workflows/ci.yaml

@@ -12,10 +12,10 @@ jobs:
 
     steps:
       - name: checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
 
@@ -28,5 +28,5 @@ jobs:
         run: tox
 
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v1
+        uses: codecov/codecov-action@v3
 

pywb/apps/rewriterapp.py

@@ -248,8 +248,8 @@ def _check_range(self, inputreq, wb_url):
         range_start = start
         range_end = end
 
-        # if start with 0, load from upstream, but add range after
-        if start == 0:
+        # if start with 0 and no end, load from upstream
+        if start == 0 and not end:
             del inputreq.env['HTTP_RANGE']
         else:
             skip_record = True

pywb/version.py

@@ -1,4 +1,4 @@
-__version__ = '2.7.3'
+__version__ = '2.7.4'
 
 if __name__ == '__main__':
     print(__version__)

pywb/warcserver/test/test_handlers.py

@@ -385,7 +385,8 @@ def test_url_agnost(self):
         assert resp.headers['Warcserver-Source-Coll'] == 'url-agnost'
         assert resp.headers['Memento-Datetime'] == 'Mon, 29 Jul 2013 19:51:51 GMT'
 
-    @pytest.mark.skipif(os.environ.get('CI') is not None, reason='Skip Test on CI')
+    #@pytest.mark.skipif(os.environ.get('CI') is not None, reason='Skip Test on CI')
+    @pytest.mark.skip("youtube-dl update needed")
     def test_live_video_loader(self):
         pytest.importorskip('youtube_dl')
         params = {'url': 'http://www.youtube.com/v/BfBgWtAIbRc',
@@ -404,7 +405,8 @@ def test_live_video_loader(self):
         assert b'WARC-Type: metadata' in resp.body
         assert b'Content-Type: application/vnd.youtube-dl_formats+json' in resp.body
 
-    @pytest.mark.skipif(os.environ.get('CI') is not None, reason='Skip Test on CI')
+    #@pytest.mark.skipif(os.environ.get('CI') is not None, reason='Skip Test on CI')
+    @pytest.mark.skip("youtube-dl update needed")
     def test_live_video_loader_post(self):
         pytest.importorskip('youtube_dl')
         req_data = """\

tests/test_live_rewriter.py

@@ -169,7 +169,8 @@ def test_live_invalid_2(self, fmod_sl):
         resp = resp.follow(status=400)
         assert resp.status_int == 400
 
-    @pytest.mark.skipif(os.environ.get('CI') is not None, reason='Skip Test on CI')
+    #@pytest.mark.skipif(os.environ.get('CI') is not None, reason='Skip Test on CI')
+    @pytest.mark.skip("youtube-dl update needed")
     def test_live_video_info(self):
         pytest.importorskip('youtube_dl')
         resp = self.testapp.get('/live/vi_/https://www.youtube.com/watch?v=DjFZyFWSt1M')

tests/test_range.py

@@ -29,7 +29,7 @@ def test_replay_range_start_end(self, fmod):
         assert resp.headers['Content-Range'] == 'bytes 0-200/1270', resp.headers['Content-Range']
         assert resp.content_length == 201, resp.content_length
 
-        assert self.recorder_skip == None
+        assert self.recorder_skip == '1'
 
         assert 'wombat.js' not in resp.text