Merge branch 'develop' into client-side-tests

Author: ikreymer

pywb/rewrite/test/test_rewrite_live.py

@@ -200,7 +200,7 @@ def test_local_unclosed_script():
 
 
 def test_example_1():
-    status_headers, buff = get_rewritten('http://example.com/', urlrewriter, req_headers={'Connection': 'close'})
+    status_headers, buff = get_rewritten('http://example.com/', urlrewriter, req_headers={'Connection': 'close', 'Accept-Encoding': 'identity'})
 
     # verify header rewriting
     assert status_headers.get_header('x-archive-orig-content-length') == '1270', status_headers

pywb/warc/archiveiterator.py

@@ -43,7 +43,14 @@ class ArchiveIterator(object):
     package which will create a properly chunked gzip file:
 
     warc2warc -Z myfile.{0} > myfile.{0}.gz
-    """
+"""
+
+    INC_RECORD = """\
+    WARNING: Record not followed by newline, perhaps Content-Length is invalid
+    Offset: {0}
+    Remainder: {1}
+"""
+
 
     def __init__(self, fileobj, no_record_parse=False,
                  verify_http=False):
@@ -130,15 +137,30 @@ def _consume_blanklines(self):
 
           count empty_size so that it can be substracted from
           the record length for uncompressed
+
+          if first line read is not blank, likely error in WARC/ARC,
+          display a warning
         """
         empty_size = 0
+        first_line = True
+
         while True:
             line = self.reader.readline()
             if len(line) == 0:
                 return None, empty_size
 
-            if line.rstrip() == '':
+            stripped = line.rstrip()
+
+            if stripped == '' or first_line:
                 empty_size += len(line)
+
+                if stripped != '':
+                    # if first line is not blank,
+                    # likely content-length was invalid, display warning
+                    err_offset = self.fh.tell() - self.reader.rem_length() - empty_size
+                    sys.stderr.write(self.INC_RECORD.format(err_offset, line))
+
+                first_line = False
                 continue
 
             return line, empty_size

setup.py

@@ -21,13 +21,9 @@
 
 class PyTest(TestCommand):
     user_options = []
-    def initialize_options(self):
-        TestCommand.initialize_options(self)
-        self._argv = []
-
     def finalize_options(self):
         TestCommand.finalize_options(self)
-        self.test_suite = True
+        self.test_suite = ' '
 
     def run_tests(self):
         import pytest