Merge branch 'develop' for 0.7.5

Author: ikreymer

.gitattributes

@@ -0,0 +1,4 @@
+*.arc  -text
+*.warc -text
+*.cdx  -text
+*.gz   -text

CHANGES.rst

@@ -1,3 +1,16 @@
+pywb 0.7.5 changelist
+~~~~~~~~~~~~~~~~~~~~~
+
+* Cross platform fixes to support Windows -- all tests pass on Linux, OS X and Windows now. Improved cross-platform support includes:
+  - read all files as binary to avoid line ending issues
+  - properly convert url <-> file
+  - avoid platform dependent apis
+
+* Change any unhandled exceptions to result in a 500 error, instead of 400.
+
+* More compresensive client side ``src`` attribute rewriting (via wombat.js), additional server-side HTML tag rewriting.
+
+
 pywb 0.7.2 changelist
 ~~~~~~~~~~~~~~~~~~~~~
 

README.rst

@@ -1,4 +1,4 @@
-PyWb 0.7.2
+PyWb 0.7.5
 ==========
 
 .. image:: https://travis-ci.org/ikreymer/pywb.png?branch=master
@@ -13,7 +13,7 @@ pywb is a python implementation of web archival replay tools, sometimes also kno
 pywb allows high-quality replay (browsing) of archived web data stored in standardized `ARC <http://en.wikipedia.org/wiki/ARC_(file_format)>`_ and `WARC <http://en.wikipedia.org/wiki/Web_ARChive>`_.
 The replay system is designed to accurately replay complex dynamic sites, including video and audio content.
 
-pywb can be used as a traditional web application or an HTTP or HTTPS proxy server.
+pywb can be used as a traditional web application or an HTTP or HTTPS proxy server, and has been tested on Linux, OS X and Windows platforms.
 
 pywb is also fully compliant with the `Memento <http://mementoweb.org/>`_ protocol (`RFC-7089 <http://tools.ietf.org/html/rfc7089>`_).
 

pywb/cdx/cdxsource.py

@@ -30,7 +30,7 @@ def __init__(self, filename):
     def load_cdx(self, query):
         def do_open():
             try:
-                source = open(self.filename)
+                source = open(self.filename, 'rb')
                 gen = iter_range(source, query.key, query.end_key)
                 for line in gen:
                     yield line

pywb/cdx/test/test_redis_source.py

@@ -26,7 +26,7 @@
 
 def load_cdx_into_redis(source, filename, key=None):
     # load a cdx into mock redis
-    with open(test_cdx_dir + filename) as fh:
+    with open(test_cdx_dir + filename, 'rb') as fh:
         for line in fh:
             zadd_cdx(source, line, key)
 

pywb/cdx/zipnum.py

@@ -84,7 +84,7 @@ def load_loc(self):
         self.loc_mtime = new_mtime
 
         logging.debug('Loading loc from: ' + self.loc_filename)
-        with open(self.loc_filename) as fh:
+        with open(self.loc_filename, 'rb') as fh:
             for line in fh:
                 parts = line.rstrip().split('\t')
                 self.loc_map[parts[0]] = parts[1:]
@@ -112,7 +112,7 @@ def lookup_loc(self, part):
     def load_cdx(self, query):
         self.load_loc()
 
-        reader = open(self.summary)
+        reader = open(self.summary, 'rb')
 
         idx_iter = iter_range(reader,
                               query.key,

pywb/framework/certauth.py

@@ -13,8 +13,8 @@
 
 
 #=================================================================
-# Duration of 100 years
-CERT_DURATION = 100 * 365 * 24 * 60 * 60
+# Duration of 10 years
+CERT_DURATION = 10 * 365 * 24 * 60 * 60
 
 CERTS_DIR = './ca/certs/'
 

pywb/framework/proxy.py

@@ -334,7 +334,7 @@ def handle_cert_install(self, env):
                 return None
 
             buff = ''
-            with open(self.ca.ca_file) as fh:
+            with open(self.ca.ca_file, 'rb') as fh:
                 buff = fh.read()
 
             content_type = 'application/x-x509-ca-cert'

pywb/framework/test/test_certauth.py

@@ -5,8 +5,8 @@
 
 from pywb.framework.certauth import main, CertificateAuthority
 
-TEST_CA_DIR = './pywb/framework/test/pywb_test_ca_certs'
-TEST_CA_ROOT = './pywb/framework/test/pywb_test_ca.pem'
+TEST_CA_DIR = os.path.join('.', 'pywb', 'framework', 'test', 'pywb_test_ca_certs')
+TEST_CA_ROOT = os.path.join('.', 'pywb', 'framework', 'test', 'pywb_test_ca.pem')
 
 def setup_module():
     openssl_support = pytest.importorskip("OpenSSL")

pywb/framework/test/test_wsgi_wrapper.py

@@ -14,7 +14,7 @@ def response(env, start_response):
 
 class TestErrApp:
     def __call__(self, env):
-        raise Exception('Test Error')
+        raise Exception('Test Unexpected Error')
 
 class TestCustomErrApp:
     def __call__(self, env):
@@ -41,8 +41,8 @@ def test_err_app():
     testapp = webtest.TestApp(the_app)
     resp = testapp.get('/abc', expect_errors=True)
 
-    assert resp.status_int == 400
-    assert '400 Bad Request Error: Test Error' in resp.body
+    assert resp.status_int == 500
+    assert '500 Internal Server Error Error: Test Unexpected Error' in resp.body
 
 def test_custom_err_app():
     the_app = init_app(initer(TestCustomErrApp), load_yaml=False)

pywb/framework/wsgi_wrappers.py

@@ -118,7 +118,7 @@ def handle_exception(self, env, exc, print_trace):
         if hasattr(exc, 'status'):
             status = exc.status()
         else:
-            status = '400 Bad Request'
+            status = '500 Internal Server Error'
 
         if hasattr(exc, 'url'):
             err_url = exc.url

pywb/rewrite/html_rewriter.py

@@ -30,14 +30,17 @@ def _init_rewrite_tags(defmod):
             'base':    {'href': defmod},
             'blockquote': {'cite': defmod},
             'body':    {'background': 'im_'},
+            'button':  {'formaction': defmod},
+            'command': {'icon': 'im_'},
             'del':     {'cite': defmod},
             'embed':   {'src': 'oe_'},
             'head':    {'': defmod},  # for head rewriting
             'iframe':  {'src': 'if_'},
             'img':     {'src': 'im_',
                         'srcset': 'im_'},
             'ins':     {'cite': defmod},
-            'input':   {'src': 'im_'},
+            'input':   {'src': 'im_',
+                        'formaction': defmod},
             'form':    {'action': defmod},
             'frame':   {'src': 'fr_'},
             'link':    {'href': 'oe_'},
@@ -49,7 +52,8 @@ def _init_rewrite_tags(defmod):
             'ref':     {'href': 'oe_'},
             'script':  {'src': 'js_'},
             'source':  {'src': 'oe_'},
-            'video':   {'src': 'oe_'},
+            'video':   {'src': 'oe_',
+                        'poster': 'im_'},
 
             'div':     {'data-src': defmod,
                         'data-uri': defmod},

pywb/rewrite/rewrite_live.py

@@ -6,10 +6,11 @@
 import datetime
 import mimetypes
 import logging
+import os
 
 from urlparse import urlsplit
 
-from pywb.utils.loaders import is_http, LimitReader, BlockLoader
+from pywb.utils.loaders import is_http, LimitReader, BlockLoader, to_file_url
 from pywb.utils.loaders import extract_client_cookie
 from pywb.utils.timeutils import datetime_to_timestamp
 from pywb.utils.statusandheaders import StatusAndHeaders
@@ -180,11 +181,18 @@ def fetch_request(self, url, urlrewriter,
         if url.startswith('//'):
             url = 'http:' + url
 
+        if is_http(url):
+            is_remote = True
+        else:
+            is_remote = False
+            if not url.startswith('file:'):
+                url = to_file_url(url)
+
         # explicit urlkey may be passed in (say for testing)
         if not urlkey:
             urlkey = canonicalize(url)
 
-        if is_http(url):
+        if is_remote:
             (status_headers, stream) = self.fetch_http(url, urlkey, env,
                                                        req_headers,
                                                        follow_redirects,

pywb/rewrite/test/test_url_rewriter.py

@@ -103,12 +103,17 @@
 'http://example.com/file.html?param=https://example.com/filename.html&other=value&a=b&param2=http://test.example.com'
 
 # HttpsUrlRewriter tests
->>> HttpsUrlRewriter('http://example.com/', None).rewrite('https://example.com/abc')
+>>> httpsrewriter = HttpsUrlRewriter('http://example.com/', None)
+>>> httpsrewriter.rewrite('https://example.com/abc')
 'http://example.com/abc'
 
->>> HttpsUrlRewriter('http://example.com/', None).rewrite('http://example.com/abc')
+>>> httpsrewriter.rewrite('http://example.com/abc')
 'http://example.com/abc'
 
+# rebase is identity
+>>> httpsrewriter.rebase_rewriter('https://example.com/') == httpsrewriter
+True
+
 """
 
 

pywb/static/wombat.js

@@ -105,6 +105,8 @@ _WBWombat = (function() {
                         "http:/" + prefix, "https:/" + prefix];
     }
 
+    var SRC_TAGS = ["IMG", "SCRIPT", "VIDEO", "AUDIO", "SOURCE", "EMBED", "INPUT"];
+
     //============================================
     function rewrite_url_(url) {
         // If undefined, just return it
@@ -692,12 +694,9 @@ _WBWombat = (function() {
                     }
 
                     override_attr(created, "src");
-                } else if (created.tagName == "IMG" || created.tagName == "VIDEO" || created.tagName == "AUDIO") {
+                } else if (created.tagName && starts_with(created.tagName, SRC_TAGS)) {
                     override_attr(created, "src");
                 }
-//                } else if (created.tagName == "A") {
-//                    override_attr(created, "href");
-//                }
 
                 return created;
             }

pywb/utils/bufferedreaders.py

@@ -46,9 +46,6 @@ def __init__(self, stream, block_size=1024,
         self.buff_size = 0
 
     def set_decomp(self, decomp_type):
-        if self.num_read > 0:
-            raise Exception('Attempting to change decompression mid-stream')
-
         self._init_decomp(decomp_type)
 
     def _init_decomp(self, decomp_type):

pywb/utils/loaders.py

@@ -7,6 +7,7 @@
 import hmac
 import urllib
 import urllib2
+import urlparse
 import time
 import pkg_resources
 from io import open
@@ -17,6 +18,15 @@ def is_http(filename):
     return filename.startswith(('http://', 'https://'))
 
 
+#=================================================================
+def to_file_url(filename):
+    """ Convert a filename to a file:// url
+    """
+    url = os.path.abspath(filename)
+    url = urlparse.urljoin('file:', urllib.pathname2url(url))
+    return url
+
+
 #=================================================================
 def load_yaml_config(config_file):
     import yaml
@@ -39,12 +49,12 @@ def extract_post_query(method, mime, length, stream):
          not mime.lower().startswith('application/x-www-form-urlencoded'))):
         return None
 
-    if not length or length == '0':
-        return None
-
     try:
         length = int(length)
-    except ValueError:
+    except (ValueError, TypeError):
+        return None
+
+    if length <= 0:
         return None
 
     #todo: encoding issues?
@@ -129,9 +139,10 @@ def load_file_or_resource(self, url, offset=0, length=-1):
         # if starting with . or /, can only be a file path..
         file_only = url.startswith(('/', '.'))
 
+        # convert to filename
         if url.startswith('file://'):
-            url = url[len('file://'):]
             file_only = True
+            url = urllib.url2pathname(url[len('file://'):])
 
         try:
             # first, try as file

pywb/utils/test/test_binsearch.py

@@ -66,12 +66,12 @@
 test_cdx_dir = get_test_dir() + 'cdx/'
 
 def print_binsearch_results(key, iter_func):
-    with open(test_cdx_dir + 'iana.cdx') as cdx:
+    with open(test_cdx_dir + 'iana.cdx', 'rb') as cdx:
         for line in iter_func(cdx, key):
             print line
 
 def print_binsearch_results_range(key, end_key, iter_func, prev_size=0):
-    with open(test_cdx_dir + 'iana.cdx') as cdx:
+    with open(test_cdx_dir + 'iana.cdx', 'rb') as cdx:
         for line in iter_func(cdx, key, end_key, prev_size=prev_size):
             print line
 

pywb/utils/test/test_loaders.py

@@ -25,7 +25,7 @@
 100
 
 # no length specified, read full amount requested
->>> len(BlockLoader().load('file://' + test_cdx_dir + 'example.cdx', 0, -1).read(400))
+>>> len(BlockLoader().load(to_file_url(test_cdx_dir + 'example.cdx'), 0, -1).read(400))
 400
 
 # HMAC Cookie Maker
@@ -56,14 +56,41 @@
 >>> extract_client_cookie(dict(HTTP_COOKIE='x'), 'x')
 
 >>> extract_client_cookie({}, 'y')
+
+
+# extract_post_query tests
+
+# correct POST data
+>>> post_data = 'foo=bar&dir=%2Fbaz'
+>>> extract_post_query('POST', 'application/x-www-form-urlencoded', len(post_data), BytesIO(post_data))
+'foo=bar&dir=/baz'
+
+# unsupported method
+>>> extract_post_query('PUT', 'application/x-www-form-urlencoded', len(post_data), BytesIO(post_data))
+
+# unsupported type
+>>> extract_post_query('POST', 'text/plain', len(post_data), BytesIO(post_data))
+
+# invalid length
+>>> extract_post_query('POST', 'application/x-www-form-urlencoded', 'abc', BytesIO(post_data))
+>>> extract_post_query('POST', 'application/x-www-form-urlencoded', 0, BytesIO(post_data))
+
+# length too short
+>>> extract_post_query('POST', 'application/x-www-form-urlencoded', len(post_data) - 4, BytesIO(post_data))
+'foo=bar&dir=%2'
+
+# length too long
+>>> extract_post_query('POST', 'application/x-www-form-urlencoded', len(post_data) + 4, BytesIO(post_data))
+'foo=bar&dir=/baz'
 """
 
 
 #=================================================================
 import re
+import os
 from io import BytesIO
-from pywb.utils.loaders import BlockLoader, HMACCookieMaker
-from pywb.utils.loaders import LimitReader, extract_client_cookie
+from pywb.utils.loaders import BlockLoader, HMACCookieMaker, to_file_url
+from pywb.utils.loaders import LimitReader, extract_client_cookie, extract_post_query
 
 from pywb import get_test_dir
 
@@ -82,7 +109,6 @@ def seek_read_full(seekable_reader, offset):
     return seekable_reader.readline()
 
 
-
 if __name__ == "__main__":
     import doctest
     doctest.testmod()

pywb/utils/wbexception.py

@@ -6,8 +6,9 @@ def __init__(self, msg=None, url=None):
         Exception.__init__(self, msg)
         self.url = url
 
-    def status(self):
-        return '500 Internal Server Error'
+# Default Error Code
+#    def status(self):
+#        return '500 Internal Server Error'
 
 
 #=================================================================