Merge branch 'develop'

Author: ikreymer

CHANGES.rst

@@ -1,3 +1,20 @@
+pywb 0.6.6 changelist
+~~~~~~~~~~~~~~~~~~~~~
+
+* JS client side improvements: check for double-inits, preserve anchor in wb.js top location redirect
+
+* JS Rewriters: add mixins for link + location (default), link only, location only rewriting by setting ``js_rewrite_location`` to ``all``, ``urls``, ``location``, respectively.
+
+  (New: location only rewriting does not change JS urls)
+
+* Beginning of new rewrite options, settable per collections and stored in UrlRewriter. Available options:
+
+  - ``rewrite_base`` - set to False to disable rewriting ``<base href="...">`` tag
+  - ``rewrite_rel_canon`` - set to false to disable rewriting ``<link rel=canon href="...">``
+
+* JS rewrite: Don't rewrite location if starting with '$'
+
+
 pywb 0.6.5 changelist
 ~~~~~~~~~~~~~~~~~~~~~
 
@@ -40,17 +57,17 @@ pywb 0.6.3 changelist
 pywb 0.6.2 changelist
 ~~~~~~~~~~~~~~~~~~~~~
 
-* Invert framed replay paradigm: Canonical page is always without a modifier (instead of with `mp_`), if using frames, the page redirects to `tf_`, and uses replaceState() to change url back to canonical form.
+* Invert framed replay paradigm: Canonical page is always without a modifier (instead of with ``mp_``), if using frames, the page redirects to ``tf_``, and uses replaceState() to change url back to canonical form.
 
 * Enable Memento support for framed replay, include Memento headers in top frame
 
-* Easier to customize just the banner html, via `banner_html` setting in the config. Default banner uses ui/banner.html and inserts the script default_banner.js, which creates the banner.
+* Easier to customize just the banner html, via ``banner_html`` setting in the config. Default banner uses ui/banner.html and inserts the script default_banner.js, which creates the banner.
 
-  Other implementations may create banner via custom JS or directly insert HTML, as needed. Setting `banner_html: False` will disable the banner.
+  Other implementations may create banner via custom JS or directly insert HTML, as needed. Setting ``banner_html: False`` will disable the banner.
 
 * Small improvements to streaming response, read in fixed chunks to allow better streaming from live.
 
-* Improved cookie and csrf-token rewriting, including: ability to set `cookie_scope: root` per collection to have all replayed cookies have their Path set to application root.
+* Improved cookie and csrf-token rewriting, including: ability to set ``cookie_scope: root`` per collection to have all replayed cookies have their Path set to application root.
 
   This is useful for replaying sites which share cookies amongst different pages and across archived time ranges.
 

README.rst

@@ -1,4 +1,4 @@
-PyWb 0.6.5
+PyWb 0.6.6
 ==========
 
 .. image:: https://travis-ci.org/ikreymer/pywb.png?branch=master
@@ -44,7 +44,7 @@ This README contains a basic overview of using pywb. After reading this intro, c
 pywb Tools Overview
 -----------------------------
 
-In addition to the standard wayback machine (explained further below), pywb tool suite includes a 
+In addition to the standard wayback machine (explained further below), pywb tool suite includes a
 number of useful command-line and web server tools. The tools should be available to run after
 running ``python setup.py install``:
 
@@ -58,10 +58,10 @@ running ``python setup.py install``:
   for all options.
 
 
-* ``cdx-server`` -- a CDX API only server which returns a responses about CDX captures in bulk. 
+* ``cdx-server`` -- a CDX API only server which returns a responses about CDX captures in bulk.
   Includes most of the features of the `original cdx server implementation <https://github.com/internetarchive/wayback/tree/master/wayback-cdx-server>`_,
   updated documentation coming soon.
-  
+
 * ``proxy-cert-auth`` -- a utility to support proxy mode. It can be used in CA root certificate, or per-host certificate with an existing root cert.
 
 
@@ -151,7 +151,7 @@ If you would like to use non-SURT ordered .cdx files, simply add this field to t
 ::
 
       surt_ordered: false
-      
+
 UI Customization
 """""""""""""""""""""
 

pywb/framework/archivalrouter.py

@@ -62,7 +62,8 @@ def parse_request(self, route, env, matcher, coll, request_uri,
                               use_abs_prefix=use_abs_prefix,
                               wburl_class=route.handler.get_wburl_type(),
                               urlrewriter_class=UrlRewriter,
-                              cookie_scope=route.cookie_scope)
+                              cookie_scope=route.cookie_scope,
+                              rewrite_opts=route.rewrite_opts)
 
         # Allow for applying of additional filters
         route.apply_filters(wbrequest, matcher)
@@ -101,6 +102,7 @@ def __init__(self, regex, handler, coll_group=0, config={},
         # collection id from regex group (default 0)
         self.coll_group = coll_group
         self.cookie_scope = config.get('cookie_scope')
+        self.rewrite_opts = config.get('rewrite_opts', {})
         self._custom_init(config)
 
     def is_handling(self, request_uri):

pywb/framework/wbrequestresponse.py

@@ -38,7 +38,8 @@ def __init__(self, env,
                  wburl_class=None,
                  urlrewriter_class=None,
                  is_proxy=False,
-                 cookie_scope=None):
+                 cookie_scope=None,
+                 rewrite_opts={}):
 
         self.env = env
 
@@ -77,7 +78,8 @@ def __init__(self, env,
                                                  host_prefix + rel_prefix,
                                                  rel_prefix,
                                                  env.get('SCRIPT_NAME', '/'),
-                                                 cookie_scope)
+                                                 cookie_scope,
+                                                 rewrite_opts)
 
             self.urlrewriter.deprefix_url()
         else:

pywb/rewrite/html_rewriter.py

@@ -92,6 +92,9 @@ def __init__(self, url_rewriter,
 
         self.rewrite_tags = self._init_rewrite_tags(defmod)
 
+        # get opts from urlrewriter
+        self.opts = url_rewriter.rewrite_opts
+
     # ===========================
     META_REFRESH_REGEX = re.compile('^[\\d.]+\\s*;\\s*url\\s*=\\s*(.+?)\\s*$',
                                     re.IGNORECASE | re.MULTILINE)
@@ -174,9 +177,11 @@ def _rewrite_tag_attrs(self, tag, tag_attrs):
             elif attr_name == 'crossorigin':
                 attr_name = '_crossorigin'
 
-            # special case: link don't rewrite canonical
+            # special case: if rewrite_canon not set,
+            # don't rewrite rel=canonical
             elif tag == 'link' and attr_name == 'href':
-                if not self.has_attr(tag_attrs, ('rel', 'canonical')):
+                if (self.opts.get('rewrite_rel_canon', True) or
+                    not self.has_attr(tag_attrs, ('rel', 'canonical'))):
                     rw_mod = handler.get(attr_name)
                     attr_value = self._rewrite_url(attr_value, rw_mod)
 
@@ -191,17 +196,21 @@ def _rewrite_tag_attrs(self, tag, tag_attrs):
                     rw_mod = 'oe_'
                     attr_value = self._rewrite_url(attr_value, rw_mod)
 
+            # special case: base tag
+            elif (tag == 'base') and (attr_name == 'href') and attr_value:
+                rw_mod = handler.get(attr_name)
+                base_value = self._rewrite_url(attr_value, rw_mod)
+                if self.opts.get('rewrite_base', True):
+                    attr_value = base_value
+                self.url_rewriter = (self.url_rewriter.
+                                     rebase_rewriter(base_value))
+
             else:
                 # rewrite url using tag handler
                 rw_mod = handler.get(attr_name)
                 if rw_mod is not None:
                     attr_value = self._rewrite_url(attr_value, rw_mod)
 
-                # special case: base tag
-                if (tag == 'base') and (attr_name == 'href') and attr_value:
-                    self.url_rewriter = (self.url_rewriter.
-                                         rebase_rewriter(attr_value))
-
             # write the attr!
             self._write_attr(attr_name, attr_value)
 

pywb/rewrite/regex_rewriters.py

@@ -35,7 +35,7 @@ def archival_rewrite(rewriter):
 
     #DEFAULT_OP = add_prefix
 
-    def __init__(self, rules):
+    def __init__(self, rewriter, rules):
         #rules = self.create_rules(http_prefix)
 
         # Build regexstr, concatenating regex list
@@ -106,7 +106,7 @@ def parse_rule(obj):
 
 
 #=================================================================
-class JSLinkOnlyRewriter(RegexRewriter):
+class JSLinkRewriterMixin(object):
     """
     JS Rewriter which rewrites absolute http://, https:// and // urls
     at the beginning of a string
@@ -118,19 +118,20 @@ def __init__(self, rewriter, rules=[]):
         rules = rules + [
             (self.JS_HTTPX, RegexRewriter.archival_rewrite(rewriter), 0)
         ]
-        super(JSLinkOnlyRewriter, self).__init__(rules)
+        super(JSLinkRewriterMixin, self).__init__(rewriter, rules)
 
 
 #=================================================================
-class JSLinkAndLocationRewriter(JSLinkOnlyRewriter):
+class JSLocationRewriterMixin(object):
+#class JSLinkAndLocationRewriter(JSLinkOnlyRewriter):
     """
-    JS Rewriter which also rewrites location and domain to the
+    JS Rewriter mixin which rewrites location and domain to the
     specified prefix (default: 'WB_wombat_')
     """
 
     def __init__(self, rewriter, rules=[], prefix='WB_wombat_'):
         rules = rules + [
-             (r'(?<!/)\blocation\b(?!\":)', RegexRewriter.add_prefix(prefix), 0),
+             (r'(?<![/$])\blocation\b(?!\":)', RegexRewriter.add_prefix(prefix), 0),
              (r'(?<=document\.)domain', RegexRewriter.add_prefix(prefix), 0),
              (r'(?<=document\.)referrer', RegexRewriter.add_prefix(prefix), 0),
              (r'(?<=document\.)cookie', RegexRewriter.add_prefix(prefix), 0),
@@ -148,7 +149,23 @@ def __init__(self, rewriter, rules=[], prefix='WB_wombat_'):
              #(r'\b(?:self|window)\b[!=\W]+\b(top)\b',
              #RegexRewriter.add_prefix(prefix), 1),
         ]
-        super(JSLinkAndLocationRewriter, self).__init__(rewriter, rules)
+        super(JSLocationRewriterMixin, self).__init__(rewriter, rules)
+
+
+#=================================================================
+class JSLocationOnlyRewriter(JSLocationRewriterMixin, RegexRewriter):
+    pass
+
+
+#=================================================================
+class JSLinkOnlyRewriter(JSLinkRewriterMixin, RegexRewriter):
+    pass
+
+#=================================================================
+class JSLinkAndLocationRewriter(JSLocationRewriterMixin,
+                                JSLinkRewriterMixin,
+                                RegexRewriter):
+    pass
 
 
 #=================================================================
@@ -161,7 +178,7 @@ class XMLRewriter(RegexRewriter):
     def __init__(self, rewriter, extra=[]):
         rules = self._create_rules(rewriter)
 
-        super(XMLRewriter, self).__init__(rules)
+        super(XMLRewriter, self).__init__(rewriter, rules)
 
     # custom filter to reject 'xmlns' attr
     def filter(self, m):
@@ -189,7 +206,7 @@ class CSSRewriter(RegexRewriter):
 
     def __init__(self, rewriter):
         rules = self._create_rules(rewriter)
-        super(CSSRewriter, self).__init__(rules)
+        super(CSSRewriter, self).__init__(rewriter, rules)
 
     def _create_rules(self, rewriter):
         return [

pywb/rewrite/rewriterules.py

@@ -1,7 +1,7 @@
 from pywb.utils.dsrules import BaseRule
 
 from regex_rewriters import RegexRewriter, CSSRewriter, XMLRewriter
-from regex_rewriters import JSLinkAndLocationRewriter, JSLinkOnlyRewriter
+from regex_rewriters import JSLinkAndLocationRewriter, JSLinkOnlyRewriter, JSLocationOnlyRewriter
 
 from header_rewriter import HeaderRewriter
 from html_rewriter import HTMLRewriter
@@ -27,12 +27,13 @@ def __init__(self, url_prefix, config={}):
         self.parse_comments = config.get('parse_comments', False)
 
         # Custom handling for js rewriting, often the most complex
-        self.js_rewrite_location = config.get('js_rewrite_location', True)
-        self.js_rewrite_location = bool(self.js_rewrite_location)
+        self.js_rewrite_location = config.get('js_rewrite_location', 'all')
 
         # ability to toggle rewriting
-        if self.js_rewrite_location:
+        if self.js_rewrite_location == 'all':
             js_default_class = JSLinkAndLocationRewriter
+        elif self.js_rewrite_location == 'location':
+            js_default_class = JSLocationOnlyRewriter
         else:
             js_default_class = JSLinkOnlyRewriter
 

pywb/rewrite/test/test_html_rewriter.py

@@ -20,13 +20,22 @@
 #>>> parse('<input "selected"><img src></div>')
 #<input "selected"=""><img src=""></div>
 
-# Base Tests
+# Base Tests -- w/ rewrite (default)
 >>> parse('<html><head><base href="http://example.com/diff/path/file.html"/>')
 <html><head><base href="/web/20131226101010/http://example.com/diff/path/file.html"/>
 
 >>> parse('<base href="static/"/><img src="image.gif"/>')
 <base href="/web/20131226101010/http://example.com/some/path/static/"/><img src="/web/20131226101010im_/http://example.com/some/path/static/image.gif"/>
 
+# Base Tests -- no rewrite
+>>> parse('<html><head><base href="http://example.com/diff/path/file.html"/>', urlrewriter=no_base_canon_rewriter)
+<html><head><base href="http://example.com/diff/path/file.html"/>
+
+>>> parse('<base href="static/"/><img src="image.gif"/>', urlrewriter=no_base_canon_rewriter)
+<base href="static/"/><img src="/web/20131226101010im_/http://example.com/some/path/static/image.gif"/>
+
+
+
 # HTML Entities
 >>> parse('<a href="">&rsaquo; &nbsp; &#62; &#63</div>')
 <a href="">&rsaquo; &nbsp; &#62; &#63</div>
@@ -102,8 +111,12 @@
 >>> parse('<link href="abc.txt"><div>SomeTest</div>', head_insert = '<script>load_stuff();</script>')
 <link href="/web/20131226101010oe_/http://example.com/some/path/abc.txt"><script>load_stuff();</script><div>SomeTest</div>
 
-# don't rewrite rel=canonical
+# rel=canonical: rewrite (default)
 >>> parse('<link rel=canonical href="http://example.com/">')
+<link rel="canonical" href="/web/20131226101010oe_/http://example.com/">
+
+# rel=canonical: no_rewrite
+>>> parse('<link rel=canonical href="http://example.com/">', urlrewriter=no_base_canon_rewriter)
 <link rel="canonical" href="http://example.com/">
 
 # doctype
@@ -143,7 +156,12 @@
 
 urlrewriter = UrlRewriter('20131226101010/http://example.com/some/path/index.html', '/web/')
 
-def parse(data, head_insert = None):
+no_base_canon_rewriter = UrlRewriter('20131226101010/http://example.com/some/path/index.html',
+                                     '/web/',
+                                     rewrite_opts=dict(rewrite_rel_canon=False,
+                                                       rewrite_base=False))
+
+def parse(data, head_insert=None, urlrewriter=urlrewriter):
     parser = HTMLRewriter(urlrewriter, head_insert = head_insert)
     #data = data.decode('utf-8')
     result = parser.rewrite(data) + parser.close()

pywb/rewrite/test/test_regex_rewriters.py

@@ -3,7 +3,7 @@
 # Custom Regex
 #=================================================================
 # Test https->http converter (other tests below in subclasses)
->>> RegexRewriter([(RegexRewriter.HTTPX_MATCH_STR, RegexRewriter.remove_https, 0)]).rewrite('a = https://example.com; b = http://example.com; c = https://some-url/path/https://embedded.example.com')
+>>> RegexRewriter(urlrewriter, [(RegexRewriter.HTTPX_MATCH_STR, RegexRewriter.remove_https, 0)]).rewrite('a = https://example.com; b = http://example.com; c = https://some-url/path/https://embedded.example.com')
 'a = http://example.com; b = http://example.com; c = http://some-url/path/http://embedded.example.com'