proxy: change HttpsUrlRewriter to SchemeOnlyUrlRewriter, which fixes http->https or https->http to match

the scheme of the current page.
url-rewrite-only mode: add uo_ mod and use that to rewrite only urls (no banner, no client side rewrite)
addresses #142

Author: ikreymer

pywb/framework/proxy.py

@@ -9,7 +9,7 @@
 import socket
 import ssl
 
-from pywb.rewrite.url_rewriter import HttpsUrlRewriter
+from pywb.rewrite.url_rewriter import SchemeOnlyUrlRewriter
 from pywb.utils.wbexception import BadRequestException
 
 from pywb.utils.bufferedreaders import BufferedReader
@@ -204,7 +204,7 @@ def __call__(self, env):
                               host_prefix=host_prefix,
                               rel_prefix=rel_prefix,
                               wburl_class=route.handler.get_wburl_type(),
-                              urlrewriter_class=HttpsUrlRewriter,
+                              urlrewriter_class=SchemeOnlyUrlRewriter,
                               use_abs_prefix=False,
                               is_proxy=True)
 
@@ -219,7 +219,7 @@ def __call__(self, env):
             wbrequest.wb_url.mod = 'bn_'
         else:
         # unaltered, no rewrite or banner
-            wbrequest.wb_url.mod = 'id_'
+            wbrequest.wb_url.mod = 'uo_'
 
         response = route.handler(wbrequest)
 

pywb/rewrite/rewrite_content.py

@@ -16,6 +16,8 @@
 from pywb.utils.bufferedreaders import DecompressingBufferedReader
 from pywb.utils.bufferedreaders import ChunkedDataReader, BufferedReader
 
+from regex_rewriters import JSNoneRewriter, JSLinkOnlyRewriter
+
 
 #=================================================================
 class RewriteContent:
@@ -159,9 +161,8 @@ def rewrite_content(self, urlrewriter, status_headers, stream,
                 charset = self._extract_html_charset(first_buff,
                                                      status_headers)
 
-            if head_insert_func:
+            if head_insert_func and not wb_url.is_url_rewrite_only:
                 head_insert_orig = head_insert_func(rule, cdx)
-                head_insert_str = None
 
                 if charset:
                     try:
@@ -191,9 +192,15 @@ def rewrite_content(self, urlrewriter, status_headers, stream,
 
                 return (status_headers, gen, False)
 
+            js_rewriter_class = rule.rewriters['js']
+            css_rewriter_class = rule.rewriters['css']
+
+            if wb_url.is_url_rewrite_only:
+                js_rewriter_class = JSNoneRewriter
+
             rewriter = rewriter_class(urlrewriter,
-                                      js_rewriter_class=rule.rewriters['js'],
-                                      css_rewriter_class=rule.rewriters['css'],
+                                      js_rewriter_class=js_rewriter_class,
+                                      css_rewriter_class=css_rewriter_class,
                                       head_insert=head_insert_str,
                                       url=wb_url.url,
                                       defmod=self.defmod,
@@ -203,6 +210,11 @@ def rewrite_content(self, urlrewriter, status_headers, stream,
             if wb_url.is_banner_only:
                 return (status_headers, self.stream_to_gen(stream), False)
 
+            # url-only rewriter, but not rewriting urls in JS, so return
+            if wb_url.is_url_rewrite_only and text_type == 'js':
+                #return (status_headers, self.stream_to_gen(stream), False)
+                rewriter_class = JSLinkOnlyRewriter
+
             # apply one of (js, css, xml) rewriters
             rewriter = rewriter_class(urlrewriter)
 

pywb/rewrite/test/test_url_rewriter.py

@@ -125,22 +125,33 @@
 >>> do_deprefix('http://example.com/file.html?foo=bar&url=' + urllib.quote_plus('http://localhost:8080/pywb/extra/path/http://example.com/filename.html') + '&foo2=bar2', '/pywb/', 'http://localhost:8080/pywb/')
 'http://example.com/file.html?foo=bar&url=http://example.com/filename.html&foo2=bar2'
 
-# HttpsUrlRewriter tests
->>> httpsrewriter = HttpsUrlRewriter('http://example.com/', None)
->>> httpsrewriter.rewrite('https://example.com/abc')
+# SchemeOnlyUrlRewriter tests
+>>> SchemeOnlyUrlRewriter('http://example.com/').rewrite('https://example.com/abc')
 'http://example.com/abc'
 
->>> httpsrewriter.rewrite('http://example.com/abc')
+>>> SchemeOnlyUrlRewriter('http://example.com/abc').rewrite('http://example.com/abc')
 'http://example.com/abc'
 
+>>> SchemeOnlyUrlRewriter('https://example.com/abc').rewrite('http://example.com/abc')
+'https://example.com/abc'
+
+>>> SchemeOnlyUrlRewriter('https://example.com/abc').rewrite('https://example.com/abc')
+'https://example.com/abc'
+
+>>> SchemeOnlyUrlRewriter('http://example.com/abc').rewrite('//example.com/abc')
+'//example.com/abc'
+
+>>> SchemeOnlyUrlRewriter('https://example.com/abc').rewrite('//example.com/abc')
+'//example.com/abc'
+
 # rebase is identity
->>> httpsrewriter.rebase_rewriter('https://example.com/') == httpsrewriter
+>>> x = SchemeOnlyUrlRewriter('http://example.com'); x.rebase_rewriter('https://example.com/') == x
 True
 
 """
 
 
-from pywb.rewrite.url_rewriter import UrlRewriter, HttpsUrlRewriter
+from pywb.rewrite.url_rewriter import UrlRewriter, SchemeOnlyUrlRewriter
 import urllib
 
 def do_rewrite(rel_url, base_url, prefix, mod=None, full_prefix=None):

pywb/rewrite/url_rewriter.py

@@ -19,7 +19,7 @@ class UrlRewriter(object):
 
     REL_SCHEME = ('//', r'\/\/', r'\\/\\/')
 
-    def __init__(self, wburl, prefix, full_prefix=None, rel_prefix=None,
+    def __init__(self, wburl, prefix='', full_prefix=None, rel_prefix=None,
                  root_path=None, cookie_scope=None, rewrite_opts={}):
         self.wburl = wburl if isinstance(wburl, WbUrl) else WbUrl(wburl)
         self.prefix = prefix
@@ -152,17 +152,26 @@ def urljoin(orig_url, url):
 
 
 #=================================================================
-class HttpsUrlRewriter(UrlRewriter):
+class SchemeOnlyUrlRewriter(UrlRewriter):
     """
-    A url rewriter which urls that start with https:// to http://
+    A url rewriter which ensures that any urls have the same
+    scheme (http or https) as the base url.
     Other urls/input is unchanged.
     """
 
-    HTTP = 'http://'
-    HTTPS = 'https://'
+    def __init__(self, *args, **kwargs):
+        super(SchemeOnlyUrlRewriter, self).__init__(*args, **kwargs)
+        self.url_scheme = self.wburl.url.split(':')[0]
+        if self.url_scheme == 'https':
+            self.opposite_scheme = 'http'
+        else:
+            self.opposite_scheme = 'https'
 
     def rewrite(self, url, mod=None):
-        return self.remove_https(url)
+        if url.startswith(self.opposite_scheme + '://'):
+            url = self.url_scheme + url[len(self.opposite_scheme):]
+
+        return url
 
     def get_new_url(self, **kwargs):
         return kwargs.get('url', self.wburl.url)
@@ -175,12 +184,3 @@ def get_cookie_rewriter(self, scope=None):
 
     def deprefix_url(self):
         return self.wburl.url
-
-    @staticmethod
-    def remove_https(url):
-        rw = HttpsUrlRewriter
-        if url.startswith(rw.HTTPS):
-            result = rw.HTTP + url[len(rw.HTTPS):]
-            return result
-        else:
-            return url

pywb/rewrite/wburl.py

@@ -325,6 +325,10 @@ def is_embed(self):
     def is_banner_only(self):
         return (self.mod == 'bn_')
 
+    @property
+    def is_url_rewrite_only(self):
+        return (self.mod == 'uo_')
+
     @property
     def is_identity(self):
         return (self.mod == 'id_')

pywb/webapp/live_rewrite_handler.py

@@ -3,7 +3,6 @@
 
 from pywb.rewrite.rewrite_live import LiveRewriter
 from pywb.rewrite.wburl import WbUrl
-from pywb.rewrite.url_rewriter import HttpsUrlRewriter
 
 from handlers import StaticHandler, SearchPageWbUrlHandler
 from views import HeadInsertView
@@ -235,7 +234,8 @@ def _get_video_info(self, wbrequest, info_url=None, video_url=None):
             headers = self._live_request_headers(wbrequest)
             headers['Content-Type'] = content_type
 
-            info_url = HttpsUrlRewriter.remove_https(info_url)
+            if info_url.startswith('https://'):
+                info_url = info_url.replace('https', 'http', 1)
 
             response = self.live_fetcher.add_metadata(info_url, headers, metadata)