Merge branch 'develop'

Author: ikreymer

README.rst

@@ -1,4 +1,4 @@
-PyWb 0.30.1
+PyWb 0.31.0
 ===========
 
 .. image:: https://travis-ci.org/ikreymer/pywb.svg?branch=master

pywb/__init__.py

@@ -1,4 +1,4 @@
-__version__ = '0.30.1'
+__version__ = '0.31.0'
 
 DEFAULT_CONFIG = 'pywb/default_config.yaml'
 

pywb/cdx/cdxobject.py

@@ -153,7 +153,7 @@ def __init__(self, cdxline=b''):
             raise CDXException(msg)
 
         for header, field in zip(cdxformat, fields):
-            self[header] = field.decode('utf-8')
+            self[header] = to_native_str(field, 'utf-8')
 
         self.cdxline = cdxline
 
@@ -213,7 +213,7 @@ def conv_to_json(obj, fields=None):
 
     def __str__(self):
         if self.cdxline:
-            return self.cdxline.decode('utf-8')
+            return to_native_str(self.cdxline, 'utf-8')
 
         if not self._from_json:
             return ' '.join(str(val) for val in six.itervalues(self))
@@ -263,7 +263,7 @@ def __init__(self, idxline):
             raise CDXException(msg.format(len(fields), self.NUM_REQ_FIELDS))
 
         for header, field in zip(self.FORMAT, fields):
-            self[header] = field.decode('utf-8')
+            self[header] = to_native_str(field, 'utf-8')
 
         self['offset'] = int(self['offset'])
         self['length'] = int(self['length'])
@@ -285,4 +285,4 @@ def to_json(self, fields=None):
         return json_encode(self) + '\n'
 
     def __str__(self):
-        return self.idxline.decode('utf-8')
+        return to_native_str(self.idxline, 'utf-8')

pywb/framework/wbrequestresponse.py

@@ -184,14 +184,15 @@ def normalize_post_query(self):
         if not self.wb_url:
             return
 
-        mime = self.env.get('CONTENT_TYPE', '').split(';')[0]
+        mime = self.env.get('CONTENT_TYPE', '')
         length = self.env.get('CONTENT_LENGTH')
         stream = self.env['wsgi.input']
 
         buffered_stream = BytesIO()
 
         post_query = extract_post_query('POST', mime, length, stream,
-                                        buffered_stream=buffered_stream)
+                                        buffered_stream=buffered_stream,
+                                        environ=self.env)
 
         if post_query:
             self.env['wsgi.input'] = buffered_stream

pywb/rewrite/html_rewriter.py

@@ -120,7 +120,7 @@ def __init__(self, url_rewriter,
 
     def _rewrite_meta_refresh(self, meta_refresh):
         if not meta_refresh:
-            return None
+            return ''
 
         m = self.META_REFRESH_REGEX.match(meta_refresh)
         if not m:
@@ -133,6 +133,9 @@ def _rewrite_meta_refresh(self, meta_refresh):
         return meta_refresh
 
     def _rewrite_base(self, url, mod=''):
+        if not url:
+            return ''
+
         url = self._ensure_url_has_path(url)
 
         base_url = self._rewrite_url(url, mod)
@@ -183,11 +186,11 @@ def _ensure_url_has_path(self, url):
 
     def _rewrite_url(self, value, mod=None):
         if not value:
-            return None
+            return ''
 
         value = value.strip()
         if not value:
-            return None
+            return ''
 
         value = self.try_unescape(value)
         return self.url_rewriter.rewrite(value, mod)
@@ -209,21 +212,24 @@ def try_unescape(self, value):
         return new_value
 
     def _rewrite_srcset(self, value, mod=''):
+        if not value:
+            return ''
+
         values = value.split(',')
-        values = map(lambda x: self._rewrite_url(x.strip()), values)
+        values = [self._rewrite_url(v.strip()) for v in values]
         return ', '.join(values)
 
     def _rewrite_css(self, css_content):
         if css_content:
             return self.css_rewriter.rewrite(css_content)
         else:
-            return None
+            return ''
 
     def _rewrite_script(self, script_content):
         if script_content:
             return self.js_rewriter.rewrite(script_content)
         else:
-            return None
+            return ''
 
     def has_attr(self, tag_attrs, attr):
         name, value = attr
@@ -252,6 +258,11 @@ def _rewrite_tag_attrs(self, tag, tag_attrs):
         self.out.write('<' + tag)
 
         for attr_name, attr_value in tag_attrs:
+            empty_attr = False
+            if attr_value is None:
+                attr_value = ''
+                empty_attr = True
+
             # special case: inline JS/event handler
             if ((attr_value and attr_value.startswith('javascript:'))
                  or attr_name.startswith('on')):
@@ -324,7 +335,7 @@ def _rewrite_tag_attrs(self, tag, tag_attrs):
                     attr_value = self._rewrite_url(attr_value, rw_mod)
 
             # write the attr!
-            self._write_attr(attr_name, attr_value)
+            self._write_attr(attr_name, attr_value, empty_attr)
 
         return True
 
@@ -347,11 +358,17 @@ def _rewrite_head(self, start_end):
 
         return True
 
-    def _write_attr(self, name, value):
-        # parser doesn't differentiate between 'attr=""' and just 'attr'
-        # 'attr=""' is more common, so use that form
-        if value:
+    def _write_attr(self, name, value, empty_attr):
+        # if empty_attr is set, just write 'attr'!
+        if empty_attr:
+            self.out.write(' ' + name)
+
+        # write with value, if set
+        elif value:
+
             self.out.write(' ' + name + '="' + value.replace('"', '&quot;') + '"')
+
+        # otherwise, 'attr=""' is more common, so use that form
         else:
             self.out.write(' ' + name + '=""')
 
@@ -421,8 +438,9 @@ def clear_cdata_mode(self):
     def feed(self, string):
         try:
             HTMLParser.feed(self, string)
-        except Exception:  # pragma: no cover
-            # only raised in 2.6
+        except Exception as e:  # pragma: no cover
+            import traceback
+            traceback.print_exc()
             self.out.write(string)
 
     def _internal_close(self):

pywb/rewrite/rewrite_amf.py

@@ -0,0 +1,52 @@
+from io import BytesIO
+from six.moves import zip
+from pywb.rewrite.rewrite_content import RewriteContent
+
+
+# ============================================================================
+# Expiermental: not fully tested
+class RewriteContentAMF(RewriteContent):  #pragma: no cover
+    def handle_custom_rewrite(self, text_type, status_headers, stream, env):
+
+        if status_headers.get_header('Content-Type') == 'application/x-amf':
+            stream = self.rewrite_amf(stream, env)
+
+        return (super(RewriteContentAMF, self).
+                handle_custom_rewrite(text_type, status_headers, stream, env))
+
+    def rewrite_amf(self, stream, env):
+        try:
+            from pyamf import remoting
+
+            iobuff = BytesIO()
+            while True:
+                buff = stream.read()
+                if not buff:
+                    break
+                iobuff.write(buff)
+
+            iobuff.seek(0)
+            res = remoting.decode(iobuff)
+
+            if env and env.get('pywb.inputdata'):
+                inputdata = env.get('pywb.inputdata')
+
+                new_list = []
+
+                for src, target in zip(inputdata.bodies, res.bodies):
+                    #print(target[0] + ' = ' + src[0])
+
+                    #print('messageId => corrId ' + target[1].body.correlationId + ' => ' + src[1].body[0].messageId)
+                    target[1].body.correlationId = src[1].body[0].messageId
+
+                    new_list.append((src[0], target[1]))
+
+                res.bodies = new_list
+
+            return BytesIO(remoting.encode(res).getvalue())
+
+        except Exception as e:
+            import traceback
+            traceback.print_exc()
+            print(e)
+            return stream

pywb/rewrite/rewrite_content.py

@@ -4,7 +4,7 @@
 import yaml
 import re
 
-from chardet.universaldetector import UniversalDetector
+#from chardet.universaldetector import UniversalDetector
 from io import BytesIO
 
 from pywb.rewrite.header_rewriter import RewrittenStatusAndHeaders
@@ -21,7 +21,7 @@
 
 
 #=================================================================
-class RewriteContent:
+class RewriteContent(object):
     HEAD_REGEX = re.compile(b'<\s*head\\b[^>]*[>]+', re.I)
 
     TAG_REGEX = re.compile(b'^\s*\<')
@@ -77,6 +77,7 @@ def _rewrite_headers(self, urlrewriter, rule, status_headers, stream,
 
 
     def _check_encoding(self, rewritten_headers, stream, enc):
+        matched = False
         if (rewritten_headers.
              contains_removed_header('content-encoding', enc)):
 
@@ -87,14 +88,15 @@ def _check_encoding(self, rewritten_headers, stream, enc):
                 stream = DecompressingBufferedReader(stream, decomp_type=enc)
 
             rewritten_headers.status_headers.remove_header('content-length')
+            matched = True
 
-        return stream
+        return matched, stream
 
 
 
     def rewrite_content(self, urlrewriter, status_headers, stream,
                         head_insert_func=None, urlkey='',
-                        cdx=None, cookie_rewriter=None):
+                        cdx=None, cookie_rewriter=None, env=None):
 
         wb_url = urlrewriter.wburl
 
@@ -118,9 +120,12 @@ def rewrite_content(self, urlrewriter, status_headers, stream,
 
         status_headers = rewritten_headers.status_headers
 
-        # use rewritten headers, but no further rewriting needed
-        if rewritten_headers.text_type is None:
-            return (status_headers, self.stream_to_gen(stream), False)
+        res = self.handle_custom_rewrite(rewritten_headers.text_type,
+                                         status_headers,
+                                         stream,
+                                         env)
+        if res:
+            return res
 
         # Handle text content rewriting
         # ====================================================================
@@ -136,8 +141,12 @@ def rewrite_content(self, urlrewriter, status_headers, stream,
         encoding = None
         first_buff = b''
 
-        stream = self._check_encoding(rewritten_headers, stream, 'gzip')
-        stream = self._check_encoding(rewritten_headers, stream, 'deflate')
+        for decomp_type in BufferedReader.get_supported_decompressors():
+            matched, stream = self._check_encoding(rewritten_headers,
+                                                   stream,
+                                                   decomp_type)
+            if matched:
+                break
 
         if mod == 'js_':
             text_type, stream = self._resolve_text_type('js',
@@ -237,6 +246,11 @@ def rewrite_content(self, urlrewriter, status_headers, stream,
 
         return (status_headers, gen, True)
 
+    def handle_custom_rewrite(self, text_type, status_headers, stream, env):
+        # use rewritten headers, but no further rewriting needed
+        if text_type is None:
+            return (status_headers, self.stream_to_gen(stream), False)
+
     @staticmethod
     def _extract_html_charset(buff, status_headers):
         charset = None
@@ -360,3 +374,5 @@ def rewrite_text_stream_to_gen(stream, rewrite_func,
 
         finally:
             stream.close()
+
+

pywb/rewrite/test/test_html_rewriter.py

@@ -49,6 +49,12 @@
 >>> parse('<base href="static/"/><img src="image.gif"/>', urlrewriter=no_base_canon_rewriter)
 <base href="static/"/><img src="/web/20131226101010im_/http://example.com/some/path/static/image.gif"/>
 
+# Empty url
+>>> parse('<base href="">')
+<base href="">
+
+>>> parse('<base href>')
+<base href>
 
 
 # HTML Entities
@@ -66,6 +72,10 @@
 >>> parse('<input value="&amp;X&amp;&quot;">X</input>')
 <input value="&amp;X&amp;&quot;">X</input>
 
+# Empty values should be ignored
+>>> parse('<input name="foo" value>')
+<input name="foo" value>
+
 # SKIPPED
 # Unicode -- default with %-encoding
 #>>> parse(u'<a href="http://испытание.испытание/">испытание</a>')
@@ -92,7 +102,7 @@
 <meta http-equiv="refresh" content="text/html; charset=utf-8"/>
 
 >>> parse('<META http-equiv="refresh" content>')
-<meta http-equiv="refresh" content="">
+<meta http-equiv="refresh" content>
 
 >>> parse('<meta property="og:image" content="http://example.com/example.jpg">')
 <meta property="og:image" content="/web/20131226101010/http://example.com/example.jpg">
@@ -115,6 +125,10 @@
 >>> parse('<img srcset="//example.com/1x 1x, //example.com/foo 2x, https://example.com/bar 4x">')
 <img srcset="/web/20131226101010///example.com/1x 1x, /web/20131226101010///example.com/foo 2x, /web/20131226101010/https://example.com/bar 4x">
 
+# empty srcset attrib
+>>> parse('<img srcset="">')
+<img srcset="">
+
 # Script tag
 >>> parse('<script>window.location = "http://example.com/a/b/c.html"</script>')
 <script>window.WB_wombat_location = "/web/20131226101010/http://example.com/a/b/c.html"</script>
@@ -131,7 +145,7 @@
 <script>/*<![CDATA[*/window.WB_wombat_location = "/web/20131226101010/http://example.com/a/b/c.html;/*]]>*/"</script>
 
 >>> parse('<div style="background: url(\'abc.html\')" onblah onclick="location = \'redirect.html\'"></div>')
-<div style="background: url('/web/20131226101010/http://example.com/some/path/abc.html')" onblah="" onclick="WB_wombat_location = 'redirect.html'"></div>
+<div style="background: url('/web/20131226101010/http://example.com/some/path/abc.html')" onblah onclick="WB_wombat_location = 'redirect.html'"></div>
 
 >>> parse('<i style="background-image: url(http://foo-.bar_.example.com/)"></i>')
 <i style="background-image: url(/web/20131226101010/http://foo-.bar_.example.com/)"></i>