Compare images lazy-loaded with JS properly (#39)

When JS is used to lazy-load images, they do not have `src` or `srcset` attributes, which means we give the images an empty list of URLs. Unfortunately, we don't always know where the lazy-loaded image's URL is stored, so we can't solve this case perfectly, but the *vast* majority of the time, `data-src` and `data-srcset` are used, so that's what we fall back on here.

Additionally, if an image has no source URLs, that should still be OK, and we'll treat two images with no URLs as the same (we previously treated them differently).

Fixes #37.

Author: Mr0grog

docs/source/release-history.rst

@@ -9,6 +9,8 @@ In Development
 
 - Fixes :func:`web_monitoring_diff.html_diff_render` to make sure the spacing of text and tags in the HTML source code of the diff matches the original. This resolves display issues on pages where CSS is used to treat spacing as significant. (`#36 <https://github.com/edgi-govdata-archiving/web-monitoring-diff/issues/36>`_)
 
+- Improve handling of lazy-loaded images in :func:`web_monitoring_diff.html_diff_render`. When images are lazy-loaded via JS, they usually use the ``data-src`` or ``data-srcset`` attributes, and we now check those, too. Additionally, if two images have no detectable URLs, we now treat them as the same, rather than different. (`#37 <https://github.com/edgi-govdata-archiving/web-monitoring-diff/issues/37>`_)
+
 
 Version 0.1.0
 -------------

web_monitoring_diff/html_render_diff.py

@@ -268,6 +268,9 @@ class UrlRules:
 
     @classmethod
     def compare_array(cls, url_list_a, url_list_b, comparator):
+        if len(url_list_a) == 0 == len(url_list_b):
+            return True
+
         for url_a in url_list_a:
             for url_b in url_list_b:
                 if comparator:
@@ -866,14 +869,20 @@ def flatten_el(el, include_hrefs, skip_tag=False):
     if not skip_tag:
         if el.tag == 'img':
             src_array = []
-            el_src = el.get('src')
+            # The `data-src` attribute is very commonly used for JS to lazy-
+            # load images, so allow it in lieu of `src`.
+            el_src = el.get('src') or el.get('data-src')
             if el_src is not None:
                 src_array.append(el_src)
-            srcset = el.get('srcset')
+
+            # Same as above with `data-srcset` here.
+            srcset = el.get('srcset') or el.get('data-srcset')
             if srcset is not None:
                 for src in srcset.split(','):
                     src_array.append(src.split(' ', maxsplit=1)[0])
+
             yield (TokenType.img, src_array, start_tag(el))
+
         elif el.tag in undiffable_content_tags:
             element_source = etree.tostring(el, encoding=str, method='html')
             yield (TokenType.undiffable, element_source)

web_monitoring_diff/tests/test_html_diff_validity.py

@@ -301,6 +301,52 @@ def test_html_diff_works_with_srcset():
     assert results['change_count'] == 0
 
 
+def test_html_diff_works_with_images_without_src_srcset():
+    results = html_diff_render(
+        '<img alt="OSIRIS Mars true color.jpg">',
+        '<img alt="OSIRIS Mars true color.jpg">',
+        include='all')
+
+    assert results['change_count'] == 0
+
+
+def test_html_diff_works_with_data_src():
+    results = html_diff_render(
+        '''
+        <img
+            alt="OSIRIS Mars true color.jpg"
+            data-src="https://upload.wikimedia.org/wikipedia/commons/thumb/0/02/OSIRIS_Mars_true_color.jpg/413px-OSIRIS_Mars_true_color.jpg">
+        ''',
+        '''
+        <img
+            alt="OSIRIS Mars true color.jpg"
+            data-src="https://upload.wikimedia.org/wikipedia/commons/thumb/0/02/OSIRIS_Mars_true_color.jpg/275px-OSIRIS_Mars_true_color.jpg">
+        ''',
+        include='all')
+
+    assert results['change_count'] == 2
+
+
+def test_html_diff_works_with_data_srcset():
+    results = html_diff_render(
+        '''
+        <img
+            alt="OSIRIS Mars true color.jpg"
+            data-src="https://upload.wikimedia.org/wikipedia/commons/thumb/0/02/OSIRIS_Mars_true_color.jpg/413px-OSIRIS_Mars_true_color.jpg">
+        ''',
+        '''
+        <img
+            alt="OSIRIS Mars true color.jpg"
+            data-src="https://upload.wikimedia.org/wikipedia/commons/thumb/0/02/OSIRIS_Mars_true_color.jpg/275px-OSIRIS_Mars_true_color.jpg"
+            data-srcset="https://upload.wikimedia.org/wikipedia/commons/thumb/0/02/OSIRIS_Mars_true_color.jpg/413px-OSIRIS_Mars_true_color.jpg 1.5x, https://upload.wikimedia.org/wikipedia/commons/thumb/0/02/OSIRIS_Mars_true_color.jpg/550px-OSIRIS_Mars_true_color.jpg 2x"
+            width="275"
+            height="275">
+        ''',
+        include='all')
+
+    assert results['change_count'] == 0
+
+
 def test_html_diff_works_with_jsessionid():
     results = html_diff_render(
         '<a href="https://www.ncdc.noaa.gov/homr/api;jsessionid=A2DECB66D2648BFED11FC721FC3043A1"></a>',