[#715] Forward custom headers for cdx queries (#813)

In particular the X-Pywb-ACL-User header must be forwarded in order
for it to be able to control CDX-queries

Author: krakan

pywb/apps/frontendapp.py

@@ -434,7 +434,11 @@ def serve_cdx(self, environ, coll='$root'):
             cdx_url += 'limit=' + str(self.query_limit)
 
         try:
-            res = requests.get(cdx_url, stream=True)
+            headers = {}
+            for key in environ.keys():
+                if key.startswith("HTTP_X_"):
+                    headers[key[5:].replace("_", "-")] = environ[key]
+            res = requests.get(cdx_url, stream=True, headers=headers)
 
             status_line = '{} {}'.format(res.status_code, res.reason)
             content_type = res.headers.get('Content-Type')

sample_archive/access/pywb.aclj

@@ -5,6 +5,8 @@ org,iana)/_css/2013.1/fonts/opensans-semibold.ttf - {"access": "allow"}
 org,iana)/_css - {"access": "exclude"}
 org,iana)/### - {"access": "allow"}
 org,iana)/ - {"access": "exclude"}
+com,example)/?example=3 - {"access": "block", "user": "staff"}
+com,example)/?example=3 - {"access": "exclude", "user": "staff2"}
 org,example)/?example=1 - {"access": "block"}
 com,example)/?example=2 - {"access": "allow_ignore_embargo"}
 com,example)/?example=1 - {"access": "allow_ignore_embargo", "user": "staff2"}

tests/test_acl.py

@@ -41,12 +41,23 @@ def test_blocked_url(self):
         assert 'Access Blocked' in resp.text
 
     def test_allow_via_acl_header(self):
-        resp = self.query('http://www.iana.org/about/')
-
+        resp = self.testapp.get('/pywb/cdx?url=http://www.iana.org/about/', headers={"X-Pywb-Acl-User": "staff"})
         assert len(resp.text.splitlines()) == 1
 
         resp = self.testapp.get('/pywb/mp_/http://www.iana.org/about/', headers={"X-Pywb-Acl-User": "staff"}, status=200)
 
+    def test_block_via_acl_header(self):
+        resp = self.testapp.get('/pywb/cdx?url=http://example.com/?example=3', headers={"X-Pywb-Acl-User": "staff"})
+        assert len(resp.text.splitlines()) > 0
+
+        resp = self.testapp.get('/pywb/mp_/http://example.com/?example=3', headers={"X-Pywb-Acl-User": "staff"}, status=451)
+
+    def test_exclude_via_acl_header(self):
+        resp = self.testapp.get('/pywb/cdx?url=http://example.com/?example=3', headers={"X-Pywb-Acl-User": "staff2"})
+        assert len(resp.text.splitlines()) == 0
+
+        resp = self.testapp.get('/pywb/mp_/http://example.com/?example=3', headers={"X-Pywb-Acl-User": "staff2"}, status=404)
+
     def test_allowed_more_specific(self):
         resp = self.query('http://www.iana.org/_css/2013.1/fonts/opensans-semibold.ttf')
 

tests/test_embargo.py

@@ -46,8 +46,12 @@ def test_embargo_ignore_acl(self):
     def test_embargo_ignore_acl_with_header_only(self):
         # ignore embargo with custom header only
         headers = {"X-Pywb-ACL-User": "staff2"}
-        resp = self.testapp.get('/pywb-embargo-acl/20140126201054mp_/http://example.com/?example=1', status=200, headers=headers)
 
+        resp = self.testapp.get('/pywb-embargo-acl/cdx?url=http://example.com/?example=1', headers=headers)
+        assert len(resp.text.splitlines()) > 0
+        resp = self.testapp.get('/pywb-embargo-acl/20140126201054mp_/http://example.com/?example=1', status=200, headers=headers)
+        resp = self.testapp.get('/pywb-embargo-acl/cdx?url=http://example.com/?example=1')
+        assert len(resp.text.splitlines()) == 0
         resp = self.testapp.get('/pywb-embargo-acl/20140126201054mp_/http://example.com/?example=1', status=404)