Merge branch 'main' into miketheman/localdev

Author: miketheman

inspector/distribution.py

@@ -7,6 +7,8 @@
 
 from flask import abort
 
+from .utilities import requests_session
+
 # Lightweight datastore ;)
 dists = {}
 
@@ -59,7 +61,7 @@ def _get_dist(first, second, rest, distname):
 
     url = f"https://files.pythonhosted.org/packages/{first}/{second}/{rest}/{distname}"
     try:
-        resp = requests.get(url, stream=True)
+        resp = requests_session().get(url, stream=True)
         resp.raise_for_status()
     except requests.HTTPError as exc:
         abort(exc.response.status_code)

inspector/main.py

@@ -2,7 +2,6 @@
 import urllib.parse
 
 import gunicorn.http.errors
-import requests
 import sentry_sdk
 
 from flask import Flask, Response, abort, redirect, render_template, request, url_for
@@ -13,7 +12,7 @@
 from .deob import decompile, disassemble
 from .distribution import _get_dist
 from .legacy import parse
-from .utilities import pypi_report_form
+from .utilities import pypi_report_form, requests_session
 
 
 def traces_sampler(sampling_context):
@@ -62,7 +61,7 @@ def versions(project_name):
             url_for("versions", project_name=canonicalize_name(project_name)), 301
         )
 
-    resp = requests.get(f"https://pypi.org/pypi/{project_name}/json")
+    resp = requests_session().get(f"https://pypi.org/pypi/{project_name}/json")
     pypi_project_url = f"https://pypi.org/project/{project_name}"
 
     # Self-host 404 page to mitigate iframe embeds
@@ -99,7 +98,9 @@ def distributions(project_name, version):
             301,
         )
 
-    resp = requests.get(f"https://pypi.org/pypi/{project_name}/{version}/json")
+    resp = requests_session().get(
+        f"https://pypi.org/pypi/{project_name}/{version}/json"
+    )
     if resp.status_code != 200:
         return redirect(f"/project/{project_name}/")
 
@@ -142,12 +143,14 @@ def distribution(project_name, version, first, second, rest, distname):
     dist = _get_dist(first, second, rest, distname)
 
     h2_paren = "View this project on PyPI"
-    resp = requests.get(f"https://pypi.org/pypi/{project_name}/json")
+    resp = requests_session().get(f"https://pypi.org/pypi/{project_name}/json")
     if resp.status_code == 404:
         h2_paren = "❌ Project no longer on PyPI"
 
     h3_paren = "View this release on PyPI"
-    resp = requests.get(f"https://pypi.org/pypi/{project_name}/{version}/json")
+    resp = requests_session().get(
+        f"https://pypi.org/pypi/{project_name}/{version}/json"
+    )
     if resp.status_code == 404:
         h3_paren = "❌ Release no longer on PyPI"
 
@@ -193,12 +196,14 @@ def file(project_name, version, first, second, rest, distname, filepath):
         )
 
     h2_paren = "View this project on PyPI"
-    resp = requests.get(f"https://pypi.org/pypi/{project_name}/json")
+    resp = requests_session().get(f"https://pypi.org/pypi/{project_name}/json")
     if resp.status_code == 404:
         h2_paren = "❌ Project no longer on PyPI"
 
     h3_paren = "View this release on PyPI"
-    resp = requests.get(f"https://pypi.org/pypi/{project_name}/{version}/json")
+    resp = requests_session().get(
+        f"https://pypi.org/pypi/{project_name}/{version}/json"
+    )
     if resp.status_code == 404:
         h3_paren = "❌ Release no longer on PyPI"
 

inspector/utilities.py

@@ -1,5 +1,7 @@
 import urllib.parse
 
+import requests
+
 
 def mailto_report_link(project_name, version, file_path, request_url):
     """
@@ -39,3 +41,22 @@ def pypi_report_form(project_name, version, file_path, request_url):
         f"?inspector_link={request_url}"
         f"&summary={urllib.parse.quote(summary)}"
     )
+
+
+def requests_session(custom_user_agent: str = "inspector.pypi.io") -> requests.Session:
+    """
+    Custom `requests` session with default headers applied.
+
+    Usage:
+
+    >>> from inspector.utilities import requests_session
+    >>> response = requests_session().get(<url>)
+    """
+    session = requests.Session()
+    session.headers.update(
+        {
+            "User-Agent": custom_user_agent,
+        }
+    )
+
+    return session

tests/test_main.py

@@ -10,7 +10,9 @@ def test_versions(monkeypatch):
         json=lambda: stub_json,
     )
     get = pretend.call_recorder(lambda a: stub_response)
-    monkeypatch.setattr(inspector.main.requests, "get", get)
+    monkeypatch.setattr(
+        inspector.main, "requests_session", lambda: pretend.stub(get=get)
+    )
 
     render_template = pretend.call_recorder(lambda *a, **kw: None)
     monkeypatch.setattr(inspector.main, "render_template", render_template)