diff --git a/seed_intersphinx_mapping/__init__.py b/seed_intersphinx_mapping/__init__.py index a983f40..17d0eff 100644 --- a/seed_intersphinx_mapping/__init__.py +++ b/seed_intersphinx_mapping/__init__.py @@ -33,13 +33,14 @@ # stdlib import functools import json +import os.path import re -from typing import Dict, Optional, Tuple, Union +from typing import Dict, List, Optional, Tuple, Union +from urllib.parse import urlparse # 3rd party import dist_meta import requests -from dist_meta.metadata_mapping import MetadataMapping from domdf_python_tools.compat import importlib_resources from domdf_python_tools.utils import stderr_writer from packaging.requirements import Requirement @@ -60,7 +61,7 @@ _DOCUMENTATION_RE = re.compile(r"^[dD]oc(s|umentation)") -def _get_project_links(project_name: str) -> MetadataMapping: +def _get_project_links(project_name: str) -> List[str]: """ Returns the web links for the given project. @@ -69,29 +70,36 @@ def _get_project_links(project_name: str) -> MetadataMapping: :param project_name: """ - urls = MetadataMapping() + urls = [] # Try a local package first try: dist = dist_meta.distributions.get_distribution(project_name) - raw_urls = dist.get_metadata().get_all("Project-URL", default=()) + metadata = dist.get_metadata() + raw_urls = metadata.get_all("Project-URL", default=()) for url in raw_urls: - label, url, *_ = map(str.strip, url.split(',')) + label, url = url.split(',', 1) if _DOCUMENTATION_RE.match(label): - urls[label] = url + urls.append(url) + + urls.append(metadata.get("Home-Page", '')) except dist_meta.distributions.DistributionNotFoundError: # Fall back to PyPI with PyPIJSON() as client: - metadata = client.get_metadata(project_name).info + pypi_metadata = client.get_metadata(project_name).info + + if "project_urls" in pypi_metadata and pypi_metadata["project_urls"]: + + for label, url in pypi_metadata["project_urls"].items(): + if _DOCUMENTATION_RE.match(label): + urls.append(url) - if "project_urls" in metadata and metadata["project_urls"]: - for label, url in metadata["project_urls"].items(): - if _DOCUMENTATION_RE.match(label): - urls[label] = url + urls.append(pypi_metadata["home_page"]) + urls = [url.strip() for url in filter(None, urls)] return urls @@ -127,25 +135,24 @@ def get_sphinx_doc_url(pypi_name: str) -> str: :exc:`apeye.slumber_url.exceptions.HttpNotFoundError` if the project could not be found on PyPI. """ - for key, value in _get_project_links(pypi_name).items(): - + docs_urls = [] + for value in _get_project_links(pypi_name): # Follow redirects to get actual URL r = requests.head(value, allow_redirects=True, timeout=10) - if r.status_code != 200: # pragma: no cover - raise ValueError(f"Documentation URL not found: HTTP Status {r.status_code}.") - docs_url = r.url + if r.status_code == 200: + has_extension = os.path.splitext(urlparse(r.url).path)[-1] + url = os.path.dirname(r.url) if has_extension else r.url + docs_urls.append(url) - if docs_url.endswith('/'): - objects_inv_url = f"{docs_url}objects.inv" - else: # pragma: no cover - objects_inv_url = f"{docs_url}/objects.inv" + for docs_url in docs_urls: + objects_inv_url = f"{docs_url.rstrip('/')}/objects.inv" r = requests.head(objects_inv_url, allow_redirects=True, timeout=10) if r.status_code != 200: - raise ValueError(f"objects.inv not found at url {objects_inv_url}: HTTP Status {r.status_code}.") - - return docs_url + stderr_writer(f"WARNING: objects.inv not found at url {objects_inv_url}: HTTP Status {r.status_code}.") + else: + return docs_url raise ValueError("Documentation URL not found in data from PyPI.") diff --git a/seed_intersphinx_mapping/fallback_mapping.json b/seed_intersphinx_mapping/fallback_mapping.json index 42472b1..7544199 100644 --- a/seed_intersphinx_mapping/fallback_mapping.json +++ b/seed_intersphinx_mapping/fallback_mapping.json @@ -29,7 +29,7 @@ "scikit-learn": "https://scikit-learn.org/stable/", "six": "https://six.readthedocs.io/", "slumber": "https://slumber.readthedocs.io/en/v0.6.0/", - "sphinx": "https://www.sphinx-doc.org/en/3.x/", + "sphinx": "https://www.sphinx-doc.org/en/master/", "typing": "https://docs.python.org/3/", "typing-extensions": "https://typing-extensions.readthedocs.io/en/latest/", "typing_extensions": "https://typing-extensions.readthedocs.io/en/latest/", diff --git a/tests/test_core.py b/tests/test_core.py index 77059aa..8f5e1fb 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -24,7 +24,7 @@ def test_get_sphinx_doc_url(): with pytest.raises(ValueError, match="Documentation URL not found in data from PyPI."): get_sphinx_doc_url("slumber") - with pytest.raises(ValueError, match="objects.inv not found at url .*: HTTP Status 404"): + with pytest.raises(ValueError, match="Documentation URL not found in data from PyPI."): get_sphinx_doc_url("autodoc_pydantic") assert cache.clear(get_sphinx_doc_url) @@ -35,10 +35,7 @@ def test_get_sphinx_doc_url(): if sys.version_info[:2] != (3, 8): # Latest numpy's "Documentation" url doesn't point to Sphinx docs. - with pytest.raises( - ValueError, - match="objects.inv not found at url https://numpy.org/doc/objects.inv: HTTP Status 404." - ): + with pytest.raises(ValueError, match="Documentation URL not found in data from PyPI."): get_sphinx_doc_url("numpy") else: assert re.match(r"https://numpy\.org/doc/1\.\d\d/", get_sphinx_doc_url("numpy")) diff --git a/tests/test_seeding.py b/tests/test_seeding.py index 0200435..cd34121 100644 --- a/tests/test_seeding.py +++ b/tests/test_seeding.py @@ -25,12 +25,12 @@ "packaging": ("https://packaging.pypa.io/en/stable/", None), "requests": ("https://requests.readthedocs.io/en/latest/", None), "slumber": ("https://slumber.readthedocs.io/en/v0.6.0/", None), - "sphinx": ("https://www.sphinx-doc.org/en/3.x/", None), + "sphinx": ("https://www.sphinx-doc.org/en/master/", None), } bad_expected_mapping = { "domdf-python-tools": ("https://domdf-python-tools.readthedocs.io/en/latest/", None), "packaging": ("https://packaging.pypa.io/en/stable/", None), - "sphinx": ("https://www.sphinx-doc.org/en/3.x/", None), + "sphinx": ("https://www.sphinx-doc.org/en/master/", None), } @@ -50,12 +50,12 @@ def test_seed_intersphinx_mapping( assert seed_intersphinx_mapping(*parse_requirements_txt(tmp_pathplus)) == expects err = capsys.readouterr().err - assert err == "WARNING: Unable to determine documentation url for project sphinxcontrib-domaintools\n" + assert "WARNING: Unable to determine documentation url for project sphinxcontrib-domaintools\n" in err requirements, comments, invalid = read_requirements(tmp_pathplus / "requirements.txt", include_invalid=True) assert seed_intersphinx_mapping(*requirements) == expects err = capsys.readouterr().err - assert err == "WARNING: Unable to determine documentation url for project sphinxcontrib-domaintools\n" + assert "WARNING: Unable to determine documentation url for project sphinxcontrib-domaintools\n" in err @pytest.mark.parametrize( @@ -70,7 +70,7 @@ def test_seed_intersphinx_mapping_pyproject(tmp_pathplus: PathPlus, contents: st assert seed_intersphinx_mapping(*parse_pyproject_toml(tmp_pathplus)) == expects err = capsys.readouterr().err - assert err == "WARNING: Unable to determine documentation url for project sphinxcontrib-domaintools\n" + assert err.endswith("WARNING: Unable to determine documentation url for project sphinxcontrib-domaintools\n") @pytest.mark.parametrize( @@ -85,7 +85,7 @@ def test_seed_intersphinx_mapping_flit(tmp_pathplus: PathPlus, contents: str, ex assert seed_intersphinx_mapping(*parse_flit_requirements(tmp_pathplus)) == expects err = capsys.readouterr().err - assert err == "WARNING: Unable to determine documentation url for project sphinxcontrib-domaintools\n" + assert err.endswith("WARNING: Unable to determine documentation url for project sphinxcontrib-domaintools\n") @pytest.mark.parametrize("pkg_requirements_source", ["requirements", "flit", "pyproject", "pyproject.toml"]) @@ -125,7 +125,8 @@ def test_sphinx_seed_intersphinx_mapping_mocked( advanced_data_regression.check(config.intersphinx_mapping) err = capsys.readouterr().err - assert err == "WARNING: Unable to determine documentation url for project sphinxcontrib-domaintools\n" + + assert err.endswith("WARNING: Unable to determine documentation url for project sphinxcontrib-domaintools\n") def test_sphinx_seed_intersphinx_mapping_list_mocked( diff --git a/tests/test_seeding_/test_sphinx_seed_intersphinx_mapping_list_mocked.yml b/tests/test_seeding_/test_sphinx_seed_intersphinx_mapping_list_mocked.yml index 8a05a51..3f71b14 100644 --- a/tests/test_seeding_/test_sphinx_seed_intersphinx_mapping_list_mocked.yml +++ b/tests/test_seeding_/test_sphinx_seed_intersphinx_mapping_list_mocked.yml @@ -16,5 +16,5 @@ slumber: - - null sphinx: - sphinx -- - https://www.sphinx-doc.org/en/3.x/ +- - https://www.sphinx-doc.org/en/master/ - - null diff --git a/tests/test_seeding_/test_sphinx_seed_intersphinx_mapping_mocked_bad_example_requirements_flit_.yml b/tests/test_seeding_/test_sphinx_seed_intersphinx_mapping_mocked_bad_example_requirements_flit_.yml index 58fcd50..4c54c69 100644 --- a/tests/test_seeding_/test_sphinx_seed_intersphinx_mapping_mocked_bad_example_requirements_flit_.yml +++ b/tests/test_seeding_/test_sphinx_seed_intersphinx_mapping_mocked_bad_example_requirements_flit_.yml @@ -8,5 +8,5 @@ packaging: - - null sphinx: - sphinx -- - https://www.sphinx-doc.org/en/3.x/ +- - https://www.sphinx-doc.org/en/master/ - - null diff --git a/tests/test_seeding_/test_sphinx_seed_intersphinx_mapping_mocked_bad_example_requirements_pyproject_.yml b/tests/test_seeding_/test_sphinx_seed_intersphinx_mapping_mocked_bad_example_requirements_pyproject_.yml index 58fcd50..4c54c69 100644 --- a/tests/test_seeding_/test_sphinx_seed_intersphinx_mapping_mocked_bad_example_requirements_pyproject_.yml +++ b/tests/test_seeding_/test_sphinx_seed_intersphinx_mapping_mocked_bad_example_requirements_pyproject_.yml @@ -8,5 +8,5 @@ packaging: - - null sphinx: - sphinx -- - https://www.sphinx-doc.org/en/3.x/ +- - https://www.sphinx-doc.org/en/master/ - - null diff --git a/tests/test_seeding_/test_sphinx_seed_intersphinx_mapping_mocked_bad_example_requirements_pyproject_toml_.yml b/tests/test_seeding_/test_sphinx_seed_intersphinx_mapping_mocked_bad_example_requirements_pyproject_toml_.yml index 58fcd50..4c54c69 100644 --- a/tests/test_seeding_/test_sphinx_seed_intersphinx_mapping_mocked_bad_example_requirements_pyproject_toml_.yml +++ b/tests/test_seeding_/test_sphinx_seed_intersphinx_mapping_mocked_bad_example_requirements_pyproject_toml_.yml @@ -8,5 +8,5 @@ packaging: - - null sphinx: - sphinx -- - https://www.sphinx-doc.org/en/3.x/ +- - https://www.sphinx-doc.org/en/master/ - - null diff --git a/tests/test_seeding_/test_sphinx_seed_intersphinx_mapping_mocked_bad_example_requirements_requirements_.yml b/tests/test_seeding_/test_sphinx_seed_intersphinx_mapping_mocked_bad_example_requirements_requirements_.yml index 58fcd50..4c54c69 100644 --- a/tests/test_seeding_/test_sphinx_seed_intersphinx_mapping_mocked_bad_example_requirements_requirements_.yml +++ b/tests/test_seeding_/test_sphinx_seed_intersphinx_mapping_mocked_bad_example_requirements_requirements_.yml @@ -8,5 +8,5 @@ packaging: - - null sphinx: - sphinx -- - https://www.sphinx-doc.org/en/3.x/ +- - https://www.sphinx-doc.org/en/master/ - - null diff --git a/tests/test_seeding_/test_sphinx_seed_intersphinx_mapping_mocked_example_requirements_a_flit_.yml b/tests/test_seeding_/test_sphinx_seed_intersphinx_mapping_mocked_example_requirements_a_flit_.yml index 8a05a51..3f71b14 100644 --- a/tests/test_seeding_/test_sphinx_seed_intersphinx_mapping_mocked_example_requirements_a_flit_.yml +++ b/tests/test_seeding_/test_sphinx_seed_intersphinx_mapping_mocked_example_requirements_a_flit_.yml @@ -16,5 +16,5 @@ slumber: - - null sphinx: - sphinx -- - https://www.sphinx-doc.org/en/3.x/ +- - https://www.sphinx-doc.org/en/master/ - - null diff --git a/tests/test_seeding_/test_sphinx_seed_intersphinx_mapping_mocked_example_requirements_a_pyproject_.yml b/tests/test_seeding_/test_sphinx_seed_intersphinx_mapping_mocked_example_requirements_a_pyproject_.yml index 8a05a51..3f71b14 100644 --- a/tests/test_seeding_/test_sphinx_seed_intersphinx_mapping_mocked_example_requirements_a_pyproject_.yml +++ b/tests/test_seeding_/test_sphinx_seed_intersphinx_mapping_mocked_example_requirements_a_pyproject_.yml @@ -16,5 +16,5 @@ slumber: - - null sphinx: - sphinx -- - https://www.sphinx-doc.org/en/3.x/ +- - https://www.sphinx-doc.org/en/master/ - - null diff --git a/tests/test_seeding_/test_sphinx_seed_intersphinx_mapping_mocked_example_requirements_a_pyproject_toml_.yml b/tests/test_seeding_/test_sphinx_seed_intersphinx_mapping_mocked_example_requirements_a_pyproject_toml_.yml index 8a05a51..3f71b14 100644 --- a/tests/test_seeding_/test_sphinx_seed_intersphinx_mapping_mocked_example_requirements_a_pyproject_toml_.yml +++ b/tests/test_seeding_/test_sphinx_seed_intersphinx_mapping_mocked_example_requirements_a_pyproject_toml_.yml @@ -16,5 +16,5 @@ slumber: - - null sphinx: - sphinx -- - https://www.sphinx-doc.org/en/3.x/ +- - https://www.sphinx-doc.org/en/master/ - - null diff --git a/tests/test_seeding_/test_sphinx_seed_intersphinx_mapping_mocked_example_requirements_a_requirements_.yml b/tests/test_seeding_/test_sphinx_seed_intersphinx_mapping_mocked_example_requirements_a_requirements_.yml index 8a05a51..3f71b14 100644 --- a/tests/test_seeding_/test_sphinx_seed_intersphinx_mapping_mocked_example_requirements_a_requirements_.yml +++ b/tests/test_seeding_/test_sphinx_seed_intersphinx_mapping_mocked_example_requirements_a_requirements_.yml @@ -16,5 +16,5 @@ slumber: - - null sphinx: - sphinx -- - https://www.sphinx-doc.org/en/3.x/ +- - https://www.sphinx-doc.org/en/master/ - - null