diff --git a/.github/workflows/docs-ci.yml b/.github/workflows/docs-ci.yml index 511b7c28..10ba5faa 100644 --- a/.github/workflows/docs-ci.yml +++ b/.github/workflows/docs-ci.yml @@ -4,34 +4,29 @@ on: [push, pull_request] jobs: build: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 strategy: max-parallel: 4 matrix: - python-version: [3.9] + python-version: [3.12] steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Give permission to run scripts - run: chmod +x ./docs/scripts/doc8_style_check.sh - - name: Install Dependencies - run: pip install -e .[docs] + run: ./configure --dev - - name: Check Sphinx Documentation build minimally - working-directory: ./docs - run: sphinx-build -E -W source build + - name: Check documentation and HTML for errors and dead links + run: make docs-check - - name: Check for documentation style errors - working-directory: ./docs - run: ./scripts/doc8_style_check.sh + - name: Check documentation for style errors + run: make doc8 diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index 0cb67402..cf0579a7 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -21,21 +21,24 @@ on: jobs: build-pypi-distribs: name: Build and publish library to PyPI - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: 3.12 - - name: Install pypa/build - run: python -m pip install build --user + - name: Install pypa/build and twine + run: python -m pip install --user build twine - name: Build a binary wheel and a source tarball run: python -m build --sdist --wheel --outdir dist/ + - name: Validate wheel and sdis for Pypi + run: python -m twine check dist/* + - name: Upload built archives uses: actions/upload-artifact@v4 with: @@ -47,7 +50,7 @@ jobs: name: Create GH release needs: - build-pypi-distribs - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - name: Download built archives @@ -57,7 +60,7 @@ jobs: path: dist - name: Create GH release - uses: softprops/action-gh-release@v1 + uses: softprops/action-gh-release@v2 with: draft: true files: dist/* @@ -67,7 +70,7 @@ jobs: name: Create PyPI release needs: - create-gh-release - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - name: Download built archives diff --git a/.gitignore b/.gitignore index 5288022b..d2eaf8f1 100644 --- a/.gitignore +++ b/.gitignore @@ -72,3 +72,5 @@ tcl # Ignore Jupyter Notebook related temp files .ipynb_checkpoints/ +/.ruff_cache/ +.env \ No newline at end of file diff --git a/.readthedocs.yml b/.readthedocs.yml index 8ab23688..683f3a82 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -26,4 +26,4 @@ python: - method: pip path: . extra_requirements: - - docs + - dev diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 50090cfe..0f4ea140 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,11 +1,14 @@ Changelog ========= -v0.13.2 +v0.14.0 ----------- - Speed up downloads with asyncio +- Introduce file-based locking when reading and writing + python package archives to avoid accessing partial zip and tarballs. + - New settings featuring environment variables and .env file to store settings and defaults. - This also changes the CACHE_THIRDPARTY_DIR environment variable: it used to default first @@ -36,6 +39,8 @@ v0.13.2 - Merge latest skeleton and adopt ruff for code formatting. +- Fix misc bugs. + v0.13.1 ----------- @@ -100,7 +105,8 @@ v0.9.5 v0.9.4 ------ -- Create PyPI cache location in the home directory if a cache directory cannot be made at the project root. +- Create PyPI cache location in the home directory if a cache directory cannot be made at the + project root. - Replace packaging with packvers. - Prevent duplicated package versions. diff --git a/MANIFEST.in b/MANIFEST.in index ef3721e8..0f197075 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,6 @@ graft src +graft docs +graft etc include *.LICENSE include NOTICE @@ -6,10 +8,18 @@ include *.ABOUT include *.toml include *.yml include *.rst +include *.png include setup.* include configure* include requirements* -include .git* +include .dockerignore +include .gitignore +include .readthedocs.yml +include manage.py +include Dockerfile* +include Makefile +include MANIFEST.in -global-exclude *.py[co] __pycache__ *.*~ +include .VERSION +global-exclude *.py[co] __pycache__ *.*~ diff --git a/Makefile b/Makefile index a88f4c68..98f5eddc 100644 --- a/Makefile +++ b/Makefile @@ -13,32 +13,33 @@ PYTHON_EXE?=python3 VENV=venv ACTIVATE?=. ${VENV}/bin/activate; -dev: - @echo "-> Configure the development envt." - ./configure --dev -isort: - @echo "-> Apply isort changes to ensure proper imports ordering" - ${VENV}/bin/isort --sl -l 100 src tests setup.py --skip-glob "*/_packagedcode/*" +conf: + @echo "-> Install dependencies" + ./configure -black: - @echo "-> Apply black code formatter" - ${VENV}/bin/black -l 100 src tests setup.py --exclude "_packagedcode/.*" +dev: + @echo "-> Configure and install development dependencies" + ./configure --dev doc8: @echo "-> Run doc8 validation" - @${ACTIVATE} doc8 --max-line-length 100 --ignore-path docs/_build/ --quiet docs/ + @${ACTIVATE} doc8 --config pyproject.toml --ignore-path docs/_build --max-line-length 100 docs/ *.rst -valid: isort black +valid: + @echo "-> Run Ruff format" + @${ACTIVATE} ruff format + @echo "-> Run Ruff linter" + @${ACTIVATE} ruff check --fix check: - @echo "-> Run pycodestyle (PEP8) validation" - @${ACTIVATE} pycodestyle --max-line-length=110 \ - --exclude=.eggs,etc/scripts,src/_packagedcode,venv/,lib/,thirdparty/,docs/,.cache/ . - @echo "-> Run isort imports ordering validation" - @${ACTIVATE} isort --sl --check-only -l 100 setup.py src tests --skip-glob "*/_packagedcode/*" - @echo "-> Run black validation" - @${ACTIVATE} black --check -l 100 src tests setup.py --exclude "_packagedcode/.*" + @echo "-> Run Ruff linter validation (pycodestyle, bandit, isort, and more)" + @${ACTIVATE} ruff check + @echo "-> Run Ruff format validation" + @${ACTIVATE} ruff format --check + @$(MAKE) doc8 + @echo "-> Run ABOUT files validation" + @${ACTIVATE} about check etc/ clean: @echo "-> Clean the Python env" @@ -50,6 +51,10 @@ test: docs: rm -rf docs/_build/ - @${ACTIVATE} sphinx-build docs/ docs/_build/ + @${ACTIVATE} sphinx-build docs/source docs/_build/ + +docs-check: + @${ACTIVATE} sphinx-build -E -W -b html docs/source docs/_build/ + @${ACTIVATE} sphinx-build -E -W -b linkcheck docs/source docs/_build/ -.PHONY: conf dev check valid black isort clean test docs +.PHONY: conf dev check valid clean test docs docs-check diff --git a/NOTICE b/NOTICE index 41efd68e..cbdaef79 100644 --- a/NOTICE +++ b/NOTICE @@ -1,39 +1,19 @@ # - # Copyright (c) nexB Inc. and others. - # SPDX-License-Identifier: Apache-2.0 - # - -# Visit https://aboutcode.org and https://github.com/aboutcode-org/python-inspector - -# for support and download. - +# Visit https://aboutcode.org and https://github.com/aboutcode-org/ for support and download. # ScanCode is a trademark of nexB Inc. - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - -# http://www.apache.org/licenses/LICENSE-2.0 - +# http://www.apache.org/licenses/LICENSE-2.0 # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - # diff --git a/README.rst b/README.rst index 5f0d6de9..999a7c59 100644 --- a/README.rst +++ b/README.rst @@ -81,19 +81,19 @@ This project is funded, supported and sponsored by: - Generous support and contributions from users like you! - the European Commission NGI programme -- the NLnet Foundation +- the NLnet Foundation - the Swiss State Secretariat for Education, Research and Innovation (SERI) - Google, including the Google Summer of Code and the Google Seasons of Doc programmes - Mercedes-Benz Group - Microsoft and Microsoft Azure - AboutCode ASBL -- nexB Inc. +- nexB Inc. -|europa| |dgconnect| +|europa| |dgconnect| -|ngi| |nlnet| +|ngi| |nlnet| |aboutcode| |nexb| @@ -177,11 +177,3 @@ Communications Networks, Content and Technology under grant agreement No 1010929 :target: https://nlnet.nl/discovery/ :height: 40 :alt: NGI Discovery logo - - - - - - - - diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 03b68641..f57e2dae 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,3 +1,4 @@ + ################################################################################ # We use Azure to run the full tests suites on multiple Python 3.x # on multiple Windows, macOS and Linux versions all on 64 bits @@ -5,50 +6,99 @@ ################################################################################ jobs: - - template: etc/ci/azure-posix.yml - parameters: - job_name: ubuntu20_cpython - image_name: ubuntu-20.04 - python_versions: ["3.9", "3.10", "3.11", "3.12"] - test_suites: - all: venv/bin/pytest -n 2 -vvs --reruns 2 - - - template: etc/ci/azure-posix.yml - parameters: - job_name: ubuntu22_cpython - image_name: ubuntu-22.04 - python_versions: ["3.9", "3.10", "3.11", "3.12"] - test_suites: - all: venv/bin/pytest -n 2 -vvs --reruns 2 - - - template: etc/ci/azure-posix.yml - parameters: - job_name: macos14_cpython - image_name: macOS-14 - python_versions: ["3.9", "3.10", "3.11", "3.12"] - test_suites: - all: venv/bin/pytest -n 2 -vvs --reruns 2 - - - template: etc/ci/azure-posix.yml - parameters: - job_name: macos13_cpython - image_name: macos-13 - python_versions: ["3.9", "3.10", "3.11", "3.12"] - test_suites: - all: venv/bin/pytest -n 2 -vvs --reruns 2 - - - template: etc/ci/azure-win.yml - parameters: - job_name: win2019_cpython - image_name: windows-2019 - python_versions: ["3.9", "3.10", "3.11", "3.12"] - test_suites: - all: venv\Scripts\pytest -n 2 -vvs --reruns 2 - - - template: etc/ci/azure-win.yml - parameters: - job_name: win2022_cpython - image_name: windows-2022 - python_versions: ["3.9", "3.10", "3.11", "3.12"] - test_suites: - all: venv\Scripts\pytest -n 2 -vvs --reruns 2 + + - template: etc/ci/azure-posix.yml + parameters: + job_name: run_code_checks + image_name: ubuntu-24.04 + python_versions: ['3.12'] + test_suites: + all: make check + + - template: etc/ci/azure-posix.yml + parameters: + job_name: online_ubuntu24_cpython + image_name: ubuntu-24.04 + python_versions: ['3.10'] + test_suites: + all: venv/bin/pytest -n 2 -vvs -m "online" + + - template: etc/ci/azure-win.yml + parameters: + job_name: online_win2022_cpython + image_name: windows-2022 + python_versions: ['3.10'] + test_suites: + all: venv\Scripts\pytest -n 2 -vvs -m "online" + + - template: etc/ci/azure-posix.yml + parameters: + job_name: online_macos14_cpython + image_name: macOS-14 + python_versions: ['3.10'] + test_suites: + all: venv/bin/pytest -n 2 -vvs -m "online" + + - template: etc/ci/azure-posix.yml + parameters: + job_name: ubuntu22_cpython + image_name: ubuntu-22.04 + python_versions: ['3.9', '3.10', '3.11', '3.12'] + test_suites: + all: venv/bin/pytest -n 2 -vvs -m "not online" + + - template: etc/ci/azure-posix.yml + parameters: + job_name: ubuntu24_cpython + image_name: ubuntu-24.04 + python_versions: ['3.9', '3.10', '3.11', '3.12'] + test_suites: + all: venv/bin/pytest -n 2 -vvs -m "not online" + + - template: etc/ci/azure-posix.yml + parameters: + job_name: macos13_cpython + image_name: macOS-13 + python_versions: ['3.9', '3.10', '3.11', '3.12'] + test_suites: + all: venv/bin/pytest -n 2 -vvs -m "not online" + + - template: etc/ci/azure-posix.yml + parameters: + job_name: macos14_cpython + image_name: macOS-14 + python_versions: ['3.9', '3.10', '3.11', '3.12'] + test_suites: + all: venv/bin/pytest -n 2 -vvs -m "not online" + + - template: etc/ci/azure-posix.yml + parameters: + job_name: macos15_cpython + image_name: macOS-15 + python_versions: ['3.9', '3.10', '3.11', '3.12'] + test_suites: + all: venv/bin/pytest -n 2 -vvs -m "not online" + + - template: etc/ci/azure-win.yml + parameters: + job_name: win2019_cpython + image_name: windows-2019 + python_versions: ['3.9', '3.10', '3.11', '3.12'] + test_suites: + all: venv\Scripts\pytest -n 2 -vvs -m "not online" + + - template: etc/ci/azure-win.yml + parameters: + job_name: win2022_cpython + image_name: windows-2022 + python_versions: ['3.9', '3.10', '3.11', '3.12'] + test_suites: + all: venv\Scripts\pytest -n 2 -vvs -m "not online" + + - template: etc/ci/azure-win.yml + parameters: + job_name: win2025_cpython + image_name: windows-2025 + python_versions: ['3.9', '3.10', '3.11', '3.12'] + test_suites: + all: venv\Scripts\pytest -n 2 -vvs -m "not online" diff --git a/configure b/configure index 22d92885..5ef0e063 100755 --- a/configure +++ b/configure @@ -29,14 +29,13 @@ CLI_ARGS=$1 # Requirement arguments passed to pip and used by default or with --dev. REQUIREMENTS="--editable . --constraint requirements.txt" -DEV_REQUIREMENTS="--editable .[testing] --constraint requirements.txt --constraint requirements-dev.txt" -DOCS_REQUIREMENTS="--editable .[docs] --constraint requirements.txt" +DEV_REQUIREMENTS="--editable .[dev] --constraint requirements.txt --constraint requirements-dev.txt" # where we create a virtualenv VIRTUALENV_DIR=venv # Cleanable files and directories to delete with the --clean option -CLEANABLE="build dist venv .cache .eggs" +CLEANABLE="build dist venv .cache .eggs *.egg-info docs/_build/ pip-selfcheck.json" # extra arguments passed to pip PIP_EXTRA_ARGS=" " @@ -168,6 +167,7 @@ clean() { for cln in $CLEANABLE; do rm -rf "${CFG_ROOT_DIR:?}/${cln:?}"; done + find . -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete set +e exit } @@ -185,7 +185,6 @@ while getopts :-: optchar; do help ) cli_help;; clean ) find_python && clean;; dev ) CFG_REQUIREMENTS="$DEV_REQUIREMENTS";; - docs ) CFG_REQUIREMENTS="$DOCS_REQUIREMENTS";; esac;; esac done diff --git a/configure.bat b/configure.bat index 5b9a9d68..3e9881fb 100644 --- a/configure.bat +++ b/configure.bat @@ -27,8 +27,7 @@ @rem # Requirement arguments passed to pip and used by default or with --dev. set "REQUIREMENTS=--editable . --constraint requirements.txt" -set "DEV_REQUIREMENTS=--editable .[testing] --constraint requirements.txt --constraint requirements-dev.txt" -set "DOCS_REQUIREMENTS=--editable .[docs] --constraint requirements.txt" +set "DEV_REQUIREMENTS=--editable .[dev] --constraint requirements.txt --constraint requirements-dev.txt" @rem # where we create a virtualenv set "VIRTUALENV_DIR=venv" @@ -76,9 +75,6 @@ if not "%1" == "" ( if "%1" EQU "--dev" ( set "CFG_REQUIREMENTS=%DEV_REQUIREMENTS%" ) - if "%1" EQU "--docs" ( - set "CFG_REQUIREMENTS=%DOCS_REQUIREMENTS%" - ) shift goto again ) diff --git a/docs/Makefile b/docs/Makefile index d0c3cbf1..94f686b2 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -5,8 +5,9 @@ # from the environment for the first two. SPHINXOPTS ?= SPHINXBUILD ?= sphinx-build +SPHINXAUTOBUILD = sphinx-autobuild SOURCEDIR = source -BUILDDIR = build +BUILDDIR = _build # Put it first so that "make" without argument is like "make help". help: @@ -14,6 +15,13 @@ help: .PHONY: help Makefile +# Run the development server using sphinx-autobuild +docs: + @echo + @echo "Starting up the docs server..." + @echo + $(SPHINXAUTOBUILD) --port 8000 --watch ${SOURCEDIR} $(SOURCEDIR) "$(BUILDDIR)/html" $(SPHINXOPTS) $(O) + # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile diff --git a/docs/make.bat b/docs/make.bat index 6247f7e2..4a3c1a48 100644 --- a/docs/make.bat +++ b/docs/make.bat @@ -7,11 +7,16 @@ REM Command file for Sphinx documentation if "%SPHINXBUILD%" == "" ( set SPHINXBUILD=sphinx-build ) +if "%SPHINXAUTOBUILD%" == "" ( + set SPHINXAUTOBUILD=sphinx-autobuild +) set SOURCEDIR=source set BUILDDIR=build if "%1" == "" goto help +if "%1" == "docs" goto docs + %SPHINXBUILD% >NUL 2>NUL if errorlevel 9009 ( echo. @@ -28,6 +33,13 @@ if errorlevel 9009 ( %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% goto end +:docs +@echo +@echo Starting up the docs server... +@echo +%SPHINXAUTOBUILD% --port 8000 --watch %SOURCEDIR% %SOURCEDIR% %BUILDDIR%\html %SPHINXOPTS% %O% +goto end + :help %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% diff --git a/docs/scripts/doc8_style_check.sh b/docs/scripts/doc8_style_check.sh deleted file mode 100755 index 94163239..00000000 --- a/docs/scripts/doc8_style_check.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -# halt script on error -set -e -# Check for Style Code Violations -doc8 --max-line-length 100 source --ignore D000 --quiet \ No newline at end of file diff --git a/docs/scripts/sphinx_build_link_check.sh b/docs/scripts/sphinx_build_link_check.sh deleted file mode 100644 index c5426863..00000000 --- a/docs/scripts/sphinx_build_link_check.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -# halt script on error -set -e -# Build locally, and then check links -sphinx-build -E -W -b linkcheck source build \ No newline at end of file diff --git a/docs/source/_static/theme_overrides.css b/docs/source/_static/theme_overrides.css index 9662d63a..5863ccf5 100644 --- a/docs/source/_static/theme_overrides.css +++ b/docs/source/_static/theme_overrides.css @@ -1,353 +1,26 @@ -body { - color: #000000; -} - -p { - margin-bottom: 10px; -} - -.wy-plain-list-disc, .rst-content .section ul, .rst-content .toctree-wrapper ul, article ul { - margin-bottom: 10px; -} - -.custom_header_01 { - color: #cc0000; - font-size: 22px; - font-weight: bold; - line-height: 50px; -} - -h1, h2, h3, h4, h5, h6 { - margin-bottom: 20px; - margin-top: 20px; -} - -h5 { - font-size: 18px; - color: #000000; - font-style: italic; - margin-bottom: 10px; -} - -h6 { - font-size: 15px; - color: #000000; - font-style: italic; - margin-bottom: 10px; -} - -/* custom admonitions */ -/* success */ -.custom-admonition-success .admonition-title { - color: #000000; - background: #ccffcc; - border-radius: 5px 5px 0px 0px; -} -div.custom-admonition-success.admonition { - color: #000000; - background: #ffffff; - border: solid 1px #cccccc; - border-radius: 5px; - box-shadow: 1px 1px 5px 3px #d8d8d8; - margin: 20px 0px 30px 0px; -} - -/* important */ -.custom-admonition-important .admonition-title { - color: #000000; - background: #ccffcc; - border-radius: 5px 5px 0px 0px; - border-bottom: solid 1px #000000; -} -div.custom-admonition-important.admonition { - color: #000000; - background: #ffffff; - border: solid 1px #cccccc; - border-radius: 5px; - box-shadow: 1px 1px 5px 3px #d8d8d8; - margin: 20px 0px 30px 0px; -} - -/* caution */ -.custom-admonition-caution .admonition-title { - color: #000000; - background: #ffff99; - border-radius: 5px 5px 0px 0px; - border-bottom: solid 1px #e8e8e8; -} -div.custom-admonition-caution.admonition { - color: #000000; - background: #ffffff; - border: solid 1px #cccccc; - border-radius: 5px; - box-shadow: 1px 1px 5px 3px #d8d8d8; - margin: 20px 0px 30px 0px; -} - -/* note */ -.custom-admonition-note .admonition-title { - color: #ffffff; - background: #006bb3; - border-radius: 5px 5px 0px 0px; -} -div.custom-admonition-note.admonition { - color: #000000; - background: #ffffff; - border: solid 1px #cccccc; - border-radius: 5px; - box-shadow: 1px 1px 5px 3px #d8d8d8; - margin: 20px 0px 30px 0px; -} - -/* todo */ -.custom-admonition-todo .admonition-title { - color: #000000; - background: #cce6ff; - border-radius: 5px 5px 0px 0px; - border-bottom: solid 1px #99ccff; -} -div.custom-admonition-todo.admonition { - color: #000000; - background: #ffffff; - border: solid 1px #99ccff; - border-radius: 5px; - box-shadow: 1px 1px 5px 3px #d8d8d8; - margin: 20px 0px 30px 0px; -} - -/* examples */ -.custom-admonition-examples .admonition-title { - color: #000000; - background: #ffe6cc; - border-radius: 5px 5px 0px 0px; - border-bottom: solid 1px #d8d8d8; -} -div.custom-admonition-examples.admonition { - color: #000000; - background: #ffffff; - border: solid 1px #cccccc; - border-radius: 5px; - box-shadow: 1px 1px 5px 3px #d8d8d8; - margin: 20px 0px 30px 0px; -} - +/* this is the container for the pages */ .wy-nav-content { max-width: 100%; - padding-right: 100px; - padding-left: 100px; - background-color: #f2f2f2; -} - -div.rst-content { - background-color: #ffffff; - border: solid 1px #e5e5e5; - padding: 20px 40px 20px 40px; -} - -.rst-content .guilabel { - border: 1px solid #ffff99; - background: #ffff99; - font-size: 100%; - font-weight: normal; - border-radius: 4px; - padding: 2px 0px; - margin: auto 2px; - vertical-align: middle; -} - -.rst-content kbd { - font-family: SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",Courier,monospace; - border: solid 1px #d8d8d8; - background-color: #f5f5f5; - padding: 0px 3px; - border-radius: 3px; -} - -.wy-nav-content-wrap a { - color: #0066cc; - text-decoration: none; -} -.wy-nav-content-wrap a:hover { - color: #0099cc; - text-decoration: underline; -} - -.wy-nav-top a { - color: #ffffff; -} - -/* Based on numerous similar approaches e.g., https://github.com/readthedocs/sphinx_rtd_theme/issues/117 and https://rackerlabs.github.io/docs-rackspace/tools/rtd-tables.html -- but remove form-factor limits to enable table wrap on full-size and smallest-size form factors */ -.wy-table-responsive table td { - white-space: normal !important; -} - -.rst-content table.docutils td, -.rst-content table.docutils th { - padding: 5px 10px 5px 10px; -} -.rst-content table.docutils td p, -.rst-content table.docutils th p { - font-size: 14px; - margin-bottom: 0px; -} -.rst-content table.docutils td p cite, -.rst-content table.docutils th p cite { - font-size: 14px; - background-color: transparent; -} - -.colwidths-given th { - border: solid 1px #d8d8d8 !important; -} -.colwidths-given td { - border: solid 1px #d8d8d8 !important; -} - -/*handles single-tick inline code*/ -.wy-body-for-nav cite { - color: #000000; - background-color: transparent; - font-style: normal; - font-family: "Courier New"; - font-size: 13px; - padding: 3px 3px 3px 3px; -} - -.rst-content pre.literal-block, .rst-content div[class^="highlight"] pre, .rst-content .linenodiv pre { - font-family: SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",Courier,monospace; - font-size: 13px; - overflow: visible; - white-space: pre-wrap; - color: #000000; -} - -.rst-content pre.literal-block, .rst-content div[class^='highlight'] { - background-color: #f8f8f8; - border: solid 1px #e8e8e8; -} - -/* This enables inline code to wrap. */ -code, .rst-content tt, .rst-content code { - white-space: pre-wrap; - padding: 2px 3px 1px; - border-radius: 3px; - font-size: 13px; - background-color: #ffffff; -} - -/* use this added class for code blocks attached to bulleted list items */ -.highlight-top-margin { - margin-top: 20px !important; -} - -/* change color of inline code block */ -span.pre { - color: #e01e5a; -} - -.wy-body-for-nav blockquote { - margin: 1em 0; - padding-left: 1em; - border-left: 4px solid #ddd; - color: #000000; -} - -/* Fix the unwanted top and bottom padding inside a nested bulleted/numbered list */ -.rst-content .section ol p, .rst-content .section ul p { - margin-bottom: 0px; -} - -/* add spacing between bullets for legibility */ -.rst-content .section ol li, .rst-content .section ul li { - margin-bottom: 5px; -} - -.rst-content .section ol li:first-child, .rst-content .section ul li:first-child { - margin-top: 5px; -} - -/* but exclude the toctree bullets */ -.rst-content .toctree-wrapper ul li, .rst-content .toctree-wrapper ul li:first-child { + padding: 0px 40px 0px 0px; margin-top: 0px; - margin-bottom: 0px; } -/* remove extra space at bottom of multine list-table cell */ -.rst-content .line-block { - margin-left: 0px; - margin-bottom: 0px; - line-height: 24px; +.wy-nav-content-wrap { + border-right: solid 1px; } -/* fix extra vertical spacing in page toctree */ -.rst-content .toctree-wrapper ul li ul, article ul li ul { - margin-top: 0; - margin-bottom: 0; -} - -/* this is used by the genindex added via layout.html (see source/_templates/) to sidebar toc */ -.reference.internal.toc-index { - color: #d9d9d9; -} - -.reference.internal.toc-index.current { - background-color: #ffffff; - color: #000000; - font-weight: bold; -} - -.toc-index-div { - border-top: solid 1px #000000; - margin-top: 10px; - padding-top: 5px; -} - -.indextable ul li { - font-size: 14px; - margin-bottom: 5px; -} - -/* The next 2 fix the poor vertical spacing in genindex.html (the alphabetized index) */ -.indextable.genindextable { - margin-bottom: 20px; -} - -div.genindex-jumpbox { - margin-bottom: 10px; -} - -/* rst image classes */ - -.clear-both { - clear: both; - } - -.float-left { - float: left; - margin-right: 20px; -} - -img { - border: solid 1px #e8e8e8; -} - -/* These are custom and need to be defined in conf.py to access in all pages, e.g., '.. role:: red' */ -.img-title { - color: #000000; - /* neither padding nor margin works for vertical spacing bc it's a span -- line-height does, sort of */ - line-height: 3.0; - font-style: italic; - font-weight: 600; -} - -.img-title-para { - color: #000000; - margin-top: 20px; - margin-bottom: 0px; - font-style: italic; - font-weight: 500; -} - -.red { - color: red; +div.rst-content { + max-width: 1300px; + border: 0; + padding: 10px 80px 10px 80px; + margin-left: 50px; +} + +@media (max-width: 768px) { + div.rst-content { + max-width: 1300px; + border: 0; + padding: 0px 10px 10px 10px; + margin-left: 0px; + } } diff --git a/docs/source/conf.py b/docs/source/conf.py index 184453bd..8d1a2886 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -18,7 +18,7 @@ # -- Project information ----------------------------------------------------- project = "python-inspector" -copyright = "nexB Inc. and others." +copyright = "nexB Inc., AboutCode and others." author = "AboutCode.org authors and contributors" @@ -30,6 +30,10 @@ extensions = [ "sphinx.ext.intersphinx", "sphinx_reredirects", + "sphinx_rtd_theme", + "sphinx_rtd_dark_mode", + "sphinx.ext.extlinks", + "sphinx_copybutton", ] @@ -39,11 +43,14 @@ # This points to aboutcode.readthedocs.io # In case of "undefined label" ERRORS check docs on intersphinx to troubleshoot -# Link was created at commit - https://github.com/nexB/aboutcode/commit/faea9fcf3248f8f198844fe34d43833224ac4a83 +# Link was created at commit - https://github.com/aboutcode-org/aboutcode/commit/faea9fcf3248f8f198844fe34d43833224ac4a83 intersphinx_mapping = { "aboutcode": ("https://aboutcode.readthedocs.io/en/latest/", None), - "scancode-workbench": ("https://scancode-workbench.readthedocs.io/en/develop/", None), + "scancode-workbench": ( + "https://scancode-workbench.readthedocs.io/en/develop/", + None, + ), } @@ -78,14 +85,17 @@ "conf_py_path": "/docs/source/", # path in the checkout to the docs root } -html_css_files = ["_static/theme_overrides.css"] +html_css_files = [ + "theme_overrides.css", +] # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. html_show_sphinx = True # Define CSS and HTML abbreviations used in .rst files. These are examples. -# .. role:: is used to refer to styles defined in _static/theme_overrides.css and is used like this: :red:`text` +# .. role:: is used to refer to styles defined in _static/theme_overrides.css +# and is used like this: :red:`text` rst_prolog = """ .. |psf| replace:: Python Software Foundation @@ -104,6 +114,4 @@ # -- Options for LaTeX output ------------------------------------------------- -latex_elements = { - 'classoptions': ',openany,oneside' -} \ No newline at end of file +latex_elements = {"classoptions": ",openany,oneside"} diff --git a/docs/source/contribute/contrib_doc.rst b/docs/source/contribute/contrib_doc.rst new file mode 100644 index 00000000..2a719a52 --- /dev/null +++ b/docs/source/contribute/contrib_doc.rst @@ -0,0 +1,271 @@ +.. _contrib_doc_dev: + +Contributing to the Documentation +================================= + +.. _contrib_doc_setup_local: + +Setup Local Build +----------------- + +To get started, check out and configure the repository for development:: + + git clone https://github.com/aboutcode-org/.git + + cd your-repo + ./configure --dev + +(Or use "make dev") + +.. note:: + + In case of windows, run ``configure --dev``. + +This will install and configure all requirements foer development including for docs development. + +Now you can build the HTML documentation locally:: + + source venv/bin/activate + make docs + +This will build a local instance of the ``docs/_build`` directory:: + + open docs/_build/index.html + + +To validate the documentation style and content, use:: + + source venv/bin/activate + make doc8 + make docs-check + + +.. _doc_ci: + +Continuous Integration +---------------------- + +The documentations are checked on every new commit, so that common errors are avoided and +documentation standards are enforced. We checks for these aspects of the documentation: + +1. Successful Builds (By using ``sphinx-build``) +2. No Broken Links (By Using ``linkcheck``) +3. Linting Errors (By Using ``doc8``) + +You myst run these scripts locally before creating a pull request:: + + make doc8 + make check-docs + + +.. _doc_style_docs8: + +Style Checks Using ``doc8`` +--------------------------- + +How To Run Style Tests +^^^^^^^^^^^^^^^^^^^^^^ + +In the project root, run the following commands:: + + make doc8 + +A sample output is:: + + Scanning... + Validating... + docs/source/misc/licence_policy_plugin.rst:37: D002 Trailing whitespace + docs/source/misc/faq.rst:45: D003 Tabulation used for indentation + docs/source/misc/faq.rst:9: D001 Line too long + docs/source/misc/support.rst:6: D005 No newline at end of file + ======== + Total files scanned = 34 + Total files ignored = 0 + Total accumulated errors = 326 + Detailed error counts: + - CheckCarriageReturn = 0 + - CheckIndentationNoTab = 75 + - CheckMaxLineLength = 190 + - CheckNewlineEndOfFile = 13 + - CheckTrailingWhitespace = 47 + - CheckValidity = 1 + +Now fix the errors and run again till there isn't any style error in the documentation. + + +What is Checked? +^^^^^^^^^^^^^^^^ + +PyCQA is an Organization for code quality tools (and plugins) for the Python programming language. +Doc8 is a sub-project of the same Organization. Refer this +`README `_ for more details. + +What is checked: + + - invalid rst format - D000 + - lines should not be longer than 100 characters - D001 + + - RST exception: line with no whitespace except in the beginning + - RST exception: lines with http or https URLs + - RST exception: literal blocks + - RST exception: rst target directives + + - no trailing whitespace - D002 + - no tabulation for indentation - D003 + - no carriage returns (use UNIX newlines) - D004 + - no newline at end of file - D005 + + +.. _doc_interspinx: + +Interspinx +---------- + +AboutCode documentation uses +`Intersphinx `_ +to link to other Sphinx Documentations, to maintain links to other Aboutcode Projects. + +To link sections in the same documentation, standart reST labels are used. Refer +`Cross-Referencing `_ +for more information. + +For example:: + + .. _my-reference-label: + + Section to cross-reference + -------------------------- + + This is the text of the section. + + It refers to the section itself, see :ref:`my-reference-label`. + +Now, using Intersphinx, you can create these labels in one Sphinx Documentation and then referance +these labels from another Sphinx Documentation, hosted in different locations. + +You just have to add the following in the ``conf.py`` file for your Sphinx Documentation, where you +want to add the links:: + + extensions = [ + 'sphinx.ext.intersphinx' + ] + + intersphinx_mapping = {'aboutcode': ('https://aboutcode.readthedocs.io/en/latest/', None)} + +To show all Intersphinx links and their targets of an Intersphinx mapping file, run:: + + python -msphinx.ext.intersphinx https://aboutcode.readthedocs.io/en/latest/objects.inv + +.. WARNING:: + + ``python -msphinx.ext.intersphinx https://aboutcode.readthedocs.io/objects.inv`` will give + error. + +This enables you to create links to the ``aboutcode`` Documentation in your own Documentation, +where you modified the configuration file. Links can be added like this:: + + For more details refer :ref:`aboutcode:doc_style_guide`. + +You can also not use the ``aboutcode`` label assigned to all links from aboutcode.readthedocs.io, +if you don't have a label having the same name in your Sphinx Documentation. Example:: + + For more details refer :ref:`doc_style_guide`. + +If you have a label in your documentation which is also present in the documentation linked by +Intersphinx, and you link to that label, it will create a link to the local label. + +For more information, refer this tutorial named +`Using Intersphinx `_. + + +.. _doc_style_conv: + +Style Conventions for the Documentaion +-------------------------------------- + +1. Headings + + (`Refer `_) + Normally, there are no heading levels assigned to certain characters as the structure is + determined from the succession of headings. However, this convention is used in Python’s Style + Guide for documenting which you may follow: + + # with overline, for parts + + * with overline, for chapters + + =, for sections + + -, for subsections + + ^, for sub-subsections + + ", for paragraphs + +2. Heading Underlines + + Do not use underlines that are longer/shorter than the title headline itself. As in: + + :: + + Correct : + + Extra Style Checks + ------------------ + + Incorrect : + + Extra Style Checks + ------------------------ + +.. note:: + + Underlines shorter than the Title text generates Errors on sphinx-build. + + +3. Internal Links + + Using ``:ref:`` is advised over standard reStructuredText links to sections (like + ```Section title`_``) because it works across files, when section headings are changed, will + raise warnings if incorrect, and works for all builders that support cross-references. + However, external links are created by using the standard ```Section title`_`` method. + +4. Eliminate Redundancy + + If a section/file has to be repeated somewhere else, do not write the exact same section/file + twice. Use ``.. include: ../README.rst`` instead. Here, ``../`` refers to the documentation + root, so file location can be used accordingly. This enables us to link documents from other + upstream folders. + +5. Using ``:ref:`` only when necessary + + Use ``:ref:`` to create internal links only when needed, i.e. it is referenced somewhere. + Do not create references for all the sections and then only reference some of them, because + this created unnecessary references. This also generates ERROR in ``restructuredtext-lint``. + +6. Spelling + + You should check for spelling errors before you push changes. `Aspell `_ + is a GNU project Command Line tool you can use for this purpose. Download and install Aspell, + then execute ``aspell check `` for all the files changed. Be careful about not + changing commands or other stuff as Aspell gives prompts for a lot of them. Also delete the + temporary ``.bak`` files generated. Refer the `manual `_ for more + information on how to use. + +7. Notes and Warning Snippets + + Every ``Note`` and ``Warning`` sections are to be kept in ``rst_snippets/note_snippets/`` and + ``rst_snippets/warning_snippets/`` and then included to eliminate redundancy, as these are + frequently used in multiple files. + + +Converting from Markdown +------------------------ + +If you want to convert a ``.md`` file to a ``.rst`` file, this +`tool `_ does it pretty well. +You will still have to clean up and check for errors as this contains a lot of bugs. But this is +definitely better than converting everything by yourself. + +This will be helpful in converting GitHub wiki's (Markdown Files) to reStructuredtext files for +Sphinx/ReadTheDocs hosting. diff --git a/docs/source/dependencies-design.rst b/docs/source/dependencies-design.rst index 41ccd9c3..b16a3070 100644 --- a/docs/source/dependencies-design.rst +++ b/docs/source/dependencies-design.rst @@ -19,16 +19,16 @@ the dependencies of each dependency. For each dependency, there can be a version "requirement" that can be an exact version, a version expression (aka. version specifier) as defined -in `pep-0440 `__ +in `pep-0440 `__ with additional OS and environment tags and constraints as specified in -`pep-0508 `__ . +`pep-0508 `__ . In particular the required Python version of a package (or version specifier) can be set for the whole package (with the``python_requires`` attribute) or as a marker for a given direct or indirect dependency. pip processes requirement specifiers and constraints from a "requirements" file and internally resolves dependency versions recursively by querying the PyPI Python package -index repository at https://PyPI.org +index repository at https://pypi.org/ *************** Problem diff --git a/docs/source/index.rst b/docs/source/index.rst index c5ac78f0..f7b76352 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -7,6 +7,7 @@ Welcome to python-inspector's documentation! dependencies-design test-protocol + contribute/contrib_doc Indices and tables ================== diff --git a/etc/scripts/check_thirdparty.py b/etc/scripts/check_thirdparty.py index b052f25b..65ae595e 100644 --- a/etc/scripts/check_thirdparty.py +++ b/etc/scripts/check_thirdparty.py @@ -1,11 +1,10 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # # Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/skeleton for support or download. +# See https://github.com/aboutcode-org/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import click @@ -42,8 +41,7 @@ def check_thirdparty_dir( """ Check a thirdparty directory for problems and print these on screen. """ - # check for problems - print(f"==> CHECK FOR PROBLEMS") + print("==> CHECK FOR PROBLEMS") utils_thirdparty.find_problems( dest_dir=dest, report_missing_sources=sdists, diff --git a/etc/scripts/fetch_thirdparty.py b/etc/scripts/fetch_thirdparty.py index eedf05c6..76a19a60 100644 --- a/etc/scripts/fetch_thirdparty.py +++ b/etc/scripts/fetch_thirdparty.py @@ -1,23 +1,21 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # # Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/skeleton for support or download. +# See https://github.com/aboutcode-org/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import itertools -import os import sys from collections import defaultdict import click -import utils_thirdparty import utils_requirements +import utils_thirdparty TRACE = False TRACE_DEEP = False @@ -109,7 +107,8 @@ @click.option( "--use-cached-index", is_flag=True, - help="Use on disk cached PyPI indexes list of packages and versions and do not refetch if present.", + help="Use on disk cached PyPI indexes list of packages and versions and " + "do not refetch if present.", ) @click.option( "--sdist-only", @@ -120,7 +119,7 @@ show_default=False, multiple=True, help="Package name(s) that come only in sdist format (no wheels). " - "The command will not fail and exit if no wheel exists for these names", + "The command will not fail and exit if no wheel exists for these names", ) @click.option( "--wheel-only", @@ -131,7 +130,7 @@ show_default=False, multiple=True, help="Package name(s) that come only in wheel format (no sdist). " - "The command will not fail and exit if no sdist exists for these names", + "The command will not fail and exit if no sdist exists for these names", ) @click.option( "--no-dist", @@ -142,7 +141,7 @@ show_default=False, multiple=True, help="Package name(s) that do not come either in wheel or sdist format. " - "The command will not fail and exit if no distribution exists for these names", + "The command will not fail and exit if no distribution exists for these names", ) @click.help_option("-h", "--help") def fetch_thirdparty( @@ -248,7 +247,6 @@ def fetch_thirdparty( print(f"Processing: {name} @ {version}") if wheels: for environment in environments: - if TRACE: print(f" ==> Fetching wheel for envt: {environment}") @@ -262,11 +260,9 @@ def fetch_thirdparty( if not fetched: wheels_or_sdist_not_found[f"{name}=={version}"].append(environment) if TRACE: - print(f" NOT FOUND") + print(" NOT FOUND") - if (sdists or - (f"{name}=={version}" in wheels_or_sdist_not_found and name in sdist_only) - ): + if sdists or (f"{name}=={version}" in wheels_or_sdist_not_found and name in sdist_only): if TRACE: print(f" ==> Fetching sdist: {name}=={version}") @@ -279,17 +275,17 @@ def fetch_thirdparty( if not fetched: wheels_or_sdist_not_found[f"{name}=={version}"].append("sdist") if TRACE: - print(f" NOT FOUND") + print(" NOT FOUND") mia = [] for nv, dists in wheels_or_sdist_not_found.items(): name, _, version = nv.partition("==") if name in no_dist: continue - sdist_missing = sdists and "sdist" in dists and not name in wheel_only + sdist_missing = sdists and "sdist" in dists and name not in wheel_only if sdist_missing: mia.append(f"SDist missing: {nv} {dists}") - wheels_missing = wheels and any(d for d in dists if d != "sdist") and not name in sdist_only + wheels_missing = wheels and any(d for d in dists if d != "sdist") and name not in sdist_only if wheels_missing: mia.append(f"Wheels missing: {nv} {dists}") @@ -298,12 +294,12 @@ def fetch_thirdparty( print(m) raise Exception(mia) - print(f"==> FETCHING OR CREATING ABOUT AND LICENSE FILES") + print("==> FETCHING OR CREATING ABOUT AND LICENSE FILES") utils_thirdparty.fetch_abouts_and_licenses(dest_dir=dest_dir, use_cached_index=use_cached_index) utils_thirdparty.clean_about_files(dest_dir=dest_dir) # check for problems - print(f"==> CHECK FOR PROBLEMS") + print("==> CHECK FOR PROBLEMS") utils_thirdparty.find_problems( dest_dir=dest_dir, report_missing_sources=sdists, diff --git a/etc/scripts/gen_pypi_simple.py b/etc/scripts/gen_pypi_simple.py index 214d90dc..89d06265 100644 --- a/etc/scripts/gen_pypi_simple.py +++ b/etc/scripts/gen_pypi_simple.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # SPDX-License-Identifier: BSD-2-Clause-Views AND MIT # Copyright (c) 2010 David Wolever . All rights reserved. @@ -69,7 +68,6 @@ def get_package_name_from_filename(filename): raise InvalidDistributionFilename(filename) elif filename.endswith(wheel_ext): - wheel_info = get_wheel_from_filename(filename) if not wheel_info: @@ -133,7 +131,7 @@ def build_links_package_index(packages_by_package_name, base_url): Return an HTML document as string which is a links index of all packages """ document = [] - header = f""" + header = """ Links for all packages @@ -178,13 +176,13 @@ def simple_index_entry(self, base_url): def build_pypi_index(directory, base_url="https://thirdparty.aboutcode.org/pypi"): """ - Using a ``directory`` directory of wheels and sdists, create the a PyPI - simple directory index at ``directory``/simple/ populated with the proper - PyPI simple index directory structure crafted using symlinks. + Create the a PyPI simple directory index using a ``directory`` directory of wheels and sdists in + the direvctory at ``directory``/simple/ populated with the proper PyPI simple index directory + structure crafted using symlinks. - WARNING: The ``directory``/simple/ directory is removed if it exists. - NOTE: in addition to the a PyPI simple index.html there is also a links.html - index file generated which is suitable to use with pip's --find-links + WARNING: The ``directory``/simple/ directory is removed if it exists. NOTE: in addition to the a + PyPI simple index.html there is also a links.html index file generated which is suitable to use + with pip's --find-links """ directory = Path(directory) @@ -200,11 +198,10 @@ def build_pypi_index(directory, base_url="https://thirdparty.aboutcode.org/pypi" simple_html_index = [ "", "PyPI Simple Index", - '' '', + '', ] for pkg_file in directory.iterdir(): - pkg_filename = pkg_file.name if ( diff --git a/etc/scripts/gen_requirements.py b/etc/scripts/gen_requirements.py index 07e26f77..1b879442 100644 --- a/etc/scripts/gen_requirements.py +++ b/etc/scripts/gen_requirements.py @@ -1,11 +1,10 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # # Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/skeleton for support or download. +# See https://github.com/aboutcode-org/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import argparse @@ -34,7 +33,8 @@ def gen_requirements(): type=pathlib.Path, required=True, metavar="DIR", - help="Path to the 'site-packages' directory where wheels are installed such as lib/python3.6/site-packages", + help="Path to the 'site-packages' directory where wheels are installed " + "such as lib/python3.12/site-packages", ) parser.add_argument( "-r", diff --git a/etc/scripts/gen_requirements_dev.py b/etc/scripts/gen_requirements_dev.py index 12cc06d3..85482056 100644 --- a/etc/scripts/gen_requirements_dev.py +++ b/etc/scripts/gen_requirements_dev.py @@ -1,11 +1,10 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # # Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/skeleton for support or download. +# See https://github.com/aboutcode-org/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import argparse @@ -36,7 +35,8 @@ def gen_dev_requirements(): type=pathlib.Path, required=True, metavar="DIR", - help='Path to the "site-packages" directory where wheels are installed such as lib/python3.6/site-packages', + help="Path to the 'site-packages' directory where wheels are installed " + "such as lib/python3.12/site-packages", ) parser.add_argument( "-d", diff --git a/etc/scripts/test_utils_pip_compatibility_tags.py b/etc/scripts/test_utils_pip_compatibility_tags.py index 98187c56..0e9c360a 100644 --- a/etc/scripts/test_utils_pip_compatibility_tags.py +++ b/etc/scripts/test_utils_pip_compatibility_tags.py @@ -1,4 +1,5 @@ -"""Generate and work with PEP 425 Compatibility Tags. +""" +Generate and work with PEP 425 Compatibility Tags. copied from pip-20.3.1 pip/tests/unit/test_utils_compatibility_tags.py download_url: https://raw.githubusercontent.com/pypa/pip/20.3.1/tests/unit/test_utils_compatibility_tags.py @@ -25,8 +26,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ -from unittest.mock import patch import sysconfig +from unittest.mock import patch import pytest @@ -51,7 +52,7 @@ def test_version_info_to_nodot(version_info, expected): assert actual == expected -class Testcompatibility_tags(object): +class Testcompatibility_tags: def mock_get_config_var(self, **kwd): """ Patch sysconfig.get_config_var for arbitrary keys. @@ -82,7 +83,7 @@ def test_no_hyphen_tag(self): assert "-" not in tag.platform -class TestManylinux2010Tags(object): +class TestManylinux2010Tags: @pytest.mark.parametrize( "manylinux2010,manylinux1", [ @@ -105,7 +106,7 @@ def test_manylinux2010_implies_manylinux1(self, manylinux2010, manylinux1): assert arches[:2] == [manylinux2010, manylinux1] -class TestManylinux2014Tags(object): +class TestManylinux2014Tags: @pytest.mark.parametrize( "manylinuxA,manylinuxB", [ diff --git a/etc/scripts/update_skeleton.py b/etc/scripts/update_skeleton.py new file mode 100644 index 00000000..374c06f2 --- /dev/null +++ b/etc/scripts/update_skeleton.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python +# +# Copyright (c) nexB Inc. AboutCode, and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/skeleton for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from pathlib import Path +import os +import subprocess + +import click + + +ABOUTCODE_PUBLIC_REPO_NAMES = [ + "aboutcode-toolkit", + "ahocode", + "bitcode", + "clearcode-toolkit", + "commoncode", + "container-inspector", + "debian-inspector", + "deltacode", + "elf-inspector", + "extractcode", + "fetchcode", + "gemfileparser2", + "gh-issue-sandbox", + "go-inspector", + "heritedcode", + "license-expression", + "license_copyright_pipeline", + "nuget-inspector", + "pip-requirements-parser", + "plugincode", + "purldb", + "pygmars", + "python-inspector", + "sanexml", + "saneyaml", + "scancode-analyzer", + "scancode-toolkit-contrib", + "scancode-toolkit-reference-scans", + "thirdparty-toolkit", + "tracecode-toolkit", + "tracecode-toolkit-strace", + "turbo-spdx", + "typecode", + "univers", +] + + +@click.command() +@click.help_option("-h", "--help") +def update_skeleton_files(repo_names=ABOUTCODE_PUBLIC_REPO_NAMES): + """ + Update project files of AboutCode projects that use the skeleton + + This script will: + - Clone the repo + - Add the skeleton repo as a new origin + - Create a new branch named "update-skeleton-files" + - Merge in the new skeleton files into the "update-skeleton-files" branch + + The user will need to save merge commit messages that pop up when running + this script in addition to resolving the merge conflicts on repos that have + them. + """ + + # Create working directory + work_dir_path = Path("/tmp/update_skeleton/") + if not os.path.exists(work_dir_path): + os.makedirs(work_dir_path, exist_ok=True) + + for repo_name in repo_names: + # Move to work directory + os.chdir(work_dir_path) + + # Clone repo + repo_git = f"git@github.com:aboutcode-org/{repo_name}.git" + subprocess.run(["git", "clone", repo_git]) + + # Go into cloned repo + os.chdir(work_dir_path / repo_name) + + # Add skeleton as an origin + subprocess.run( + ["git", "remote", "add", "skeleton", "git@github.com:aboutcode-org/skeleton.git"] + ) + + # Fetch skeleton files + subprocess.run(["git", "fetch", "skeleton"]) + + # Create and checkout new branch + subprocess.run(["git", "checkout", "-b", "update-skeleton-files"]) + + # Merge skeleton files into the repo + subprocess.run(["git", "merge", "skeleton/main", "--allow-unrelated-histories"]) + + +if __name__ == "__main__": + update_skeleton_files() diff --git a/etc/scripts/utils_dejacode.py b/etc/scripts/utils_dejacode.py index c42e6c93..b6bff518 100644 --- a/etc/scripts/utils_dejacode.py +++ b/etc/scripts/utils_dejacode.py @@ -1,11 +1,10 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # # Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/skeleton for support or download. +# See https://github.com/aboutcode-org/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import io @@ -14,7 +13,6 @@ import requests import saneyaml - from packvers import version as packaging_version """ @@ -26,7 +24,7 @@ DEJACODE_API_URL_PACKAGES = f"{DEJACODE_API_URL}packages/" DEJACODE_API_HEADERS = { - "Authorization": "Token {}".format(DEJACODE_API_KEY), + "Authorization": f"Token {DEJACODE_API_KEY}", "Accept": "application/json; indent=4", } @@ -51,6 +49,7 @@ def fetch_dejacode_packages(params): DEJACODE_API_URL_PACKAGES, params=params, headers=DEJACODE_API_HEADERS, + timeout=10, ) return response.json()["results"] @@ -94,7 +93,7 @@ def update_with_dejacode_about_data(distribution): if package_data: package_api_url = package_data["api_url"] about_url = f"{package_api_url}about" - response = requests.get(about_url, headers=DEJACODE_API_HEADERS) + response = requests.get(about_url, headers=DEJACODE_API_HEADERS, timeout=10) # note that this is YAML-formatted about_text = response.json()["about_data"] about_data = saneyaml.load(about_text) @@ -114,7 +113,7 @@ def fetch_and_save_about_files(distribution, dest_dir="thirdparty"): if package_data: package_api_url = package_data["api_url"] about_url = f"{package_api_url}about_files" - response = requests.get(about_url, headers=DEJACODE_API_HEADERS) + response = requests.get(about_url, headers=DEJACODE_API_HEADERS, timeout=10) about_zip = response.content with io.BytesIO(about_zip) as zf: with zipfile.ZipFile(zf) as zi: @@ -153,7 +152,7 @@ def find_latest_dejacode_package(distribution): with_versions = sorted(with_versions) latest_version, latest_package_version = sorted(with_versions)[-1] print( - f"Found DejaCode latest version: {latest_version} " f"for dist: {distribution.package_url}", + f"Found DejaCode latest version: {latest_version} for dist: {distribution.package_url}", ) return latest_package_version @@ -179,7 +178,7 @@ def create_dejacode_package(distribution): } fields_to_carry_over = [ - "download_url" "type", + "download_urltype", "namespace", "name", "version", @@ -202,10 +201,11 @@ def create_dejacode_package(distribution): DEJACODE_API_URL_PACKAGES, data=new_package_payload, headers=DEJACODE_API_HEADERS, + timeout=10, ) new_package_data = response.json() if response.status_code != 201: raise Exception(f"Error, cannot create package for: {distribution}") - print(f'New Package created at: {new_package_data["absolute_url"]}') + print(f"New Package created at: {new_package_data['absolute_url']}") return new_package_data diff --git a/etc/scripts/utils_pip_compatibility_tags.py b/etc/scripts/utils_pip_compatibility_tags.py index af42a0cd..dd954bca 100644 --- a/etc/scripts/utils_pip_compatibility_tags.py +++ b/etc/scripts/utils_pip_compatibility_tags.py @@ -1,4 +1,5 @@ -"""Generate and work with PEP 425 Compatibility Tags. +""" +Generate and work with PEP 425 Compatibility Tags. copied from pip-20.3.1 pip/_internal/utils/compatibility_tags.py download_url: https://github.com/pypa/pip/blob/20.3.1/src/pip/_internal/utils/compatibility_tags.py @@ -27,14 +28,12 @@ import re -from packvers.tags import ( - compatible_tags, - cpython_tags, - generic_tags, - interpreter_name, - interpreter_version, - mac_platforms, -) +from packvers.tags import compatible_tags +from packvers.tags import cpython_tags +from packvers.tags import generic_tags +from packvers.tags import interpreter_name +from packvers.tags import interpreter_version +from packvers.tags import mac_platforms _osx_arch_pat = re.compile(r"(.+)_(\d+)_(\d+)_(.+)") @@ -132,7 +131,7 @@ def _get_custom_interpreter(implementation=None, version=None): implementation = interpreter_name() if version is None: version = interpreter_version() - return "{}{}".format(implementation, version) + return f"{implementation}{version}" def get_supported( @@ -142,7 +141,8 @@ def get_supported( abis=None, # type: Optional[List[str]] ): # type: (...) -> List[Tag] - """Return a list of supported tags for each version specified in + """ + Return a list of supported tags for each version specified in `versions`. :param version: a string version, of the form "33" or "32", diff --git a/etc/scripts/utils_requirements.py b/etc/scripts/utils_requirements.py index db7e0ee2..3b056614 100644 --- a/etc/scripts/utils_requirements.py +++ b/etc/scripts/utils_requirements.py @@ -1,11 +1,10 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # # Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/skeleton for support or download. +# See https://github.com/aboutcode-org/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import os @@ -39,7 +38,7 @@ def get_required_name_versions(requirement_lines, with_unpinned=False): req_line = req_line.strip() if not req_line or req_line.startswith("#"): continue - if req_line.startswith("-") or (not with_unpinned and not "==" in req_line): + if req_line.startswith("-") or (not with_unpinned and "==" not in req_line): print(f"Requirement line is not supported: ignored: {req_line}") continue yield get_required_name_version(requirement=req_line, with_unpinned=with_unpinned) @@ -56,21 +55,25 @@ def get_required_name_version(requirement, with_unpinned=False): >>> assert get_required_name_version("fooA==1.2.3.DEV1") == ("fooa", "1.2.3.dev1") >>> assert get_required_name_version("foo==1.2.3", with_unpinned=False) == ("foo", "1.2.3") >>> assert get_required_name_version("foo", with_unpinned=True) == ("foo", "") - >>> assert get_required_name_version("foo>=1.2", with_unpinned=True) == ("foo", ""), get_required_name_version("foo>=1.2") + >>> expected = ("foo", ""), get_required_name_version("foo>=1.2") + >>> assert get_required_name_version("foo>=1.2", with_unpinned=True) == expected >>> try: ... assert not get_required_name_version("foo", with_unpinned=False) ... except Exception as e: ... assert "Requirement version must be pinned" in str(e) """ requirement = requirement and "".join(requirement.lower().split()) - assert requirement, f"specifier is required is empty:{requirement!r}" + if not requirement: + raise ValueError(f"specifier is required is empty:{requirement!r}") name, operator, version = split_req(requirement) - assert name, f"Name is required: {requirement}" + if not name: + raise ValueError(f"Name is required: {requirement}") is_pinned = operator == "==" if with_unpinned: version = "" else: - assert is_pinned and version, f"Requirement version must be pinned: {requirement}" + if not is_pinned and version: + raise ValueError(f"Requirement version must be pinned: {requirement}") return name, version @@ -116,7 +119,7 @@ def get_installed_reqs(site_packages_dir): # Also include these packages in the output with --all: wheel, distribute, # setuptools, pip args = ["pip", "freeze", "--exclude-editable", "--all", "--path", site_packages_dir] - return subprocess.check_output(args, encoding="utf-8") + return subprocess.check_output(args, encoding="utf-8") # noqa: S603 comparators = ( @@ -146,9 +149,11 @@ def split_req(req): >>> assert split_req("foo >= 1.2.3 ") == ("foo", ">=", "1.2.3"), split_req("foo >= 1.2.3 ") >>> assert split_req("foo>=1.2") == ("foo", ">=", "1.2"), split_req("foo>=1.2") """ - assert req + if not req: + raise ValueError("req is required") # do not allow multiple constraints and tags - assert not any(c in req for c in ",;") + if not any(c in req for c in ",;"): + raise Exception(f"complex requirements with : or ; not supported: {req}") req = "".join(req.split()) if not any(c in req for c in comparators): return req, "", "" diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index addf8e5e..aafc1d69 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -1,11 +1,10 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # # Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/skeleton for support or download. +# See https://github.com/aboutcode-org/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import email @@ -25,14 +24,13 @@ import packageurl import requests import saneyaml +import utils_pip_compatibility_tags from commoncode import fileutils from commoncode.hash import multi_checksums from commoncode.text import python_safe_name from packvers import tags as packaging_tags from packvers import version as packaging_version -import utils_pip_compatibility_tags - """ Utilities to manage Python thirparty libraries source, binaries and metadata in local directories and remote repositories. @@ -93,7 +91,8 @@ - parse requirement file - create a TODO queue of requirements to process -- done: create an empty map of processed binary requirements as {package name: (list of versions/tags} +- done: create an empty map of processed binary requirements as + {package name: (list of versions/tags} - while we have package reqs in TODO queue, process one requirement: @@ -355,7 +354,6 @@ def sorted(cls, namevers): @attr.attributes class Distribution(NameVer): - # field names that can be updated from another Distribution or mapping updatable_fields = [ "license_expression", @@ -555,7 +553,8 @@ def download(self, dest_dir=THIRDPARTY_DIR): Download this distribution into `dest_dir` directory. Return the fetched filename. """ - assert self.filename + if not self.filename: + raise ValueError(f"self.filename has no value but is required: {self.filename!r}") if TRACE_DEEP: print( f"Fetching distribution of {self.name}=={self.version}:", @@ -823,9 +822,9 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False): """ urls = LinksRepository.from_url(use_cached_index=use_cached_index).links errors = [] - extra_lic_names = [l.get("file") for l in self.extra_data.get("licenses", {})] + extra_lic_names = [lic.get("file") for lic in self.extra_data.get("licenses", {})] extra_lic_names += [self.extra_data.get("license_file")] - extra_lic_names = [ln for ln in extra_lic_names if ln] + extra_lic_names = [eln for eln in extra_lic_names if eln] lic_names = [f"{key}.LICENSE" for key in self.get_license_keys()] for filename in lic_names + extra_lic_names: floc = os.path.join(dest_dir, filename) @@ -845,7 +844,7 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False): if TRACE: print(f"Fetched license from remote: {lic_url}") - except: + except Exception: try: # try licensedb second lic_url = f"{LICENSEDB_API_URL}/{filename}" @@ -858,8 +857,9 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False): if TRACE: print(f"Fetched license from licensedb: {lic_url}") - except: - msg = f'No text for license {filename} in expression "{self.license_expression}" from {self}' + except Exception: + msg = f"No text for license {filename} in expression " + f"{self.license_expression!r} from {self}" print(msg) errors.append(msg) @@ -999,7 +999,7 @@ def get_license_link_for_filename(filename, urls): exception if no link is found or if there are more than one link for that file name. """ - path_or_url = [l for l in urls if l.endswith(f"/{filename}")] + path_or_url = [url for url in urls if url.endswith(f"/{filename}")] if not path_or_url: raise Exception(f"Missing link to file: {filename}") if not len(path_or_url) == 1: @@ -1091,7 +1091,6 @@ def get_sdist_name_ver_ext(filename): @attr.attributes class Sdist(Distribution): - extension = attr.ib( repr=False, type=str, @@ -1129,7 +1128,6 @@ def to_filename(self): @attr.attributes class Wheel(Distribution): - """ Represents a wheel file. @@ -1290,7 +1288,7 @@ def is_pure(self): def is_pure_wheel(filename): try: return Wheel.from_filename(filename).is_pure() - except: + except Exception: return False @@ -1486,8 +1484,7 @@ def get_distributions(self): """ if self.sdist: yield self.sdist - for wheel in self.wheels: - yield wheel + yield from self.wheels def get_url_for_filename(self, filename): """ @@ -1616,7 +1613,8 @@ class PypiSimpleRepository: type=dict, default=attr.Factory(lambda: defaultdict(dict)), metadata=dict( - help="Mapping of {name: {version: PypiPackage, version: PypiPackage, etc} available in this repo" + help="Mapping of {name: {version: PypiPackage, version: PypiPackage, etc} " + "available in this repo" ), ) @@ -1630,7 +1628,8 @@ class PypiSimpleRepository: type=bool, default=False, metadata=dict( - help="If True, use any existing on-disk cached PyPI index files. Otherwise, fetch and cache." + help="If True, use any existing on-disk cached PyPI index files. " + "Otherwise, fetch and cache." ), ) @@ -1639,7 +1638,8 @@ def _get_package_versions_map(self, name): Return a mapping of all available PypiPackage version for this package name. The mapping may be empty. It is ordered by version from oldest to newest """ - assert name + if not name: + raise ValueError(f"name is required: {name!r}") normalized_name = NameVer.normalize_name(name) versions = self.packages[normalized_name] if not versions and normalized_name not in self.fetched_package_normalized_names: @@ -1694,7 +1694,7 @@ def fetch_links(self, normalized_name): ) links = collect_urls(text) # TODO: keep sha256 - links = [l.partition("#sha256=") for l in links] + links = [link.partition("#sha256=") for link in links] links = [url for url, _, _sha256 in links] return links @@ -1915,7 +1915,7 @@ def get_remote_file_content( # several redirects and that we can ignore content there. A HEAD request may # not get us this last header print(f" DOWNLOADING: {url}") - with requests.get(url, allow_redirects=True, stream=True, headers=headers) as response: + with requests.get(url, allow_redirects=True, stream=True, headers=headers) as response: # noqa: S113 status = response.status_code if status != requests.codes.ok: # NOQA if status == 429 and _delay < 20: @@ -2134,10 +2134,9 @@ def call(args, verbose=TRACE): """ if TRACE_DEEP: print("Calling:", " ".join(args)) - with subprocess.Popen( + with subprocess.Popen( # noqa: S603 args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8" ) as process: - stdouts = [] while True: line = process.stdout.readline() @@ -2200,7 +2199,7 @@ def download_wheels_with_pip( cli_args.extend(["--requirement", req_file]) if TRACE: - print(f"Downloading wheels using command:", " ".join(cli_args)) + print("Downloading wheels using command:", " ".join(cli_args)) existing = set(os.listdir(dest_dir)) error = False @@ -2233,7 +2232,7 @@ def download_wheels_with_pip( def check_about(dest_dir=THIRDPARTY_DIR): try: - subprocess.check_output(f"venv/bin/about check {dest_dir}".split()) + subprocess.check_output(f"venv/bin/about check {dest_dir}".split()) # noqa: S603 except subprocess.CalledProcessError as cpe: print() print("Invalid ABOUT files:") @@ -2284,5 +2283,5 @@ def get_license_expression(declared_licenses): return get_only_expression_from_extracted_license(declared_licenses) except ImportError: # Scancode is not installed, clean and join all the licenses - lics = [python_safe_name(l).lower() for l in declared_licenses] + lics = [python_safe_name(lic).lower() for lic in declared_licenses] return " AND ".join(lics).lower() diff --git a/pyproject.toml b/pyproject.toml index 3ca4f153..e7fcd5d8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [tool.setuptools_scm] # this is used populated when creating a git archive # and when there is .git dir and/or there is no git installed -fallback_version = "9999.9999.99" +fallback_version = "9999.$Format:%h-%cs$" [tool.pytest.ini_options] norecursedirs = [ @@ -14,7 +14,6 @@ norecursedirs = [ "dist", "build", "_build", - "dist", "etc", "local", "ci", @@ -34,7 +33,9 @@ norecursedirs = [ "thirdparty", "tmp", "venv", + ".venv", "tests/data", + "*/tests/test_data", ".eggs", "src/*/data", "tests/*/data" @@ -50,3 +51,81 @@ addopts = [ "--strict-markers", "--doctest-modules" ] + +[tool.ruff] +line-length = 100 +extend-exclude = [] +target-version = "py310" +include = [ + "pyproject.toml", + "src/**/*.py", + "etc/**/*.py", + "test/**/*.py", + "tests/**/*.py", + "doc/**/*.py", + "docs/**/*.py", + "*.py", + "." + +] +# ignore test data and testfiles: they should never be linted nor formatted +exclude = [ +# main style + "**/tests/data/**/*", +# scancode-toolkit + "**/tests/*/data/**/*", +# dejacode, purldb + "**/tests/testfiles/**/*", +# vulnerablecode, fetchcode + "**/tests/*/test_data/**/*", + "**/tests/test_data/**/*", +# django migrations + "**/migrations/**/*", +# exclude vendored code from ScanCode + "src/_packagedcode", +] + +[tool.ruff.lint] +# Rules: https://docs.astral.sh/ruff/rules/ +select = [ +# "E", # pycodestyle +# "W", # pycodestyle warnings + "D", # pydocstyle +# "F", # Pyflakes +# "UP", # pyupgrade +# "S", # flake8-bandit + "I", # isort +# "C9", # McCabe complexity +] +ignore = ["D1", "D200", "D202", "D203", "D205", "D212", "D400", "D415", "I001"] + + +[tool.ruff.lint.isort] +force-single-line = true +lines-after-imports = 1 +default-section = "first-party" +known-first-party = ["src", "tests", "etc/scripts/**/*.py"] +known-third-party = ["click", "pytest"] + +sections = { django = ["django"] } +section-order = [ + "future", + "standard-library", + "django", + "third-party", + "first-party", + "local-folder", +] + +[tool.ruff.lint.mccabe] +max-complexity = 10 + +[tool.ruff.lint.per-file-ignores] +# Place paths of files to be ignored by ruff here +"tests/*" = ["S101"] +"test_*.py" = ["S101"] + + +[tool.doc8] +ignore-path = ["docs/build", "doc/build", "docs/_build", "doc/_build"] +max-line-length=100 diff --git a/setup.cfg b/setup.cfg index c40a94af..c4a15475 100644 --- a/setup.cfg +++ b/setup.cfg @@ -41,8 +41,11 @@ license_files = AUTHORS.rst CHANGELOG.rst CODE_OF_CONDUCT.rst + README.rst [options] +python_requires = >=3.9 + package_dir = =src packages = find: @@ -51,7 +54,6 @@ zip_safe = false setup_requires = setuptools_scm[toml] >= 4 -python_requires = >=3.7 install_requires = attrs >= 18.1, !=20.1.0 @@ -83,20 +85,18 @@ console_scripts = python-inspector = python_inspector.resolve_cli:resolve_dependencies [options.extras_require] -testing = - pytest >= 6, != 7.0.0 +dev = + pytest >= 7.0.1 pytest-xdist >= 2 aboutcode-toolkit >= 7.0.2 - pycodestyle >= 2.8.0 twine - black - isort + ruff pytest-rerunfailures pytest-asyncio >= 0.21 - -docs = Sphinx>=5.0.2 sphinx-rtd-theme>=1.0.0 sphinx-reredirects >= 0.1.2 doc8>=0.11.2 - + sphinx-autobuild + sphinx-rtd-dark-mode>=1.3.0 + sphinx-copybutton diff --git a/src/_packagedcode/pypi.py b/src/_packagedcode/pypi.py index 4f5b2a61..e8fe5171 100644 --- a/src/_packagedcode/pypi.py +++ b/src/_packagedcode/pypi.py @@ -457,7 +457,7 @@ def parse_metadata(location, datasource_id, package_type): name=name, version=version, description=get_description(metainfo=meta, location=str(location)), - #TODO: https://github.com/aboutcode-org/scancode-toolkit/issues/3014 + # TODO: https://github.com/aboutcode-org/scancode-toolkit/issues/3014 declared_license=get_declared_license(meta), keywords=get_keywords(meta), parties=get_parties(meta), @@ -521,19 +521,25 @@ class PypiWheelHandler(BasePypiHandler): @classmethod def parse(cls, location): - with zipfile.ZipFile(location) as zf: - for path in ZipPath(zf).iterdir(): - if not path.name.endswith(META_DIR_SUFFIXES): - continue - for metapath in path.iterdir(): - if not metapath.name.endswith('METADATA'): - continue - yield parse_metadata( - location=metapath, - datasource_id=cls.datasource_id, - package_type=cls.default_package_type, - ) + from python_inspector import lockfile + from python_inspector.utils_pypi import PYINSP_CACHE_LOCK_TIMEOUT + lock_file = os.path.join(f"{location}.lockfile") + with lockfile.FileLock(lock_file).locked(timeout=PYINSP_CACHE_LOCK_TIMEOUT): + + with zipfile.ZipFile(location) as zf: + for path in ZipPath(zf).iterdir(): + if not path.name.endswith(META_DIR_SUFFIXES): + continue + for metapath in path.iterdir(): + if not metapath.name.endswith('METADATA'): + continue + + yield parse_metadata( + location=metapath, + datasource_id=cls.datasource_id, + package_type=cls.default_package_type, + ) class PypiEggHandler(BasePypiHandler): @@ -547,20 +553,26 @@ class PypiEggHandler(BasePypiHandler): @classmethod def parse(cls, location): - with zipfile.ZipFile(location) as zf: - for path in ZipPath(zf).iterdir(): - if not path.name.endswith(META_DIR_SUFFIXES): - continue - for metapath in path.iterdir(): - if not metapath.name.endswith('PKG-INFO'): + from python_inspector import lockfile + from python_inspector.utils_pypi import PYINSP_CACHE_LOCK_TIMEOUT + lock_file = os.path.join(f"{location}.lockfile") + with lockfile.FileLock(lock_file).locked(timeout=PYINSP_CACHE_LOCK_TIMEOUT): + + with zipfile.ZipFile(location) as zf: + for path in ZipPath(zf).iterdir(): + if not path.name.endswith(META_DIR_SUFFIXES): continue - yield parse_metadata( - location=metapath, - datasource_id=cls.datasource_id, - package_type=cls.default_package_type, - ) + for metapath in path.iterdir(): + if not metapath.name.endswith('PKG-INFO'): + continue + + yield parse_metadata( + location=metapath, + datasource_id=cls.datasource_id, + package_type=cls.default_package_type, + ) class PypiSdistArchiveHandler(BasePypiHandler): @@ -765,7 +777,6 @@ def parse(cls, location): ) ] - yield models.PackageData( datasource_id=cls.datasource_id, type=cls.default_package_type, @@ -818,6 +829,7 @@ def get_resolved_purl(purl: PackageURL, specifiers: SpecifierSet): is_resolved=is_resolved, ) + class PipfileHandler(BaseDependencyFileHandler): datasource_id = 'pipfile' path_patterns = ('*Pipfile',) diff --git a/src/python_inspector/resolution.py b/src/python_inspector/resolution.py index 6dca3a5d..39202374 100644 --- a/src/python_inspector/resolution.py +++ b/src/python_inspector/resolution.py @@ -12,6 +12,7 @@ import operator import os import tarfile +from traceback import format_exc from typing import Dict from typing import Generator from typing import Iterable @@ -87,9 +88,15 @@ def get_requirements_from_distribution( if not os.path.exists(location): return [] reqs = [] - for package_data in handler.parse(location): - dependencies = package_data.dependencies - reqs.extend(get_requirements_from_dependencies(dependencies=dependencies)) + try: + for package_data in handler.parse(location): + dependencies = package_data.dependencies + reqs.extend(get_requirements_from_dependencies(dependencies=dependencies)) + except Exception as e: + trace = format_exc() + raise Exception( + f"Failed to get_requirements_from_distribution for: {location!r}\n{trace}" + ) from e return reqs @@ -174,6 +181,8 @@ def is_valid_version( def get_python_version_from_env_tag(python_version: str) -> str: """ + Return the python version extracted from an environment tag. + >>> assert get_python_version_from_env_tag("310") == "3.10" >>> assert get_python_version_from_env_tag("39") == "3.9" """ diff --git a/src/python_inspector/setup_py_live_eval.py b/src/python_inspector/setup_py_live_eval.py index 7ba51c52..e955f34e 100755 --- a/src/python_inspector/setup_py_live_eval.py +++ b/src/python_inspector/setup_py_live_eval.py @@ -29,7 +29,7 @@ def minver_error(pkg_name): """Report error about missing minimum version constraint and exit.""" print( - 'ERROR: specify minimal version of "{0}" using ' '">=" or "=="'.format(pkg_name), + 'ERROR: specify minimal version of "{0}" using ">=" or "=="'.format(pkg_name), file=sys.stderr, ) sys.exit(1) @@ -164,7 +164,7 @@ def iter_requirements(level, extras, setup_file): specs = {s.operator: s.version for s in specs._specs} if ((">=" in specs) and (">" in specs)) or (("<=" in specs) and ("<" in specs)): print( - "ERROR: Do not specify such weird constraints! " '("{0}")'.format(pkg), + 'ERROR: Do not specify such weird constraints! ("{0}")'.format(pkg), file=sys.stderr, ) sys.exit(1) diff --git a/src/python_inspector/utils_pip_compatibility_tags.py b/src/python_inspector/utils_pip_compatibility_tags.py index afe3fb3b..cb52c353 100644 --- a/src/python_inspector/utils_pip_compatibility_tags.py +++ b/src/python_inspector/utils_pip_compatibility_tags.py @@ -1,4 +1,5 @@ -"""Generate and work with PEP 425 Compatibility Tags. +""" +Generate and work with PEP 425 Compatibility Tags. copied from pip-20.3.1 pip/_internal/utils/compatibility_tags.py download_url: https://github.com/pypa/pip/blob/20.3.1/src/pip/_internal/utils/compatibility_tags.py @@ -141,7 +142,8 @@ def get_supported( abis=None, # type: Optional[List[str]] ): # type: (...) -> List[Tag] - """Return a list of supported tags for each version specified in + """ + Return a list of supported tags for each version specified in `versions`. :param version: a string version, of the form "33" or "32", diff --git a/src/python_inspector/utils_pypi.py b/src/python_inspector/utils_pypi.py index 2ca750ff..f2757cd6 100644 --- a/src/python_inspector/utils_pypi.py +++ b/src/python_inspector/utils_pypi.py @@ -153,6 +153,8 @@ def get_python_dot_version(version): "manylinux1_x86_64", "manylinux2010_x86_64", "manylinux2014_x86_64", + "manylinux_2_27_x86_64", + "manylinux_2_28_x86_64", "manylinux2014_aarch6", "musllinux_1_2_x86_64", "manylinux_2_33_aarch64", @@ -196,7 +198,6 @@ def get_python_dot_version(version): DEFAULT_PYTHON_VERSION = settings.DEFAULT_PYTHON_VERSION CACHE_THIRDPARTY_DIR = settings.CACHE_THIRDPARTY_DIR - ################################################################################ EXTENSIONS_SDIST = ( @@ -316,9 +317,10 @@ async def get_supported_and_valid_wheels( ): continue if TRACE_DEEP: + durl = await wheel.download_url(repo) print( f""" get_supported_and_valid_wheels: Getting wheel from index (or cache): - {await wheel.download_url(repo)}""" + {durl}""" ) wheels.append(wheel) return wheels @@ -430,7 +432,6 @@ class Link(NamedTuple): @attr.attributes class Distribution(NameVer): - """ A Distribution is either either a Wheel or Sdist and is identified by and created from its filename as well as its name and version. A Distribution is @@ -1003,7 +1004,6 @@ def to_filename(self): @attr.attributes class Wheel(Distribution): - """ Represents a wheel file. @@ -1687,6 +1687,7 @@ async def get( # the cache key is a hash of the normalized path cache_key = self.sha256_hash(quote_plus(path_or_url.strip("/"))) cached = os.path.join(self.directory, cache_key) + lock_file = f"{cached}.lockfile" if force or not os.path.exists(cached): if TRACE_DEEP: @@ -1701,8 +1702,6 @@ async def get( wmode = "w" if as_text else "wb" # acquire lock and wait until timeout to get a lock or die - lock_file = os.path.join(self.directory, f"{cache_key}.lockfile") - with lockfile.FileLock(lock_file).locked(timeout=PYINSP_CACHE_LOCK_TIMEOUT): async with aiofiles.open(cached, mode=wmode) as fo: await fo.write(content) @@ -1710,7 +1709,9 @@ async def get( else: if TRACE_DEEP: print(f" FILE CACHE HIT: {path_or_url}") - return await get_local_file_content(path=cached, as_text=as_text), cached + # also lock on read to avoid race conditions + with lockfile.FileLock(lock_file).locked(timeout=PYINSP_CACHE_LOCK_TIMEOUT): + return await get_local_file_content(path=cached, as_text=as_text), cached CACHE = Cache() diff --git a/tests/data/pinned-pdt-requirements.txt b/tests/data/pinned-pdt-requirements.txt index f3c5ff11..12d7f144 100644 --- a/tests/data/pinned-pdt-requirements.txt +++ b/tests/data/pinned-pdt-requirements.txt @@ -27,3 +27,4 @@ typing==3.6.6 typing_extensions==4.1.1 urllib3==1.26.9 zipp==3.6.0 +boolean.py==4.0.0 diff --git a/tests/data/pinned-pdt-requirements.txt-expected.json b/tests/data/pinned-pdt-requirements.txt-expected.json index 32df5608..62dcb3f4 100644 --- a/tests/data/pinned-pdt-requirements.txt-expected.json +++ b/tests/data/pinned-pdt-requirements.txt-expected.json @@ -656,6 +656,27 @@ "is_name_at_url": false, "is_local_path": null } + }, + { + "purl": "pkg:pypi/boolean-py@4.0.0", + "extracted_requirement": "boolean.py==4.0.0", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {}, + "extra_data": { + "is_editable": false, + "link": null, + "hash_options": [], + "is_constraint": false, + "is_archive": null, + "is_wheel": false, + "is_url": null, + "is_vcs_url": null, + "is_name_at_url": false, + "is_local_path": null + } } ], "repository_homepage_url": null, @@ -879,12 +900,12 @@ "type": "pypi", "namespace": null, "name": "boolean-py", - "version": "5.0", + "version": "4.0", "qualifiers": {}, "subpath": null, "primary_language": "Python", "description": "Define boolean algebras, create and parse boolean expressions and create custom boolean DSL.\nThis library helps you deal with boolean expressions and algebra with variables\nand the boolean functions AND, OR, NOT.\n\nYou can parse expressions from strings and simplify and compare expressions.\nYou can also easily create your custom algreba and mini DSL and create custom\ntokenizers to handle custom expressions.\n\nFor extensive documentation look either into the docs directory or view it online, at\nhttps://booleanpy.readthedocs.org/en/latest/\n\nhttps://github.com/bastikr/boolean.py\n\nCopyright (c) 2009-2020 Sebastian Kraemer, basti.kr@gmail.com and others\nSPDX-License-Identifier: BSD-2-Clause", - "release_date": "2025-04-03T10:39:48", + "release_date": "2022-05-05T08:18:58", "parties": [ { "type": "person", @@ -910,11 +931,11 @@ "Topic :: Utilities" ], "homepage_url": "https://github.com/bastikr/boolean.py", - "download_url": "https://files.pythonhosted.org/packages/e5/ca/78d423b324b8d77900030fa59c4aa9054261ef0925631cd2501dd015b7b7/boolean_py-5.0-py3-none-any.whl", - "size": 26577, + "download_url": "https://files.pythonhosted.org/packages/3f/02/6389ef0529af6da0b913374dedb9bbde8eabfe45767ceec38cc37801b0bd/boolean.py-4.0-py3-none-any.whl", + "size": 25909, "sha1": null, - "md5": "df9060a88bfb6ba3b9314783b16a0faa", - "sha256": "ef28a70bd43115208441b53a045d1549e2f0ec6e3d08a9d142cbc41c1938e8d9", + "md5": "73208a6fc38d6904a1d7e793e8da1292", + "sha256": "2876f2051d7d6394a531d82dc6eb407faa0b01a0a0b3083817ccd7323b8d96bd", "sha512": null, "bug_tracking_url": null, "code_view_url": null, @@ -931,9 +952,9 @@ "dependencies": [], "repository_homepage_url": null, "repository_download_url": null, - "api_data_url": "https://pypi.org/pypi/boolean-py/5.0/json", + "api_data_url": "https://pypi.org/pypi/boolean-py/4.0/json", "datasource_id": null, - "purl": "pkg:pypi/boolean-py@5.0" + "purl": "pkg:pypi/boolean-py@4.0" }, { "type": "pypi", @@ -2820,7 +2841,7 @@ { "key": "boolean-py", "package_name": "boolean-py", - "installed_version": "5.0", + "installed_version": "4.0", "dependencies": [] }, { @@ -2856,7 +2877,7 @@ { "key": "boolean-py", "package_name": "boolean-py", - "installed_version": "5.0", + "installed_version": "4.0", "dependencies": [] } ] diff --git a/tests/data/pinned-requirements.txt b/tests/data/pinned-requirements.txt index f3c5ff11..12d7f144 100644 --- a/tests/data/pinned-requirements.txt +++ b/tests/data/pinned-requirements.txt @@ -27,3 +27,4 @@ typing==3.6.6 typing_extensions==4.1.1 urllib3==1.26.9 zipp==3.6.0 +boolean.py==4.0.0 diff --git a/tests/data/pinned-requirements.txt-expected.json b/tests/data/pinned-requirements.txt-expected.json index 19780ecd..a5848345 100644 --- a/tests/data/pinned-requirements.txt-expected.json +++ b/tests/data/pinned-requirements.txt-expected.json @@ -656,6 +656,27 @@ "is_name_at_url": false, "is_local_path": null } + }, + { + "purl": "pkg:pypi/boolean-py@4.0.0", + "extracted_requirement": "boolean.py==4.0.0", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {}, + "extra_data": { + "is_editable": false, + "link": null, + "hash_options": [], + "is_constraint": false, + "is_archive": null, + "is_wheel": false, + "is_url": null, + "is_vcs_url": null, + "is_name_at_url": false, + "is_local_path": null + } } ], "repository_homepage_url": null, @@ -879,12 +900,12 @@ "type": "pypi", "namespace": null, "name": "boolean-py", - "version": "5.0", + "version": "4.0", "qualifiers": {}, "subpath": null, "primary_language": "Python", "description": "Define boolean algebras, create and parse boolean expressions and create custom boolean DSL.\nThis library helps you deal with boolean expressions and algebra with variables\nand the boolean functions AND, OR, NOT.\n\nYou can parse expressions from strings and simplify and compare expressions.\nYou can also easily create your custom algreba and mini DSL and create custom\ntokenizers to handle custom expressions.\n\nFor extensive documentation look either into the docs directory or view it online, at\nhttps://booleanpy.readthedocs.org/en/latest/\n\nhttps://github.com/bastikr/boolean.py\n\nCopyright (c) 2009-2020 Sebastian Kraemer, basti.kr@gmail.com and others\nSPDX-License-Identifier: BSD-2-Clause", - "release_date": "2025-04-03T10:39:48", + "release_date": "2022-05-05T08:18:58", "parties": [ { "type": "person", @@ -910,11 +931,11 @@ "Topic :: Utilities" ], "homepage_url": "https://github.com/bastikr/boolean.py", - "download_url": "https://files.pythonhosted.org/packages/e5/ca/78d423b324b8d77900030fa59c4aa9054261ef0925631cd2501dd015b7b7/boolean_py-5.0-py3-none-any.whl", - "size": 26577, + "download_url": "https://files.pythonhosted.org/packages/3f/02/6389ef0529af6da0b913374dedb9bbde8eabfe45767ceec38cc37801b0bd/boolean.py-4.0-py3-none-any.whl", + "size": 25909, "sha1": null, - "md5": "df9060a88bfb6ba3b9314783b16a0faa", - "sha256": "ef28a70bd43115208441b53a045d1549e2f0ec6e3d08a9d142cbc41c1938e8d9", + "md5": "73208a6fc38d6904a1d7e793e8da1292", + "sha256": "2876f2051d7d6394a531d82dc6eb407faa0b01a0a0b3083817ccd7323b8d96bd", "sha512": null, "bug_tracking_url": null, "code_view_url": null, @@ -931,9 +952,9 @@ "dependencies": [], "repository_homepage_url": null, "repository_download_url": null, - "api_data_url": "https://pypi.org/pypi/boolean-py/5.0/json", + "api_data_url": "https://pypi.org/pypi/boolean-py/4.0/json", "datasource_id": null, - "purl": "pkg:pypi/boolean-py@5.0" + "purl": "pkg:pypi/boolean-py@4.0" }, { "type": "pypi", @@ -2810,7 +2831,7 @@ "package": "pkg:pypi/aboutcode-toolkit@7.0.2", "dependencies": [ "pkg:pypi/attrs@21.4.0", - "pkg:pypi/boolean-py@5.0", + "pkg:pypi/boolean-py@4.0", "pkg:pypi/certifi@2022.5.18.1", "pkg:pypi/click@8.0.4", "pkg:pypi/jinja2@3.1.6", @@ -2831,7 +2852,7 @@ ] }, { - "package": "pkg:pypi/boolean-py@5.0", + "package": "pkg:pypi/boolean-py@4.0", "dependencies": [] }, { @@ -2897,7 +2918,7 @@ { "package": "pkg:pypi/license-expression@30.3.1", "dependencies": [ - "pkg:pypi/boolean-py@5.0" + "pkg:pypi/boolean-py@4.0" ] }, { diff --git a/tests/test_api.py b/tests/test_api.py index 79cc7170..29f4afde 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -22,6 +22,7 @@ test_env.test_data_dir = os.path.join(os.path.dirname(__file__), "data") +@pytest.mark.online def test_api_with_specifier(): expected_file = test_env.get_test_loc("test-api-expected.json", must_exist=False) results = resolver_api( @@ -32,6 +33,7 @@ def test_api_with_specifier(): check_data_results(results=results.to_dict(generic_paths=True), expected_file=expected_file) +@pytest.mark.online def test_api_with_specifier_pdt(): expected_file = test_env.get_test_loc("test-api-pdt-expected.json", must_exist=False) results = resolver_api( @@ -43,6 +45,7 @@ def test_api_with_specifier_pdt(): check_data_results(results=results.to_dict(generic_paths=True), expected_file=expected_file) +@pytest.mark.online def test_api_with_requirement_file(): expected_file = test_env.get_test_loc("test-api-with-requirement-file.json", must_exist=False) results = resolver_api( @@ -53,6 +56,7 @@ def test_api_with_requirement_file(): check_data_results(results=results.to_dict(generic_paths=True), expected_file=expected_file) +@pytest.mark.online def test_api_with_prefer_source(): expected_file = test_env.get_test_loc("test-api-with-prefer-source.json", must_exist=False) results = resolver_api( @@ -64,6 +68,7 @@ def test_api_with_prefer_source(): check_data_results(results=results.to_dict(generic_paths=True), expected_file=expected_file) +@pytest.mark.online def test_api_with_recursive_requirement_file(): requirement_file = test_env.get_test_loc("recursive_requirements/r.txt") expected_file = test_env.get_test_loc( @@ -97,6 +102,7 @@ def test_api_with_wrong_pyver(): resolver_api(specifiers=["flask==2.1.2"], python_version="3.14", operating_system="linux") +@pytest.mark.online def test_api_with_python_311(): expected_file = test_env.get_test_loc("test-api-with-python-311.json", must_exist=False) results = resolver_api( @@ -108,6 +114,7 @@ def test_api_with_python_311(): check_data_results(results=results.to_dict(generic_paths=True), expected_file=expected_file) +@pytest.mark.online def test_api_with_lief_python_312(): expected_file = test_env.get_test_loc("test-api-with-lief-python-312.json", must_exist=False) results = resolver_api( @@ -119,6 +126,7 @@ def test_api_with_lief_python_312(): check_data_results(results=results.to_dict(generic_paths=True), expected_file=expected_file) +@pytest.mark.online def test_api_with_partial_setup_py(): expected_file = test_env.get_test_loc("test-api-with-partial-setup-py.json", must_exist=False) results = resolver_api( @@ -132,7 +140,6 @@ def test_api_with_partial_setup_py(): def test_get_index_urls(): - # pass as a tuple index_urls = get_index_urls( index_urls=("https://pypi.org/simple",), diff --git a/tests/test_cli.py b/tests/test_cli.py index af8d2748..41a52f87 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -230,9 +230,9 @@ def test_cli_with_single_env_var_index_url_except_pypi_simple(): def test_cli_with_multiple_env_var_index_url_and_tilde_req(): expected_file = test_env.get_test_loc("tilde_req-expected-env.json", must_exist=False) specifier = "zipp~=3.8.0" - os.environ[ - "PYINSP_INDEX_URL" - ] = "https://pypi.org/simple https://thirdparty.aboutcode.org/pypi/simple/" + os.environ["PYINSP_INDEX_URL"] = ( + "https://pypi.org/simple https://thirdparty.aboutcode.org/pypi/simple/" + ) check_specs_resolution( specifier=specifier, expected_file=expected_file, @@ -440,7 +440,13 @@ def test_cli_with_insecure_option(): ) -@pytest.mark.skipif(sys.version_info[:2] == (3, 12), reason="Skipping test for Python 3.12") +@pytest.mark.skipif( + sys.version_info[:2] >= (3, 12), + reason="Skipping test for Python 3.12+ because of " + "https://github.com/aboutcode-org/python-inspector/issues/212" + "to avoid error AttributeError: module 'configparser' has no attribute " + "'SafeConfigParser'. Did you mean: 'RawConfigParser'?", +) @pytest.mark.online def test_cli_with_insecure_option_testpkh(): setup_py_file = test_env.get_test_loc("insecure-setup-2/setup.py") @@ -676,6 +682,9 @@ def clean_results(results): headers = results.get("headers", {}) or {} if "tool_version" in headers: del headers["tool_version"] + options = headers.get("options", []) or [] + if "--verbose" in options: + options.remove("--verbose") return results diff --git a/tests/test_codestyle.py b/tests/test_codestyle.py deleted file mode 100644 index b86afa9c..00000000 --- a/tests/test_codestyle.py +++ /dev/null @@ -1,33 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# ScanCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/skeleton for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import subprocess -import sys -import unittest - -import pytest - - -class BaseTests(unittest.TestCase): - @pytest.mark.skipif(sys.version_info[:2] == (3, 12), reason="Skipping test for Python 3.12") - def test_codestyle(self): - """ - This test shouldn't run in proliferated repositories. - """ - args = "make check" - try: - subprocess.check_output(args.split()) - except subprocess.CalledProcessError as e: - print("===========================================================") - print(e.output) - print("===========================================================") - raise Exception( - "Code style check failed!", - e.output, - ) from e diff --git a/tests/test_setup_py_live_eval.py b/tests/test_setup_py_live_eval.py index 92c69f4c..5166df39 100755 --- a/tests/test_setup_py_live_eval.py +++ b/tests/test_setup_py_live_eval.py @@ -8,6 +8,7 @@ # file for more details. # """Tests for `requirements-builder` module.""" + from os.path import abspath from os.path import dirname from os.path import join diff --git a/tests/test_skeleton_codestyle.py b/tests/test_skeleton_codestyle.py deleted file mode 100644 index 95fcb9f5..00000000 --- a/tests/test_skeleton_codestyle.py +++ /dev/null @@ -1,36 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# ScanCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/skeleton for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import configparser -import subprocess -import unittest - - -class BaseTests(unittest.TestCase): - def test_skeleton_codestyle(self): - """ - This test shouldn't run in proliferated repositories. - """ - setup_cfg = configparser.ConfigParser() - setup_cfg.read("setup.cfg") - if setup_cfg["metadata"]["name"] != "skeleton": - return - - args = "venv/bin/black --check -l 100 setup.py etc tests" - try: - subprocess.check_output(args.split()) - except subprocess.CalledProcessError as e: - print("===========================================================") - print(e.output) - print("===========================================================") - raise Exception( - "Black style check failed; please format the code using:\n" - " python -m black -l 100 setup.py etc tests", - e.output, - ) from e diff --git a/tests/test_utils.py b/tests/test_utils.py index 00ff4d51..17e7e897 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -87,6 +87,7 @@ def test_parse_reqs(): check_json_file_results(result_file, expected_file) +@pytest.mark.online @pytest.mark.asyncio async def test_get_sdist_file(): sdist_file = await fetch_and_extract_sdist( diff --git a/tests/test_utils_pip_compatibility_tags.py b/tests/test_utils_pip_compatibility_tags.py index a492912c..69771352 100644 --- a/tests/test_utils_pip_compatibility_tags.py +++ b/tests/test_utils_pip_compatibility_tags.py @@ -1,4 +1,5 @@ -"""Generate and work with PEP 425 Compatibility Tags. +""" +Generate and work with PEP 425 Compatibility Tags. copied from pip-20.3.1 pip/tests/unit/test_utils_compatibility_tags.py download_url: https://raw.githubusercontent.com/pypa/pip/20.3.1/tests/unit/test_utils_compatibility_tags.py diff --git a/tests/test_utils_pypi.py b/tests/test_utils_pypi.py index de6d4db9..1932d9c1 100644 --- a/tests/test_utils_pypi.py +++ b/tests/test_utils_pypi.py @@ -13,7 +13,9 @@ import pytest +from python_inspector import utils_pypi from python_inspector.utils_pypi import Distribution +from python_inspector.utils_pypi import PypiPackage from python_inspector.utils_pypi import Sdist from python_inspector.utils_pypi import Wheel @@ -138,6 +140,15 @@ def check(self, using=Distribution): ), ] +linux_platforms = [ + "linux_x86_64", + "manylinux1_x86_64", + "manylinux2010_x86_64", + "manylinux2014_x86_64", + "manylinux_2_27_x86_64", + "manylinux_2_28_x86_64", +] + @pytest.mark.parametrize("dist_test", sdist_tests + wheel_tests) def test_Distribution_from_filename(dist_test): @@ -152,3 +163,14 @@ def test_Sdist_from_filename(dist_test): @pytest.mark.parametrize("dist_test", wheel_tests) def test_Wheel_from_filename(dist_test): dist_test.check(using=Wheel) + + +@pytest.mark.parametrize("linux_platform", linux_platforms) +def test_PypiPackage_get_supported_wheels(linux_platform): + whl = Wheel.from_filename(f"onnxruntime-1.19.2-cp311-cp311-{linux_platform}.whl") + pkg = PypiPackage.package_from_dists(dists=[whl]) + env = utils_pypi.Environment.from_pyver_and_os(python_version="311", operating_system="linux") + + supported_wheels = list(pkg.get_supported_wheels(environment=env)) + + assert supported_wheels == [whl]