Skip to content

Commit 988a063

Browse files
authored
Merge pull request #226 from aboutcode-org/215-lock
Introduce lockfile for download integrity
2 parents 1dcd761 + 829d2a5 commit 988a063

16 files changed

+960
-56
lines changed

requirements-dev.txt

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ cryptography==37.0.2
77
dataclasses==0.8
88
docutils==0.18.1
99
et-xmlfile==1.1.0
10-
execnet==1.9.0
10+
execnet==2.1.1
1111
importlib-resources==5.4.0
1212
iniconfig==1.1.1
1313
isort==5.10.1
@@ -21,14 +21,15 @@ openpyxl==3.0.10
2121
pathspec==0.9.0
2222
pkginfo==1.8.3
2323
platformdirs==2.4.0
24-
pluggy==1.0.0
24+
pluggy==1.6.0
2525
py==1.11.0
2626
pycodestyle==2.8.0
2727
pycparser==2.21
2828
pygments==2.12.0
29-
pytest==7.0.1
30-
pytest-forked==1.4.0
31-
pytest-xdist==2.5.0
29+
pytest==8.4.0
30+
pytest-xdist==3.7.0
31+
pytest-forked==1.6.0
32+
pytest-rerunfailures==15.1
3233
readme-renderer==34.0
3334
requests-toolbelt==0.9.1
3435
rfc3986==1.5.0
@@ -39,4 +40,4 @@ tqdm==4.64.0
3940
twine==3.8.0
4041
typed-ast==1.5.4
4142
webencodings==0.5.1
42-
pytest-asyncio==0.21.1
43+
pytest-asyncio==1.0.0

requirements.txt

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ click==8.1.3
66
colorama==0.4.5
77
commoncode==30.2.0
88
dparse2==0.7.0
9+
fasteners==0.17.3
910
idna==3.3
1011
importlib-metadata==4.12.0
1112
intbitset==3.1.0
@@ -26,5 +27,5 @@ text-unidecode==1.3
2627
toml==0.10.2
2728
urllib3==1.26.11
2829
zipp==3.8.1
29-
aiohttp==3.11.14
30-
aiofiles==23.2.1
30+
aiohttp==3.12.7
31+
aiofiles==24.1.0

setup.cfg

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ install_requires =
5959
colorama >= 0.3.9
6060
commoncode >= 30.0.0
6161
dparse2 >= 0.7.0
62+
fasteners >= 0.17.3
6263
importlib_metadata >= 4.12.0
6364
packageurl_python >= 0.9.0
6465
pkginfo2 >= 30.0.0

src/python_inspector/api.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@
2929
from _packagedcode.pypi import can_process_dependent_package
3030
from _packagedcode.pypi import get_resolved_purl
3131
from python_inspector import dependencies
32-
from python_inspector import pyinspector_settings as settings
32+
from python_inspector import pyinspector_settings
3333
from python_inspector import utils
3434
from python_inspector import utils_pypi
3535
from python_inspector.package_data import get_pypi_data_from_purl
@@ -82,7 +82,7 @@ def resolve_dependencies(
8282
specifiers=tuple(),
8383
python_version=None,
8484
operating_system=None,
85-
index_urls: tuple[str, ...] = settings.INDEX_URL,
85+
index_urls: tuple[str, ...] = pyinspector_settings.INDEX_URL,
8686
pdt_output=None,
8787
netrc_file=None,
8888
max_rounds=200000,
@@ -251,10 +251,10 @@ def resolve_dependencies(
251251
repos_by_url = {}
252252
if not use_pypi_json_api:
253253
# Collect PyPI repos
254-
use_only_confed = settings.USE_ONLY_CONFIGURED_INDEX_URLS
254+
use_only_confed = pyinspector_settings.USE_ONLY_CONFIGURED_INDEX_URLS
255255
for index_url in index_urls:
256256
index_url = index_url.strip("/")
257-
if use_only_confed and index_url not in settings.INDEX_URL:
257+
if use_only_confed and index_url not in pyinspector_settings.INDEX_URL:
258258
if verbose:
259259
printer(f"Skipping index URL unknown in settings: {index_url!r}")
260260
continue

src/python_inspector/lockfile.py

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
#
2+
# Copyright (c) nexB Inc. and others. All rights reserved.
3+
# ScanCode is a trademark of nexB Inc.
4+
# SPDX-License-Identifier: Apache-2.0
5+
# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
6+
# See https://github.com/nexB/scancode-toolkit for support or download.
7+
# See https://aboutcode.org for more information about nexB OSS projects.
8+
#
9+
10+
from contextlib import contextmanager
11+
12+
import fasteners
13+
14+
"""
15+
An interprocess lockfile with a timeout.
16+
"""
17+
18+
19+
class LockTimeout(Exception):
20+
pass
21+
22+
23+
class FileLock(fasteners.InterProcessLock):
24+
@contextmanager
25+
def locked(self, timeout):
26+
acquired = self.acquire(timeout=timeout)
27+
if not acquired:
28+
raise LockTimeout(timeout)
29+
try:
30+
yield
31+
finally:
32+
self.release()

src/python_inspector/resolve_cli.py

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -13,16 +13,17 @@
1313

1414
import click
1515

16-
from python_inspector import pyinspector_settings as settings
16+
from python_inspector import settings
1717
from python_inspector import utils_pypi
1818
from python_inspector.cli_utils import FileOptionType
1919
from python_inspector.utils import write_output_in_file
2020

2121
TRACE = False
2222

23-
__version__ = "0.13.0"
23+
__version__ = "0.14.0"
2424

2525
DEFAULT_PYTHON_VERSION = settings.DEFAULT_PYTHON_VERSION
26+
PYPI_SIMPLE_URL = settings.PYPI_SIMPLE_URL
2627

2728

2829
def print_version(ctx, param, value):
@@ -71,7 +72,6 @@ def print_version(ctx, param, value):
7172
"python_version",
7273
type=click.Choice(utils_pypi.valid_python_versions),
7374
metavar="PYVER",
74-
default=settings.DEFAULT_PYTHON_VERSION,
7575
show_default=True,
7676
required=True,
7777
help="Python version to use for dependency resolution. One of "
@@ -83,19 +83,18 @@ def print_version(ctx, param, value):
8383
"operating_system",
8484
type=click.Choice(utils_pypi.PLATFORMS_BY_OS),
8585
metavar="OS",
86-
default=settings.DEFAULT_OS,
8786
show_default=True,
8887
required=True,
8988
help="OS to use for dependency resolution. One of " + ", ".join(utils_pypi.PLATFORMS_BY_OS),
9089
)
9190
@click.option(
9291
"--index-url",
9392
"index_urls",
94-
envvar="PYINSP_INDEX_URL",
9593
type=str,
9694
metavar="INDEX",
9795
show_default=True,
98-
default=tuple(settings.INDEX_URL),
96+
# since multiple is True, this is a sequence
97+
default=[settings.PYPI_SIMPLE_URL],
9998
multiple=True,
10099
help="PyPI simple index URL(s) to use in order of preference. "
101100
"This option can be used multiple times.",
@@ -123,7 +122,6 @@ def print_version(ctx, param, value):
123122
"--netrc",
124123
"netrc_file",
125124
type=click.Path(exists=True, readable=True, path_type=str, dir_okay=False),
126-
envvar="PYINSP_NETRC_FILE",
127125
metavar="NETRC-FILE",
128126
hidden=True,
129127
required=False,
@@ -165,7 +163,6 @@ def print_version(ctx, param, value):
165163
)
166164
@click.option(
167165
"--verbose",
168-
envvar="PYINSP_VERBOSE",
169166
is_flag=True,
170167
help="Enable verbose debug output.",
171168
)

src/python_inspector/settings.py

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,9 @@
1313
from pydantic_settings import BaseSettings
1414
from pydantic_settings import SettingsConfigDict
1515

16+
DEFAULT_PYTHON_VERSION = "39"
17+
PYPI_SIMPLE_URL = "https://pypi.org/simple"
18+
1619

1720
class Settings(BaseSettings):
1821
"""
@@ -27,28 +30,30 @@ class Settings(BaseSettings):
2730
env_prefix="PYINSP_",
2831
case_sensitive=True,
2932
extra="allow",
33+
# never treat data as JSON
34+
enable_decoding=False,
3035
)
3136

3237
# the default Python version to use if none is provided
33-
DEFAULT_PYTHON_VERSION: str = "39"
38+
DEFAULT_PYTHON_VERSION: str = DEFAULT_PYTHON_VERSION
3439

3540
# the default OS to use if none is provided
3641
DEFAULT_OS: str = "linux"
3742

38-
# a list of PyPI simple index URLs. Use a JSON array to represent multiple URLs
39-
INDEX_URL: tuple[str, ...] = ("https://pypi.org/simple",)
43+
# a string with a tuple of PyPI simple index URLs, each separated by a space
44+
INDEX_URL: tuple[str, ...] = (PYPI_SIMPLE_URL,)
4045

4146
# If True, only uses configured INDEX_URLs listed above and ignore other URLs found in requirements
4247
USE_ONLY_CONFIGURED_INDEX_URLS: bool = False
4348

4449
# a path string where to store the cached downloads. Will be created if it does not exists.
4550
CACHE_THIRDPARTY_DIR: str = str(Path(Path.home() / ".cache/python_inspector"))
4651

47-
@field_validator("INDEX_URL")
52+
@field_validator("INDEX_URL", mode="before")
4853
@classmethod
4954
def validate_index_url(cls, value):
5055
if isinstance(value, str):
51-
return (value,)
56+
return tuple(value.split())
5257
elif isinstance(value, (tuple, list)):
5358
return tuple(value)
5459
else:

src/python_inspector/utils_pypi.py

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
from packvers import version as packaging_version
4242
from packvers.specifiers import SpecifierSet
4343

44+
from python_inspector import lockfile
4445
from python_inspector import pyinspector_settings as settings
4546
from python_inspector import utils_pip_compatibility_tags
4647

@@ -1650,6 +1651,8 @@ def resolve_relative_url(package_url, url):
16501651
#
16511652
################################################################################
16521653

1654+
PYINSP_CACHE_LOCK_TIMEOUT = 120 # in seconds
1655+
16531656

16541657
@attr.attributes
16551658
class Cache:
@@ -1681,6 +1684,7 @@ async def get(
16811684
True otherwise as treat as binary. `path_or_url` can be a path or a URL
16821685
to a file.
16831686
"""
1687+
# the cache key is a hash of the normalized path
16841688
cache_key = self.sha256_hash(quote_plus(path_or_url.strip("/")))
16851689
cached = os.path.join(self.directory, cache_key)
16861690

@@ -1695,8 +1699,13 @@ async def get(
16951699
echo_func=echo_func,
16961700
)
16971701
wmode = "w" if as_text else "wb"
1698-
async with aiofiles.open(cached, mode=wmode) as fo:
1699-
await fo.write(content)
1702+
1703+
# acquire lock and wait until timeout to get a lock or die
1704+
lock_file = os.path.join(self.directory, f"{cache_key}.lockfile")
1705+
1706+
with lockfile.FileLock(lock_file).locked(timeout=PYINSP_CACHE_LOCK_TIMEOUT):
1707+
async with aiofiles.open(cached, mode=wmode) as fo:
1708+
await fo.write(content)
17001709
return content, cached
17011710
else:
17021711
if TRACE_DEEP:

tests/data/pinned-pdt-requirements.txt-expected.json

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22
"headers": {
33
"tool_name": "python-inspector",
44
"tool_homepageurl": "https://github.com/aboutcode-org/python-inspector",
5-
"tool_version": "0.13.0",
65
"options": [
76
"--index-url https://pypi.org/simple",
87
"--json-pdt <file>",

tests/data/pinned-requirements.txt-expected.json

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22
"headers": {
33
"tool_name": "python-inspector",
44
"tool_homepageurl": "https://github.com/aboutcode-org/python-inspector",
5-
"tool_version": "0.13.0",
65
"options": [
76
"--index-url https://pypi.org/simple",
87
"--json <file>",

0 commit comments

Comments
 (0)