From 3e6896e3118cc1335232fa95e71ea112070406b3 Mon Sep 17 00:00:00 2001 From: Helio Chissini de Castro Date: Sat, 10 May 2025 17:11:17 +0200 Subject: [PATCH 1/3] feat: Add extended standard logger support CustomLogger to replace uneven print/trace solution. Add trace level support, set standard level to warning/error. Using --verbose or -v options, set the app logger to INFO, -vv set the app logger to DEBUG, -vvv set the app logger to TRACE status and last -vvvv to DEEP status. Signed-off-by: Helio Chissini de Castro --- .gitignore | 1 + src/python_inspector/__init__.py | 1 + src/python_inspector/logging.py | 106 ++++++++++++++++++++++++++++ src/python_inspector/resolve_cli.py | 22 ++++-- 4 files changed, 124 insertions(+), 6 deletions(-) create mode 100644 src/python_inspector/logging.py diff --git a/.gitignore b/.gitignore index d2eaf8f1..7f776707 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ pip3 # Python compiled files *.py[cod] # virtualenv and other misc bits +.venv /src/*.egg-info *.egg-info /dist diff --git a/src/python_inspector/__init__.py b/src/python_inspector/__init__.py index 8d70e3d6..52999540 100644 --- a/src/python_inspector/__init__.py +++ b/src/python_inspector/__init__.py @@ -13,3 +13,4 @@ pyinspector_settings = settings.Settings() settings.create_cache_directory(pyinspector_settings.CACHE_THIRDPARTY_DIR) + diff --git a/src/python_inspector/logging.py b/src/python_inspector/logging.py new file mode 100644 index 00000000..c56e65c9 --- /dev/null +++ b/src/python_inspector/logging.py @@ -0,0 +1,106 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/python-inspector for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import logging +from types import TracebackType +from typing import Any, Optional, Tuple, Type, Union + +# Add TRACE custom level to be third leve, as verbose will match +# -v == logLevel INFO +# -vv == logLevel DEBUG +# -vvv == logLevel TRACE +TRACE_LEVEL: int = 5 +DEEP_LEVEL: int = 4 + + +class CustomLogger(logging.Logger): + def trace( + self: logging.Logger, + msg: Any, + *args: Any, + exc_info: Union[ + bool, + BaseException, + Tuple[Type[BaseException], BaseException, TracebackType], + None, + ] = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: Optional[dict[str, Any]] = None, + ) -> None: + if self.isEnabledFor(TRACE_LEVEL): + self._log( + TRACE_LEVEL, + msg, + args, + exc_info=exc_info, + extra=extra, + stack_info=stack_info, + stacklevel=stacklevel, + ) + + def deep( + self: logging.Logger, + msg: Any, + *args: Any, + exc_info: Union[ + bool, + BaseException, + Tuple[Type[BaseException], BaseException, TracebackType], + None, + ] = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: Optional[dict[str, Any]] = None, + ) -> None: + if self.isEnabledFor(DEEP_LEVEL): + self._log( + DEEP_LEVEL, + msg, + args, + exc_info=exc_info, + extra=extra, + stack_info=stack_info, + stacklevel=stacklevel, + ) + + + logging.Logger.trace = trace + logging.Logger.deep = deep + + +def setup_logger(level: str = "WARNING") -> None: + """ + Configures the logger for the 'python-inspector' application. + + This function sets up a custom logging level, assigns a custom logger class, + and configures the logger with the specified logging level. If no handlers are present, + it adds a stream handler with a simple formatter. + + Args: + level (str): The logging level to set for the logger (e.g., 'DEBUG', 'INFO', 'WARNING', "TRACE"). + """ + # Setup out trace level + logging.addLevelName(TRACE_LEVEL, "TRACE") + logging.addLevelName(DEEP_LEVEL, "DEEP") + logging.setLoggerClass(CustomLogger) + + _logger = logging.getLogger("python-inspector") + _logger.setLevel(level) + _logger.propagate = False + + if not _logger.hasHandlers(): + handler = logging.StreamHandler() + formatter = logging.Formatter("[%(levelname)s] %(message)s") + handler.setFormatter(formatter) + _logger.addHandler(handler) + + +# Logger as a singleton +logger: logging.Logger = logging.getLogger("python-inspector") diff --git a/src/python_inspector/resolve_cli.py b/src/python_inspector/resolve_cli.py index 299625cc..70a909ba 100644 --- a/src/python_inspector/resolve_cli.py +++ b/src/python_inspector/resolve_cli.py @@ -13,8 +13,8 @@ import click -from python_inspector import settings -from python_inspector import utils_pypi +from python_inspector import logging, utils_pypi +from python_inspector import pyinspector_settings as settings from python_inspector.cli_utils import FileOptionType from python_inspector.utils import write_output_in_file @@ -162,9 +162,10 @@ def print_version(ctx, param, value): "distribution is available then binary distributions are used", ) @click.option( + "-v", "--verbose", - is_flag=True, - help="Enable verbose debug output.", + count=True, + help="Increase verbosity: -v=INFO, -vv=DEBUG, -vvv=TRACE.", ) @click.option( "-V", @@ -198,11 +199,11 @@ def resolve_dependencies( pdt_output, netrc_file, max_rounds, + verbose, use_cached_index=False, use_pypi_json_api=False, analyze_setup_py_insecurely=False, prefer_source=False, - verbose=TRACE, generic_paths=False, ignore_errors=False, ): @@ -237,6 +238,16 @@ def resolve_dependencies( click.secho("Only one of --json or --json-pdt can be used.", err=True) ctx.exit(1) + # Setup verbose level + if verbose >= 3: + logging.setup_logger("TRACE") + elif verbose == 2: + logging.setup_logger("DEBUG") + elif verbose == 1: + logging.setup_logger("INFO") + else: + logging.setup_logger() + options = get_pretty_options(ctx, generic_paths=generic_paths) notice = ( @@ -268,7 +279,6 @@ def resolve_dependencies( max_rounds=max_rounds, use_cached_index=use_cached_index, use_pypi_json_api=use_pypi_json_api, - verbose=verbose, analyze_setup_py_insecurely=analyze_setup_py_insecurely, printer=click.secho, prefer_source=prefer_source, From 3b60d740938df4f40c62fb184ea566c50fc830a9 Mon Sep 17 00:00:00 2001 From: Helio Chissini de Castro Date: Sat, 10 May 2025 17:53:49 +0200 Subject: [PATCH 2/3] feat: Replace old print/echo calls to new logger function Signed-off-by: Helio Chissini de Castro --- src/python_inspector/api.py | 59 +++---- src/python_inspector/dependencies.py | 9 +- src/python_inspector/resolution.py | 9 +- src/python_inspector/resolve_cli.py | 5 +- src/python_inspector/setup_py_live_eval.py | 17 +- src/python_inspector/utils_pypi.py | 177 ++++++--------------- 6 files changed, 88 insertions(+), 188 deletions(-) diff --git a/src/python_inspector/api.py b/src/python_inspector/api.py index 47b2f272..8aa7b622 100644 --- a/src/python_inspector/api.py +++ b/src/python_inspector/api.py @@ -9,6 +9,7 @@ # See https://aboutcode.org for more information about nexB OSS projects. # import asyncio +import logging import os from netrc import netrc from typing import Dict @@ -32,6 +33,7 @@ from python_inspector import pyinspector_settings from python_inspector import utils from python_inspector import utils_pypi +from python_inspector.logging import logger from python_inspector.package_data import get_pypi_data_from_purl from python_inspector.resolution import PythonInputProvider from python_inspector.resolution import format_pdt_tree @@ -88,10 +90,8 @@ def resolve_dependencies( max_rounds=200000, use_cached_index=False, use_pypi_json_api=False, - verbose=False, analyze_setup_py_insecurely=False, prefer_source=False, - printer=print, generic_paths=False, ignore_errors=False, ): @@ -124,8 +124,7 @@ def resolve_dependencies( f"Must be one of: {', '.join(valid_python_versions)}" ) - if verbose: - printer("Resolving dependencies...") + logger.info("Resolving dependencies...") if netrc_file: if not os.path.exists(netrc_file): @@ -139,8 +138,7 @@ def resolve_dependencies( netrc_file = None if netrc_file: - if verbose: - printer(f"Using netrc file {netrc_file}") + logger.info(f"Using netrc file {netrc_file}") parsed_netrc = netrc(netrc_file) else: parsed_netrc = None @@ -235,18 +233,17 @@ def resolve_dependencies( files=files, ) - if verbose: - printer("direct_dependencies:") + logger.info("direct_dependencies:") + if logger.level <= logging.INFO: for dep in direct_dependencies: - printer(f" {dep}") + logging.info(f" {dep}") # create a resolution environments environment = utils_pypi.Environment.from_pyver_and_os( python_version=python_version, operating_system=operating_system ) - if verbose: - printer(f"environment: {environment}") + logging.info(f"environment: {environment}") repos_by_url = {} if not use_pypi_json_api: @@ -254,9 +251,10 @@ def resolve_dependencies( use_only_confed = pyinspector_settings.USE_ONLY_CONFIGURED_INDEX_URLS for index_url in index_urls: index_url = index_url.strip("/") + if use_only_confed and index_url not in pyinspector_settings.INDEX_URL: - if verbose: - printer(f"Skipping index URL unknown in settings: {index_url!r}") + logger.info(f"Skipping index URL unknown in settings: {index_url!r}") + continue if index_url in repos_by_url: continue @@ -274,10 +272,10 @@ def resolve_dependencies( repos_by_url[index_url] = repo repos = repos_by_url.values() - if verbose: - printer("repos:") + logger.info("repos:") + if logger.level <= logging.INFO: for repo in repos: - printer(f" {repo}") + logger.info(f" {repo}") # resolve dependencies proper resolution, purls = resolve( @@ -289,8 +287,6 @@ def resolve_dependencies( pdt_output=pdt_output, analyze_setup_py_insecurely=analyze_setup_py_insecurely, ignore_errors=ignore_errors, - verbose=verbose, - printer=printer, ) async def gather_pypi_data(): @@ -299,20 +295,17 @@ async def get_pypi_data(package): package, repos=repos, environment=environment, prefer_source=prefer_source ) - if verbose: - printer(f" retrieved package '{package}'") + logger.info(f" retrieved package '{package}'") return data - if verbose: - printer(f"retrieve package data from pypi:") + logger.info(f"retrieve package data from pypi:") return await asyncio.gather(*[get_pypi_data(package) for package in purls]) packages = [pkg.to_dict() for pkg in asyncio.run(gather_pypi_data()) if pkg is not None] - if verbose: - printer("done!") + logger.info("done!") return Resolution( packages=packages, @@ -352,8 +345,6 @@ def resolve( pdt_output: bool = False, analyze_setup_py_insecurely: bool = False, ignore_errors: bool = False, - verbose: bool = False, - printer=print, ): """ Resolve dependencies given a ``direct_dependencies`` list of @@ -380,8 +371,6 @@ def resolve( pdt_output=pdt_output, analyze_setup_py_insecurely=analyze_setup_py_insecurely, ignore_errors=ignore_errors, - verbose=verbose, - printer=printer, ) return resolved_dependencies, packages @@ -396,8 +385,6 @@ def get_resolved_dependencies( pdt_output: bool = False, analyze_setup_py_insecurely: bool = False, ignore_errors: bool = False, - verbose: bool = False, - printer=print, ) -> Tuple[List[Dict], List[str]]: """ Return resolved dependencies of a ``requirements`` list of Requirement for @@ -420,13 +407,11 @@ async def gather_version_data(): async def get_version_data(name: str): versions = await provider.fill_versions_for_package(name) - if verbose: - printer(f" retrieved versions for package '{name}'") + logger.info(f" retrieved versions for package '{name}'") return versions - if verbose: - printer(f"versions:") + logger.info(f"versions:") return await asyncio.gather( *[get_version_data(requirement.name) for requirement in requirements] @@ -446,11 +431,9 @@ async def get_dependencies(requirement: Requirement): candidate = Candidate(requirement.name, purl.version, requirement.extras) await provider.fill_requirements_for_package(purl, candidate) - if verbose: - printer(f" retrieved dependencies for requirement '{str(purl)}'") + logger.info(f" retrieved dependencies for requirement '{str(purl)}'") - if verbose: - printer(f"dependencies:") + logger.info(f"dependencies:") return await asyncio.gather( *[get_dependencies(requirement) for requirement in requirements] diff --git a/src/python_inspector/dependencies.py b/src/python_inspector/dependencies.py index 0c491304..545262a5 100644 --- a/src/python_inspector/dependencies.py +++ b/src/python_inspector/dependencies.py @@ -19,6 +19,8 @@ from _packagedcode.pypi import PipRequirementsFileHandler from _packagedcode.pypi import get_requirements_txt_dependencies +from python_inspector.logging import logger + """ Utilities to resolve dependencies. """ @@ -36,10 +38,9 @@ def get_dependencies_from_requirements( location=requirements_file, include_nested=True ) for dependent_package in dependent_packages: - if TRACE: - print( - "dependent_package.extracted_requirement:", - dependent_package.extracted_requirement, + logger.debug( + "dependent_package.extracted_requirement: " + f"{dependent_package.extracted_requirement}", ) yield dependent_package diff --git a/src/python_inspector/resolution.py b/src/python_inspector/resolution.py index 39202374..5b59eb6d 100644 --- a/src/python_inspector/resolution.py +++ b/src/python_inspector/resolution.py @@ -43,6 +43,7 @@ from python_inspector import pyinspector_settings as settings from python_inspector import utils_pypi from python_inspector.error import NoVersionsFound +from python_inspector.logging import logger from python_inspector.setup_py_live_eval import iter_requirements from python_inspector.utils import Candidate from python_inspector.utils import contain_string @@ -330,8 +331,8 @@ def get_requirements_from_python_manifest( ) ] if len(setup_fct) > 1: - print( - f"Warning: identified multiple definitions of 'setup()' in {setup_py_location}, " + logger.warning( + f"Identified multiple definitions of 'setup()' in {setup_py_location}, " "defaulting to the first occurrence" ) setup_fct = setup_fct[0] @@ -340,8 +341,8 @@ def get_requirements_from_python_manifest( ] if install_requires: if len(install_requires) > 1: - print( - f"Warning: identified multiple definitions of 'install_requires' in " + logger.warning( + f"Identified multiple definitions of 'install_requires' in " "{setup_py_location}, defaulting to the first occurrence" ) install_requires = install_requires[0].elts diff --git a/src/python_inspector/resolve_cli.py b/src/python_inspector/resolve_cli.py index 70a909ba..55394b7d 100644 --- a/src/python_inspector/resolve_cli.py +++ b/src/python_inspector/resolve_cli.py @@ -239,7 +239,9 @@ def resolve_dependencies( ctx.exit(1) # Setup verbose level - if verbose >= 3: + if verbose >= 4: + logging.setup_logger("DEEP") + elif verbose == 3: logging.setup_logger("TRACE") elif verbose == 2: logging.setup_logger("DEBUG") @@ -280,7 +282,6 @@ def resolve_dependencies( use_cached_index=use_cached_index, use_pypi_json_api=use_pypi_json_api, analyze_setup_py_insecurely=analyze_setup_py_insecurely, - printer=click.secho, prefer_source=prefer_source, ignore_errors=ignore_errors, generic_paths=generic_paths, diff --git a/src/python_inspector/setup_py_live_eval.py b/src/python_inspector/setup_py_live_eval.py index e955f34e..01cd3c15 100755 --- a/src/python_inspector/setup_py_live_eval.py +++ b/src/python_inspector/setup_py_live_eval.py @@ -24,12 +24,13 @@ import setuptools from commoncode.command import pushd from packvers.requirements import Requirement +from python_inspector.logging import logger def minver_error(pkg_name): """Report error about missing minimum version constraint and exit.""" - print( - 'ERROR: specify minimal version of "{0}" using ">=" or "=="'.format(pkg_name), + logger.error( + 'Specify minimal version of "{0}" using ' '">=" or "=="'.format(pkg_name), file=sys.stderr, ) sys.exit(1) @@ -98,16 +99,16 @@ def iter_requirements(level, extras, setup_file): imports.append(name) setup_providers = [i for i in imports if i in ["distutils.core", "setuptools"]] if len(setup_providers) == 0: - print( - f"Warning: unable to recognize setup provider in {setup_file}: " + logger.warning( + f"Unable to recognize setup provider in {setup_file}: " "defaulting to 'distutils.core'." ) setup_provider = "distutils.core" elif len(setup_providers) == 1: setup_provider = setup_providers[0] else: - print( - f"Warning: ambiguous setup provider in {setup_file}: candidates are {setup_providers}" + logger.warning( + f"Ambiguous setup provider in {setup_file}: candidates are {setup_providers}" "defaulting to 'distutils.core'." ) setup_provider = "distutils.core" @@ -163,8 +164,8 @@ def iter_requirements(level, extras, setup_file): specs = pkg.specifier specs = {s.operator: s.version for s in specs._specs} if ((">=" in specs) and (">" in specs)) or (("<=" in specs) and ("<" in specs)): - print( - 'ERROR: Do not specify such weird constraints! ("{0}")'.format(pkg), + logger.error( + "Do not specify such weird constraints! " '("{0}")'.format(pkg), file=sys.stderr, ) sys.exit(1) diff --git a/src/python_inspector/utils_pypi.py b/src/python_inspector/utils_pypi.py index f2757cd6..65cf0888 100644 --- a/src/python_inspector/utils_pypi.py +++ b/src/python_inspector/utils_pypi.py @@ -44,6 +44,7 @@ from python_inspector import lockfile from python_inspector import pyinspector_settings as settings from python_inspector import utils_pip_compatibility_tags +from python_inspector.logging import logger """ Utilities to manage Python thirparty libraries source, binaries and metadata in @@ -105,10 +106,6 @@ """ -TRACE = False -TRACE_DEEP = False -TRACE_ULTRA_DEEP = False - # Supported environments PYTHON_VERSIONS = "27", "36", "37", "38", "39", "310", "311", "312", "313" @@ -224,8 +221,6 @@ async def download_wheel( environment, dest_dir=CACHE_THIRDPARTY_DIR, repos=tuple(), - verbose=False, - echo_func=None, python_version=DEFAULT_PYTHON_VERSION, ): """ @@ -236,8 +231,7 @@ async def download_wheel( Use the first PyPI simple repository from a list of ``repos`` that contains this wheel. """ - if TRACE_DEEP: - print(f" download_wheel: {name}=={version} for envt: {environment}") + logger.trace(f" download_wheel: {name}=={version} for envt: {environment}") if not repos: raise ValueError("download_wheel: missing repos") @@ -248,17 +242,14 @@ async def download_wheel( repo, name, version, environment, python_version ) if not supported_and_valid_wheels: - if TRACE_DEEP: - print( - f" download_wheel: No supported and valid wheel for {name}=={version}: {environment} " - ) + logger.trace( + f" download_wheel: No supported and valid wheel for {name}=={version}: {environment}" + ) continue for wheel in supported_and_valid_wheels: wheel.credentials = repo.credentials fetched_wheel_filename = await wheel.download( dest_dir=dest_dir, - verbose=verbose, - echo_func=echo_func, ) fetched_wheel_filenames.append(fetched_wheel_filename) @@ -271,22 +262,17 @@ async def download_wheel( async def get_valid_sdist(repo, name, version, python_version=DEFAULT_PYTHON_VERSION): package = await repo.get_package_version(name=name, version=version) if not package: - if TRACE_DEEP: - print( - print(f" get_valid_sdist: No package in {repo.index_url} for {name}=={version}") - ) + logger.trace(f" get_valid_sdist: No package in {repo.index_url} for {name}=={version}") return sdist = package.sdist if not sdist: - if TRACE_DEEP: - print(f" get_valid_sdist: No sdist for {name}=={version}") + logger.trace(f" get_valid_sdist: No sdist for {name}=={version}") return if not valid_python_version( python_requires=sdist.python_requires, python_version=python_version ): return - if TRACE_DEEP: - print(f" get_valid_sdist: Getting sdist from index (or cache): {sdist.download_url}") + logger.trace(f" get_valid_sdist: Getting sdist from index (or cache): {sdist.download_url}") return sdist @@ -298,17 +284,15 @@ async def get_supported_and_valid_wheels( """ package = await repo.get_package_version(name=name, version=version) if not package: - if TRACE_DEEP: - print( - f" get_supported_and_valid_wheels: No package in {repo.index_url} for {name}=={version}" - ) + logger.trace( + f" get_supported_and_valid_wheels: No package in {repo.index_url} for {name}=={version}" + ) return [] supported_wheels = list(package.get_supported_wheels(environment=environment)) if not supported_wheels: - if TRACE_DEEP: - print( - f" get_supported_and_valid_wheels: No supported wheel for {name}=={version}: {environment}" - ) + logger.trace( + f" get_supported_and_valid_wheels: No supported wheel for {name}=={version}: {environment}" + ) return [] wheels = [] for wheel in supported_wheels: @@ -316,9 +300,7 @@ async def get_supported_and_valid_wheels( python_requires=wheel.python_requires, python_version=python_version ): continue - if TRACE_DEEP: - durl = await wheel.download_url(repo) - print( + logger.trace( f""" get_supported_and_valid_wheels: Getting wheel from index (or cache): {durl}""" ) @@ -340,8 +322,6 @@ async def download_sdist( version, dest_dir=CACHE_THIRDPARTY_DIR, repos=tuple(), - verbose=False, - echo_func=None, python_version=DEFAULT_PYTHON_VERSION, ): """ @@ -351,8 +331,7 @@ async def download_sdist( Use the first PyPI simple repository from a list of ``repos`` that contains this sdist. """ - if TRACE: - print(f" download_sdist: {name}=={version}") + logger.debug(f" download_sdist: {name}=={version}") if not repos: raise ValueError("download_sdist: missing repos") @@ -362,13 +341,10 @@ async def download_sdist( for repo in repos: sdist = await get_valid_sdist(repo, name, version, python_version=python_version) if not sdist: - if TRACE_DEEP: - print(f" download_sdist: No valid sdist for {name}=={version}") + logger.trace(f" download_sdist: No valid sdist for {name}=={version}") continue fetched_sdist_filename = await sdist.download( dest_dir=dest_dir, - verbose=verbose, - echo_func=echo_func, ) if fetched_sdist_filename: @@ -629,8 +605,7 @@ async def get_best_download_url(self, repos=tuple()): for repo in repos: package = await repo.get_package_version(name=self.name, version=self.version) if not package: - if TRACE: - print( + logger.debug( f" get_best_download_url: {self.name}=={self.version} " f"not found in {repo.index_url}" ) @@ -639,26 +614,21 @@ async def get_best_download_url(self, repos=tuple()): if pypi_url: return pypi_url else: - if TRACE: - print( + logger.debug( f" get_best_download_url: {self.filename} not found in {repo.index_url}" ) async def download( self, dest_dir=CACHE_THIRDPARTY_DIR, - verbose=False, - echo_func=None, ): """ Download this distribution into `dest_dir` directory. Return the fetched filename. """ assert self.filename - if TRACE_DEEP: - print( - f"Fetching distribution of {self.name}=={self.version}:", - self.filename, + logger.trace( + f"Fetching distribution of {self.name}=={self.version}: {self.filename}", ) # FIXME: @@ -668,8 +638,6 @@ async def download( credentials=self.credentials, filename=self.filename, as_text=False, - verbose=verbose, - echo_func=echo_func, ) return self.filename @@ -782,7 +750,7 @@ def load_pkginfo_data(self, dest_dir=CACHE_THIRDPARTY_DIR): """ pkginfo_text = self.extract_pkginfo(dest_dir=dest_dir) if not pkginfo_text: - print(f"!!!!PKG-INFO/METADATA not found in {self.filename}") + logger.warning(f"PKG-INFO/METADATA not found in {self.filename}") return raw_data = email.message_from_string(pkginfo_text) @@ -834,7 +802,7 @@ def update(self, data, overwrite=False, keep_extra=True): purl_from_data = packageurl.PackageURL.from_string(package_url) purl_from_self = packageurl.PackageURL.from_string(self.package_url) if purl_from_data != purl_from_self: - print( + logger.warning( f"Invalid dist update attempt, no same same purl with dist: " f"{self} using data {data}." ) @@ -1118,10 +1086,9 @@ def is_supported_by_tags(self, tags): """ Return True is this wheel is compatible with one of a list of PEP 425 tags. """ - if TRACE_DEEP: - print() - print("is_supported_by_tags: tags:", tags) - print("self.tags:", self.tags) + logger.deep("") + logger.deep(f"is_supported_by_tags: {tags}") + logger.deep(f"self.tags: {self.tags}") return not self.tags.isdisjoint(tags) def to_filename(self): @@ -1190,14 +1157,13 @@ class PypiPackage(NameVer): metadata=dict(help="List of Wheel for this package"), ) - def get_supported_wheels(self, environment, verbose=TRACE_ULTRA_DEEP): + def get_supported_wheels(self, environment): """ Yield all the Wheel of this package supported and compatible with the Environment `environment`. """ envt_tags = environment.tags() - if verbose: - print("get_supported_wheels: envt_tags:", envt_tags) + logger.deep(f"get_supported_wheels: envt_tags: {envt_tags}") for wheel in self.wheels: if wheel.is_supported_by_tags(envt_tags): yield wheel @@ -1223,8 +1189,7 @@ def package_from_dists(cls, dists): >>> assert package.wheels == [w1, w2] """ dists = list(dists) - if TRACE_DEEP: - print(f"package_from_dists: {dists}") + logger.deep(f"package_from_dists: {dists}") if not dists: return @@ -1236,8 +1201,7 @@ def package_from_dists(cls, dists): for dist in dists: if dist.normalized_name != normalized_name: - if TRACE: - print( + logger.debug( f" Skipping inconsistent dist name: expected {normalized_name} got {dist}" ) continue @@ -1245,8 +1209,7 @@ def package_from_dists(cls, dists): dv = packaging_version.parse(dist.version) v = packaging_version.parse(version) if dv != v: - if TRACE: - print( + logger.debug( f" Skipping inconsistent dist version: expected {version} got {dist}" ) continue @@ -1260,8 +1223,7 @@ def package_from_dists(cls, dists): else: raise Exception(f"Unknown distribution type: {dist}") - if TRACE_DEEP: - print(f"package_from_dists: {package}") + logger.trace(f"package_from_dists: {package}") return package @@ -1272,8 +1234,7 @@ async def packages_from_links(cls, links: List[Link]): These are sorted by name and then by version from oldest to newest. """ dists = await PypiPackage.dists_from_links(links) - if TRACE_ULTRA_DEEP: - print("packages_from_many_paths_or_urls: dists:", dists) + logger.deep(f"packages_from_many_paths_or_urls: dists: {dists}") dists = NameVer.sorted(dists) @@ -1282,8 +1243,7 @@ async def packages_from_links(cls, links: List[Link]): key=NameVer.sortable_name_version, ): package = PypiPackage.package_from_dists(dists_of_package) - if TRACE_ULTRA_DEEP: - print("packages_from_many_paths_or_urls", package) + logger.trace(f"packages_from_many_paths_or_urls: {package}") yield package @classmethod @@ -1319,24 +1279,16 @@ async def dists_from_links(cls, links: List[Link]): Sdist bitarray 0.8.1 """ dists = [] - if TRACE_ULTRA_DEEP: - print(" ###paths_or_urls:", links) + logger.deep(f" ###paths_or_urls: {links}") installable: List[Link] = [link for link in links if link.url.endswith(EXTENSIONS)] for link in installable: try: dist = Distribution.from_link(link=link) dists.append(dist) - if TRACE_DEEP: - print( - " ===> dists_from_paths_or_urls:", - dist, - "\n ", - "from URL:", - link.url, - ) + logger.trace(f" ===> dists_from_paths_or_urls: {dist}") + logger.trace(f" from URL: {link.url}") except InvalidDistributionFilename: - if TRACE_DEEP: - print(f" Skipping invalid distribution from: {link.url}") + logger.trace(f" Skipping invalid distribution from: {link.url}") continue return dists @@ -1503,8 +1455,6 @@ class PypiSimpleRepository: async def _get_package_versions_map( self, name, - verbose=False, - echo_func=None, ): """ Return a mapping of all available PypiPackage version for this package name. @@ -1518,8 +1468,6 @@ async def _get_package_versions_map( try: links = await self.fetch_links( normalized_name=normalized_name, - verbose=verbose, - echo_func=echo_func, ) # note that this is sorted so the mapping is also sorted versions = { @@ -1528,19 +1476,16 @@ async def _get_package_versions_map( } self.packages[normalized_name] = versions except RemoteNotFetchedException as e: - if TRACE: - print(f"failed to fetch package name: {name} from: {self.index_url}:\n{e}") + logger.trace(f"failed to fetch package name: {name} from: {self.index_url}:\n{e}") - if not versions and TRACE: - print(f"WARNING: package {name} not found in repo: {self.index_url}") + if not versions: + logger.debug(f"Package {name} not found in repo: {self.index_url}") return versions async def get_package_versions( self, name, - verbose=False, - echo_func=None, ) -> Dict: """ Return a mapping of all available PypiPackage version as{version: @@ -1550,8 +1495,6 @@ async def get_package_versions( return dict( await self._get_package_versions_map( name=name, - verbose=verbose, - echo_func=echo_func, ) ) @@ -1559,8 +1502,6 @@ async def get_package_version( self, name, version=None, - verbose=False, - echo_func=None, ): """ Return the PypiPackage with name and version or None. @@ -1571,8 +1512,6 @@ async def get_package_version( ( await self._get_package_versions_map( name=name, - verbose=verbose, - echo_func=echo_func, ) ).values() ) @@ -1582,16 +1521,12 @@ async def get_package_version( return ( await self._get_package_versions_map( name=name, - verbose=verbose, - echo_func=echo_func, ) ).get(version) async def fetch_links( self, normalized_name, - verbose=False, - echo_func=None, ): """ Return a list of download link URLs found in a PyPI simple index for package @@ -1603,8 +1538,6 @@ async def fetch_links( credentials=self.credentials, as_text=True, force=not self.use_cached_index, - verbose=verbose, - echo_func=echo_func, ) soup = BeautifulSoup(text, features="html.parser") anchor_tags = soup.find_all("a") @@ -1675,8 +1608,6 @@ async def get( path_or_url, as_text=True, force=False, - verbose=False, - echo_func=None, ) -> Tuple[Union[str, bytes], str]: """ Return the content fetched from a ``path_or_url`` through the cache. @@ -1690,14 +1621,11 @@ async def get( lock_file = f"{cached}.lockfile" if force or not os.path.exists(cached): - if TRACE_DEEP: - print(f" FILE CACHE MISS: {path_or_url}") + logger.trace(f" FILE CACHE MISS: {path_or_url}") content = await get_file_content( path_or_url=path_or_url, credentials=credentials, as_text=as_text, - verbose=verbose, - echo_func=echo_func, ) wmode = "w" if as_text else "wb" @@ -1707,8 +1635,7 @@ async def get( await fo.write(content) return content, cached else: - if TRACE_DEEP: - print(f" FILE CACHE HIT: {path_or_url}") + logger.trace(f" FILE CACHE HIT: {path_or_url}") # also lock on read to avoid race conditions with lockfile.FileLock(lock_file).locked(timeout=PYINSP_CACHE_LOCK_TIMEOUT): return await get_local_file_content(path=cached, as_text=as_text), cached @@ -1721,22 +1648,17 @@ async def get_file_content( path_or_url, credentials, as_text=True, - verbose=False, - echo_func=None, ): """ Fetch and return the content at `path_or_url` from either a local path or a remote URL. Return the content as bytes is `as_text` is False. """ if path_or_url.startswith("https://"): - if TRACE_DEEP: - print(f"Fetching: {path_or_url}") + logger.trace(f"Fetching: {path_or_url}") _headers, content = await get_remote_file_content( url=path_or_url, credentials=credentials, as_text=as_text, - verbose=verbose, - echo_func=echo_func, ) return content @@ -1773,8 +1695,6 @@ async def get_remote_file_content( headers_only=False, headers=None, _delay=0, - verbose=False, - echo_func=None, ): """ Fetch and return a tuple of (headers, content) at `url`. Return content as a @@ -1790,10 +1710,7 @@ async def get_remote_file_content( # using a GET with stream=True ensure we get the final header from # several redirects and that we can ignore content there. A HEAD request may # not get us this last header - if verbose and not echo_func: - echo_func = print - if verbose: - echo_func(f"DOWNLOADING: {url}") + logger.info(f"DOWNLOADING: {url}") auth = None if credentials: @@ -1830,8 +1747,6 @@ async def fetch_and_save( filename, credentials, as_text=True, - verbose=False, - echo_func=None, ): """ Fetch content at ``path_or_url`` URL or path and save this to @@ -1843,8 +1758,6 @@ async def fetch_and_save( path_or_url=path_or_url, credentials=credentials, as_text=as_text, - verbose=verbose, - echo_func=echo_func, ) output = os.path.join(dest_dir, filename) From 0af65bdbf5c57e284b2201c08a842bbb22890342 Mon Sep 17 00:00:00 2001 From: Helio Chissini de Castro Date: Sat, 10 May 2025 18:39:49 +0200 Subject: [PATCH 3/3] fea: Add file handler for log output Signed-off-by: Helio Chissini de Castro --- src/python_inspector/logging.py | 8 +++++++- src/python_inspector/resolve_cli.py | 17 +++++++++++------ 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/python_inspector/logging.py b/src/python_inspector/logging.py index c56e65c9..90c13e41 100644 --- a/src/python_inspector/logging.py +++ b/src/python_inspector/logging.py @@ -8,6 +8,7 @@ # import logging +from pathlib import Path from types import TracebackType from typing import Any, Optional, Tuple, Type, Union @@ -75,7 +76,7 @@ def deep( logging.Logger.deep = deep -def setup_logger(level: str = "WARNING") -> None: +def setup_logger(level: str = "WARNING", log_file: Optional[Path] = None) -> None: """ Configures the logger for the 'python-inspector' application. @@ -101,6 +102,11 @@ def setup_logger(level: str = "WARNING") -> None: handler.setFormatter(formatter) _logger.addHandler(handler) + if log_file: + file_handler = logging.FileHandler(log_file, encoding="utf-8") + file_handler.setFormatter(formatter) + logger.addHandler(file_handler) + # Logger as a singleton logger: logging.Logger = logging.getLogger("python-inspector") diff --git a/src/python_inspector/resolve_cli.py b/src/python_inspector/resolve_cli.py index 55394b7d..e737feea 100644 --- a/src/python_inspector/resolve_cli.py +++ b/src/python_inspector/resolve_cli.py @@ -9,7 +9,8 @@ # See https://aboutcode.org for more information about nexB OSS projects. # -from typing import Dict +from pathlib import Path +from typing import Dict, Optional import click @@ -187,6 +188,9 @@ def print_version(ctx, param, value): help="Use generic or truncated paths in the JSON output header and files sections. " "Used only for testing to avoid absolute paths and paths changing at each run.", ) +@click.option( + "--log-file", type=click.Path(path_type=Path), help="Write logs to a file." +) def resolve_dependencies( ctx, requirement_files, @@ -200,6 +204,7 @@ def resolve_dependencies( netrc_file, max_rounds, verbose, + log_file: Optional[Path], use_cached_index=False, use_pypi_json_api=False, analyze_setup_py_insecurely=False, @@ -240,15 +245,15 @@ def resolve_dependencies( # Setup verbose level if verbose >= 4: - logging.setup_logger("DEEP") + logging.setup_logger("DEEP", log_file=log_file) elif verbose == 3: - logging.setup_logger("TRACE") + logging.setup_logger("TRACE", log_file=log_file) elif verbose == 2: - logging.setup_logger("DEBUG") + logging.setup_logger("DEBUG", log_file=log_file) elif verbose == 1: - logging.setup_logger("INFO") + logging.setup_logger("INFO", log_file=log_file) else: - logging.setup_logger() + logging.setup_logger(log_file=log_file) options = get_pretty_options(ctx, generic_paths=generic_paths)