Improve thirdparty scripts

Ensure that site-package dir exists.
Other minor adjustments from a scancode-toolkit release

Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>

Author: pombredanne

etc/scripts/fetch_thirdparty.py

@@ -18,7 +18,8 @@
 import utils_thirdparty
 import utils_requirements
 
-TRACE = True
+TRACE = False
+TRACE_DEEP = False
 
 
 @click.command()
@@ -204,7 +205,7 @@ def fetch_thirdparty(
                     existing_wheels = None
 
                 if existing_wheels:
-                    if TRACE:
+                    if TRACE_DEEP:
                         print(
                             f"====> Wheels already available: {name}=={version} on: {environment}: {existing_package.wheels!r}"
                         )
@@ -213,7 +214,7 @@ def fetch_thirdparty(
                     else:
                         continue
 
-                if TRACE:
+                if TRACE_DEEP:
                     print(f"Fetching wheel for: {name}=={version} on: {environment}")
 
                 try:

etc/scripts/utils_requirements.py

@@ -8,6 +8,7 @@
 # See https://github.com/nexB/skeleton for support or download.
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
+import os
 import re
 import subprocess
 
@@ -110,6 +111,8 @@ def get_installed_reqs(site_packages_dir):
     Return the installed pip requirements as text found in `site_packages_dir`
     as a text.
     """
+    if not os.path.exists(site_packages_dir):
+        raise Exception(f"site_packages directort: {site_packages_dir!r} does not exists")
     # Also include these packages in the output with --all: wheel, distribute,
     # setuptools, pip
     args = ["pip", "freeze", "--exclude-editable", "--all", "--path", site_packages_dir]

etc/scripts/utils_thirdparty.py

@@ -8,8 +8,8 @@
 # See https://github.com/nexB/skeleton for support or download.
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
-from collections import defaultdict
 import email
+import functools
 import itertools
 import os
 import re
@@ -18,6 +18,8 @@
 import tempfile
 import time
 import urllib
+from collections import defaultdict
+from urllib.parse import quote_plus
 
 import attr
 import license_expression
@@ -29,7 +31,6 @@
 from commoncode.text import python_safe_name
 from packaging import tags as packaging_tags
 from packaging import version as packaging_version
-from urllib.parse import quote_plus
 
 import utils_pip_compatibility_tags
 from utils_requirements import load_requirements
@@ -111,7 +112,7 @@
 
 """
 
-TRACE = True
+TRACE = False
 TRACE_DEEP = False
 TRACE_ULTRA_DEEP = False
 
@@ -233,7 +234,7 @@ def download_wheel(
     tuple of lists of (fetched_wheel_filenames, existing_wheel_filenames)
     """
     if TRACE_DEEP:
-        print(f"  download_wheel: {name}=={version}: {environment}")
+        print(f"  download_wheel: {name}=={version}: {environment} and index_urls: {index_urls}")
 
     fetched_wheel_filenames = []
     existing_wheel_filenames = []
@@ -311,6 +312,7 @@ def download_sdist(
     raise DistributionNotFound(f"Failed to fetch sdist: {name}=={version}: No sources found")
 
 
+@functools.cache
 def get_package_versions(
     name,
     version=None,
@@ -321,15 +323,28 @@ def get_package_versions(
     repository ``index_urls`` list of URLs.
     If ``version`` is not provided, return the latest available versions.
     """
+    found = []
+    not_found = []
     for index_url in index_urls:
         try:
             repo = get_pypi_repo(index_url)
             package = repo.get_package(name, version)
+
             if package:
-                yield package
+                found.append((package, index_url))
+            else:
+                not_found.append((name, version, index_url))
         except RemoteNotFetchedException as e:
-            print(f"Failed to fetch PyPI package {name} @ {version} info from {index_url}: {e}")
+            if TRACE_ULTRA_DEEP:
+                print(f"Failed to fetch PyPI package {name} @ {version} info from {index_url}: {e}")
+            not_found.append((name, version, index_url))
+
+    if not found:
+        raise Exception(f"No PyPI package {name} @ {version} found!")
 
+    for package, index_url in found:
+        print(f"Fetched PyPI package {package.name} @ {package.version} info from {index_url}")
+        yield package
 
 ################################################################################
 #
@@ -546,14 +561,14 @@ def get_best_download_url(
         If none is found, return a synthetic remote URL.
         """
         for index_url in index_urls:
-            pypi_package = get_pypi_package(
+            pypi_package = get_pypi_package_data(
                 name=self.normalized_name,
                 version=self.version,
                 index_url=index_url,
             )
             if pypi_package:
                 if isinstance(pypi_package, tuple):
-                    raise Exception("############", repr(pypi_package))
+                    raise Exception("############", repr(pypi_package), self.normalized_name, self.version, index_url)
                 try:
                     pypi_url = pypi_package.get_url_for_filename(self.filename)
                 except Exception as e:
@@ -1450,7 +1465,7 @@ def get_name_version(cls, name, version, packages):
         nvs = [p for p in cls.get_versions(name, packages) if p.version == version]
 
         if not nvs:
-            return name, version
+            return
 
         if len(nvs) == 1:
             return nvs[0]
@@ -1494,8 +1509,8 @@ def dists_from_paths_or_urls(cls, paths_or_urls):
         >>> assert expected == result
         """
         dists = []
-        if TRACE_DEEP:
-            print("   ###paths_or_urls:", paths_or_urls)
+        if TRACE_ULTRA_DEEP:
+            print("     ###paths_or_urls:", paths_or_urls)
         installable = [f for f in paths_or_urls if f.endswith(EXTENSIONS_INSTALLABLE)]
         for path_or_url in installable:
             try:
@@ -1618,7 +1633,6 @@ def tags(self):
             )
         )
 
-
 ################################################################################
 #
 # PyPI repo and link index for package wheels and sources
@@ -1657,7 +1671,10 @@ def get_versions(self, name):
         The list may be empty.
         """
         name = name and NameVer.normalize_name(name)
-        self._populate_links_and_packages(name)
+        try:
+            self._populate_links_and_packages(name)
+        except Exception as e:
+            print(f"        ==> Cannot find versions of {name}: {e}")
         return self.packages_by_normalized_name.get(name, [])
 
     def get_latest_version(self, name):
@@ -1703,7 +1720,7 @@ def _fetch_links(self, name, _LINKS={}):
             _LINKS[index_url] = [l for l in links if l.endswith(EXTENSIONS)]
 
         links = _LINKS[index_url]
-        if TRACE_DEEP:
+        if TRACE_ULTRA_DEEP:
             print(f"          Found links {links!r}")
         return links
 
@@ -1803,7 +1820,6 @@ def from_url(cls, url=ABOUT_BASE_URL, _LINKS_REPO={}):
             _LINKS_REPO[url] = cls(url=url)
         return _LINKS_REPO[url]
 
-
 ################################################################################
 # Globals for remote repos to be lazily created and cached on first use for the
 # life of the session together with some convenience functions.
@@ -1824,19 +1840,21 @@ def get_pypi_repo(index_url, _PYPI_REPO={}):
     return _PYPI_REPO[index_url]
 
 
-def get_pypi_package(name, version, index_url, verbose=TRACE_DEEP):
+@functools.cache
+def get_pypi_package_data(name, version, index_url, verbose=TRACE_DEEP):
     """
     Return a PypiPackage or None.
     """
     try:
+        if verbose:
+            print(f"    get_pypi_package_data: Fetching {name} @ {version} info from {index_url}")
         package = get_pypi_repo(index_url).get_package(name, version)
         if verbose:
-            print(f"    get_pypi_package: {name} @ {version} info from {index_url}: {package}")
+            print(f"      get_pypi_package_data: Fetched: {package}")
         return package
 
     except RemoteNotFetchedException as e:
-        print(f"Failed to fetch PyPI package {name} @ {version} info from {index_url}: {e}")
-
+        print(f"      get_pypi_package_data: Failed to fetch PyPI package {name} @ {version} info from {index_url}: {e}")
 
 ################################################################################
 #
@@ -1998,7 +2016,6 @@ def fetch_and_save_path_or_url(
         fo.write(content)
     return content
 
-
 ################################################################################
 # Requirements processing
 ################################################################################
@@ -2036,7 +2053,6 @@ def get_required_packages(
             print("  get_required_packages: name:", name, "version:", version)
         yield repo.get_package(name, version)
 
-
 ################################################################################
 # Functions to update or fetch ABOUT and license files
 ################################################################################
@@ -2115,7 +2131,6 @@ def get_other_dists(_package, _dist):
                 for p in packages_by_name[local_package.name]
                 if p.version != local_package.version
             ]
-
             other_local_version = other_local_packages and other_local_packages[-1]
             if other_local_version:
                 latest_local_dists = list(other_local_version.get_distributions())
@@ -2187,7 +2202,6 @@ def get_other_dists(_package, _dist):
                 lic_errs = "\n".join(lic_errs)
                 print(f"Failed to fetch some licenses:: {lic_errs}")
 
-
 ################################################################################
 #
 # Functions to build new Python wheels including native on multiple OSes
@@ -2318,9 +2332,9 @@ def build_wheels_locally_if_pure_python(
             "--wheel-dir",
             wheel_dir,
         ]
-        + deps
-        + verbose
-        + [requirements_specifier]
+        +deps
+        +verbose
+        +[requirements_specifier]
     )
 
     print(f"Building local wheels for: {requirements_specifier}")