Release 34.6.1 (#1260)

* Remove print statements from migration files.

Signed-off-by: tdruez <tdruez@nexb.com>

* Display full traceback on error in execute command

Signed-off-by: tdruez <tdruez@nexb.com>

* Refactor the ``get_env_from_config_file`` to support empty config file

Signed-off-by: tdruez <tdruez@nexb.com>

---------

Signed-off-by: tdruez <tdruez@nexb.com>

Author: tdruez

CHANGELOG.rst

@@ -1,6 +1,14 @@
 Changelog
 =========
 
+v34.6.1 (2024-06-07)
+--------------------
+
+- Remove print statements from migration files.
+- Display full traceback on error in the ``execute`` management command.
+- Log the Project message creation.
+- Refactor the ``get_env_from_config_file`` to support empty config file.
+
 v34.6.0 (2024-06-07)
 --------------------
 

scancodeio/__init__.py

@@ -28,7 +28,7 @@
 
 import git
 
-VERSION = "34.6.0"
+VERSION = "34.6.1"
 
 PROJECT_DIR = Path(__file__).resolve().parent
 ROOT_DIR = PROJECT_DIR.parent

scanpipe/management/commands/__init__.py

@@ -22,6 +22,7 @@
 
 import shutil
 import sys
+import traceback
 from pathlib import Path
 
 from django.apps import apps
@@ -405,9 +406,10 @@ def execute_project(project, run_async=False, command=None):  # noqa: C901
     except KeyboardInterrupt:
         run.set_task_stopped()
         raise CommandError("Pipeline execution stopped.")
-    except Exception as e:
-        run.set_task_ended(exitcode=1, output=str(e))
-        raise CommandError(e)
+    except Exception:
+        traceback_str = traceback.format_exc()
+        run.set_task_ended(exitcode=1, output=traceback_str)
+        raise CommandError(traceback_str)
 
     run.refresh_from_db()
 

scanpipe/migrations/0031_scancode_toolkit_v32_data_updates.py

@@ -1,9 +1,13 @@
 # Generated by Django 4.2 on 2023-05-05 10:11
 
+import logging
+
 from django.db import migrations
 from django.db.models import Q
 from django.conf import settings
 
+logger = logging.getLogger("django")
+
 
 def compute_package_declared_license_expression_spdx(apps, schema_editor):
     """
@@ -21,7 +25,7 @@ def compute_package_declared_license_expression_spdx(apps, schema_editor):
     ).only("declared_license_expression")
 
     object_count = queryset.count()
-    print(f"\nCompute declared_license_expression_spdx for {object_count:,} packages.")
+    logger.info(f"\nCompute declared_license_expression_spdx for {object_count:,} packages.")
 
     chunk_size = 2000
     iterator = queryset.iterator(chunk_size=chunk_size)
@@ -33,9 +37,9 @@ def compute_package_declared_license_expression_spdx(apps, schema_editor):
             unsaved_objects.append(package)
 
         if not (index % chunk_size) and unsaved_objects:
-            print(f"  {index:,} / {object_count:,} computed")
+            logger.info(f"  {index:,} / {object_count:,} computed")
 
-    print("Updating DB objects...")
+    logger.info("Updating DB objects...")
     DiscoveredPackage.objects.bulk_update(
         objs=unsaved_objects,
         fields=["declared_license_expression_spdx"],
@@ -61,7 +65,7 @@ def compute_resource_detected_license_expression(apps, schema_editor):
     )
 
     object_count = queryset.count()
-    print(f"\nCompute detected_license_expression for {object_count:,} resources.")
+    logger.info(f"\nCompute detected_license_expression for {object_count:,} resources.")
 
     chunk_size = 2000
     iterator = queryset.iterator(chunk_size=chunk_size)
@@ -88,9 +92,9 @@ def compute_resource_detected_license_expression(apps, schema_editor):
         unsaved_objects.append(resource)
 
         if not (index % chunk_size) and unsaved_objects:
-            print(f"  {index:,} / {object_count:,} computed")
+            logger.info(f"  {index:,} / {object_count:,} computed")
 
-    print("Updating DB objects...")
+    logger.info("Updating DB objects...")
     CodebaseResource.objects.bulk_update(
         objs=unsaved_objects,
         fields=[
@@ -164,7 +168,7 @@ def compute_resource_license_detections(apps, schema_editor):
     queryset = CodebaseResource.objects.filter(~Q(licenses=[])).only("licenses")
 
     object_count = queryset.count()
-    print(f"\nCompute license_detections for {object_count:,} resources.")
+    logger.info(f"\nCompute license_detections for {object_count:,} resources.")
 
     chunk_size = 2000
     iterator = queryset.iterator(chunk_size=chunk_size)
@@ -176,9 +180,9 @@ def compute_resource_license_detections(apps, schema_editor):
         unsaved_objects.append(resource)
 
         if not (index % chunk_size):
-            print(f"  {index:,} / {object_count:,} computed")
+            logger.info(f"  {index:,} / {object_count:,} computed")
 
-    print("Updating DB objects...")
+    logger.info("Updating DB objects...")
     # Keeping the batch_size small as the `license_detections` content is often large,
     # and it may raise `django.db.utils.OperationalError: out of memory`
     CodebaseResource.objects.bulk_update(

scanpipe/migrations/0055_discoveredpackage_datafile_paths.py

@@ -1,8 +1,12 @@
 # Generated by Django 5.0.2 on 2024-03-01 16:09
 
+import logging
+
 from django.db import migrations, models
 from django.db.models import Q
 
+logger = logging.getLogger("django")
+
 
 def update_package_datasource_ids(apps, schema_editor):
     """
@@ -14,7 +18,7 @@ def update_package_datasource_ids(apps, schema_editor):
 
     object_count = queryset.count()
     if object_count:
-        print(f"\nCompute datasource_ids for {object_count:,} packages.")
+        logger.info(f"\nCompute datasource_ids for {object_count:,} packages.")
 
     chunk_size = 2000
     iterator = queryset.iterator(chunk_size=chunk_size)
@@ -26,7 +30,7 @@ def update_package_datasource_ids(apps, schema_editor):
         unsaved_objects.append(package)
 
         if not (index % chunk_size) and unsaved_objects:
-            print(f"  {index:,} / {object_count:,} computed")
+            logger.info(f"  {index:,} / {object_count:,} computed")
 
     DiscoveredPackage.objects.bulk_update(
         objs=unsaved_objects,

scanpipe/models.py

@@ -804,6 +804,21 @@ def get_enabled_settings(self):
             if value not in EMPTY_VALUES
         }
 
+    def get_env_from_config_file(self):
+        """Return ``env`` dict loaded from the ``scancode-config.yml`` config file."""
+        config_file = self.get_input_config_file()
+        if not config_file:
+            return
+
+        logger.info(f"Loading env from {config_file.relative_to(self.work_path)}")
+        try:
+            return saneyaml.load(config_file.read_text())
+        except (saneyaml.YAMLError, Exception):
+            self.add_error(
+                f'Failed to load configuration from "{config_file}". '
+                f"The file format is invalid."
+            )
+
     def get_env(self, field_name=None):
         """
         Return the project environment loaded from the ``scancode-config.yml`` config
@@ -814,15 +829,8 @@ def get_env(self, field_name=None):
         env = {}
 
         # 1. Load settings from config file when available.
-        if config_file := self.get_input_config_file():
-            logger.info(f"Loading env from {config_file.relative_to(self.work_path)}")
-            try:
-                env = saneyaml.load(config_file.read_text())
-            except saneyaml.YAMLError:
-                self.add_error(
-                    f'Failed to load configuration from "{config_file}". '
-                    f"The file format is invalid."
-                )
+        if env_from_config_file := self.get_env_from_config_file():
+            env = env_from_config_file
 
         # 2. Update with defined values from the Project ``settings`` field.
         env.update(self.get_enabled_settings())
@@ -1173,6 +1181,8 @@ def add_message(
         A ``resource`` can be provided to keep track of the codebase resource that was
         analyzed when the error occurred.
         """
+        logger.info(f"[{severity}] {description}")
+
         if inspect.isclass(model):
             model = model.__name__
 

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = scancodeio
-version = 34.6.0
+version = 34.6.1
 license = Apache-2.0
 description = Automate software composition analysis pipelines
 long_description = file:README.rst
@@ -170,7 +170,7 @@ ignore = D1,D203,D205,D212,D400,D415
 
 [bumpver]
 version_pattern = "MAJOR.MINOR.PATCH"
-current_version = "34.6.0"
+current_version = "34.6.1"
 
 [bumpver:file_patterns]
 setup.cfg =