Add children_packages m2m and rename resolved_to_package #1066 (#1252)

Signed-off-by: tdruez <tdruez@nexb.com>

Author: tdruez

CHANGELOG.rst

@@ -17,6 +17,14 @@ v34.6.0 (unreleased)
   Project error message.
   https://github.com/nexB/scancode.io/issues/1249
 
+- Rename DiscoveredDependency ``resolved_to`` to ``resolved_to_package``, and
+  ``resolved_dependencies`` to ``resolved_from_dependencies`` for clarity and
+  consistency.
+  Add ``children_packages`` and ``parent_packages`` ManyToMany field on the
+  DiscoveredPackage model.
+  Add full dependency tree in the CycloneDX output.
+  https://github.com/nexB/scancode.io/issues/1066
+
 v34.5.0 (2024-05-22)
 --------------------
 

scanpipe/api/serializers.py

@@ -392,7 +392,7 @@ class Meta:
 class DiscoveredDependencySerializer(serializers.ModelSerializer):
     purl = serializers.ReadOnlyField()
     for_package_uid = serializers.ReadOnlyField()
-    resolved_to_uid = serializers.ReadOnlyField()
+    resolved_to_package_uid = serializers.ReadOnlyField()
     datafile_path = serializers.ReadOnlyField()
     package_type = serializers.ReadOnlyField(source="type")
 
@@ -407,7 +407,7 @@ class Meta:
             "is_resolved",
             "dependency_uid",
             "for_package_uid",
-            "resolved_to_uid",
+            "resolved_to_package_uid",
             "datafile_path",
             "datasource_id",
             "package_type",

scanpipe/filters.py

@@ -748,7 +748,7 @@ class DependencyFilterSet(FilterSetUtilsMixin, django_filters.FilterSet):
             "is_optional",
             "is_resolved",
             "for_package",
-            "resolved_to",
+            "resolved_to_package",
             "datafile_resource",
             "datasource_id",
         ],

scanpipe/migrations/0060_discovereddependency_renames.py

@@ -0,0 +1,41 @@
+# Generated by Django 5.0.6 on 2024-06-03 11:32
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("scanpipe", "0059_alter_codebaseresource_status"),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name="discovereddependency",
+            old_name="resolved_to",
+            new_name="resolved_to_package",
+        ),
+        migrations.AlterField(
+            model_name="discovereddependency",
+            name="resolved_to_package",
+            field=models.ForeignKey(
+                blank=True,
+                editable=False,
+                help_text="The resolved package for this dependency. If empty, it indicates the dependency is unresolved.",
+                null=True,
+                on_delete=django.db.models.deletion.SET_NULL,
+                related_name="resolved_from_dependencies",
+                to="scanpipe.discoveredpackage",
+            ),
+        ),
+        migrations.AddField(
+            model_name="discoveredpackage",
+            name="children_packages",
+            field=models.ManyToManyField(
+                related_name="parent_packages",
+                through="scanpipe.DiscoveredDependency",
+                to="scanpipe.discoveredpackage",
+            ),
+        ),
+    ]

scanpipe/models.py

@@ -3034,6 +3034,13 @@ class DiscoveredPackage(
     codebase_resources = models.ManyToManyField(
         "CodebaseResource", related_name="discovered_packages"
     )
+    children_packages = models.ManyToManyField(
+        "self",
+        through="DiscoveredDependency",
+        symmetrical=False,
+        related_name="parent_packages",
+        through_fields=("for_package", "resolved_to_package"),
+    )
     missing_resources = models.JSONField(default=list, blank=True)
     modified_resources = models.JSONField(default=list, blank=True)
     package_uid = models.CharField(
@@ -3232,6 +3239,15 @@ def as_spdx(self):
             external_refs=external_refs,
         )
 
+    @property
+    def cyclonedx_bom_ref(self):
+        """
+        Use the package_uid when available to ensure having unique bom_ref
+        in the SBOM when several instances of the same DiscoveredPackage
+        (i.e. same purl) are present in the project.
+        """
+        return self.package_uid or str(self.get_package_url())
+
     def as_cyclonedx(self):
         """Return this DiscoveredPackage as an CycloneDX Component entry."""
         licenses = []
@@ -3298,17 +3314,12 @@ def as_cyclonedx(self):
                 ],
             )
 
-        package_url = self.get_package_url()
-        # Use the package_uid when available to ensure having unique bom_ref
-        # in the SBOM when several instances of the same DiscoveredPackage
-        # (i.e. same purl) are present in the project.
-        bom_ref = self.package_uid or str(package_url)
-
         return cyclonedx_component.Component(
             name=self.name,
             version=self.version,
-            bom_ref=bom_ref,
-            purl=package_url,  # Warning: Use the real purl and not package_uid here.
+            bom_ref=self.cyclonedx_bom_ref,
+            # Warning: Use the real purl and not package_uid here.
+            purl=self.get_package_url(),
             licenses=licenses,
             copyright=self.copyright,
             description=self.description,
@@ -3332,6 +3343,10 @@ def prefetch_for_serializer(self):
             Prefetch(
                 "for_package", queryset=DiscoveredPackage.objects.only("package_uid")
             ),
+            Prefetch(
+                "resolved_to_package",
+                queryset=DiscoveredPackage.objects.only("package_uid"),
+            ),
             Prefetch(
                 "datafile_resource", queryset=CodebaseResource.objects.only("path")
             ),
@@ -3373,9 +3388,9 @@ class DiscoveredDependency(
         blank=True,
         null=True,
     )
-    resolved_to = models.ForeignKey(
+    resolved_to_package = models.ForeignKey(
         DiscoveredPackage,
-        related_name="resolved_dependencies",
+        related_name="resolved_from_dependencies",
         help_text=_(
             "The resolved package for this dependency. "
             "If empty, it indicates the dependency is unresolved."
@@ -3468,9 +3483,9 @@ def for_package_uid(self):
             return self.for_package.package_uid
 
     @cached_property
-    def resolved_to_uid(self):
-        if self.resolved_to:
-            return self.resolved_to.package_uid
+    def resolved_to_package_uid(self):
+        if self.resolved_to_package:
+            return self.resolved_to_package.package_uid
 
     @cached_property
     def datafile_path(self):

scanpipe/pipes/output.py

@@ -675,23 +675,36 @@ def get_cyclonedx_bom(project):
         ],
     )
 
-    components = []
     vulnerabilities = []
-    for package in get_queryset(project, "discoveredpackage"):
+    dependencies = {}
+
+    package_qs = get_queryset(project, "discoveredpackage")
+    package_qs = package_qs.prefetch_related("children_packages")
+
+    for package in package_qs:
         component = package.as_cyclonedx()
-        components.append(component)
+        bom.components.add(component)
+        bom.register_dependency(project_as_root_component, [component])
+
+        # Store the component dependencies to be added later since all components need
+        # to be added on the BOM first.
+        dependencies[component] = [
+            package.cyclonedx_bom_ref for package in package.children_packages.all()
+        ]
 
         for vulnerability_data in package.affected_by_vulnerabilities:
             vulnerabilities.append(
-                vulnerability_as_cyclonedx(
-                    vulnerability_data=vulnerability_data,
-                    component_bom_ref=component.bom_ref,
-                )
+                vulnerability_as_cyclonedx(vulnerability_data, component.bom_ref)
             )
 
-    for component in components:
-        bom.components.add(component)
-        bom.register_dependency(project_as_root_component, [component])
+    for component, depends_on_bom_refs in dependencies.items():
+        if not depends_on_bom_refs:
+            continue
+        # Craft disposable Component instances for registering dependencies
+        dependencies = [
+            cdx_component.Component(name="", bom_ref=ref) for ref in depends_on_bom_refs
+        ]
+        bom.register_dependency(component, dependencies)
 
     bom.vulnerabilities = vulnerabilities
 

scanpipe/templates/scanpipe/dependency_list.html

@@ -55,9 +55,9 @@
               {% endif %}
             </td>
             <td>
-              {% if dependency.resolved_to %}
+              {% if dependency.resolved_to_package %}
                 {# CAUTION: Avoid relying on get_absolute_url to prevent unnecessary query triggers #}
-                <a href="{% url 'package_detail' project.slug dependency.for_package.uuid %}" title="{{ dependency.resolved_to.purl }}">{{ dependency.resolved_to.purl }}</a>
+                <a href="{% url 'package_detail' project.slug dependency.resolved_to_package.uuid %}" title="{{ dependency.resolved_to_package.purl }}">{{ dependency.resolved_to_package.purl }}</a>
               {% endif %}
             </td>
             <td>

scanpipe/tests/__init__.py

@@ -27,6 +27,8 @@
 from django.apps import apps
 
 from scanpipe.models import CodebaseResource
+from scanpipe.models import DiscoveredDependency
+from scanpipe.models import DiscoveredPackage
 from scanpipe.tests.pipelines.do_nothing import DoNothing
 from scanpipe.tests.pipelines.profile_step import ProfileStep
 from scanpipe.tests.pipelines.raise_exception import RaiseException
@@ -65,6 +67,17 @@ def make_resource_directory(project, path, **extra):
     )
 
 
+def make_package(project, package_url, **extra):
+    package = DiscoveredPackage(project=project, **extra)
+    package.set_package_url(package_url)
+    package.save()
+    return package
+
+
+def make_dependency(project, **extra):
+    return DiscoveredDependency.objects.create(project=project, **extra)
+
+
 resource_data1 = {
     "path": "notice.NOTICE",
     "type": "file",

scanpipe/tests/data/asgiref-3.3.0_load_inventory_expected.json

@@ -277,7 +277,7 @@
       "is_resolved": false,
       "dependency_uid": "pkg:pypi/pytest?uuid=fixed-uid-done-for-testing-5642512d1758",
       "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=fixed-uid-done-for-testing-5642512d1758",
-      "resolved_to_uid": null,
+      "resolved_to_package_uid": null,
       "datafile_path": "asgiref-3.3.0-py3-none-any.whl",
       "datasource_id": "pypi_wheel",
       "package_type": "pypi",
@@ -292,7 +292,7 @@
       "is_resolved": false,
       "dependency_uid": "pkg:pypi/pytest?uuid=fixed-uid-done-for-testing-5642512d1758",
       "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=fixed-uid-done-for-testing-5642512d1758",
-      "resolved_to_uid": null,
+      "resolved_to_package_uid": null,
       "datafile_path": "asgiref-3.3.0-py3-none-any.whl-extract/asgiref-3.3.0.dist-info/METADATA",
       "datasource_id": "pypi_wheel_metadata",
       "package_type": "pypi",
@@ -307,7 +307,7 @@
       "is_resolved": false,
       "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=fixed-uid-done-for-testing-5642512d1758",
       "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=fixed-uid-done-for-testing-5642512d1758",
-      "resolved_to_uid": null,
+      "resolved_to_package_uid": null,
       "datafile_path": "asgiref-3.3.0-py3-none-any.whl",
       "datasource_id": "pypi_wheel",
       "package_type": "pypi",
@@ -322,7 +322,7 @@
       "is_resolved": false,
       "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=fixed-uid-done-for-testing-5642512d1758",
       "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=fixed-uid-done-for-testing-5642512d1758",
-      "resolved_to_uid": null,
+      "resolved_to_package_uid": null,
       "datafile_path": "asgiref-3.3.0-py3-none-any.whl-extract/asgiref-3.3.0.dist-info/METADATA",
       "datasource_id": "pypi_wheel_metadata",
       "package_type": "pypi",

scanpipe/tests/data/daglib-0.6.0-py3-none-any.whl_scan_codebase.json

@@ -257,7 +257,7 @@
       "is_resolved": false,
       "dependency_uid": "pkg:pypi/dask?uuid=fixed-uid-done-for-testing-5642512d1758",
       "for_package_uid": "pkg:pypi/daglib@0.6.0?uuid=fixed-uid-done-for-testing-5642512d1758",
-      "resolved_to_uid": null,
+      "resolved_to_package_uid": null,
       "datafile_path": "daglib-0.6.0-py3-none-any.whl",
       "datasource_id": "pypi_wheel",
       "package_type": "pypi",
@@ -272,7 +272,7 @@
       "is_resolved": false,
       "dependency_uid": "pkg:pypi/dask?uuid=fixed-uid-done-for-testing-5642512d1758",
       "for_package_uid": "pkg:pypi/daglib@0.6.0?uuid=fixed-uid-done-for-testing-5642512d1758",
-      "resolved_to_uid": null,
+      "resolved_to_package_uid": null,
       "datafile_path": "daglib-0.6.0-py3-none-any.whl-extract/daglib-0.6.0.dist-info/METADATA",
       "datasource_id": "pypi_wheel_metadata",
       "package_type": "pypi",
@@ -287,7 +287,7 @@
       "is_resolved": false,
       "dependency_uid": "pkg:pypi/graphviz?uuid=fixed-uid-done-for-testing-5642512d1758",
       "for_package_uid": "pkg:pypi/daglib@0.6.0?uuid=fixed-uid-done-for-testing-5642512d1758",
-      "resolved_to_uid": null,
+      "resolved_to_package_uid": null,
       "datafile_path": "daglib-0.6.0-py3-none-any.whl",
       "datasource_id": "pypi_wheel",
       "package_type": "pypi",
@@ -302,7 +302,7 @@
       "is_resolved": false,
       "dependency_uid": "pkg:pypi/graphviz?uuid=fixed-uid-done-for-testing-5642512d1758",
       "for_package_uid": "pkg:pypi/daglib@0.6.0?uuid=fixed-uid-done-for-testing-5642512d1758",
-      "resolved_to_uid": null,
+      "resolved_to_package_uid": null,
       "datafile_path": "daglib-0.6.0-py3-none-any.whl-extract/daglib-0.6.0.dist-info/METADATA",
       "datasource_id": "pypi_wheel_metadata",
       "package_type": "pypi",
@@ -317,7 +317,7 @@
       "is_resolved": false,
       "dependency_uid": "pkg:pypi/ipycytoscape?uuid=fixed-uid-done-for-testing-5642512d1758",
       "for_package_uid": "pkg:pypi/daglib@0.6.0?uuid=fixed-uid-done-for-testing-5642512d1758",
-      "resolved_to_uid": null,
+      "resolved_to_package_uid": null,
       "datafile_path": "daglib-0.6.0-py3-none-any.whl",
       "datasource_id": "pypi_wheel",
       "package_type": "pypi",
@@ -332,7 +332,7 @@
       "is_resolved": false,
       "dependency_uid": "pkg:pypi/ipycytoscape?uuid=fixed-uid-done-for-testing-5642512d1758",
       "for_package_uid": "pkg:pypi/daglib@0.6.0?uuid=fixed-uid-done-for-testing-5642512d1758",
-      "resolved_to_uid": null,
+      "resolved_to_package_uid": null,
       "datafile_path": "daglib-0.6.0-py3-none-any.whl-extract/daglib-0.6.0.dist-info/METADATA",
       "datasource_id": "pypi_wheel_metadata",
       "package_type": "pypi",
@@ -347,7 +347,7 @@
       "is_resolved": false,
       "dependency_uid": "pkg:pypi/networkx?uuid=fixed-uid-done-for-testing-5642512d1758",
       "for_package_uid": "pkg:pypi/daglib@0.6.0?uuid=fixed-uid-done-for-testing-5642512d1758",
-      "resolved_to_uid": null,
+      "resolved_to_package_uid": null,
       "datafile_path": "daglib-0.6.0-py3-none-any.whl",
       "datasource_id": "pypi_wheel",
       "package_type": "pypi",
@@ -362,7 +362,7 @@
       "is_resolved": false,
       "dependency_uid": "pkg:pypi/networkx?uuid=fixed-uid-done-for-testing-5642512d1758",
       "for_package_uid": "pkg:pypi/daglib@0.6.0?uuid=fixed-uid-done-for-testing-5642512d1758",
-      "resolved_to_uid": null,
+      "resolved_to_package_uid": null,
       "datafile_path": "daglib-0.6.0-py3-none-any.whl-extract/daglib-0.6.0.dist-info/METADATA",
       "datasource_id": "pypi_wheel_metadata",
       "package_type": "pypi",