Associate resolved packages with their source codebase resource #1140 (#1141)

Signed-off-by: tdruez <tdruez@nexb.com>

Author: tdruez

CHANGELOG.rst

@@ -1,6 +1,12 @@
 Changelog
 =========
 
+v34.3.0 (unreleased)
+--------------------
+
+- Associate resolved packages with their source codebase resource.
+  https://github.com/nexB/scancode.io/issues/1140
+
 v34.2.0 (2024-03-28)
 --------------------
 

scanpipe/pipes/__init__.py

@@ -192,7 +192,7 @@ def update_or_create_package(project, package_data, codebase_resources=None):
     else:
         package = DiscoveredPackage.create_from_data(project, package_data)
 
-    if codebase_resources:
+    if package and codebase_resources:
         package.add_resources(codebase_resources)
 
     return package

scanpipe/pipes/resolve.py

@@ -58,10 +58,12 @@ def get_packages(project, package_registry, manifest_resources, model=None):
 
     for resource in manifest_resources:
         if packages := get_packages_from_manifest(resource.location, package_registry):
+            for package_data in packages:
+                package_data["codebase_resources"] = [resource]
             resolved_packages.extend(packages)
         else:
             project.add_error(
-                description="No packages could be resolved for",
+                description="No packages could be resolved",
                 model=model,
                 resource=resource,
             )
@@ -80,7 +82,8 @@ def create_packages_and_dependencies(project, packages, resolved=False):
     for package_data in packages:
         package_data = set_license_expression(package_data)
         dependencies = package_data.pop("dependencies", [])
-        update_or_create_package(project, package_data)
+        codebase_resources = package_data.pop("codebase_resources", [])
+        update_or_create_package(project, package_data, codebase_resources)
 
         for dependency_data in dependencies:
             if resolved:

scanpipe/tests/pipes/test_pipes.py

@@ -116,6 +116,9 @@ def test_scanpipe_pipes_update_or_create_package(self):
         self.assertIn(package2, resource1.discovered_packages.all())
         self.assertIn(package2, resource2.discovered_packages.all())
 
+        # Make sure the following does not raise an exception
+        self.assertIsNone(pipes.update_or_create_package(p1, {}, [resource1]))
+
     def test_scanpipe_pipes_update_or_create_package_codebase_resources(self):
         p1 = Project.objects.create(name="Analysis")
         resource1 = make_resource_file(project=p1, path="filename.ext")

scanpipe/tests/pipes/test_resolve.py

@@ -196,7 +196,10 @@ def test_scanpipe_resolve_get_packages_from_sbom(self):
             resources,
         )
         self.assertEqual(1, len(packages))
-        self.assertEqual("toml", packages[0]["name"])
+        package = packages[0]
+        self.assertEqual("toml", package["name"])
+        resource1 = project1.codebaseresources.get(name="toml.spdx.json")
+        self.assertEqual([resource1], package.get("codebase_resources"))
 
     def test_scanpipe_resolve_create_packages_and_dependencies(self):
         project1 = Project.objects.create(name="Analysis")
@@ -214,3 +217,7 @@ def test_scanpipe_resolve_create_packages_and_dependencies(self):
         resolve.create_packages_and_dependencies(project1, packages)
         self.assertEqual(1, project1.discoveredpackages.count())
         self.assertEqual(0, project1.discovereddependencies.count())
+
+        resource1 = project1.codebaseresources.get(name="toml.spdx.json")
+        package = project1.discoveredpackages.get()
+        self.assertEqual(resource1, package.codebase_resources.get())

scanpipe/tests/test_pipelines.py

@@ -915,7 +915,7 @@ def test_scanpipe_resolve_dependencies_pipeline_integration_empty_manifest(self)
         self.assertEqual(1, project1.projectmessages.count())
         message = project1.projectmessages.get()
         self.assertEqual("get_packages_from_manifest", message.model)
-        expected = "No packages could be resolved for"
+        expected = "No packages could be resolved"
         self.assertIn(expected, message.description)
 
     def test_scanpipe_resolve_dependencies_pipeline_integration_misc(self):