Add full test coverage for the enrich_with_purldb Pipeline (#1331)

Signed-off-by: tdruez <tdruez@nexb.com>

Author: tdruez

CHANGELOG.rst

@@ -4,6 +4,10 @@ Changelog
 v34.7.2 (unreleased)
 --------------------
 
+- Add a new ``enrich_with_purldb`` add-on pipeline to enrich the discovered packages
+  with data available in the PurlDB.
+  https://github.com/nexB/scancode.io/issues/1182
+
 - Add the ability to define a results_url on the Pipeline class.
   When available, that link is displayed in the UI to easily reach the results view
   related to the Pipeline run.

docs/built-in-pipelines.rst

@@ -74,6 +74,20 @@ Collect symbols and string with Tree-Sitter (addon)
     :members:
     :member-order: bysource
 
+.. _pipeline_enrich_with_purldb:
+
+Enrich With PurlDB (addon)
+--------------------------
+
+.. warning::
+    This pipeline requires access to a PurlDB service.
+    Refer to :ref:`scancodeio_settings_purldb` to configure access to PurlDB in your
+    ScanCode.io instance.
+
+.. autoclass:: scanpipe.pipelines.enrich_with_purldb.EnrichWithPurlDB()
+    :members:
+    :member-order: bysource
+
 .. _pipeline_find_vulnerabilities:
 
 Find Vulnerabilities (addon)

scanpipe/pipelines/enrich_with_purldb.py

@@ -29,6 +29,7 @@ class EnrichWithPurlDB(Pipeline):
 
     download_inputs = False
     is_addon = True
+    results_url = "/project/{slug}/packages/?extra_data=" + purldb.ENRICH_EXTRA_DATA_KEY
 
     @classmethod
     def steps(cls):
@@ -39,4 +40,4 @@ def steps(cls):
 
     def enrich_discovered_packages_with_purldb(self):
         """Lookup discovered packages in PurlDB."""
-        purldb.enrich_discovered_packages_with_purldb(self.project, logger=self.log)
+        purldb.enrich_discovered_packages(self.project, logger=self.log)

scanpipe/pipes/purldb.py

@@ -24,6 +24,7 @@
 import logging
 
 from django.conf import settings
+from django.template.defaultfilters import pluralize
 from django.utils.text import slugify
 
 import requests
@@ -461,25 +462,28 @@ def get_run_status(run, **kwargs):
     return run.status
 
 
-def enrich_package_with_purldb_data(package):
+def enrich_package(package):
     """Enrich the provided ``package`` with the PurlDB data."""
     purldb_entry = get_package_by_purl(package.package_url)
     if purldb_entry:
         package_data = _clean_package_data(purldb_entry)
-        updated_fields = package.update_from_data(package_data)
-        return updated_fields
+        if updated_fields := package.update_from_data(package_data):
+            package.update_extra_data({ENRICH_EXTRA_DATA_KEY: updated_fields})
+            return updated_fields
 
 
-def enrich_discovered_packages_with_purldb(project, logger=logger.info):
+def enrich_discovered_packages(project, logger=logger.info):
     """Enrich all project discovered packages with the PurlDB data."""
     packages = project.discoveredpackages.all()
 
     updated_package_count = 0
     for package in packages:
-        updated_fields = enrich_package_with_purldb_data(package)
-        if updated_fields:
-            package.update_extra_data({ENRICH_EXTRA_DATA_KEY: updated_fields})
+        if updated_fields := enrich_package(package):
             logger(f"{package} {updated_fields}")
             updated_package_count += 1
 
-    logger(f"{updated_package_count} discovered packages enriched with the PurlDB.")
+    logger(
+        f"{updated_package_count} discovered package{pluralize(updated_package_count)} "
+        f"enriched with the PurlDB."
+    )
+    return updated_package_count

scanpipe/tests/data/purldb/csvtojson-2.0.10.json

@@ -0,0 +1,50 @@
+{
+  "url": "https://purldb.io/api/packages/f4300f8b-e5a8-4258-8eb1-3d01aeae8623/?format=json",
+  "uuid": "f4300f8b-e5a8-4258-8eb1-3d01aeae8623",
+  "filename": null,
+  "package_sets": [],
+  "package_content": null,
+  "purl": "pkg:npm/csvtojson@2.0.10",
+  "type": "npm",
+  "namespace": "",
+  "name": "csvtojson",
+  "version": "2.0.10",
+  "qualifiers": "",
+  "subpath": "",
+  "primary_language": null,
+  "description": null,
+  "release_date": "2019-06-26T00:00:00Z",
+  "parties": [],
+  "keywords": [],
+  "homepage_url": "https://github.com/Keyang/node-csvtojson",
+  "download_url": "https://registry.npmjs.com/csvtojson/-/csvtojson-2.0.10.tgz",
+  "bug_tracking_url": null,
+  "code_view_url": null,
+  "vcs_url": null,
+  "repository_homepage_url": null,
+  "repository_download_url": null,
+  "api_data_url": null,
+  "size": null,
+  "md5": null,
+  "sha1": "11e7242cc630da54efce7958a45f443210357574",
+  "sha256": "41ab7fecdc9cf7007696196d927560741cecdf7fc28c47565221178bfb3ae592",
+  "sha512": null,
+  "copyright": "Copyright (c) 2013 Keyang Xiang\nCopyright Joyent, Inc. and other Node contributors.\nCopyright JS Foundation and other contributors https://js.foundation\nCopyright Jeremy Ashkenas, DocumentCloud and Investigative Reporters Editors",
+  "holder": null,
+  "declared_license_expression": "mit",
+  "declared_license_expression_spdx": "MIT",
+  "license_detections": [],
+  "other_license_expression": null,
+  "other_license_expression_spdx": null,
+  "other_license_detections": [],
+  "extracted_license_statement": "MIT",
+  "notice_text": null,
+  "source_packages": [],
+  "extra_data": {},
+  "package_uid": "pkg:npm/csvtojson@2.0.10?uuid=f4300f8b-e5a8-4258-8eb1-3d01aeae8623",
+  "datasource_id": null,
+  "file_references": [],
+  "dependencies": [],
+  "resources": "https://purldb.io/api/packages/f4300f8b-e5a8-4258-8eb1-3d01aeae8623/resources/?format=json",
+  "history": "https://purldb.io/api/packages/f4300f8b-e5a8-4258-8eb1-3d01aeae8623/history/?format=json"
+}

scanpipe/tests/pipes/test_purldb.py

@@ -21,6 +21,7 @@
 # Visit https://github.com/nexB/scancode.io for support and download.
 
 import io
+import json
 from pathlib import Path
 from unittest import mock
 
@@ -35,6 +36,7 @@
 from scanpipe.pipes import purldb
 from scanpipe.tests import dependency_data2
 from scanpipe.tests import dependency_data3
+from scanpipe.tests import make_package
 from scanpipe.tests import package_data1
 
 
@@ -234,3 +236,59 @@ def test_scanpipe_pipes_purldb_create_project_name(self):
         scannable_uri_uuid = "52b2930d-6e85-4b3e-ba3e-17dd9a618650"
         project_name = purldb.create_project_name(download_url, scannable_uri_uuid)
         self.assertEqual("httpsregistrynpmjsorgasdf-asdf-101tgz-52b2930d", project_name)
+
+    @mock.patch("scanpipe.pipes.purldb.get_package_by_purl")
+    def test_scanpipe_pipes_purldb_enrich_package(self, mock_get_package_by_purl):
+        package1 = make_package(self.project1, package_url="pkg:npm/csvtojson@2.0.10")
+
+        mock_get_package_by_purl.return_value = {}
+        updated_fields = purldb.enrich_package(package=package1)
+        self.assertIsNone(updated_fields)
+
+        purldb_entry_file = self.data / "purldb" / "csvtojson-2.0.10.json"
+        purldb_entry = json.loads(purldb_entry_file.read_text())
+        mock_get_package_by_purl.return_value = purldb_entry
+        updated_fields = purldb.enrich_package(package=package1)
+        self.assertTrue(updated_fields)
+        self.assertIn("homepage_url", updated_fields)
+
+        package1.refresh_from_db()
+        self.assertTrue(package1.extra_data.get("enrich_with_purldb"))
+        self.assertEqual(purldb_entry.get("sha1"), package1.sha1)
+        self.assertEqual(purldb_entry.get("sha256"), package1.sha256)
+        self.assertEqual(purldb_entry.get("copyright"), package1.copyright)
+
+    @mock.patch("scanpipe.pipes.purldb.get_package_by_purl")
+    def test_scanpipe_pipes_purldb_enrich_discovered_packages(
+        self, mock_get_package_by_purl
+    ):
+        package1 = make_package(self.project1, package_url="pkg:npm/csvtojson@2.0.10")
+
+        mock_get_package_by_purl.return_value = {}
+        buffer = io.StringIO()
+        updated_package_count = purldb.enrich_discovered_packages(
+            project=self.project1,
+            logger=buffer.write,
+        )
+        self.assertEqual(0, updated_package_count)
+        expected_log = buffer.getvalue()
+        self.assertIn("0 discovered packages enriched with the PurlDB.", expected_log)
+
+        purldb_entry_file = self.data / "purldb" / "csvtojson-2.0.10.json"
+        purldb_entry = json.loads(purldb_entry_file.read_text())
+        mock_get_package_by_purl.return_value = purldb_entry
+        buffer = io.StringIO()
+        updated_package_count = purldb.enrich_discovered_packages(
+            project=self.project1,
+            logger=buffer.write,
+        )
+        self.assertEqual(1, updated_package_count)
+        package1.refresh_from_db()
+        self.assertEqual(purldb_entry.get("sha1"), package1.sha1)
+        self.assertEqual(purldb_entry.get("sha256"), package1.sha256)
+        self.assertEqual(purldb_entry.get("copyright"), package1.copyright)
+        self.assertTrue(package1.extra_data.get("enrich_with_purldb"))
+
+        expected_log = buffer.getvalue()
+        self.assertIn("pkg:npm/csvtojson@2.0.10 ['release_date'", expected_log)
+        self.assertIn("1 discovered package enriched with the PurlDB.", expected_log)

scanpipe/tests/test_pipelines.py

@@ -49,6 +49,7 @@
 from scanpipe.pipes import scancode
 from scanpipe.pipes.input import copy_input
 from scanpipe.tests import FIXTURES_REGEN
+from scanpipe.tests import make_package
 from scanpipe.tests import package_data1
 from scanpipe.tests.pipelines.do_nothing import DoNothing
 from scanpipe.tests.pipelines.profile_step import ProfileStep
@@ -845,6 +846,7 @@ def test_scanpipe_inspect_packages_creates_packages_pypi(self):
         self.assertEqual(0, project1.discoveredpackages.count())
         self.assertEqual(26, project1.discovereddependencies.count())
 
+    @skipIf(sys.platform == "darwin", "Not supported on macOS")
     def test_scanpipe_inspect_packages_with_resolved_dependencies_npm(self):
         pipeline_name = "inspect_packages"
         project1 = Project.objects.create(name="Analysis")
@@ -872,6 +874,7 @@ def test_scanpipe_inspect_packages_with_resolved_dependencies_npm(self):
         )
         self.assertPipelineResultEqual(expected_file, result_file)
 
+    @skipIf(sys.platform == "darwin", "Not supported on macOS")
     def test_scanpipe_inspect_packages_with_resolved_dependencies_poetry(self):
         pipeline_name = "inspect_packages"
         project1 = Project.objects.create(name="Analysis")
@@ -1711,3 +1714,33 @@ def test_scanpipe_collect_symbols_tree_sitter_pipeline_integration(self):
             expected_extra_data = json.load(f)
 
         self.assertDictEqual(expected_extra_data, result_extra_data)
+
+    @mock.patch("scanpipe.pipes.purldb.is_available")
+    @mock.patch("scanpipe.pipes.purldb.is_configured")
+    @mock.patch("scanpipe.pipes.purldb.get_package_by_purl")
+    def test_scanpipe_enrich_with_purldb_pipeline_integration(
+        self, mock_get_package, mock_is_configured, mock_is_available
+    ):
+        pipeline_name = "enrich_with_purldb"
+        project1 = Project.objects.create(name="Analysis")
+        package1 = make_package(project1, package_url="pkg:npm/csvtojson@2.0.10")
+
+        mock_is_configured.return_value = True
+        mock_is_available.return_value = True
+
+        purldb_entry_file = self.data / "purldb" / "csvtojson-2.0.10.json"
+        purldb_entry = json.loads(purldb_entry_file.read_text())
+        mock_get_package.return_value = purldb_entry
+
+        run = project1.add_pipeline(pipeline_name)
+        pipeline = run.make_pipeline_instance()
+
+        exitcode, out = pipeline.execute()
+        self.assertEqual(0, exitcode, msg=out)
+
+        package1.refresh_from_db()
+        self.assertTrue(package1.extra_data.get("enrich_with_purldb"))
+
+        run.refresh_from_db()
+        self.assertIn("pkg:npm/csvtojson@2.0.10 ['release_date'", run.log)
+        self.assertIn("1 discovered package enriched with the PurlDB.", run.log)