Fingerprint codebase resources (#1163)

* Create fingerprint_codebase_resources

Signed-off-by: Jono Yang <jyang@nexb.com>

* Update docstring

    * Use specific matchcode-toolkit branch in setup.cfg

Signed-off-by: Jono Yang <jyang@nexb.com>

* Add test for fingerprint_codebase_resources

Signed-off-by: Jono Yang <jyang@nexb.com>

* Update CHANGELOG.rst

Signed-off-by: Jono Yang <jyang@nexb.com>

* Use scancode API to run fingerprinting function

Signed-off-by: Jono Yang <jyang@nexb.com>

* Move changelog entry

Signed-off-by: Jono Yang <jyang@nexb.com>

* Bump matchcode-toolkit version to v4.1.0

Signed-off-by: Jono Yang <jyang@nexb.com>

* Remove scan_func_kwargs from fingerprint_codebase_resources

Signed-off-by: Jono Yang <jyang@nexb.com>

---------

Signed-off-by: Jono Yang <jyang@nexb.com>
Co-authored-by: tdruez <489057+tdruez@users.noreply.github.com>

Author: JonoYang

CHANGELOG.rst

@@ -11,6 +11,13 @@ v34.4.0 (unreleased)
   Also, the CycloneDX outputs can be downloaded as 1.6, 1.5, and 1.4 spec versions.
   https://github.com/nexB/scancode.io/pull/1165
 
+- Update matchcode-toolkit to v4.1.0
+
+- Add a new function
+  `scanpipe.pipes.matchcode.fingerprint_codebase_resources()`, which computes
+  approximate file matching fingerprints for text files using the new
+  `get_file_fingerprint_hashes` function from matchcode-toolkit.
+
 - Rename the `purldb-scan-queue-worker` management command to `purldb-scan-worker`.
 
 - Add `docker-compose.purldb-scan-worker.yml` to run ScanCode.io as a PurlDB

scanpipe/pipes/matchcode.py

@@ -27,11 +27,15 @@
 
 import requests
 from matchcode_toolkit.fingerprinting import compute_codebase_directory_fingerprints
+from matchcode_toolkit.fingerprinting import get_file_fingerprint_hashes
+from scancode import Scanner
 
 from scanpipe.pipes import codebase
 from scanpipe.pipes import flag
 from scanpipe.pipes import poll_until_success
 from scanpipe.pipes.output import to_json
+from scanpipe.pipes.scancode import _scan_resource
+from scanpipe.pipes.scancode import scan_resources
 
 
 class MatchCodeIOException(Exception):
@@ -174,10 +178,12 @@ def save_directory_fingerprints(project, virtual_codebase, to_codebase_only=Fals
 
 def fingerprint_codebase_directories(project, to_codebase_only=False):
     """
-    Compute directory fingerprints for the directories of the to/ codebase from
-    `project`.
+    Compute directory fingerprints for the directories from `project`.
 
     These directory fingerprints are used for matching purposes on matchcode.
+
+    If `to_codebase_only` is True, the only directories from the `to/` codebase
+    are computed.
     """
     resources = project.codebaseresources.all()
     if to_codebase_only:
@@ -189,6 +195,61 @@ def fingerprint_codebase_directories(project, to_codebase_only=False):
     )
 
 
+def fingerprint_codebase_resource(location, with_threading=True, **kwargs):
+    """
+    Compute fingerprints for the resource at `location` using the
+    scancode-toolkit direct API.
+
+    Return a dictionary of scan `results` and a list of `errors`.
+    """
+    scanners = [
+        Scanner("fingerprints", get_file_fingerprint_hashes),
+    ]
+    return _scan_resource(location, scanners, with_threading=with_threading)
+
+
+def save_resource_fingerprints(resource, scan_results, scan_errors=None):
+    """
+    Save computed fingerprints from `scan_results` to `resource.extra_data`.
+    Create project errors if any occurred during the scan.
+    """
+    resource.extra_data.update(scan_results)
+    resource.save()
+
+    if scan_errors:
+        resource.add_errors(scan_errors)
+        resource.update(status=flag.SCANNED_WITH_ERROR)
+
+
+def fingerprint_codebase_resources(
+    project, resource_qs=None, progress_logger=None, to_codebase_only=False
+):
+    """
+    Compute fingerprints for the resources from `project`.
+
+    These resource fingerprints are used for matching purposes on matchcode.
+
+    Multiprocessing is enabled by default on this pipe, the number of processes can be
+    controlled through the SCANCODEIO_PROCESSES setting.
+
+    If `to_codebase_only` is True, the only resources from the `to/` codebase
+    are computed.
+    """
+    # Checking for None to make the distinction with an empty resource_qs queryset
+    if resource_qs is None:
+        resource_qs = project.codebaseresources.filter(is_text=True)
+
+    if to_codebase_only:
+        resource_qs = resource_qs.to_codebase()
+
+    scan_resources(
+        resource_qs=resource_qs,
+        scan_func=fingerprint_codebase_resource,
+        save_func=save_resource_fingerprints,
+        progress_logger=progress_logger,
+    )
+
+
 def send_project_json_to_matchcode(
     project, timeout=DEFAULT_TIMEOUT, api_url=MATCHCODEIO_API_URL
 ):

scanpipe/tests/pipes/test_matchcode.py

@@ -29,8 +29,10 @@
 from django.test import TestCase
 
 from scanpipe.models import AbstractTaskFieldsModel
+from scanpipe.models import CodebaseResource
 from scanpipe.models import Project
 from scanpipe.pipes import matchcode
+from scanpipe.pipes.input import copy_input
 from scanpipe.tests import make_resource_file
 
 
@@ -311,3 +313,23 @@ def test_scanpipe_pipes_matchcode_get_match_results(
         match_results = matchcode.get_match_results(run_url)
 
         self.assertEqual(mock_request_get_results_return, match_results)
+
+    def test_scanpipe_pipes_matchcode_fingerprint_codebase_resources(self):
+        copy_input(self.data_location / "notice.NOTICE", self.project1.codebase_path)
+        codebase_resource1 = CodebaseResource.objects.create(
+            project=self.project1, path="notice.NOTICE", is_text=True
+        )
+
+        # This resource should not have a fingerprint
+        copy_input(self.data_location / "is-npm-1.0.0.tgz", self.project1.codebase_path)
+        codebase_resource2 = CodebaseResource.objects.create(
+            project=self.project1, path="is-npm-1.0.0.tgz"
+        )
+
+        matchcode.fingerprint_codebase_resources(self.project1)
+        codebase_resource1.refresh_from_db()
+        codebase_resource2.refresh_from_db()
+
+        expected_extra_data = {"halo1": "ef420f7e84c8c74c691315f0a06ac4f0"}
+        self.assertEqual(expected_extra_data, codebase_resource1.extra_data)
+        self.assertFalse(codebase_resource2.extra_data)

setup.cfg

@@ -95,7 +95,7 @@ install_requires =
     # Font Awesome
     fontawesomefree==6.5.1
     # MatchCode-toolkit
-    matchcode-toolkit==4.0.0
+    matchcode-toolkit==4.1.0
     # Univers
     univers==30.11.0
     # Markdown