Refactor policies implementation to support more than licenses (#1718)

Signed-off-by: tdruez <tdruez@nexb.com>

Author: tdruez

CHANGELOG.rst

@@ -4,6 +4,16 @@ Changelog
 v35.2.0 (unreleased)
 --------------------
 
+- Refactor policies implementation to support more than licenses.
+  The entire ``policies`` data is now stored on the ``ScanPipeConfig`` in place of the
+  ``license_policy_index``.
+  Also, a new method ``get_policies_dict`` methods is now available on the ``Project``
+  model to easily retrieve all the policies data as a dictionary.
+  Renamed for clarity:
+  * ``policy_index`` to ``license_policy_index``
+  * ``policies_enabled`` to ``license_policies_enabled``
+  https://github.com/aboutcode-org/scancode.io/pull/1718
+
 - Add support for SPDX license identifiers as ``license_key`` in license policies
   ``policies.yml`` file.
   https://github.com/aboutcode-org/scancode.io/issues/1348

scancodeio/settings.py

@@ -292,6 +292,7 @@
     SCANCODEIO_WORKSPACE_LOCATION = tempfile.mkdtemp()
     SCANCODEIO_REQUIRE_AUTHENTICATION = True
     SCANCODEIO_SCAN_FILE_TIMEOUT = 120
+    SCANCODEIO_POLICIES_FILE = None
     # The default password hasher is rather slow by design.
     # Using a faster hashing algorithm in the testing context to speed up the run.
     PASSWORD_HASHERS = ["django.contrib.auth.hashers.MD5PasswordHasher"]

scanpipe/apps.py

@@ -39,7 +39,6 @@
 from licensedcode.models import load_licenses
 
 from scanpipe.policies import load_policies_file
-from scanpipe.policies import make_license_policy_index
 
 try:
     from importlib import metadata as importlib_metadata
@@ -61,7 +60,7 @@ def __init__(self, app_name, app_module):
 
         # Mapping of registered pipeline names to pipeline classes.
         self._pipelines = {}
-        self.license_policies_index = {}
+        self.policies = {}
 
         workspace_location = settings.SCANCODEIO_WORKSPACE_LOCATION
         self.workspace_path = Path(workspace_location).expanduser().resolve()
@@ -226,7 +225,7 @@ def get_scancode_licenses(self):
 
     def set_policies(self):
         """
-        Compute and sets the `license_policies` on the app instance.
+        Set the global app policies on the app instance.
 
         If the policies file is available but not formatted properly or doesn't
         include the proper content, we want to raise an exception while the app
@@ -240,7 +239,7 @@ def set_policies(self):
         if policies_file.exists():
             policies = load_policies_file(policies_file)
             logger.debug(style.SUCCESS(f"Loaded policies from {policies_file}"))
-            self.license_policies_index = make_license_policy_index(policies)
+            self.policies = policies
         else:
             logger.debug(style.WARNING("Policies file not found."))
 

scanpipe/models.py

@@ -1495,37 +1495,40 @@ def has_single_resource(self):
         """
         return self.resource_count == 1
 
-    def get_policy_index(self):
+    def get_policies_dict(self):
         """
-        Return the policy index for this project instance.
+        Load and return the policies from the following locations in that order:
 
-        The policies are loaded from the following locations in that order:
-        1. the project local settings
-        2. the "policies.yml" file in the project input/ directory
-        3. the global app settings license policies
+        1. project local settings (stored in the database)
+        2. "policies.yml" file in the project ``input/`` directory
+        3. global app settings policies, from SCANCODEIO_POLICIES_FILE setting
         """
         if policies_from_settings := self.get_env("policies"):
             policies_dict = policies_from_settings
             if isinstance(policies_from_settings, str):
                 policies_dict = policies.load_policies_yaml(policies_from_settings)
-            return policies.make_license_policy_index(policies_dict)
+            return policies_dict
 
-        elif policies_file := self.get_input_policies_file():
-            policies_dict = policies.load_policies_file(policies_file)
-            return policies.make_license_policy_index(policies_dict)
+        elif project_input_policies_file := self.get_input_policies_file():
+            return policies.load_policies_file(project_input_policies_file)
 
-        else:
-            return scanpipe_app.license_policies_index
+        return scanpipe_app.policies
+
+    def get_license_policy_index(self):
+        """Return the policy license index for this project instance."""
+        if policies_dict := self.get_policies_dict():
+            return policies.make_license_policy_index(policies_dict)
+        return {}
 
     @cached_property
-    def policy_index(self):
-        """Return the cached policy index for this project instance."""
-        return self.get_policy_index()
+    def license_policy_index(self):
+        """Return the cached license policy index for this project instance."""
+        return self.get_license_policy_index()
 
     @property
-    def policies_enabled(self):
-        """Return True if the policies are enabled for this project."""
-        return bool(self.policy_index)
+    def license_policies_enabled(self):
+        """Return True if the license policies are available for this project."""
+        return bool(self.license_policy_index)
 
 
 class GroupingQuerySetMixin:
@@ -2540,7 +2543,7 @@ def save(self, codebase=None, *args, **kwargs):
         ``codebase`` is not used in this context but required for compatibility
         with the commoncode.resource.Codebase class API.
         """
-        if self.policies_enabled:
+        if self.license_policies_enabled:
             loaded_license_expression = getattr(self, "_loaded_license_expression", "")
             license_expression = getattr(self, self.license_expression_field, "")
             if license_expression != loaded_license_expression:
@@ -2566,28 +2569,29 @@ def from_db(cls, db, field_names, values):
         return new
 
     @property
-    def policy_index(self):
-        return self.project.policy_index
+    def license_policy_index(self):
+        return self.project.license_policy_index
 
     @cached_property
-    def policies_enabled(self):
-        return self.project.policies_enabled
+    def license_policies_enabled(self):
+        return self.project.license_policies_enabled
 
     def compute_compliance_alert(self):
         """
         Compute and return the compliance_alert value from the license policies.
         Chooses the most severe compliance_alert found among licenses.
         """
         license_expression = getattr(self, self.license_expression_field, "")
-        policy_index = self.policy_index
-        if not license_expression or not policy_index:
+        license_policy_index = self.license_policy_index
+        if not license_expression or not license_policy_index:
             return ""
 
         licensing = get_licensing()
         parsed_symbols = licensing.parse(license_expression, simple=True).symbols
 
         alerts = [
-            self.get_alert_for_symbol(policy_index, symbol) for symbol in parsed_symbols
+            self.get_alert_for_symbol(license_policy_index, symbol)
+            for symbol in parsed_symbols
         ]
         most_severe_alert = max(alerts, key=self.COMPLIANCE_SEVERITY_MAP.get)
         return most_severe_alert or self.Compliance.OK

scanpipe/pipes/output.py

@@ -541,7 +541,7 @@ def to_xlsx(project):
     exclude_fields = XLSX_EXCLUDE_FIELDS.copy()
     output_file = project.get_output_file_path("results", "xlsx")
 
-    if not project.policies_enabled:
+    if not project.license_policies_enabled:
         exclude_fields.append("compliance_alert")
 
     model_names = [

scanpipe/templates/scanpipe/forms/project_settings_form.html

@@ -153,7 +153,7 @@
     <p id="policies" class="panel-heading">Policies</p>
     <div class="panel-block is-block px-4">
       <div>
-        {% if project.policies_enabled %}
+        {% if project.license_policies_enabled %}
           <i class="fa-solid fa-check"></i>
           Policies are <strong>enabled</strong> for this project.
         {% else %}

scanpipe/templates/scanpipe/project_detail.html

@@ -117,7 +117,7 @@
         <div hx-get="{% url 'project_resource_license_summary' project.slug %}" hx-trigger="load" hx-swap="outerHTML"></div>
       </div>
 
-      {% if policies_enabled %}
+      {% if license_policies_enabled %}
         <div class="columns">
           <div hx-get="{% url 'project_compliance_panel' project.slug %}" hx-trigger="load" hx-swap="outerHTML"></div>
         </div>

scanpipe/tests/pipes/test_scancode.py

@@ -467,8 +467,8 @@ def test_scanpipe_max_file_size_works(self):
             self.assertEqual(resource1.status, flag.IGNORED_BY_MAX_FILE_SIZE)
 
     def test_scanpipe_pipes_scancode_make_results_summary(self, regen=FIXTURES_REGEN):
-        # Ensure the policies index is empty to avoid any side effect on results
-        scanpipe_app.license_policies_index = None
+        # Ensure the policies are empty to avoid any side effect on results
+        scanpipe_app.policies = None
         # Run the scan_single_package pipeline to have a proper DB and local files setup
         pipeline_name = "scan_single_package"
         project1 = Project.objects.create(name="Analysis")

scanpipe/tests/test_apps.py

@@ -23,6 +23,7 @@
 import uuid
 from pathlib import Path
 from unittest import mock
+from unittest.mock import patch
 
 from django.apps import apps
 from django.core.exceptions import ImproperlyConfigured
@@ -33,7 +34,7 @@
 from scanpipe.models import Project
 from scanpipe.models import Run
 from scanpipe.tests import filter_warnings
-from scanpipe.tests import license_policies_index
+from scanpipe.tests import global_policies
 from scanpipe.tests.pipelines.register_from_file import RegisterFromFile
 
 scanpipe_app = apps.get_app_config("scanpipe")
@@ -43,26 +44,23 @@ class ScanPipeAppsTest(TestCase):
     data = Path(__file__).parent / "data"
     pipelines_location = Path(__file__).parent / "pipelines"
 
-    def test_scanpipe_apps_set_policies(self):
-        scanpipe_app.license_policies_index = {}
-        policies_files = None
-        with override_settings(SCANCODEIO_POLICIES_FILE=policies_files):
+    @patch.object(scanpipe_app, "policies", new_callable=dict)
+    def test_scanpipe_apps_set_policies(self, mock_policies):
+        # Case 1: No file set
+        with override_settings(SCANCODEIO_POLICIES_FILE=None):
             scanpipe_app.set_policies()
-            self.assertEqual({}, scanpipe_app.license_policies_index)
+            self.assertEqual({}, scanpipe_app.policies)
 
-        scanpipe_app.license_policies_index = {}
-        policies_files = "not_existing"
-        with override_settings(SCANCODEIO_POLICIES_FILE=policies_files):
+        # Case 2: Non-existing file
+        with override_settings(SCANCODEIO_POLICIES_FILE="not_existing"):
             scanpipe_app.set_policies()
-            self.assertEqual({}, scanpipe_app.license_policies_index)
+            self.assertEqual({}, scanpipe_app.policies)
 
-        scanpipe_app.license_policies_index = {}
+        # Case 3: Valid file
         policies_files = self.data / "policies" / "policies.yml"
         with override_settings(SCANCODEIO_POLICIES_FILE=str(policies_files)):
             scanpipe_app.set_policies()
-            self.assertEqual(
-                license_policies_index, scanpipe_app.license_policies_index
-            )
+            self.assertEqual(global_policies, scanpipe_app.policies)
 
     def test_scanpipe_apps_register_pipeline_from_file(self):
         path = self.pipelines_location / "do_nothing.py"

scanpipe/tests/test_forms.py

@@ -230,7 +230,7 @@ def test_scanpipe_forms_project_settings_form_policies(self):
         self.assertTrue(form.is_valid())
         project = form.save()
         self.assertEqual(policies_as_yaml.strip(), project.settings["policies"])
-        self.assertEqual(license_policies_index, project.get_policy_index())
+        self.assertEqual(license_policies_index, project.get_license_policy_index())
 
     def test_scanpipe_forms_project_settings_form_purl(self):
         data_invalid_purl = {