Refactor the policies related code to its own module #386 (#1430)

Signed-off-by: tdruez <tdruez@nexb.com>

Author: tdruez

CHANGELOG.rst

@@ -10,6 +10,9 @@ v34.9.0 (unreleased)
 - Add a new ``list-pipelines`` management command.
   https://github.com/aboutcode-org/scancode.io/issues/1397
 
+- Refactor the policies related code to its own module.
+  https://github.com/aboutcode-org/scancode.io/issues/386
+
 v34.8.3 (2024-10-30)
 --------------------
 

scanpipe/apps.py

@@ -35,9 +35,11 @@
 from django.utils.functional import cached_property
 from django.utils.translation import gettext_lazy as _
 
-import saneyaml
 from licensedcode.models import load_licenses
 
+from scanpipe.policies import load_policies_file
+from scanpipe.policies import make_license_policy_index
+
 try:
     from importlib import metadata as importlib_metadata
 except ImportError:
@@ -218,31 +220,21 @@ def set_policies(self):
         """
         Compute and sets the `license_policies` on the app instance.
 
-        If the policies file is available but formatted properly or doesn't
+        If the policies file is available but not formatted properly or doesn't
         include the proper content, we want to raise an exception while the app
         is loading to warn system admins about the issue.
         """
-        policies_file_location = getattr(settings, "SCANCODEIO_POLICIES_FILE", None)
-
-        if policies_file_location:
-            policies_file = Path(policies_file_location).expanduser()
-
-            if policies_file.exists():
-                logger.debug(style.SUCCESS(f"Load policies from {policies_file}"))
-                policies = saneyaml.load(policies_file.read_text())
-                license_policies = policies.get("license_policies", [])
-                self.license_policies_index = self.get_policies_index(
-                    policies_list=license_policies,
-                    key="license_key",
-                )
-
-            else:
-                logger.debug(style.WARNING("Policies file not found."))
-
-    @staticmethod
-    def get_policies_index(policies_list, key):
-        """Return an inverted index by `key` of the `policies_list`."""
-        return {policy.get(key): policy for policy in policies_list}
+        policies_file_setting = getattr(settings, "SCANCODEIO_POLICIES_FILE", None)
+        if not policies_file_setting:
+            return
+
+        policies_file = Path(policies_file_setting).expanduser()
+        if policies_file.exists():
+            policies = load_policies_file(policies_file)
+            logger.debug(style.SUCCESS(f"Loaded policies from {policies_file}"))
+            self.license_policies_index = make_license_policy_index(policies)
+        else:
+            logger.debug(style.WARNING("Policies file not found."))
 
     @property
     def policies_enabled(self):

scanpipe/policies.py

@@ -0,0 +1,65 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# http://nexb.com and https://github.com/aboutcode-org/scancode.io
+# The ScanCode.io software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode.io is provided as-is without warranties.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+# ScanCode.io should be considered or used as legal advice. Consult an Attorney
+# for any legal advice.
+#
+# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
+# Visit https://github.com/aboutcode-org/scancode.io for support and download.
+
+from django.core.exceptions import ValidationError
+
+import saneyaml
+
+
+def load_policies_yaml(policies_yaml):
+    """Load provided ``policies_yaml``."""
+    try:
+        return saneyaml.load(policies_yaml)
+    except saneyaml.YAMLError as e:
+        raise ValidationError(f"Policies format error: {e}")
+
+
+def load_policies_file(policies_file, validate=True):
+    """
+    Load provided ``policies_file`` into a Python dictionary.
+    The policies format is validated by default.
+    """
+    policies_dict = load_policies_yaml(policies_yaml=policies_file.read_text())
+    if validate:
+        validate_policies(policies_dict)
+    return policies_dict
+
+
+def validate_policies(policies_dict):
+    """Return True if the provided ``policies_dict`` is valid."""
+    if not isinstance(policies_dict, dict):
+        raise ValidationError("The `policies_dict` argument must be a dictionary.")
+
+    if "license_policies" not in policies_dict:
+        raise ValidationError(
+            "The `license_policies` key is missing from provided policies data."
+        )
+
+    return True
+
+
+def make_license_policy_index(policies_dict):
+    """Return an inverted index by ``key`` of the ``policies_list``."""
+    validate_policies(policies_dict)
+
+    license_policies = policies_dict.get("license_policies", [])
+    return {policy.get("license_key"): policy for policy in license_policies}

scanpipe/templates/scanpipe/includes/project_settings_menu.html

@@ -15,6 +15,11 @@
         <i class="fa-solid fa-forward mr-2"></i>Ignored
       </a>
     </li>
+    <li>
+      <a href="#policies">
+        <i class="fa-solid fa-certificate mr-2"></i>Policies
+      </a>
+    </li>
     <li>
       <a href="#dejacode">
         <i class="fa-solid fa-laptop-code mr-2"></i>DejaCode

scanpipe/templates/scanpipe/project_settings.html

@@ -131,6 +131,20 @@
               </div>
             </div>
 
+            <div class="panel">
+              <p id="policies" class="panel-heading py-2 is-size-6">Policies</p>
+              <div class="panel-block is-block px-4">
+                {% if project.policies_enabled %}
+                  Policies are <strong>enabled</strong> for this project.
+                {% else %}
+                  Policies are <strong>not enabled</strong> for this project.
+                {% endif %}
+                <p class="help">
+                  See <a target="_blank" href="https://scancodeio.readthedocs.io/en/latest/tutorial_license_policies.html">License Policies and Compliance Alerts documentation</a> for details.
+                </p>
+              </div>
+            </div>
+
             <div class="panel">
               <p id="dejacode" class="panel-heading py-2 is-size-6">DejaCode</p>
               <div class="panel-block is-block px-4">

scanpipe/tests/__init__.py

@@ -251,6 +251,10 @@ def make_dependency(project, **extra):
     },
 ]
 
+global_policies = {
+    "license_policies": license_policies,
+}
+
 license_policies_index = {
     "gpl-3.0": {
         "color_code": "#c83025",

scanpipe/tests/test_apps.py

@@ -30,10 +30,8 @@
 from django.test import override_settings
 from django.utils import timezone
 
-from scanpipe.apps import ScanPipeConfig
 from scanpipe.models import Project
 from scanpipe.models import Run
-from scanpipe.tests import license_policies
 from scanpipe.tests import license_policies_index
 from scanpipe.tests.pipelines.register_from_file import RegisterFromFile
 
@@ -44,14 +42,6 @@ class ScanPipeAppsTest(TestCase):
     data = Path(__file__).parent / "data"
     pipelines_location = Path(__file__).parent / "pipelines"
 
-    def test_scanpipe_apps_get_policies_index(self):
-        self.assertEqual({}, ScanPipeConfig.get_policies_index([], "license_key"))
-        policies_index = ScanPipeConfig.get_policies_index(
-            policies_list=license_policies,
-            key="license_key",
-        )
-        self.assertEqual(license_policies_index, policies_index)
-
     def test_scanpipe_apps_set_policies(self):
         scanpipe_app.license_policies_index = {}
         policies_files = None

scanpipe/tests/test_policies.py

@@ -0,0 +1,85 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# http://nexb.com and https://github.com/nexB/scancode.io
+# The ScanCode.io software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode.io is provided as-is without warranties.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+# ScanCode.io should be considered or used as legal advice. Consult an Attorney
+# for any legal advice.
+#
+# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
+# Visit https://github.com/nexB/scancode.io for support and download.
+
+from pathlib import Path
+
+from django.apps import apps
+from django.core.exceptions import ValidationError
+from django.test import TestCase
+
+from scanpipe.policies import load_policies_file
+from scanpipe.policies import load_policies_yaml
+from scanpipe.policies import make_license_policy_index
+from scanpipe.policies import validate_policies
+from scanpipe.tests import global_policies
+from scanpipe.tests import license_policies_index
+
+scanpipe_app = apps.get_app_config("scanpipe")
+
+
+class ScanPipePoliciesTest(TestCase):
+    data = Path(__file__).parent / "data"
+
+    def test_scanpipe_policies_load_policies_yaml(self):
+        policies_yaml = "{wrong format"
+        with self.assertRaises(ValidationError):
+            load_policies_yaml(policies_yaml)
+
+        policies_files = self.data / "policy" / "policies.yml"
+        policies_dict = load_policies_yaml(policies_files.read_text())
+        self.assertIn("license_policies", policies_dict)
+
+    def test_scanpipe_policies_load_policies_file(self):
+        policies_files = self.data / "policy" / "policies.yml"
+        policies_dict = load_policies_file(policies_files)
+        self.assertIn("license_policies", policies_dict)
+
+    def test_scanpipe_policies_validate_policies(self):
+        error_msg = "The `policies_dict` argument must be a dictionary."
+        policies_dict = None
+        with self.assertRaisesMessage(ValidationError, error_msg):
+            validate_policies(policies_dict)
+
+        policies_dict = []
+        with self.assertRaisesMessage(ValidationError, error_msg):
+            validate_policies(policies_dict)
+
+        error_msg = "The `license_policies` key is missing from provided policies data."
+        policies_dict = {}
+        with self.assertRaisesMessage(ValidationError, error_msg):
+            validate_policies(policies_dict)
+
+        policies_dict = {"missing": "data"}
+        with self.assertRaisesMessage(ValidationError, error_msg):
+            validate_policies(policies_dict)
+
+        policies_dict = global_policies
+        self.assertTrue(validate_policies(policies_dict))
+
+    def test_scanpipe_policies_make_license_policy_index(self):
+        policies_dict = {"missing": "data"}
+        with self.assertRaises(ValidationError):
+            make_license_policy_index(policies_dict)
+
+        self.assertEqual(
+            license_policies_index, make_license_policy_index(global_policies)
+        )