From eff3494a7c289c41e184a17d127ed920ba127d58 Mon Sep 17 00:00:00 2001 From: NucleonGodX Date: Fri, 27 Jun 2025 22:17:58 +0530 Subject: [PATCH 1/9] integrating the clarity-compliance in UI Signed-off-by: NucleonGodX --- scanpipe/pipes/license_clarity.py | 25 +++++++++++++ scanpipe/pipes/scancode.py | 16 +++++++++ .../scanpipe/panels/clarity_compliance.html | 36 +++++++++++++++++++ .../templates/scanpipe/project_detail.html | 1 + scanpipe/tests/test_views.py | 18 ++++++++++ scanpipe/urls.py | 5 +++ scanpipe/views.py | 17 +++++++++ 7 files changed, 118 insertions(+) create mode 100644 scanpipe/templates/scanpipe/panels/clarity_compliance.html diff --git a/scanpipe/pipes/license_clarity.py b/scanpipe/pipes/license_clarity.py index 0f230976d..a4b044101 100644 --- a/scanpipe/pipes/license_clarity.py +++ b/scanpipe/pipes/license_clarity.py @@ -41,6 +41,7 @@ from pathlib import Path +from django.conf import settings from django.core.exceptions import ValidationError import saneyaml @@ -163,3 +164,27 @@ def load_clarity_thresholds_from_file(file_path): return load_clarity_thresholds_from_yaml(yaml_content) except (OSError, UnicodeDecodeError) as e: raise ValidationError(f"Error reading file {file_path}: {e}") + + +def get_project_clarity_thresholds(project): + """ + Get clarity thresholds for a project, checking multiple sources. + + Returns: + ClarityThresholdsPolicy or None: Policy object if thresholds are configured + + """ + if hasattr(project, "get_input_policies_file"): + policies_file = project.get_input_policies_file() + if policies_file: + policy = load_clarity_thresholds_from_file(policies_file) + if policy: + return policy + + global_policies_file = getattr(settings, "SCANCODEIO_POLICIES_FILE", None) + if global_policies_file: + policy = load_clarity_thresholds_from_file(global_policies_file) + if policy: + return policy + + return None diff --git a/scanpipe/pipes/scancode.py b/scanpipe/pipes/scancode.py index f3501c9ec..98058041e 100644 --- a/scanpipe/pipes/scancode.py +++ b/scanpipe/pipes/scancode.py @@ -52,6 +52,7 @@ from scanpipe.models import DiscoveredDependency from scanpipe.models import DiscoveredPackage from scanpipe.pipes import flag +from scanpipe.pipes.license_clarity import get_project_clarity_thresholds logger = logging.getLogger("scanpipe.pipes") @@ -931,7 +932,10 @@ def make_results_summary(project, scan_results_location): Extract selected sections of the Scan results, such as the `summary` `license_clarity_score`, and `license_matches` related data. The `key_files` are also collected and injected in the `summary` output. + Additionally, store clarity_compliance_alert in project's extra_data. """ + import json + from scanpipe.api.serializers import CodebaseResourceSerializer from scanpipe.api.serializers import DiscoveredPackageSerializer @@ -964,4 +968,16 @@ def make_results_summary(project, scan_results_location): DiscoveredPackageSerializer(package).data for package in key_files_packages_qs ] + clarity_score = summary.get("license_clarity_score", {}).get("score") + if clarity_score is not None: + clarity_policy = get_project_clarity_thresholds(project) + if clarity_policy: + alert = clarity_policy.get_alert_for_score(clarity_score) + summary["clarity_compliance_alert"] = alert + + extra_data = project.extra_data or {} + extra_data["clarity_compliance_alert"] = alert + project.extra_data = extra_data + project.save(update_fields=["extra_data"]) + return summary diff --git a/scanpipe/templates/scanpipe/panels/clarity_compliance.html b/scanpipe/templates/scanpipe/panels/clarity_compliance.html new file mode 100644 index 000000000..24b99e7ad --- /dev/null +++ b/scanpipe/templates/scanpipe/panels/clarity_compliance.html @@ -0,0 +1,36 @@ +{% if clarity_compliance_alert %} +
+ +
+{% endif %} diff --git a/scanpipe/templates/scanpipe/project_detail.html b/scanpipe/templates/scanpipe/project_detail.html index e494166de..4b16ad85a 100644 --- a/scanpipe/templates/scanpipe/project_detail.html +++ b/scanpipe/templates/scanpipe/project_detail.html @@ -120,6 +120,7 @@ {% if policies_enabled %}
+
{% endif %} diff --git a/scanpipe/tests/test_views.py b/scanpipe/tests/test_views.py index 891962382..1a7fe05d3 100644 --- a/scanpipe/tests/test_views.py +++ b/scanpipe/tests/test_views.py @@ -1008,6 +1008,24 @@ def test_scanpipe_views_project_compliance_panel_view(self, mock_policies_enable expected = f"/project/{self.project1.slug}/packages/?compliance_alert=error" self.assertContains(response, expected) + @mock.patch.object(Project, "policies_enabled", new_callable=mock.PropertyMock) + def test_scanpipe_views_project_clarity_compliance_panel_view( + self, mock_policies_enabled + ): + url = reverse("clarity_compliance_panel", args=[self.project1.slug]) + self.project1.extra_data = {"clarity_compliance_alert": "error"} + self.project1.save(update_fields=["extra_data"]) + + mock_policies_enabled.return_value = False + response = self.client.get(url) + self.assertEqual(404, response.status_code) + + mock_policies_enabled.return_value = True + response = self.client.get(url) + self.assertContains(response, "Clarity Compliance") + self.assertContains(response, "Error") + self.assertContains(response, "License clarity is insufficient") + def test_scanpipe_views_pipeline_help_view(self): url = reverse("pipeline_help", args=["not_existing_pipeline"]) response = self.client.get(url) diff --git a/scanpipe/urls.py b/scanpipe/urls.py index c760becbf..e84a54bfa 100644 --- a/scanpipe/urls.py +++ b/scanpipe/urls.py @@ -221,6 +221,11 @@ views.ProjectCompliancePanelView.as_view(), name="project_compliance_panel", ), + path( + "projects//clarity_compliance_panel/", + views.ClarityCompliancePanelView.as_view(), + name="clarity_compliance_panel", + ), path( "project//", views.ProjectDetailView.as_view(), diff --git a/scanpipe/views.py b/scanpipe/views.py index 707d4920b..d91e9fe99 100644 --- a/scanpipe/views.py +++ b/scanpipe/views.py @@ -1212,6 +1212,23 @@ def get_context_data(self, **kwargs): return context +class ClarityCompliancePanelView(ConditionalLoginRequired, generic.DetailView): + model = Project + template_name = "scanpipe/panels/clarity_compliance.html" + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + project = self.object + + if not project.policies_enabled: + raise Http404 + + extra_data = project.extra_data or {} + context["clarity_compliance_alert"] = extra_data.get("clarity_compliance_alert") + + return context + + class ProjectArchiveView(ConditionalLoginRequired, SingleObjectMixin, FormView): model = Project http_method_names = ["post"] From a22533cf16e666570df147e95cd942370c56a8e6 Mon Sep 17 00:00:00 2001 From: NucleonGodX Date: Sat, 28 Jun 2025 08:25:04 +0530 Subject: [PATCH 2/9] minor change Signed-off-by: NucleonGodX --- .../templates/scanpipe/panels/clarity_compliance.html | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/scanpipe/templates/scanpipe/panels/clarity_compliance.html b/scanpipe/templates/scanpipe/panels/clarity_compliance.html index 24b99e7ad..5a511964a 100644 --- a/scanpipe/templates/scanpipe/panels/clarity_compliance.html +++ b/scanpipe/templates/scanpipe/panels/clarity_compliance.html @@ -1,10 +1,6 @@ {% if clarity_compliance_alert %}
-
-{% endif %} +{% endif %} \ No newline at end of file From 50eaa162ac2e348120fccba805467f9208e31cc9 Mon Sep 17 00:00:00 2001 From: NucleonGodX Date: Mon, 30 Jun 2025 19:19:38 +0530 Subject: [PATCH 3/9] have compliance alerts in the single panel Signed-off-by: NucleonGodX --- .../scanpipe/panels/clarity_compliance.html | 32 ------------------- .../scanpipe/panels/project_compliance.html | 20 ++++++++++-- .../templates/scanpipe/project_detail.html | 1 - scanpipe/tests/test_views.py | 23 +++---------- scanpipe/urls.py | 5 --- scanpipe/views.py | 13 -------- 6 files changed, 22 insertions(+), 72 deletions(-) delete mode 100644 scanpipe/templates/scanpipe/panels/clarity_compliance.html diff --git a/scanpipe/templates/scanpipe/panels/clarity_compliance.html b/scanpipe/templates/scanpipe/panels/clarity_compliance.html deleted file mode 100644 index 5a511964a..000000000 --- a/scanpipe/templates/scanpipe/panels/clarity_compliance.html +++ /dev/null @@ -1,32 +0,0 @@ -{% if clarity_compliance_alert %} -
- -
-{% endif %} \ No newline at end of file diff --git a/scanpipe/templates/scanpipe/panels/project_compliance.html b/scanpipe/templates/scanpipe/panels/project_compliance.html index 167ea1c08..0fec197f8 100644 --- a/scanpipe/templates/scanpipe/panels/project_compliance.html +++ b/scanpipe/templates/scanpipe/panels/project_compliance.html @@ -1,11 +1,11 @@ {% load humanize %} -{% if compliance_alerts %} +{% if compliance_alerts or clarity_compliance_alert %}
-{% endif %} \ No newline at end of file +{% endif %} diff --git a/scanpipe/templates/scanpipe/project_detail.html b/scanpipe/templates/scanpipe/project_detail.html index 4b16ad85a..e494166de 100644 --- a/scanpipe/templates/scanpipe/project_detail.html +++ b/scanpipe/templates/scanpipe/project_detail.html @@ -120,7 +120,6 @@ {% if policies_enabled %}
-
{% endif %} diff --git a/scanpipe/tests/test_views.py b/scanpipe/tests/test_views.py index 1a7fe05d3..fb82d9fa8 100644 --- a/scanpipe/tests/test_views.py +++ b/scanpipe/tests/test_views.py @@ -997,6 +997,9 @@ def test_scanpipe_views_project_compliance_panel_view(self, mock_policies_enable compliance_alert=DiscoveredPackage.Compliance.ERROR, ) + self.project1.extra_data = {"clarity_compliance_alert": "warning"} + self.project1.save(update_fields=["extra_data"]) + mock_policies_enabled.return_value = False response = self.client.get(url) self.assertEqual(404, response.status_code) @@ -1005,27 +1008,11 @@ def test_scanpipe_views_project_compliance_panel_view(self, mock_policies_enable response = self.client.get(url) self.assertContains(response, "Compliance alerts") self.assertContains(response, "1 Error") + self.assertContains(response, "License clarity") + self.assertContains(response, "Warning") expected = f"/project/{self.project1.slug}/packages/?compliance_alert=error" self.assertContains(response, expected) - @mock.patch.object(Project, "policies_enabled", new_callable=mock.PropertyMock) - def test_scanpipe_views_project_clarity_compliance_panel_view( - self, mock_policies_enabled - ): - url = reverse("clarity_compliance_panel", args=[self.project1.slug]) - self.project1.extra_data = {"clarity_compliance_alert": "error"} - self.project1.save(update_fields=["extra_data"]) - - mock_policies_enabled.return_value = False - response = self.client.get(url) - self.assertEqual(404, response.status_code) - - mock_policies_enabled.return_value = True - response = self.client.get(url) - self.assertContains(response, "Clarity Compliance") - self.assertContains(response, "Error") - self.assertContains(response, "License clarity is insufficient") - def test_scanpipe_views_pipeline_help_view(self): url = reverse("pipeline_help", args=["not_existing_pipeline"]) response = self.client.get(url) diff --git a/scanpipe/urls.py b/scanpipe/urls.py index e84a54bfa..c760becbf 100644 --- a/scanpipe/urls.py +++ b/scanpipe/urls.py @@ -221,11 +221,6 @@ views.ProjectCompliancePanelView.as_view(), name="project_compliance_panel", ), - path( - "projects//clarity_compliance_panel/", - views.ClarityCompliancePanelView.as_view(), - name="clarity_compliance_panel", - ), path( "project//", views.ProjectDetailView.as_view(), diff --git a/scanpipe/views.py b/scanpipe/views.py index d91e9fe99..d5b9a79a5 100644 --- a/scanpipe/views.py +++ b/scanpipe/views.py @@ -1209,19 +1209,6 @@ def get_context_data(self, **kwargs): fail_level="missing", ) context["compliance_alerts"] = compliance_alerts - return context - - -class ClarityCompliancePanelView(ConditionalLoginRequired, generic.DetailView): - model = Project - template_name = "scanpipe/panels/clarity_compliance.html" - - def get_context_data(self, **kwargs): - context = super().get_context_data(**kwargs) - project = self.object - - if not project.policies_enabled: - raise Http404 extra_data = project.extra_data or {} context["clarity_compliance_alert"] = extra_data.get("clarity_compliance_alert") From a102e00b777c03ff96c1df280686889b592f7e7b Mon Sep 17 00:00:00 2001 From: NucleonGodX Date: Mon, 30 Jun 2025 19:51:50 +0530 Subject: [PATCH 4/9] add updated check-compliance Signed-off-by: NucleonGodX --- .../management/commands/check-compliance.py | 18 ++++++-- scanpipe/tests/test_commands.py | 42 +++++++++++++++++++ 2 files changed, 57 insertions(+), 3 deletions(-) diff --git a/scanpipe/management/commands/check-compliance.py b/scanpipe/management/commands/check-compliance.py index 342b5b55e..a14119777 100644 --- a/scanpipe/management/commands/check-compliance.py +++ b/scanpipe/management/commands/check-compliance.py @@ -74,8 +74,16 @@ def check_compliance(self, fail_level): len(issues) for model in alerts.values() for issues in model.values() ) - if count and self.verbosity > 0: - self.stderr.write(f"{count} compliance issues detected.") + extra_data = self.project.extra_data or {} + clarity_alert = extra_data.get("clarity_compliance_alert") + + # Count clarity issue only if alert is not 'ok' or None + clarity_issue_count = 1 if clarity_alert and clarity_alert != "ok" else 0 + + total_issues = count + clarity_issue_count + + if total_issues and self.verbosity > 0: + self.stderr.write(f"{total_issues} compliance issues detected.") for label, model in alerts.items(): self.stderr.write(f"[{label}]") for severity, entries in model.items(): @@ -83,7 +91,11 @@ def check_compliance(self, fail_level): if self.verbosity > 1: self.stderr.write(" " + "\n ".join(entries)) - return count > 0 + if clarity_issue_count: + self.stderr.write("[License Clarity Compliance]") + self.stderr.write(f" > Alert Level: {clarity_alert}") + + return total_issues > 0 def check_vulnerabilities(self): packages = self.project.discoveredpackages.vulnerable_ordered() diff --git a/scanpipe/tests/test_commands.py b/scanpipe/tests/test_commands.py index 6dcbd5d09..97e09fde3 100644 --- a/scanpipe/tests/test_commands.py +++ b/scanpipe/tests/test_commands.py @@ -1213,6 +1213,48 @@ def test_scanpipe_management_command_check_compliance(self): ) self.assertEqual(expected, out_value) + def test_scanpipe_management_command_check_clarity_compliance_only(self): + project = make_project(name="my_project_clarity") + + project.extra_data = {"clarity_compliance_alert": "error"} + project.save(update_fields=["extra_data"]) + + out = StringIO() + options = ["--project", project.name] + with self.assertRaises(SystemExit) as cm: + call_command("check-compliance", *options, stderr=out) + self.assertEqual(cm.exception.code, 1) + out_value = out.getvalue().strip() + expected = ( + "1 compliance issues detected." + "\n[License Clarity Compliance]\n > Alert Level: error" + ) + self.assertEqual(expected, out_value) + + def test_scanpipe_management_command_check_both_compliance_and_clarity(self): + project = make_project(name="my_project_both") + + make_package( + project, + package_url="pkg:generic/name@1.0", + compliance_alert=CodebaseResource.Compliance.ERROR, + ) + project.extra_data = {"clarity_compliance_alert": "warning"} + project.save(update_fields=["extra_data"]) + + out = StringIO() + options = ["--project", project.name, "--fail-level", "WARNING"] + with self.assertRaises(SystemExit) as cm: + call_command("check-compliance", *options, stderr=out) + self.assertEqual(cm.exception.code, 1) + out_value = out.getvalue().strip() + expected = ( + "2 compliance issues detected." + "\n[packages]\n > ERROR: 1" + "\n[License Clarity Compliance]\n > Alert Level: warning" + ) + self.assertEqual(expected, out_value) + def test_scanpipe_management_command_check_compliance_vulnerabilities(self): project = make_project(name="my_project") package1 = make_package(project, package_url="pkg:generic/name@1.0") From ae1c68806e1bd4af9862b71e67f05438424a5297 Mon Sep 17 00:00:00 2001 From: NucleonGodX Date: Thu, 3 Jul 2025 21:36:46 +0530 Subject: [PATCH 5/9] add documentation and support api endpoint for clarity compliance Signed-off-by: NucleonGodX --- docs/policies.rst | 39 ++++++++++++++++++++++- docs/rest-api.rst | 23 ++++++++++++++ docs/tutorial_license_policies.rst | 51 ++++++++++++++++++++++++++++-- scanpipe/api/views.py | 16 ++++++++++ 4 files changed, 126 insertions(+), 3 deletions(-) diff --git a/docs/policies.rst b/docs/policies.rst index 52060cc94..a922eeb4f 100644 --- a/docs/policies.rst +++ b/docs/policies.rst @@ -38,6 +38,43 @@ structure similar to the following: - ``warning`` - ``error`` +Creating Clarity Thresholds Files +--------------------------------- + +A valid clarity thresholds file is required to **enable clarity compliance features**. + +The clarity thresholds file, by default named ``policies.yml``, is a **YAML file** with a +structure similar to the following: + +.. code-block:: yaml + + license_clarity_thresholds: + 91: ok + 80: warning + 0: error + +- In the example above, the keys ``91``, ``80``, and ``0`` are integer threshold values + representing **minimum clarity scores**. +- The values ``error``, ``warning``, and ``ok`` are the **compliance alert levels** that + will be triggered if the project's license clarity score meets or exceeds the + corresponding threshold. +- The thresholds must be listed in **strictly descending order**. + +How it works: + +- If the clarity score is **91 or above**, the alert is **``ok``**. +- If the clarity score is **80 to 90**, the alert is **``warning``**. +- If the clarity score is **below 80**, the alert is **``error``**. + +You can adjust the threshold values and alert levels to match your organization's +compliance requirements. + +Accepted values for the alert level: + +- ``ok`` +- ``warning`` +- ``error`` + App Policies ------------ @@ -99,7 +136,7 @@ REST API -------- For more details on retrieving compliance data through the REST API, see the -:ref:`rest_api_compliance` section. +:ref:`rest_api_compliance` section and :ref:`rest_api_clarity_compliance` section Command Line Interface ---------------------- diff --git a/docs/rest-api.rst b/docs/rest-api.rst index 7525803ed..e08ce9d0b 100644 --- a/docs/rest-api.rst +++ b/docs/rest-api.rst @@ -493,6 +493,29 @@ Data: } } +.. _rest_api_clarity_compliance: + +Clarity Compliance +^^^^^^^^^^^^^^^^^^ + +This action returns the **clarity compliance alert** for a project. + +The clarity compliance alert is a single value (``ok``, ``warning``, or ``error``) that summarizes +the project's **license clarity status**, based on the thresholds defined in the ``policies.yml`` file. + +``GET /api/projects/6461408c-726c-4b70-aa7a-c9cc9d1c9685/clarity_compliance/`` + +Data: + - ``clarity_compliance_alert``: The overall clarity compliance alert for the project. + + Possible values: ``ok``, ``warning``, ``error``. + +.. code-block:: json + + { + "clarity_compliance_alert": "warning" + } + Reset ^^^^^ diff --git a/docs/tutorial_license_policies.rst b/docs/tutorial_license_policies.rst index 9e0a6aea9..47f515ea1 100644 --- a/docs/tutorial_license_policies.rst +++ b/docs/tutorial_license_policies.rst @@ -83,6 +83,51 @@ detected license, and computed at the codebase resource level, for example: "[...]": "[...]" } +License Clarity Thresholds and Compliance +----------------------------------------- + +ScanCode.io also supports **license clarity thresholds**, allowing you to enforce +minimum standards for license detection quality in your codebase. This is managed +through the ``license_clarity_thresholds`` section in your ``policies.yml`` file. + +Defining Clarity Thresholds +--------------------------- + +Add a ``license_clarity_thresholds`` section to your ``policies.yml`` file, for example: + +.. code-block:: yaml + + license_clarity_thresholds: + 91: ok + 80: warning + 0: error + + +Clarity Compliance in Results +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When you run a pipeline with clarity thresholds defined in your ``policies.yml``, +the computed clarity compliance alert is included in the project's ``extra_data`` field. + +For example: + +.. code-block:: json + + "extra_data": { + "md5": "d23df4a4", + "sha1": "3e9b61cc98c", + "size": 3095, + "sha256": "abacfc8bcee59067", + "sha512": "208f6a83c83a4c770b3c0", + "filename": "cuckoo_filter-1.0.6.tar.gz", + "sha1_git": "3fdb0f82ad59", + "clarity_compliance_alert": "error" + } + +The ``clarity_compliance_alert`` value (e.g., ``"error"``, ``"warning"``, or ``"ok"``) +is computed automatically based on the thresholds you configured and reflects the +overall license clarity status of the scanned codebase. + Run the ``check-compliance`` command ------------------------------------ @@ -95,7 +140,7 @@ in the project: .. code-block:: bash - 4 compliance issues detected on this project. + 5 compliance issues detected on this project. [packages] > ERROR: 3 pkg:pypi/cuckoo-filter@. @@ -104,7 +149,9 @@ in the project: [resources] > ERROR: 1 cuckoo_filter-1.0.6.tar.gz-extract/cuckoo_filter-1.0.6/README.md + [License Clarity Compliance] + > Alert Level: error .. tip:: In case of compliance alerts, the command returns a non-zero exit code which - may be useful to trigger a failure in an automated process. + may be useful to trigger a failure in an automated process. \ No newline at end of file diff --git a/scanpipe/api/views.py b/scanpipe/api/views.py index 4870c21a3..8e334f948 100644 --- a/scanpipe/api/views.py +++ b/scanpipe/api/views.py @@ -481,6 +481,22 @@ def compliance(self, request, *args, **kwargs): compliance_alerts = get_project_compliance_alerts(project, fail_level) return Response({"compliance_alerts": compliance_alerts}) + @action(detail=True, methods=["get"]) + def clarity_compliance(self, request, *args, **kwargs): + """ + Retrieve the clarity compliance alert for a project. + + This endpoint returns the clarity compliance alert stored in the + project's extra_data. + + Example: + GET /api/projects/{project_id}/clarity_compliance/ + + """ + project = self.get_object() + clarity_alert = (project.extra_data or {}).get("clarity_compliance_alert") + return Response({"clarity_compliance_alert": clarity_alert}) + class RunViewSet(mixins.RetrieveModelMixin, viewsets.GenericViewSet): """Add actions to the Run viewset.""" From f205c7c63a4f87e5eed0bac895c4d08b37af2bcc Mon Sep 17 00:00:00 2001 From: NucleonGodX Date: Thu, 3 Jul 2025 21:44:31 +0530 Subject: [PATCH 6/9] fix tests Signed-off-by: NucleonGodX --- docs/policies.rst | 2 +- docs/tutorial_license_policies.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/policies.rst b/docs/policies.rst index a922eeb4f..1803b3ec9 100644 --- a/docs/policies.rst +++ b/docs/policies.rst @@ -136,7 +136,7 @@ REST API -------- For more details on retrieving compliance data through the REST API, see the -:ref:`rest_api_compliance` section and :ref:`rest_api_clarity_compliance` section +:ref:`rest_api_compliance` section and :ref:`rest_api_clarity_compliance` section Command Line Interface ---------------------- diff --git a/docs/tutorial_license_policies.rst b/docs/tutorial_license_policies.rst index 47f515ea1..31bc6b931 100644 --- a/docs/tutorial_license_policies.rst +++ b/docs/tutorial_license_policies.rst @@ -154,4 +154,4 @@ in the project: .. tip:: In case of compliance alerts, the command returns a non-zero exit code which - may be useful to trigger a failure in an automated process. \ No newline at end of file + may be useful to trigger a failure in an automated process. From 9732f9146e05832d5b3a3bf2f573dcfaaa12bc72 Mon Sep 17 00:00:00 2001 From: NucleonGodX Date: Fri, 4 Jul 2025 01:44:59 +0530 Subject: [PATCH 7/9] create separate method to retrieve compliance alert Signed-off-by: NucleonGodX --- docs/policies.rst | 2 +- .../management/commands/check-compliance.py | 19 ++++++++++++------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/docs/policies.rst b/docs/policies.rst index 1803b3ec9..47d4f163e 100644 --- a/docs/policies.rst +++ b/docs/policies.rst @@ -136,7 +136,7 @@ REST API -------- For more details on retrieving compliance data through the REST API, see the -:ref:`rest_api_compliance` section and :ref:`rest_api_clarity_compliance` section +:ref:`rest_api_compliance` section and :ref:`rest_api_clarity_compliance` section. Command Line Interface ---------------------- diff --git a/scanpipe/management/commands/check-compliance.py b/scanpipe/management/commands/check-compliance.py index a14119777..f967a4b22 100644 --- a/scanpipe/management/commands/check-compliance.py +++ b/scanpipe/management/commands/check-compliance.py @@ -74,13 +74,10 @@ def check_compliance(self, fail_level): len(issues) for model in alerts.values() for issues in model.values() ) - extra_data = self.project.extra_data or {} - clarity_alert = extra_data.get("clarity_compliance_alert") - - # Count clarity issue only if alert is not 'ok' or None - clarity_issue_count = 1 if clarity_alert and clarity_alert != "ok" else 0 + clarity_alert = self.get_clarity_compliance_alert() + has_clarity_issue = clarity_alert not in (None, "ok") - total_issues = count + clarity_issue_count + total_issues = count + (1 if has_clarity_issue else 0) if total_issues and self.verbosity > 0: self.stderr.write(f"{total_issues} compliance issues detected.") @@ -91,12 +88,20 @@ def check_compliance(self, fail_level): if self.verbosity > 1: self.stderr.write(" " + "\n ".join(entries)) - if clarity_issue_count: + if has_clarity_issue: self.stderr.write("[License Clarity Compliance]") self.stderr.write(f" > Alert Level: {clarity_alert}") return total_issues > 0 + def get_clarity_compliance_alert(self): + """ + Return the clarity compliance alert value for the project, + or None if not set. + """ + extra_data = self.project.extra_data or {} + return extra_data.get("clarity_compliance_alert") + def check_vulnerabilities(self): packages = self.project.discoveredpackages.vulnerable_ordered() dependencies = self.project.discovereddependencies.vulnerable_ordered() From 2598b0d4243c959477223233830fb6f1e761a731 Mon Sep 17 00:00:00 2001 From: NucleonGodX Date: Mon, 7 Jul 2025 19:24:24 +0530 Subject: [PATCH 8/9] used the new refactored policy code Signed-off-by: NucleonGodX --- docs/tutorial_license_policies.rst | 2 +- scanpipe/api/views.py | 2 +- .../management/commands/check-compliance.py | 12 ++------- scanpipe/models.py | 8 ++++++ scanpipe/pipes/license_clarity.py | 26 +++++++------------ scanpipe/pipes/scancode.py | 6 +---- scanpipe/tests/test_commands.py | 5 ++-- 7 files changed, 25 insertions(+), 36 deletions(-) diff --git a/docs/tutorial_license_policies.rst b/docs/tutorial_license_policies.rst index 31bc6b931..6f100c9bd 100644 --- a/docs/tutorial_license_policies.rst +++ b/docs/tutorial_license_policies.rst @@ -149,7 +149,7 @@ in the project: [resources] > ERROR: 1 cuckoo_filter-1.0.6.tar.gz-extract/cuckoo_filter-1.0.6/README.md - [License Clarity Compliance] + [License Clarity] > Alert Level: error .. tip:: diff --git a/scanpipe/api/views.py b/scanpipe/api/views.py index 8e334f948..e9053d1b3 100644 --- a/scanpipe/api/views.py +++ b/scanpipe/api/views.py @@ -494,7 +494,7 @@ def clarity_compliance(self, request, *args, **kwargs): """ project = self.get_object() - clarity_alert = (project.extra_data or {}).get("clarity_compliance_alert") + clarity_alert = project.get_clarity_compliance_alert() return Response({"clarity_compliance_alert": clarity_alert}) diff --git a/scanpipe/management/commands/check-compliance.py b/scanpipe/management/commands/check-compliance.py index f967a4b22..2a12b672f 100644 --- a/scanpipe/management/commands/check-compliance.py +++ b/scanpipe/management/commands/check-compliance.py @@ -74,7 +74,7 @@ def check_compliance(self, fail_level): len(issues) for model in alerts.values() for issues in model.values() ) - clarity_alert = self.get_clarity_compliance_alert() + clarity_alert = self.project.get_clarity_compliance_alert() has_clarity_issue = clarity_alert not in (None, "ok") total_issues = count + (1 if has_clarity_issue else 0) @@ -89,19 +89,11 @@ def check_compliance(self, fail_level): self.stderr.write(" " + "\n ".join(entries)) if has_clarity_issue: - self.stderr.write("[License Clarity Compliance]") + self.stderr.write("[License Clarity]") self.stderr.write(f" > Alert Level: {clarity_alert}") return total_issues > 0 - def get_clarity_compliance_alert(self): - """ - Return the clarity compliance alert value for the project, - or None if not set. - """ - extra_data = self.project.extra_data or {} - return extra_data.get("clarity_compliance_alert") - def check_vulnerabilities(self): packages = self.project.discoveredpackages.vulnerable_ordered() dependencies = self.project.discovereddependencies.vulnerable_ordered() diff --git a/scanpipe/models.py b/scanpipe/models.py index fd72fe711..b719ccc74 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -1514,6 +1514,14 @@ def get_policies_dict(self): return scanpipe_app.policies + def get_clarity_compliance_alert(self): + """ + Return the clarity compliance alert value for the project, + or None if not set. + """ + extra_data = self.extra_data or {} + return extra_data.get("clarity_compliance_alert") + def get_license_policy_index(self): """Return the policy license index for this project instance.""" if policies_dict := self.get_policies_dict(): diff --git a/scanpipe/pipes/license_clarity.py b/scanpipe/pipes/license_clarity.py index a4b044101..859e2e393 100644 --- a/scanpipe/pipes/license_clarity.py +++ b/scanpipe/pipes/license_clarity.py @@ -41,7 +41,6 @@ from pathlib import Path -from django.conf import settings from django.core.exceptions import ValidationError import saneyaml @@ -168,23 +167,18 @@ def load_clarity_thresholds_from_file(file_path): def get_project_clarity_thresholds(project): """ - Get clarity thresholds for a project, checking multiple sources. + Get clarity thresholds for a project using the unified policy loading logic. Returns: ClarityThresholdsPolicy or None: Policy object if thresholds are configured """ - if hasattr(project, "get_input_policies_file"): - policies_file = project.get_input_policies_file() - if policies_file: - policy = load_clarity_thresholds_from_file(policies_file) - if policy: - return policy - - global_policies_file = getattr(settings, "SCANCODEIO_POLICIES_FILE", None) - if global_policies_file: - policy = load_clarity_thresholds_from_file(global_policies_file) - if policy: - return policy - - return None + policies_dict = project.get_policies_dict() + if not policies_dict: + return None + + clarity_thresholds = policies_dict.get("license_clarity_thresholds") + if not clarity_thresholds: + return None + + return ClarityThresholdsPolicy(clarity_thresholds) diff --git a/scanpipe/pipes/scancode.py b/scanpipe/pipes/scancode.py index 98058041e..5098a78fe 100644 --- a/scanpipe/pipes/scancode.py +++ b/scanpipe/pipes/scancode.py @@ -975,9 +975,5 @@ def make_results_summary(project, scan_results_location): alert = clarity_policy.get_alert_for_score(clarity_score) summary["clarity_compliance_alert"] = alert - extra_data = project.extra_data or {} - extra_data["clarity_compliance_alert"] = alert - project.extra_data = extra_data - project.save(update_fields=["extra_data"]) - + project.update_extra_data({"clarity_compliance_alert": alert}) return summary diff --git a/scanpipe/tests/test_commands.py b/scanpipe/tests/test_commands.py index 525f2916a..7c3dfe91e 100644 --- a/scanpipe/tests/test_commands.py +++ b/scanpipe/tests/test_commands.py @@ -1226,8 +1226,7 @@ def test_scanpipe_management_command_check_clarity_compliance_only(self): self.assertEqual(cm.exception.code, 1) out_value = out.getvalue().strip() expected = ( - "1 compliance issues detected." - "\n[License Clarity Compliance]\n > Alert Level: error" + "1 compliance issues detected.\n[License Clarity]\n > Alert Level: error" ) self.assertEqual(expected, out_value) @@ -1251,7 +1250,7 @@ def test_scanpipe_management_command_check_both_compliance_and_clarity(self): expected = ( "2 compliance issues detected." "\n[packages]\n > ERROR: 1" - "\n[License Clarity Compliance]\n > Alert Level: warning" + "\n[License Clarity]\n > Alert Level: warning" ) self.assertEqual(expected, out_value) From 5fee298add0f0f5febd7b2983f5d47381bf4c6ba Mon Sep 17 00:00:00 2001 From: NucleonGodX Date: Mon, 7 Jul 2025 20:32:22 +0530 Subject: [PATCH 9/9] add test for api and replace clarity_compliance with license_clarity_compliance everywhere Signed-off-by: NucleonGodX --- docs/policies.rst | 2 +- docs/rest-api.rst | 18 ++++++++++-------- docs/tutorial_license_policies.rst | 14 +++++++------- scanpipe/api/views.py | 10 +++++----- .../management/commands/check-compliance.py | 6 +++--- scanpipe/models.py | 6 +++--- scanpipe/pipes/scancode.py | 6 +++--- .../scanpipe/panels/project_compliance.html | 12 ++++++------ scanpipe/tests/test_api.py | 18 ++++++++++++++++++ scanpipe/tests/test_commands.py | 8 ++++---- scanpipe/tests/test_views.py | 2 +- scanpipe/views.py | 4 +++- 12 files changed, 64 insertions(+), 42 deletions(-) diff --git a/docs/policies.rst b/docs/policies.rst index e49a9ae69..783942d7a 100644 --- a/docs/policies.rst +++ b/docs/policies.rst @@ -57,7 +57,7 @@ structure similar to the following: Creating Clarity Thresholds Files --------------------------------- -A valid clarity thresholds file is required to **enable clarity compliance features**. +A valid clarity thresholds file is required to **enable license clarity compliance features**. The clarity thresholds file, by default named ``policies.yml``, is a **YAML file** with a structure similar to the following: diff --git a/docs/rest-api.rst b/docs/rest-api.rst index d9a494fd5..09c2a4145 100644 --- a/docs/rest-api.rst +++ b/docs/rest-api.rst @@ -495,25 +495,27 @@ Data: .. _rest_api_clarity_compliance: -Clarity Compliance -^^^^^^^^^^^^^^^^^^ +License Clarity Compliance +^^^^^^^^^^^^^^^^^^^^^^^^^^ -This action returns the **clarity compliance alert** for a project. +This action returns the **license clarity compliance alert** for a project. -The clarity compliance alert is a single value (``ok``, ``warning``, or ``error``) that summarizes -the project's **license clarity status**, based on the thresholds defined in the ``policies.yml`` file. +The license clarity compliance alert is a single value (``ok``, ``warning``, or ``error``) +that summarizes the project's **license clarity status**, based on the thresholds defined in +the ``policies.yml`` file. -``GET /api/projects/6461408c-726c-4b70-aa7a-c9cc9d1c9685/clarity_compliance/`` +``GET /api/projects/6461408c-726c-4b70-aa7a-c9cc9d1c9685/license_clarity_compliance/`` Data: - - ``clarity_compliance_alert``: The overall clarity compliance alert for the project. + - ``license_clarity_compliance_alert``: The overall license clarity compliance alert + for the project. Possible values: ``ok``, ``warning``, ``error``. .. code-block:: json { - "clarity_compliance_alert": "warning" + "license_clarity_compliance_alert": "warning" } Reset diff --git a/docs/tutorial_license_policies.rst b/docs/tutorial_license_policies.rst index 6f100c9bd..aa7ac717c 100644 --- a/docs/tutorial_license_policies.rst +++ b/docs/tutorial_license_policies.rst @@ -103,11 +103,11 @@ Add a ``license_clarity_thresholds`` section to your ``policies.yml`` file, for 0: error -Clarity Compliance in Results -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +License Clarity Compliance in Results +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ When you run a pipeline with clarity thresholds defined in your ``policies.yml``, -the computed clarity compliance alert is included in the project's ``extra_data`` field. +the computed license clarity compliance alert is included in the project's ``extra_data`` field. For example: @@ -121,10 +121,10 @@ For example: "sha512": "208f6a83c83a4c770b3c0", "filename": "cuckoo_filter-1.0.6.tar.gz", "sha1_git": "3fdb0f82ad59", - "clarity_compliance_alert": "error" + "license_clarity_compliance_alert": "error" } -The ``clarity_compliance_alert`` value (e.g., ``"error"``, ``"warning"``, or ``"ok"``) +The ``license_clarity_compliance_alert`` value (e.g., ``"error"``, ``"warning"``, or ``"ok"``) is computed automatically based on the thresholds you configured and reflects the overall license clarity status of the scanned codebase. @@ -149,8 +149,8 @@ in the project: [resources] > ERROR: 1 cuckoo_filter-1.0.6.tar.gz-extract/cuckoo_filter-1.0.6/README.md - [License Clarity] - > Alert Level: error + [license clarity] + > Alert Level: ERROR .. tip:: In case of compliance alerts, the command returns a non-zero exit code which diff --git a/scanpipe/api/views.py b/scanpipe/api/views.py index e9053d1b3..b69796d98 100644 --- a/scanpipe/api/views.py +++ b/scanpipe/api/views.py @@ -484,18 +484,18 @@ def compliance(self, request, *args, **kwargs): @action(detail=True, methods=["get"]) def clarity_compliance(self, request, *args, **kwargs): """ - Retrieve the clarity compliance alert for a project. + Retrieve the license clarity compliance alert for a project. - This endpoint returns the clarity compliance alert stored in the + This endpoint returns the license clarity compliance alert stored in the project's extra_data. Example: - GET /api/projects/{project_id}/clarity_compliance/ + GET /api/projects/{project_id}/license_clarity_compliance/ """ project = self.get_object() - clarity_alert = project.get_clarity_compliance_alert() - return Response({"clarity_compliance_alert": clarity_alert}) + clarity_alert = project.get_license_clarity_compliance_alert() + return Response({"license_clarity_compliance_alert": clarity_alert}) class RunViewSet(mixins.RetrieveModelMixin, viewsets.GenericViewSet): diff --git a/scanpipe/management/commands/check-compliance.py b/scanpipe/management/commands/check-compliance.py index 2a12b672f..41e340684 100644 --- a/scanpipe/management/commands/check-compliance.py +++ b/scanpipe/management/commands/check-compliance.py @@ -74,7 +74,7 @@ def check_compliance(self, fail_level): len(issues) for model in alerts.values() for issues in model.values() ) - clarity_alert = self.project.get_clarity_compliance_alert() + clarity_alert = self.project.get_license_clarity_compliance_alert() has_clarity_issue = clarity_alert not in (None, "ok") total_issues = count + (1 if has_clarity_issue else 0) @@ -89,8 +89,8 @@ def check_compliance(self, fail_level): self.stderr.write(" " + "\n ".join(entries)) if has_clarity_issue: - self.stderr.write("[License Clarity]") - self.stderr.write(f" > Alert Level: {clarity_alert}") + self.stderr.write("[license clarity]") + self.stderr.write(f" > Alert Level: {clarity_alert.upper()}") return total_issues > 0 diff --git a/scanpipe/models.py b/scanpipe/models.py index b719ccc74..aeb187430 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -1514,13 +1514,13 @@ def get_policies_dict(self): return scanpipe_app.policies - def get_clarity_compliance_alert(self): + def get_license_clarity_compliance_alert(self): """ - Return the clarity compliance alert value for the project, + Return the license clarity compliance alert value for the project, or None if not set. """ extra_data = self.extra_data or {} - return extra_data.get("clarity_compliance_alert") + return extra_data.get("license_clarity_compliance_alert") def get_license_policy_index(self): """Return the policy license index for this project instance.""" diff --git a/scanpipe/pipes/scancode.py b/scanpipe/pipes/scancode.py index 5098a78fe..12d3d3272 100644 --- a/scanpipe/pipes/scancode.py +++ b/scanpipe/pipes/scancode.py @@ -932,7 +932,7 @@ def make_results_summary(project, scan_results_location): Extract selected sections of the Scan results, such as the `summary` `license_clarity_score`, and `license_matches` related data. The `key_files` are also collected and injected in the `summary` output. - Additionally, store clarity_compliance_alert in project's extra_data. + Additionally, store license_clarity_compliance_alert in project's extra_data. """ import json @@ -973,7 +973,7 @@ def make_results_summary(project, scan_results_location): clarity_policy = get_project_clarity_thresholds(project) if clarity_policy: alert = clarity_policy.get_alert_for_score(clarity_score) - summary["clarity_compliance_alert"] = alert + summary["license_clarity_compliance_alert"] = alert - project.update_extra_data({"clarity_compliance_alert": alert}) + project.update_extra_data({"license_clarity_compliance_alert": alert}) return summary diff --git a/scanpipe/templates/scanpipe/panels/project_compliance.html b/scanpipe/templates/scanpipe/panels/project_compliance.html index 0fec197f8..c1f0967b2 100644 --- a/scanpipe/templates/scanpipe/panels/project_compliance.html +++ b/scanpipe/templates/scanpipe/panels/project_compliance.html @@ -1,5 +1,5 @@ {% load humanize %} -{% if compliance_alerts or clarity_compliance_alert %} +{% if compliance_alerts or license_clarity_compliance_alert %}
{% endfor %} - {% if clarity_compliance_alert %} + {% if license_clarity_compliance_alert %}
License clarity - {{ clarity_compliance_alert|title }} + {{ license_clarity_compliance_alert|title }}
{% endif %} diff --git a/scanpipe/tests/test_api.py b/scanpipe/tests/test_api.py index 71cbe1f8d..ef621b79a 100644 --- a/scanpipe/tests/test_api.py +++ b/scanpipe/tests/test_api.py @@ -1252,6 +1252,24 @@ def test_scanpipe_api_project_action_compliance(self): } self.assertDictEqual(expected, response.data) + def test_scanpipe_api_project_action_license_clarity_compliance(self): + project = make_project() + url = reverse("project-clarity-compliance", args=[project.uuid]) + + response = self.csrf_client.get(url) + expected = {"license_clarity_compliance_alert": None} + self.assertEqual(expected, response.data) + + project.update_extra_data({"license_clarity_compliance_alert": "ok"}) + response = self.csrf_client.get(url) + expected = {"license_clarity_compliance_alert": "ok"} + self.assertEqual(expected, response.data) + + project.update_extra_data({"license_clarity_compliance_alert": "error"}) + response = self.csrf_client.get(url) + expected = {"license_clarity_compliance_alert": "error"} + self.assertEqual(expected, response.data) + def test_scanpipe_api_serializer_get_model_serializer(self): self.assertEqual( DiscoveredPackageSerializer, get_model_serializer(DiscoveredPackage) diff --git a/scanpipe/tests/test_commands.py b/scanpipe/tests/test_commands.py index 7c3dfe91e..0c9d03cda 100644 --- a/scanpipe/tests/test_commands.py +++ b/scanpipe/tests/test_commands.py @@ -1216,7 +1216,7 @@ def test_scanpipe_management_command_check_compliance(self): def test_scanpipe_management_command_check_clarity_compliance_only(self): project = make_project(name="my_project_clarity") - project.extra_data = {"clarity_compliance_alert": "error"} + project.extra_data = {"license_clarity_compliance_alert": "error"} project.save(update_fields=["extra_data"]) out = StringIO() @@ -1226,7 +1226,7 @@ def test_scanpipe_management_command_check_clarity_compliance_only(self): self.assertEqual(cm.exception.code, 1) out_value = out.getvalue().strip() expected = ( - "1 compliance issues detected.\n[License Clarity]\n > Alert Level: error" + "1 compliance issues detected.\n[license clarity]\n > Alert Level: ERROR" ) self.assertEqual(expected, out_value) @@ -1238,7 +1238,7 @@ def test_scanpipe_management_command_check_both_compliance_and_clarity(self): package_url="pkg:generic/name@1.0", compliance_alert=CodebaseResource.Compliance.ERROR, ) - project.extra_data = {"clarity_compliance_alert": "warning"} + project.extra_data = {"license_clarity_compliance_alert": "warning"} project.save(update_fields=["extra_data"]) out = StringIO() @@ -1250,7 +1250,7 @@ def test_scanpipe_management_command_check_both_compliance_and_clarity(self): expected = ( "2 compliance issues detected." "\n[packages]\n > ERROR: 1" - "\n[License Clarity]\n > Alert Level: warning" + "\n[license clarity]\n > Alert Level: WARNING" ) self.assertEqual(expected, out_value) diff --git a/scanpipe/tests/test_views.py b/scanpipe/tests/test_views.py index 7aefea9c8..518968961 100644 --- a/scanpipe/tests/test_views.py +++ b/scanpipe/tests/test_views.py @@ -1003,7 +1003,7 @@ def test_scanpipe_views_project_compliance_panel_view( compliance_alert=DiscoveredPackage.Compliance.ERROR, ) - self.project1.extra_data = {"clarity_compliance_alert": "warning"} + self.project1.extra_data = {"license_clarity_compliance_alert": "warning"} self.project1.save(update_fields=["extra_data"]) mock_license_policies_enabled.return_value = False diff --git a/scanpipe/views.py b/scanpipe/views.py index 528677c21..4e46dfeb9 100644 --- a/scanpipe/views.py +++ b/scanpipe/views.py @@ -1211,7 +1211,9 @@ def get_context_data(self, **kwargs): context["compliance_alerts"] = compliance_alerts extra_data = project.extra_data or {} - context["clarity_compliance_alert"] = extra_data.get("clarity_compliance_alert") + context["license_clarity_compliance_alert"] = extra_data.get( + "license_clarity_compliance_alert" + ) return context