Skip to content

Commit 54e5db2

Browse files
committed
integrate clarity compliance
Signed-off-by: NucleonGodX <racerpro41@gmail.com>
1 parent d161b95 commit 54e5db2

File tree

7 files changed

+193
-46
lines changed

7 files changed

+193
-46
lines changed

scanpipe/apps.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,8 @@
3737

3838
from licensedcode.models import load_licenses
3939

40-
from scanpipe.policies import load_policies_file
41-
from scanpipe.policies import make_license_policy_index
40+
from scanpipe.license_policies import load_policies_file
41+
from scanpipe.license_policies import make_license_policy_index
4242

4343
try:
4444
from importlib import metadata as importlib_metadata

scanpipe/forms.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,13 +29,13 @@
2929
from taggit.forms import TagField
3030
from taggit.forms import TagWidget
3131

32+
from scanpipe.license_policies import load_policies_yaml
33+
from scanpipe.license_policies import validate_policies
3234
from scanpipe.models import Project
3335
from scanpipe.models import Run
3436
from scanpipe.models import WebhookSubscription
3537
from scanpipe.pipelines import convert_markdown_to_html
3638
from scanpipe.pipes import fetch
37-
from scanpipe.policies import load_policies_yaml
38-
from scanpipe.policies import validate_policies
3939

4040
scanpipe_app = apps.get_app_config("scanpipe")
4141

scanpipe/policies.py renamed to scanpipe/license_policies.py

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,14 +25,21 @@
2525
import saneyaml
2626

2727

28-
def load_policies_yaml(policies_yaml):
29-
"""Load provided ``policies_yaml``."""
28+
def load_yaml_content(yaml_content):
29+
"""Load and parse YAML content into a Python dictionary."""
3030
try:
31-
return saneyaml.load(policies_yaml)
31+
return saneyaml.load(yaml_content)
3232
except saneyaml.YAMLError as e:
3333
raise ValidationError(f"Policies file format error: {e}")
3434

3535

36+
def load_policies_yaml(policies_yaml):
37+
"""Load provided ``policies_yaml``."""
38+
data = load_yaml_content(policies_yaml)
39+
validate_policies(data)
40+
return data
41+
42+
3643
def load_policies_file(policies_file, validate=True):
3744
"""
3845
Load provided ``policies_file`` into a Python dictionary.

scanpipe/models.py

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@
9494

9595
import scancodeio
9696
from scanpipe import humanize_time
97-
from scanpipe import policies
97+
from scanpipe import license_policies
9898
from scanpipe import tasks
9999

100100
logger = logging.getLogger(__name__)
@@ -1508,12 +1508,14 @@ def get_policy_index(self):
15081508
if policies_from_settings := self.get_env("policies"):
15091509
policies_dict = policies_from_settings
15101510
if isinstance(policies_from_settings, str):
1511-
policies_dict = policies.load_policies_yaml(policies_from_settings)
1512-
return policies.make_license_policy_index(policies_dict)
1511+
policies_dict = license_policies.load_policies_yaml(
1512+
policies_from_settings
1513+
)
1514+
return license_policies.make_license_policy_index(policies_dict)
15131515

15141516
elif policies_file := self.get_input_policies_file():
1515-
policies_dict = policies.load_policies_file(policies_file)
1516-
return policies.make_license_policy_index(policies_dict)
1517+
policies_dict = license_policies.load_policies_file(policies_file)
1518+
return license_policies.make_license_policy_index(policies_dict)
15171519

15181520
else:
15191521
return scanpipe_app.license_policies_index

scanpipe/pipes/license_clarity.py

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
# SPDX-License-Identifier: Apache-2.0
2+
#
3+
# http://nexb.com and https://github.com/nexB/scancode.io
4+
# The ScanCode.io software is licensed under the Apache License version 2.0.
5+
# Data generated with ScanCode.io is provided as-is without warranties.
6+
# ScanCode is a trademark of nexB Inc.
7+
#
8+
# You may not use this software except in compliance with the License.
9+
# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
10+
# Unless required by applicable law or agreed to in writing, software distributed
11+
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
12+
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
13+
# specific language governing permissions and limitations under the License.
14+
#
15+
# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
16+
# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
17+
# ScanCode.io should be considered or used as legal advice. Consult an Attorney
18+
# for any legal advice.
19+
#
20+
# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
21+
# Visit https://github.com/nexB/scancode.io for support and download.
22+
23+
# clarity_thresholds.py (updated)
24+
"""
25+
License Clarity Thresholds Management
26+
27+
This module provides an independent mechanism to read, validate, and evaluate
28+
license clarity score thresholds from policy files. Unlike license policies
29+
which are applied during scan processing, clarity thresholds are evaluated
30+
post-scan during summary generation.
31+
32+
The clarity thresholds system uses a simple key-value mapping where:
33+
- Keys are integer threshold values (minimum scores)
34+
- Values are compliance alert levels ('ok', 'warning', 'error')
35+
36+
Example policies.yml structure:
37+
38+
license_clarity_thresholds:
39+
80: ok # Scores >= 80 get 'ok' alert
40+
50: warning # Scores 50-79 get 'warning' alert
41+
"""
42+
43+
from django.core.exceptions import ValidationError
44+
45+
import saneyaml
46+
47+
48+
def load_yaml_content(yaml_content):
49+
"""Load and parse YAML content into a Python dictionary."""
50+
try:
51+
return saneyaml.load(yaml_content)
52+
except saneyaml.YAMLError as e:
53+
raise ValidationError(f"Policies file format error: {e}")
54+
55+
56+
class ClarityThresholdsPolicy:
57+
"""
58+
Manages clarity score thresholds and compliance evaluation.
59+
60+
This class reads clarity thresholds from a dictionary, validates them
61+
against threshold configurations and determines compliance alerts based on
62+
clarity scores.
63+
"""
64+
65+
def __init__(self, threshold_dict):
66+
"""Initialize with validated threshold dictionary."""
67+
self.thresholds = self.validate_thresholds(threshold_dict)
68+
69+
@staticmethod
70+
def validate_thresholds(threshold_dict):
71+
if not isinstance(threshold_dict, dict):
72+
raise ValidationError(
73+
"The `license_clarity_thresholds` must be a dictionary"
74+
)
75+
validated = {}
76+
seen = set()
77+
for key, value in threshold_dict.items():
78+
try:
79+
threshold = int(key)
80+
except (ValueError, TypeError):
81+
raise ValidationError(f"Threshold keys must be integers, got: {key}")
82+
if threshold in seen:
83+
raise ValidationError(f"Duplicate threshold key: {threshold}")
84+
seen.add(threshold)
85+
if value not in ["ok", "warning", "error"]:
86+
raise ValidationError(
87+
f"Compliance alert must be one of 'ok', 'warning', 'error', "
88+
f"got: {value}"
89+
)
90+
validated[threshold] = value
91+
sorted_keys = sorted(validated.keys(), reverse=True)
92+
if list(validated.keys()) != sorted_keys:
93+
raise ValidationError("Thresholds must be strictly descending")
94+
return validated
95+
96+
def get_alert_for_score(self, score):
97+
"""
98+
Determine compliance alert level for a given clarity score
99+
100+
Returns:
101+
str: Compliance alert level ('ok', 'warning', 'error')
102+
103+
"""
104+
if score is None:
105+
return "error"
106+
107+
# Find the highest threshold that the score meets or exceeds
108+
applicable_thresholds = [t for t in self.thresholds if score >= t]
109+
if not applicable_thresholds:
110+
return "error"
111+
112+
max_threshold = max(applicable_thresholds)
113+
return self.thresholds[max_threshold]
114+
115+
def get_thresholds_summary(self):
116+
"""
117+
Get a summary of configured thresholds for reporting
118+
119+
Returns:
120+
dict: Summary of thresholds and their alert levels
121+
122+
"""
123+
return dict(sorted(self.thresholds.items(), reverse=True))
124+
125+
126+
def load_clarity_thresholds_from_yaml(yaml_content):
127+
"""
128+
Load clarity thresholds from YAML content.
129+
130+
Returns:
131+
ClarityThresholdsPolicy: Configured policy object
132+
133+
"""
134+
data = load_yaml_content(yaml_content)
135+
136+
if not isinstance(data, dict):
137+
raise ValidationError("YAML content must be a dictionary.")
138+
139+
if "license_clarity_thresholds" not in data:
140+
raise ValidationError(
141+
"Missing 'license_clarity_thresholds' key in policies file."
142+
)
143+
144+
return ClarityThresholdsPolicy(data["license_clarity_thresholds"])
145+
146+
147+
def load_clarity_thresholds_from_file(file_path):
148+
"""
149+
Load clarity thresholds from a YAML file.
150+
151+
Returns:
152+
ClarityThresholdsPolicy: Configured policy object or None if file not found
153+
154+
"""
155+
from pathlib import Path
156+
157+
file_path = Path(file_path)
158+
159+
if not file_path.exists():
160+
return None
161+
162+
try:
163+
yaml_content = file_path.read_text(encoding="utf-8")
164+
return load_clarity_thresholds_from_yaml(yaml_content)
165+
except (OSError, UnicodeDecodeError) as e:
166+
raise ValidationError(f"Error reading file {file_path}: {e}")

scanpipe/tests/pipes/test_license_clarity.py

Lines changed: 2 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -20,15 +20,12 @@
2020
# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
2121
# Visit https://github.com/nexB/scancode.io for support and download.
2222

23-
import tempfile
24-
from pathlib import Path
2523

2624
from django.core.exceptions import ValidationError
2725
from django.test import TestCase
2826

29-
from scanpipe.pipes.license_clarity_compliance import ClarityThresholdsPolicy
30-
from scanpipe.pipes.license_clarity_compliance import load_clarity_thresholds_from_file
31-
from scanpipe.pipes.license_clarity_compliance import load_clarity_thresholds_from_yaml
27+
from scanpipe.pipes.license_clarity import ClarityThresholdsPolicy
28+
from scanpipe.pipes.license_clarity import load_clarity_thresholds_from_yaml
3229

3330

3431
class ClarityThresholdsPolicyTest(TestCase):
@@ -148,28 +145,3 @@ def test_yaml_string_invalid_yaml(self):
148145
yaml_content = "license_clarity_thresholds: [80, 50"
149146
with self.assertRaises(ValidationError):
150147
load_clarity_thresholds_from_yaml(yaml_content)
151-
152-
153-
class ClarityThresholdsFileLoadingTest(TestCase):
154-
"""Test file loading functionality."""
155-
156-
def test_load_from_existing_file(self):
157-
yaml_content = """
158-
license_clarity_thresholds:
159-
90: ok
160-
70: warning
161-
"""
162-
with tempfile.NamedTemporaryFile(mode="w", suffix=".yml", delete=False) as f:
163-
f.write(yaml_content)
164-
temp_path = f.name
165-
166-
try:
167-
policy = load_clarity_thresholds_from_file(temp_path)
168-
self.assertIsNotNone(policy)
169-
self.assertEqual(policy.get_alert_for_score(95), "ok")
170-
finally:
171-
Path(temp_path).unlink()
172-
173-
def test_load_from_nonexistent_file(self):
174-
policy = load_clarity_thresholds_from_file("/nonexistent/file.yml")
175-
self.assertIsNone(policy)

scanpipe/tests/test_policies.py renamed to scanpipe/tests/test_license_policies.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,11 +27,11 @@
2727
from django.core.exceptions import ValidationError
2828
from django.test import TestCase
2929

30+
from scanpipe.license_policies import load_policies_file
31+
from scanpipe.license_policies import load_policies_yaml
32+
from scanpipe.license_policies import make_license_policy_index
33+
from scanpipe.license_policies import validate_policies
3034
from scanpipe.pipes.input import copy_input
31-
from scanpipe.policies import load_policies_file
32-
from scanpipe.policies import load_policies_yaml
33-
from scanpipe.policies import make_license_policy_index
34-
from scanpipe.policies import validate_policies
3535
from scanpipe.tests import global_policies
3636
from scanpipe.tests import license_policies_index
3737
from scanpipe.tests import make_project

0 commit comments

Comments
 (0)