Improve alpine license tests debuggability

Provide details on each step of the Alpine expression cleanups

Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>

Author: pombredanne

src/packagedcode/alpine.py

@@ -8,6 +8,7 @@
 #
 import base64
 import codecs
+import dataclasses
 import email
 import posixpath
 import re
@@ -1362,32 +1363,76 @@ def source_handler(value, **kwargs):
 }
 
 
-def detect_declared_license(declared):
+@dataclasses.dataclass
+class ApkLicenseDetection:
+    """
+    Represent the results of an Alpine license detection, including intermediate steps.
     """
-    Return a tuple of (cleaned declared license, detected license expression)
-    strings from a ``declared`` license text. Both can be None.
+    declared_license: str
+    cleaned_license: str
+    mapped_license: str
+    license_detections: list
+    license_expression: str
+
+    def to_dict(self):
+        return dict(
+            declared_license=self.declared_license                        ,
+            cleaned_license=self.cleaned_license                         ,
+            mapped_license=self.mapped_license                          ,
+            license_detections=self.license_detections                            ,
+            license_expression=self.license_expression             ,
+        )
+
+
+def get_alpine_license_detection(declared):
+    """
+    Return an ApkLicenseDetection from a ``declared`` license text
     """
     # cleaning first to fix syntax quirks and try to get something we can parse
-    cleaned = normalize_and_cleanup_declared_license(declared)
-    if not cleaned:
-        return None, None
+    cleaned_license = normalize_and_cleanup_declared_license(declared)
+    if not cleaned_license:
+        return None
 
-    # then we apply mappings for known non-standard symbols
+    # then we apply mappings for known non-standard symbols.
     # the output should be a proper SPDX expression
-    mapped = apply_expressions_mapping(cleaned)
+    mapped_license = apply_expressions_mapping(cleaned_license)
 
     # Finally perform SPDX expressions detection: Alpine uses mostly SPDX, but
     # with some quirks such as some non standard symbols (in addition to the
     # non-standard syntax)
     extra_licenses = {}
     expression_symbols = get_license_symbols(extra_licenses=extra_licenses)
 
-    license_detections, detected_license_expression = get_license_detections_and_expression(
-        extracted_license_statement=mapped,
+    license_detections, license_expression = get_license_detections_and_expression(
+        extracted_license_statement=mapped_license,
         expression_symbols=expression_symbols,
     )
 
-    return cleaned, detected_license_expression, license_detections
+    return ApkLicenseDetection(
+        declared_license=declared,
+        cleaned_license=cleaned_license,
+        mapped_license=mapped_license,
+        license_expression=license_expression,
+        license_detections=license_detections,
+    )
+
+
+def detect_declared_license(declared):
+    """
+    Return a three-tuple of detected license data from a ``declared`` license text, with this shape:
+        (cleaned declared license, detected license expression, license_detections)
+    - cleaned declared license and detected license expression are strings.
+    - license_detections is a list of LicenseDetection.
+    - Any of these can be None.
+    """
+    if alpine_detection := get_alpine_license_detection(declared):
+        return (
+            alpine_detection.cleaned_license,
+            alpine_detection.license_expression,
+            alpine_detection.license_detections,
+        )
+    else:
+        return None, None, None
 
 
 def get_license_symbols(extra_licenses):
@@ -1416,25 +1461,19 @@ def get_license_symbols(extra_licenses):
 def normalize_and_cleanup_declared_license(declared):
     """
     Return a cleaned and normalized declared license.
-
-    The expression should be valida SPDX but are far from this in practice.
-
+    The expressions should be valid SPDX license expressions but they are far from this in practice.
     Several fixes are applied:
-
     - plain text replacemnet aka. syntax fixes are plain text replacements
       to make the expression parsable
-
     - common fixes includes also nadling space-separated and comma-separated
       lists of licenses
     """
     declared = declared or ''
 
-    # normalize spaces
+    # normalize spaces and case
     declared = ' '.join(declared.split())
-
     declared = declared.lower()
 
-    # performa replacements
     declared = apply_syntax_fixes(declared)
 
     # comma-separated as in gpl-2.0+, lgpl-2.1+, zlib
@@ -1516,15 +1555,15 @@ def normalize_and_cleanup_declared_license(declared):
 
 def apply_syntax_fixes(s):
     """
-    Fix the expression string s by aplying replacement for various quirks.
+    Fix the expression string ``s`` by aplying replacement for various quirks to get clean license
+    expression syntax.
     """
     for src, tgt in EXPRESSION_SYNTAX_FIXES.items():
         s = s.replace(src, tgt)
     return s
 
-# These are parsed expression objects replacement that make the expression SPDX compliant
-
 
+# These are parsed expression objects replacement that make the expression SPDX compliant
 # {alpine sub-expression: SPDX subexpression}
 DECLARED_TO_SPDX = {
     'openssl-exception': 'licenseref-scancode-generic-exception',

tests/packagedcode/test_alpine_license.py

@@ -25,7 +25,6 @@
 from packagedcode import alpine
 from scancode_config import REGEN_TEST_FIXTURES
 
-
 """
 Data-driven tests using tests and expectations stored in YAML files.
 Test functions are attached to test classes at module import time
@@ -53,8 +52,7 @@ def from_file(cls, data_file):
             data = saneyaml.load(df.read())
         data['data_file'] = data_file
         alptest = cls(**data)
-        alptest.license_expression = cls.licensing.parse(
-            alptest.license_expression).render()
+        alptest.license_expression = cls.licensing.parse(alptest.license_expression).render()
         return alptest
 
     def to_dict(self):
@@ -110,14 +108,19 @@ def make_test(license_test, regen=REGEN_TEST_FIXTURES):
 
     def closure_test_function(*args, **kwargs):
         declared = license_test.declared_license
-        _cleaned, detected, _license_detections = alpine.detect_declared_license(declared)
+        detection = alpine.get_alpine_license_detection(declared)
 
         if regen:
-            license_test.license_expression = detected
+            license_test.license_expression = detection.license_expression
             license_test.dump()
             return
 
-        assert detected  == license_test.license_expression
+        if detection.license_expression != license_test.license_expression:
+            assert (
+                (detection.license_expression, list(detection.to_dict().items()))
+                == 
+                (license_test.license_expression, list(license_test.to_dict().items())) 
+            )
 
     return closure_test_function