Add license_expression_spdx in Detections

Adds a new attribute license_expression_spdx to both LicenseMatch and
LicenseDetection objects, and reflected also in top-level detections and
todo items. Basically this adds the SPDX license-expression everywhere
we have a license-expression.

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

Author: AyanSinhaMahapatra

src/licensedcode/cache.py

@@ -545,11 +545,12 @@ def validate_spdx_license_keys(license_expression, licensing):
         try:
             parsed.render(template='{symbol.wrapped.spdx_license_key}')
         except AttributeError:
+            msg = f"Error rendering SPDX license key for: {key}"
             messages.append(msg)
             pass
 
     if messages:
-        raise InvalidLicenseKeyError(messages)
+        raise InvalidLicenseKeyError(f"ERROR in parsing license_expression: {license_expression}: type: {type(license_expression)} :{messages}")
 
 
 class InvalidLicenseKeyError(Exception):

src/licensedcode/detection.py

@@ -169,6 +169,12 @@ class LicenseDetection:
             'using the SPDX license expression syntax and ScanCode license keys.')
     )
 
+    license_expression_spdx = attr.ib(
+        default=None,
+        metadata=dict(
+            help='Full license expression string with SPDX license keys.')
+    )
+
     matches = attr.ib(
         default=attr.Factory(list),
         metadata=dict(
@@ -248,8 +254,17 @@ def from_matches(
             detection_log=detection_log,
         )
         detection.identifier = detection.identifier_with_expression
+        detection.license_expression_spdx = detection.spdx_license_expression()
         return detection
 
+    def spdx_license_expression(self):
+        from licensedcode.cache import build_spdx_license_expression
+        from licensedcode.cache import get_cache
+        return str(build_spdx_license_expression(
+            license_expression=self.license_expression,
+            licensing=get_cache().licensing,
+        ))
+
     def __eq__(self, other):
         return (
             isinstance(other, LicenseDetection)
@@ -515,6 +530,7 @@ def from_license_detection_mapping(
 
         detection = cls(
             license_expression=license_detection_mapping["license_expression"],
+            license_expression_spdx=license_detection_mapping["license_expression_spdx"],
             detection_log=license_detection_mapping.get("detection_log", []) or None,
             identifier=license_detection_mapping["identifier"],
             matches=matches,
@@ -665,6 +681,7 @@ def to_dict(
 
         # LicenseDB Level Information (Rule that was matched)
         result['license_expression'] = self.rule.license_expression
+        result['license_expression_spdx'] = self.rule.spdx_license_expression()
         result['rule_identifier'] = self.rule.identifier
         result['rule_relevance'] = self.rule.relevance
         result['rule_url'] = self.rule.rule_url
@@ -865,6 +882,7 @@ class UniqueDetection:
     """
     identifier = attr.ib(default=None)
     license_expression = attr.ib(default=None)
+    license_expression_spdx = attr.ib(default=None)
     detection_count = attr.ib(default=None)
     matches = attr.ib(default=attr.Factory(list))
     detection_log = attr.ib(default=attr.Factory(list))
@@ -897,12 +915,14 @@ def get_unique_detections(cls, license_detections):
                         for match in detection.matches
                     ]
                 ))
+                detection.license_expression_spdx = detection.spdx_license_expression()
                 detection.identifier = detection.identifier_with_expression
 
             unique_license_detections.append(
                 cls(
                     identifier=detection.identifier,
                     license_expression=detection.license_expression,
+                    license_expression_spdx=detection.license_expression_spdx,
                     detection_log=detection_log or [],
                     matches=detection.matches,
                     detection_count=len(file_regions),
@@ -937,6 +957,7 @@ def dict_fields(attr, value):
     def get_license_detection_object(self):
         return LicenseDetection(
             license_expression=self.license_expression,
+            license_expression_spdx=self.license_expression_spdx,
             detection_log=self.detection_log,
             matches=self.matches,
             identifier=self.identifier,

src/licensedcode/match.py

@@ -802,6 +802,7 @@ def to_dict(
 
         # LicenseDB Level Information (Rule that was matched)
         result['license_expression'] = self.rule.license_expression
+        result['spdx_license_expression'] = self.rule.spdx_license_expression()
         result['rule_identifier'] = self.rule.identifier
         result['rule_relevance'] = self.rule.relevance
         result['rule_url'] = self.rule.rule_url

src/licensedcode/models.py

@@ -1932,9 +1932,10 @@ def licensing_contains(self, other):
                 expression2=other.license_expression_object,
             )
 
-    def spdx_license_expression(self, licensing=None):
+    def spdx_license_expression(self):
         from licensedcode.cache import build_spdx_license_expression
-        return str(build_spdx_license_expression(self.license_expression, licensing=licensing))
+        from licensedcode.cache import get_cache
+        return str(build_spdx_license_expression(self.license_expression, licensing=get_cache().licensing))
 
     def get_length(self, unique=False):
         return self.length_unique if unique else self.length

src/licensedcode/plugin_license.py

@@ -297,7 +297,12 @@ def add_referenced_filenames_license_matches_for_detections(resource, codebase):
             analysis=DetectionCategory.UNKNOWN_FILE_REFERENCE_LOCAL.value,
             post_scan=True,
         )
+        license_expression_spdx = build_spdx_license_expression(
+            license_expression=str(license_expression),
+            licensing=get_cache().licensing,
+        )
         license_detection_mapping["license_expression"] = str(license_expression)
+        license_detection_mapping["license_expression_spdx"] = str(license_expression_spdx)
         license_detection_mapping["detection_log"] = detection_log
         license_detection_mapping["identifier"] = get_new_identifier_from_detections(
             initial_detection=license_detection_mapping,

src/packagedcode/licensing.py

@@ -133,7 +133,12 @@ def add_referenced_license_matches_for_package(resource, codebase):
                 analysis=DetectionCategory.PACKAGE_UNKNOWN_FILE_REFERENCE_LOCAL.value,
                 post_scan=True,
             )
+            license_expression_spdx = build_spdx_license_expression(
+                license_expression=str(license_expression),
+                licensing=get_cache().licensing,
+            )
             license_detection_mapping["license_expression"] = str(license_expression)
+            license_detection_mapping["license_expression_spdx"] = str(license_expression_spdx)
             license_detection_mapping["detection_log"] = detection_log
             license_detection_mapping["identifier"] = get_new_identifier_from_detections(
                 initial_detection=license_detection_mapping,
@@ -253,7 +258,12 @@ def add_referenced_license_detection_from_package(resource, codebase):
             analysis=analysis,
             post_scan=True,
         )
+        license_expression_spdx = build_spdx_license_expression(
+            license_expression=str(license_expression),
+            licensing=get_cache().licensing,
+        )
         license_detection_mapping["license_expression"] = str(license_expression)
+        license_detection_mapping["license_expression_spdx"] = str(license_expression_spdx)
         license_detection_mapping["detection_log"] = detection_log
         license_detection_mapping["identifier"] = get_new_identifier_from_detections(
             initial_detection=license_detection_mapping,

tests/cluecode/data/plugin_filter_clues/filtered-expected.json

@@ -3,6 +3,7 @@
     {
       "identifier": "apache_1_1-1712efcb-d696-b5e4-214d-b2ab69680c2a",
       "license_expression": "apache-1.1",
+      "license_expression_spdx": "Apache-1.1",
       "detection_count": 1,
       "sample_matches": [
         {
@@ -14,6 +15,7 @@
           "match_coverage": 100.0,
           "matcher": "3-seq",
           "license_expression": "apache-1.1",
+          "license_expression_spdx": "Apache-1.1",
           "rule_identifier": "apache-1.1_63.RULE",
           "rule_relevance": 100,
           "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/apache-1.1_63.RULE",
@@ -47,6 +49,7 @@
       "license_detections": [
         {
           "license_expression": "apache-1.1",
+          "license_expression_spdx": "Apache-1.1",
           "matches": [
             {
               "score": 96.07,
@@ -57,6 +60,7 @@
               "match_coverage": 100.0,
               "matcher": "3-seq",
               "license_expression": "apache-1.1",
+              "spdx_license_expression": "Apache-1.1",
               "rule_identifier": "apache-1.1_63.RULE",
               "rule_relevance": 100,
               "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/apache-1.1_63.RULE"

tests/cluecode/data/plugin_filter_clues/filtered-expected2.json

@@ -3,6 +3,7 @@
     {
       "identifier": "pygres_2_2-04f085e8-3db1-f9e2-8dde-7ba8a7ba619a",
       "license_expression": "pygres-2.2",
+      "license_expression_spdx": "LicenseRef-scancode-pygres-2.2",
       "detection_count": 1,
       "sample_matches": [
         {
@@ -14,6 +15,7 @@
           "match_coverage": 100.0,
           "matcher": "2-aho",
           "license_expression": "pygres-2.2",
+          "license_expression_spdx": "LicenseRef-scancode-pygres-2.2",
           "rule_identifier": "pygres-2.2_2.RULE",
           "rule_relevance": 100,
           "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/pygres-2.2_2.RULE",
@@ -47,6 +49,7 @@
       "license_detections": [
         {
           "license_expression": "pygres-2.2",
+          "license_expression_spdx": "LicenseRef-scancode-pygres-2.2",
           "matches": [
             {
               "score": 100.0,
@@ -57,6 +60,7 @@
               "match_coverage": 100.0,
               "matcher": "2-aho",
               "license_expression": "pygres-2.2",
+              "spdx_license_expression": "LicenseRef-scancode-pygres-2.2",
               "rule_identifier": "pygres-2.2_2.RULE",
               "rule_relevance": 100,
               "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/pygres-2.2_2.RULE"

tests/cluecode/data/plugin_filter_clues/filtered-expected3.json

@@ -3,6 +3,7 @@
     {
       "identifier": "pcre-c61d8210-7748-d787-5e3e-fd64c6cd6b6a",
       "license_expression": "pcre",
+      "license_expression_spdx": "LicenseRef-scancode-pcre",
       "detection_count": 1,
       "sample_matches": [
         {
@@ -14,6 +15,7 @@
           "match_coverage": 100.0,
           "matcher": "1-hash",
           "license_expression": "pcre",
+          "license_expression_spdx": "LicenseRef-scancode-pcre",
           "rule_identifier": "pcre.LICENSE",
           "rule_relevance": 100,
           "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/pcre.LICENSE",
@@ -47,6 +49,7 @@
       "license_detections": [
         {
           "license_expression": "pcre",
+          "license_expression_spdx": "LicenseRef-scancode-pcre",
           "matches": [
             {
               "score": 100.0,
@@ -57,6 +60,7 @@
               "match_coverage": 100.0,
               "matcher": "1-hash",
               "license_expression": "pcre",
+              "spdx_license_expression": "LicenseRef-scancode-pcre",
               "rule_identifier": "pcre.LICENSE",
               "rule_relevance": 100,
               "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/pcre.LICENSE"