Make license references default

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

Author: AyanSinhaMahapatra

src/formattedcode/output_debian.py

@@ -17,7 +17,7 @@
 from plugincode.output import output_impl
 from plugincode.output import OutputPlugin
 from licensedcode.detection import get_matches_from_detection_mappings
-
+from licensedcode.plugin_licenses_reference import get_matched_text_from_reference_data
 from scancode import notice
 
 """
@@ -106,7 +106,7 @@ def build_copyright_paragraphs(codebase, **kwargs):
         if scanned_file['type'] == 'directory':
             continue
         dfiles = scanned_file['path']
-        dlicense = build_license(scanned_file)
+        dlicense = build_license(codebase, scanned_file)
         dcopyright = build_copyright_field(scanned_file)
 
         file_para = CopyrightFilesParagraph.from_dict(dict(
@@ -132,7 +132,7 @@ def build_copyright_field(scanned_file):
     return '\n'.join(statements)
 
 
-def build_license(scanned_file):
+def build_license(codebase, scanned_file):
     """
     Return Debian-like text where the first line is the expression and the
     remaining lines are the license text from licenses detected in
@@ -146,11 +146,11 @@ def build_license(scanned_file):
         return
 
     licenses = scanned_file.get('license_detections', [])
-    text = '\n'.join(get_texts(licenses))
+    text = '\n'.join(get_texts(codebase, licenses))
     return f'{expression}\n{text}'
 
 
-def get_texts(detected_licenses):
+def get_texts(codebase, detected_licenses):
     """
     Yield license texts detected in this file.
 
@@ -179,8 +179,12 @@ def get_texts(detected_licenses):
     # set of (start line, end line, matched_rule identifier)
     seen = set()
     for lic in get_matches_from_detection_mappings(detected_licenses):
+        matched_text = get_matched_text_from_reference_data(
+            codebase=codebase,
+            rule_identifier=lic['rule_identifier']
+        )
         key = lic['start_line'], lic['end_line'], lic['rule_identifier']
         if key not in seen:
-            yield lic['matched_text']
+            if matched_text != None:
+                yield matched_text
             seen.add(key)
-

src/formattedcode/output_spdx.py

@@ -23,13 +23,15 @@
 from spdx.utils import SPDXNone
 from spdx.version import Version
 
+from license_expression import Licensing
 from commoncode.cliutils import OUTPUT_GROUP
 from commoncode.cliutils import PluggableCommandLineOption
 from commoncode.fileutils import file_name
 from commoncode.fileutils import parent_directory
 from commoncode.text import python_safe_name
 from formattedcode import FileOptionType
 from licensedcode.detection import get_matches_from_detection_mappings
+from licensedcode.plugin_licenses_reference import get_matched_text_from_reference_data
 from plugincode.output import output_impl
 from plugincode.output import OutputPlugin
 import scancode_config
@@ -170,6 +172,7 @@ def _process_codebase(
     package_name = build_package_name(input_path)
 
     write_spdx(
+        codebase=codebase,
         output_file=output_file,
         files=files,
         tool_name=tool_name,
@@ -208,6 +211,7 @@ def check_sha1(codebase):
 
 
 def write_spdx(
+    codebase,
     output_file,
     files,
     tool_name,
@@ -229,6 +233,10 @@ def write_spdx(
     producing this SPDX document.
     Use ``package_name`` as a Package name and as a namespace prefix base.
     """
+    from licensedcode import cache
+    licenses = cache.get_licenses_db()
+    licensing = Licensing()
+
     as_rdf = not as_tagvalue
     _patch_license_list()
 
@@ -282,11 +290,20 @@ def write_spdx(
         if license_matches:
             all_files_have_no_license = False
             for match in license_matches:
-                file_licenses = match["licenses"]
-                for file_license in file_licenses:
-                    license_key = file_license.get('key')
-
-                    spdx_id = file_license.get('spdx_license_key')
+                file_license_expression = match["license_expression"]
+                file_license_keys = licensing.license_keys(
+                    expression=file_license_expression,
+                    unique=True
+                )
+                matched_text = get_matched_text_from_reference_data(
+                    codebase=codebase,
+                    rule_identifier=match["rule_identifier"],
+                )
+                for license_key in file_license_keys:
+                    file_license = licenses.get(license_key)
+                    license_key = file_license.key
+
+                    spdx_id = file_license.spdx_license_key
                     if not spdx_id:
                         spdx_id = f'LicenseRef-scancode-{license_key}'
                     is_license_ref = spdx_id.lower().startswith('licenseref-')
@@ -295,7 +312,7 @@ def write_spdx(
                         spdx_license = License.from_identifier(spdx_id)
                     else:
                         spdx_license = ExtractedLicense(spdx_id)
-                        spdx_license.name = file_license.get('short_name')
+                        spdx_license.name = file_license.short_name
                         # FIXME: replace this with the licensedb URL
                         comment = (
                             f'See details at https://github.com/nexB/scancode-toolkit'

src/licensedcode/plugin_licenses_reference.py

@@ -51,25 +51,33 @@ class LicensesReference(PostScanPlugin):
     sort_order = 500
 
     options = [
-        PluggableCommandLineOption(('--licenses-reference',),
-            is_flag=True, default=False,
+        PluggableCommandLineOption(('--no-licenses-reference',),
+            is_flag=True,
+            default=True,
             help='Include a reference of all the licenses referenced in this '
                  'scan with the data details and full texts.',
             help_group=POST_SCAN_GROUP)
     ]
 
-    def is_enabled(self, licenses_reference, **kwargs):
-        return licenses_reference
+    def is_enabled(self, no_licenses_reference, **kwargs):
+        return no_licenses_reference
 
-    def process_codebase(self, codebase, licenses_reference, **kwargs):
+    def process_codebase(self, codebase, no_licenses_reference, **kwargs):
         """
         Get unique License and Rule data from all license detections in a codebase-level
         list and only refer to them in the resource level detections. 
         """
         licexps = []
         rules_data = []
 
+        if not hasattr(codebase.attributes, 'licenses'):
+            return
+
+        has_packages = False
         if hasattr(codebase.attributes, 'packages'):
+            has_packages = True
+
+        if has_packages:
             codebase_packages = codebase.attributes.packages
             for pkg in codebase_packages:
                 rules_data.extend(
@@ -78,40 +86,58 @@ def process_codebase(self, codebase, licenses_reference, **kwargs):
                     )
                 )
                 licexps.append(pkg['declared_license_expression'])
+        
+        # This license rules reference data is duplicate as `licenses` is a
+        # top level summary of all unique license detections but this function
+        # is called as the side effect is removing the reference attributes
+        # from license matches
+        try:
+            _discard = get_license_rules_reference_data(codebase.attributes.licenses)
+        except KeyError:
+            pass
 
         for resource in codebase.walk():
 
             # Get license_expressions from both package and license detections
             license_licexp = getattr(resource, 'detected_license_expression')
             if license_licexp:
                 licexps.append(license_licexp)
-            package_data = getattr(resource, 'package_data', []) or []
-            package_licexps = [
-                pkg['declared_license_expression']
-                for pkg in package_data
-            ]
-            licexps.extend(package_licexps)
-
-            # Get license matches from both package and license detections
-            package_license_detections = []
-            for pkg in package_data:
-                if not pkg['license_detections']:
-                    continue
-
-                package_license_detections.extend(pkg['license_detections'])
-
-            rules_data.extend(
-                get_license_rules_reference_data(license_detections=package_license_detections)
-            )
+            
+            if has_packages:
+                package_data = getattr(resource, 'package_data', []) or []
+                package_licexps = [
+                    pkg['declared_license_expression']
+                    for pkg in package_data
+                ]
+                licexps.extend(package_licexps)
+
+                # Get license matches from both package and license detections
+                package_license_detections = []
+                for pkg in package_data:
+                    if not pkg['license_detections']:
+                        continue
+
+                    package_license_detections.extend(pkg['license_detections'])
+
+                try:
+                    rules_data.extend(
+                        get_license_rules_reference_data(license_detections=package_license_detections)
+                    )
+                except KeyError:
+                    pass
 
             license_detections = getattr(resource, 'license_detections', []) or []
             license_clues = getattr(resource, 'license_clues', []) or []
-            rules_data.extend(
-                get_license_rules_reference_data(
-                    license_detections=license_detections,
-                    license_clues=license_clues,
+
+            try:
+                rules_data.extend(
+                    get_license_rules_reference_data(
+                        license_detections=license_detections,
+                        license_clues=license_clues,
+                    )
                 )
-            )
+            except KeyError:
+                pass
 
             codebase.save_resource(resource)
 
@@ -122,6 +148,12 @@ def process_codebase(self, codebase, licenses_reference, **kwargs):
         codebase.attributes.rule_references.extend(rule_references)
 
 
+def get_matched_text_from_reference_data(codebase, rule_identifier):
+    for rule_reference_data in codebase.attributes.rule_references:
+        if rule_reference_data["rule_identifier"] == rule_identifier:
+            matched_text = getattr(rule_reference_data, "matched_text", None) or None
+            return matched_text
+
 def get_license_references(license_expressions, licensing=Licensing()):
     """
     Get a list of unique License data from a list of `license_expression` strings.