Restore returning whole_lines by default

This restores returning whole_lines behaviour and also adds
attributes to functions so it can be turned off in the case of
debian copyrights where this was problematic.

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

Author: AyanSinhaMahapatra

src/licensedcode/detection.py

@@ -380,6 +380,7 @@ def to_dict(
         license_text_diagnostics=False,
         license_url_template=SCANCODE_LICENSEDB_URL,
         spdx_license_url=SPDX_LICENSE_URL,
+        whole_lines=True,
     ):
         def dict_fields(attr, value):
             if attr.name == 'file_region':
@@ -396,6 +397,7 @@ def dict_fields(attr, value):
                     license_text_diagnostics=license_text_diagnostics,
                     license_url_template=license_url_template,
                     spdx_license_url=spdx_license_url,
+                    whole_lines=whole_lines,
                 )
             )
 

src/licensedcode/match.py

@@ -759,6 +759,7 @@ def get_mapping(
         spdx_license_url,
         include_text=False,
         license_text_diagnostics=False,
+        whole_lines=True,
     ):
         """
         Return a list of "matches" scan data built from a license match.
@@ -771,7 +772,10 @@ def get_mapping(
             if license_text_diagnostics:
                 matched_text = self.matched_text(whole_lines=False, highlight=True)
             else:
-                matched_text = self.matched_text(whole_lines=False, highlight=False)
+                if whole_lines:
+                    matched_text = self.matched_text(whole_lines=True, highlight=False)
+                else:
+                    matched_text = self.matched_text(whole_lines=False, highlight=False)
 
         SCANCODE_BASE_URL = 'https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses'
         SCANCODE_LICENSE_TEXT_URL = SCANCODE_BASE_URL + '/{}.LICENSE'

src/packagedcode/debian_copyright.py

@@ -511,7 +511,8 @@ def get_license_detections_mapping(self):
         )
 
         license_detections_mapping, _expression = get_mapping_and_expression_from_detections(
-            license_detections=license_detections
+            license_detections=license_detections,
+            whole_lines=False,
         )
 
         return license_detections_mapping
@@ -661,7 +662,8 @@ def set_primary_license(self):
             detection_objects.append(detection)
 
         detections_mapping, _expression = get_mapping_and_expression_from_detections(
-            license_detections=detection_objects
+            license_detections=detection_objects,
+            whole_lines=False,
         )
 
         self.primary_license_detections = detections_mapping
@@ -955,7 +957,8 @@ def get_license_detections_mapping(self):
             detection_objects.append(detection)
 
         detections_mapping, _expression = get_mapping_and_expression_from_detections(
-            license_detections=detection_objects
+            license_detections=detection_objects,
+            whole_lines=False,
         )
 
         return detections_mapping

src/packagedcode/licensing.py

@@ -378,6 +378,7 @@ def get_mapping_and_expression_from_detections(
     unique=True,
     include_text=True,
     license_text_diagnostics=False,
+    whole_lines=True,
 ):
     detection_data = []
 
@@ -396,6 +397,7 @@ def get_mapping_and_expression_from_detections(
             license_detection.to_dict(
                 include_text=include_text,
                 license_text_diagnostics=license_text_diagnostics,
+                whole_lines=whole_lines,
             )
         )
 

tests/formattedcode/data/spdx/unicode/expected.tv

@@ -36,7 +36,7 @@ LicenseID: LicenseRef-scancode-agere-bsd
 LicenseName: Agere Systems BSD Software license
 LicenseComment: <text>See details at https://github.com/nexB/scancode-toolkit/blob/develop/src/licensedcode/data/licenses/agere-bsd.yml
 </text>
-ExtractedText: <text>SOFTWARE LICENSE
+ExtractedText: <text> * SOFTWARE LICENSE
 *
 * This software is provided subject to the following terms and conditions,
 * which you should read carefully before using the software.  Using this

tests/licensedcode/data/plugin_license/license-expression/spdx-expressions.expected.json

@@ -16,20 +16,20 @@
               "score": 100.0,
               "start_line": 2,
               "end_line": 2,
-              "matched_length": 4,
+              "matched_length": 5,
               "match_coverage": 100.0,
-              "matcher": "1-spdx-id",
+              "matcher": "2-aho",
               "license_expression": "zlib",
-              "rule_identifier": "spdx-license-identifier: zlib",
+              "rule_identifier": "zlib_101.RULE",
               "referenced_filenames": [],
               "is_license_text": false,
               "is_license_notice": false,
-              "is_license_reference": false,
-              "is_license_tag": true,
+              "is_license_reference": true,
+              "is_license_tag": false,
               "is_license_intro": false,
-              "rule_length": 4,
+              "rule_length": 5,
               "rule_relevance": 100,
-              "matched_text": "licenses.nuget.org/Zlib",
+              "matched_text": "https://licenses.nuget.org/Zlib",
               "licenses": [
                 {
                   "key": "zlib",
@@ -90,7 +90,7 @@
         }
       ],
       "license_clues": [],
-      "percentage_of_license_text": 90.91,
+      "percentage_of_license_text": 100.0,
       "scan_errors": []
     }
   ]

tests/licensedcode/data/plugin_license/mock_index/scan-unknown-intro-eclipse-foundation.expected.json

@@ -22,7 +22,7 @@
         "is_license_intro": true,
         "rule_length": 14,
         "rule_relevance": 100,
-        "matched_text": "This program and the accompanying materials are made\n * available under the terms of the",
+        "matched_text": " * This program and the accompanying materials are made\n * available under the terms of the Eclipse Public License 2.0",
         "licenses": [
           {
             "key": "unknown-license-reference",
@@ -59,7 +59,7 @@
         "is_license_intro": false,
         "rule_length": 5,
         "rule_relevance": 100,
-        "matched_text": "Eclipse Public License 2.0",
+        "matched_text": " * available under the terms of the Eclipse Public License 2.0",
         "licenses": [
           {
             "key": "epl-2.0",
@@ -96,7 +96,7 @@
         "is_license_intro": false,
         "rule_length": 8,
         "rule_relevance": 100,
-        "matched_text": "https://www.eclipse.org/legal/epl-2.0/",
+        "matched_text": " * which is available at https://www.eclipse.org/legal/epl-2.0/",
         "licenses": [
           {
             "key": "epl-2.0",
@@ -133,7 +133,7 @@
         "is_license_intro": false,
         "rule_length": 5,
         "rule_relevance": 100,
-        "matched_text": "License-Identifier: EPL-2.0",
+        "matched_text": " * SPDX-License-Identifier: EPL-2.0",
         "licenses": [
           {
             "key": "epl-2.0",

tests/licensedcode/data/plugin_license/text/scan.expected.json

@@ -117,7 +117,7 @@
               "is_license_intro": false,
               "rule_length": 35,
               "rule_relevance": 100,
-              "matched_text": "and distribution of this file, with or without modification, are\npermitted in any medium without royalties provided the copyright notice\nand this notice are preserved. This file is offered as-is, without any",
+              "matched_text": "Reproduction and distribution of this file, with or without modification, are\npermitted in any medium without royalties provided the copyright notice\nand this notice are preserved. This file is offered as-is, without any warranties.",
               "licenses": [
                 {
                   "key": "fsf-ap",