Merge pull request #4215 from aboutcode-org/4190-license-licence

Support equivalent words in license detection #4190

Author: AyanSinhaMahapatra

CHANGELOG.rst

@@ -1,15 +1,15 @@
 Changelog
 =========
 
-v33.0.0 (next next, roadmap)
-----------------------------
-
 - We now support new package manifest formats:
 
   - OpenWRT packages.
   - Yocto/BitBake .bb recipes.
 
 
+v33.0.0 (next next, roadmap)
+----------------------------
+
 - Fallback packages for non-native dependencies of SCTK.
 - Dependencies for
 - Support for copyright detection objects.
@@ -26,9 +26,14 @@ v33.0.0 (next next, roadmap)
 
 - `--unknown-licenses` is removed and this is always enabled
   and only used in case of improper detections automatically.
-  Also tag all license rules with required phrases to improve
-  license detection and reduce false positives.
-  See https://github.com/nexB/scancode-toolkit/issues/3300
+
+- All license rules have been tagged with required phrases to improve detection accuracy
+  and reduce false positives. See https://github.com/nexB/scancode-toolkit/issues/3300
+
+- Equivalent words like license and licence, as well as plurals are now treated as the same in
+  license detection. With this, many redundant rules have been deprecated.
+
+- The license detection accuracy of Maven POMS has been improved fixing corner cases.
 
 - File categorization support added, a post scan plugin tagging
   files with priority levels for review, and also take advantage
@@ -41,6 +46,7 @@ v33.0.0 (next next, roadmap)
 - Update ABOUT files to adapt the ABOUT File Specification.
   See https://github.com/aboutcode-org/scancode-toolkit/issues/4181
 
+
 v32.3.3 - 2025-03-06
 --------------------
 

etc/scripts/licenses/buildrules.py

@@ -14,7 +14,6 @@
 
 from licensedcode import cache
 from licensedcode import models
-from licensedcode import match_hash
 from licensedcode import frontmatter
 from licensedcode.models import get_rule_id_for_text
 from license_expression import Licensing
@@ -73,7 +72,13 @@ def __attrs_post_init__(self, *args, **kwargs):
             print(rdat)
             print("########################################################")
             raise
-        self.data = {k: v for k, v in self.data.items() if v is not None or (v is None and k == "license_expression")}
+        if "referenced_filenames" in self.data and not self.data["referenced_filenames"]:
+            self.data.pop("referenced_filenames")
+        self.data = {
+            k: v for k, v in self.data.items()
+            if v is not None 
+            or (v is None and k == "license_expression")
+        }
 
 
 def load_data(location="00-new-licenses.txt"):
@@ -133,7 +138,7 @@ def load_data(location="00-new-licenses.txt"):
 
 def all_rule_by_tokens():
     """
-    Return a mapping of {tuples of tokens: rule id}, with one item for each
+    Return a mapping of {(tuple of token id): rule id}, with one item for each
     existing and added rules. Used to avoid duplicates.
     """
     rule_tokens = {}
@@ -143,7 +148,7 @@ def all_rule_by_tokens():
         except Exception as e:
             rf = f"  file://{rule.rule_file()}"
             raise Exception(
-                f"Failed to to get tokens from rule:: {rule.identifier}\n" f"{rf}"
+                f"Failed to get tokens from rule:: {rule.identifier}\n" f"{rf}"
             ) from e
     return rule_tokens
 

src/formattedcode/output_cyclonedx.py

@@ -20,9 +20,9 @@
 from typing import List
 
 import attr
+from lxml import etree
 from commoncode.cliutils import OUTPUT_GROUP
 from commoncode.cliutils import PluggableCommandLineOption
-from lxml import etree
 from plugincode.output import OutputPlugin
 from plugincode.output import output_impl
 

src/licensedcode/cache.py

@@ -284,7 +284,7 @@ def build_licensing(licenses_db=None):
     from licensedcode.models import load_licenses
 
     licenses_db = licenses_db or load_licenses()
-    return Licensing((LicenseSymbolLike(lic) for lic in licenses_db.values()))
+    return Licensing(symbols=(LicenseSymbolLike(lic) for lic in licenses_db.values()))
 
 
 def build_spdx_symbols(licenses_db=None):
@@ -324,7 +324,6 @@ def get_licenses_by_spdx_key(
 
     Optionally include deprecated if ``include_deprecated`` is True.
 
-
     Optionally make the keys lowercase if ``lowercase_keys`` is True.
 
     Optionally include the license "other_spdx_license_keys" if present and

src/licensedcode/data/licenses/agpl-3.0-bacula.LICENSE

@@ -1,6 +1,14 @@
 ---
 key: agpl-3.0-bacula
 is_deprecated: yes
+replaced_by:
+    - bacula-exception
+    - bsd-simplified
+    - bsd-simplified
+    - bsd-simplified
+    - agpl-3.0-plus
+    - agpl-3.0-plus
+    - agpl-3.0
 short_name: AGPL 3.0 with Bacula exception
 name: AGPL 3.0 with Bacula exception
 category: Copyleft

src/licensedcode/data/licenses/agpl-3.0-linking-exception.LICENSE

@@ -1,13 +1,15 @@
 ---
 key: agpl-3.0-linking-exception
+is_deprecated: yes
+replaced_by:
+    - linking-exception-agpl-3.0
 short_name: AGPL 3.0 linking exception
 name: AGPL 3.0 linking exception
 category: Copyleft Limited
 owner: Unspecified
-is_exception: yes
 homepage_url: http://mo.morsi.org/blog/2009/08/13/lesser_affero_gplv3/
 notes: renamed to linking-exception-agpl-3.0
-is_deprecated: yes
+is_exception: yes
 ---
 
 Additional permission under the GNU Affero GPL version 3 section 7:

src/licensedcode/data/licenses/agpl-3.0-openssl.LICENSE

@@ -1,15 +1,16 @@
 ---
 key: agpl-3.0-openssl
+is_deprecated: yes
+replaced_by:
+    - openssl-exception-agpl-3.0
 short_name: AGPL 3.0 with OpenSSL exception
 name: AGPL 3.0 with OpenSSL exception
 category: Copyleft
 owner: MongoDB
-is_exception: yes
-is_deprecated: yes
 notes: replaced by openssl-exception-agpl-3.0
+is_exception: yes
 ---
 
-
 As a special exception, the copyright holders give permission to link the
 code of portions of this program with the OpenSSL library under certain
 conditions as described in each individual source file and distribute

src/licensedcode/data/licenses/aladdin-md5.LICENSE

@@ -1,6 +1,8 @@
 ---
 key: aladdin-md5
 is_deprecated: yes
+replaced_by:
+    - zlib
 short_name: Aladdin MD5 License
 name: Aladdin MD5 License
 category: Permissive

src/licensedcode/data/licenses/aop-pd.LICENSE

@@ -1,8 +1,10 @@
 ---
 key: aop-pd
+is_deprecated: yes
+replaced_by:
+    - cc-pd
 short_name: AOP-PD
 name: AOP Public Domain License
-is_deprecated: yes
 category: Public Domain
 owner: AOP Alliance Project
 ---

src/licensedcode/data/licenses/apache-2.0-linking-exception.LICENSE

@@ -1,12 +1,14 @@
 ---
 key: apache-2.0-linking-exception
+is_deprecated: yes
+replaced_by:
+    - compuphase-linking-exception
 short_name: Apache 2.0 with Linking Exception
 name: Apache 2.0 with Linking Exception
 category: Permissive
 owner: compuphase
 homepage_url: https://github.com/compuphase/minIni/blob/master/LICENSE
 is_exception: yes
-is_deprecated: yes
 ---
 
 EXCEPTION TO THE APACHE 2.0 LICENSE