Detect community files to ignore license detections

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

Author: AyanSinhaMahapatra

src/licensedcode/detection.py

@@ -37,6 +37,8 @@
 from licensedcode.spans import Span
 from licensedcode.tokenize import query_tokenizer
 
+from summarycode.classify import check_is_path_community_file
+
 """
 LicenseDetection data structure and processing.
 
@@ -1667,6 +1669,25 @@ def get_license_keys_from_detections(license_detections, licensing=Licensing()):
     return list(license_keys)
 
 
+def can_ignore_ambiguous_detection(license_detection):
+    """
+    Return True if the license_detection is not an ambigious detection
+    which needs to be reviewed. A few cases are:
+    1. All the locations of the license detection are community files
+    """
+    all_file_paths = [
+        file_region.path
+        for file_region in license_detection.file_regions
+    ]
+    if all(
+        check_is_path_community_file(file_path)
+        for file_path in all_file_paths
+    ):
+        return True
+
+    return False
+
+
 def get_ambiguous_license_detections_by_type(unique_license_detections):
     """
     Return a list of ambiguous unique license detections which needs review
@@ -1677,13 +1698,20 @@ def get_ambiguous_license_detections_by_type(unique_license_detections):
     ambi_license_detections = {}
 
     for detection in unique_license_detections:
+
         if not detection.license_expression:
             ambi_license_detections[DetectionCategory.LOW_QUALITY_MATCH_FRAGMENTS.value] = detection
 
+        elif can_ignore_ambiguous_detection(detection):
+            continue
+
         elif is_undetected_license_matches(license_matches=detection.matches):
             ambi_license_detections[DetectionCategory.UNDETECTED_LICENSE.value] = detection
 
-        elif has_correct_license_clue_matches(license_matches=detection.matches):
+        elif (
+            has_correct_license_clue_matches(license_matches=detection.matches) and
+            has_unknown_matches(license_matches=detection.matches)
+        ):
             ambi_license_detections[DetectionCategory.LICENSE_CLUES.value] = detection
 
         elif "unknown" in detection.license_expression:

src/summarycode/classify.py

@@ -8,6 +8,9 @@
 #
 
 
+from commoncode.fileutils import file_name
+from commoncode.fileutils import file_base_name
+
 def get_relative_path(root_path, path):
     """
     Return a path relativefrom the posix 'path' relative to a
@@ -91,6 +94,49 @@ def get_relative_path(root_path, path):
     'readme',
 )
 
+COMMUNITY_FILES = (
+    'CHANGELOG',
+    'ROADMAP',
+    'CONTRIBUTING',
+    'CODE_OF_CONDUCT',
+    'AUTHORS',
+    'SECURITY',
+    'FUNDING',
+)
+
+
+def clean_underscore_dash(filename):
+    return filename.replace('_', '').replace('-', '')
+
+
+def check_is_community_file(filename):
+    """
+    Return True if the resource is a known community filename,
+    return False otherwise.
+    """
+    community_files_cleaned = [
+        clean_underscore_dash(filename.lower())
+        for filename in COMMUNITY_FILES
+    ]
+    name = clean_underscore_dash(filename.lower())
+    if any(
+        name.startswith(comm_name) or name.endswith(comm_name)
+        for comm_name in community_files_cleaned
+    ):
+        return True
+
+    return False
+
+
+def check_is_resource_community_file(resource):
+    return check_is_community_file(resource.name) or check_is_community_file(resource.base_name)
+
+
+def check_is_path_community_file(path):
+    name = file_name(path, force_posix=True)
+    base_name = file_base_name(path, force_posix=True)
+    return check_is_community_file(name) or check_is_community_file(base_name)
+
 
 def check_resource_name_start_and_end(resource, STARTS_ENDS):
     """
@@ -119,6 +165,7 @@ def set_classification_flags(resource,
 
     resource.is_legal = is_legal = check_resource_name_start_and_end(resource, _LEGAL)
     resource.is_readme = is_readme = check_resource_name_start_and_end(resource, _README)
+    resource.is_community = check_is_resource_community_file(resource)
     # FIXME: this will never be picked up as this is NOT available in a pre-scan plugin
     has_package_data = bool(getattr(resource, 'package_data', False))
     resource.is_manifest = is_manifest = path.endswith(_MANIF) or has_package_data

src/summarycode/classify_plugin.py

@@ -77,6 +77,10 @@ class FileClassifier(PostScanPlugin):
          Boolean(help='True if this file is "top-level" file and either a '
                       'legal, readme or manifest file.')),
 
+        ('is_community',
+         Boolean(help='True if this file is a community file generally used for'
+                      'maintainance or participation in the FOSS project community.')),
+
 #         ('is_doc',
 #          Boolean(help='True if this file is likely a documentation file.')),
 #

tests/summarycode/data/classify/cli.expected.json

@@ -24,6 +24,7 @@
       "is_readme": false,
       "is_top_level": true,
       "is_key_file": false,
+      "is_community": false,
       "files_count": 8,
       "dirs_count": 1,
       "size_count": 0,
@@ -53,6 +54,7 @@
       "is_readme": true,
       "is_top_level": true,
       "is_key_file": true,
+      "is_community": false,
       "files_count": 0,
       "dirs_count": 0,
       "size_count": 0,
@@ -82,6 +84,7 @@
       "is_readme": true,
       "is_top_level": true,
       "is_key_file": true,
+      "is_community": false,
       "files_count": 0,
       "dirs_count": 0,
       "size_count": 0,
@@ -111,6 +114,7 @@
       "is_readme": false,
       "is_top_level": true,
       "is_key_file": true,
+      "is_community": false,
       "files_count": 0,
       "dirs_count": 0,
       "size_count": 0,
@@ -140,6 +144,7 @@
       "is_readme": false,
       "is_top_level": true,
       "is_key_file": true,
+      "is_community": false,
       "files_count": 0,
       "dirs_count": 0,
       "size_count": 0,
@@ -169,6 +174,7 @@
       "is_readme": false,
       "is_top_level": true,
       "is_key_file": false,
+      "is_community": false,
       "files_count": 2,
       "dirs_count": 0,
       "size_count": 0,
@@ -198,6 +204,7 @@
       "is_readme": false,
       "is_top_level": false,
       "is_key_file": false,
+      "is_community": false,
       "files_count": 0,
       "dirs_count": 0,
       "size_count": 0,
@@ -227,6 +234,7 @@
       "is_readme": false,
       "is_top_level": false,
       "is_key_file": false,
+      "is_community": false,
       "files_count": 0,
       "dirs_count": 0,
       "size_count": 0,
@@ -256,6 +264,7 @@
       "is_readme": false,
       "is_top_level": true,
       "is_key_file": false,
+      "is_community": false,
       "files_count": 0,
       "dirs_count": 0,
       "size_count": 0,
@@ -285,6 +294,7 @@
       "is_readme": false,
       "is_top_level": true,
       "is_key_file": false,
+      "is_community": false,
       "files_count": 0,
       "dirs_count": 0,
       "size_count": 0,