v1.4 - new matching model and options

Author: matthewb66

bd_sig_filter/BOMClass.py

@@ -71,3 +71,19 @@ def report_full(self):
         print(tabulate(table, headers=["Component", "Match Type", "Ignored", "Reviewed", "To be Ignored",
                                        "To be Reviewed", "Action"]))
         print()
+        if global_values.report_file != '':
+            with open(global_values.report_file, "a") as rfile:
+                # Writing data to a file
+                rfile.writelines(tabulate(table, headers=["Component", "Match Type", "Ignored", "Reviewed", "To be Ignored",
+                                       "To be Reviewed", "Action"]))
+                rfile.writelines("")
+
+    def report_unmatched(self):
+        data = self.complist.get_unmatched_list()
+        data = "UNMATCHED COMPONENTS:\n" + data
+        print(data)
+        if global_values.report_file != '':
+            with open(global_values.report_file, "a") as rfile:
+                # Writing data to a file
+                rfile.writelines(data)
+

bd_sig_filter/ComponentClass.py

@@ -18,8 +18,10 @@ def __init__(self, name, version, data):
         self.sig_match_result = -1
         self.compname_found = False
         self.compver_found = False
-        self.reason = 'No Action - compname or version not found in Sig paths'
+        self.reason = 'No Action'
         self.best_sigpath = ''
+        self.oriname_arr = self.get_origin_compnames()
+        self.unmatched = False
 
     def get_compverid(self):
         try:
@@ -54,7 +56,7 @@ def is_signature(self):
         return False
 
     def is_only_signature(self):
-        return (not self.is_dependency() and self.is_signature())
+        return not self.is_dependency() and self.is_signature()
 
     def set_ignore(self):
         self.ignore = True
@@ -80,6 +82,7 @@ def is_ignored(self):
 
     def process_signatures(self):
         all_paths_ignoreable = True
+        unmatched = False
         reason = ''
         for sigentry in self.sigentry_arr:
             ignore, reason = sigentry.filter_folders()
@@ -100,23 +103,33 @@ def process_signatures(self):
             self.sig_match_result = 0
             set_reviewed = False
             ignore = True
+            unmatched = True
+            reason = f"No Action - component name '{self.oriname_arr}' not found in signature paths"
             for sigentry in self.sigentry_arr:
+                # compname_found, compver_found,\
+                #     new_match_result = sigentry.search_component(self.filter_name, self.filter_version)
                 compname_found, compver_found,\
-                    new_match_result = sigentry.search_component(self.filter_name, self.filter_version)
+                    new_match_result = sigentry.search_component(self.oriname_arr, self.filter_version)
                 logging.debug(f"Compname in path {compname_found}, Version in path {compver_found}, "
                               f"Match result {new_match_result}, Path '{sigentry.path}'")
+
                 if compver_found:
                     self.compver_found = True
                     ignore = False
+                    unmatched = False
                 if compname_found:
                     self.compname_found = True
                 if global_values.version_match_reqd:
                     if compver_found:
                         set_reviewed = True
                         ignore = False
+                        unmatched = False
+                    else:
+                        reason = f"No Action - component version {self.filter_version} not found (and required because --version_match_reqd set)"
                 elif compname_found:
                     set_reviewed = True
                     ignore = False
+                    unmatched = False
                 if new_match_result > self.sig_match_result:
                     self.sig_match_result = new_match_result
                     self.best_sigpath = sigentry.path
@@ -125,14 +138,17 @@ def process_signatures(self):
                 if self.compver_found:
                     reason = f"Mark REVIEWED - Compname & version in path '{self.best_sigpath}', Match result {self.sig_match_result}"
                 elif self.compname_found:
-                    reason = f"Mark REVIEWED - Compname in path '{self.best_sigpath}', Match result {self.sig_match_result}"
+                    reason = f"Mark REVIEWED - Compname {self.oriname_arr} in path '{self.best_sigpath}', Match result {self.sig_match_result}"
 
-                self.reason = reason
                 logging.debug(f"- Component {self.name}/{self.version}: {reason}")
                 self.set_reviewed()
-            if ignore and global_values.ignore_no_path_matches:
-                self.set_ignore()
-                self.reason = f"Mark IGNORED - compname or version not found in paths & --ignore_no_path_matches set"
+                unmatched = False
+        if ignore and global_values.ignore_no_path_matches:
+            self.set_ignore()
+            reason = f"Mark IGNORED - compname or version not found in paths & --ignore_no_path_matches set"
+
+        self.reason = reason
+        self.unmatched = unmatched
 
     @staticmethod
     def filter_name_string(name):
@@ -148,7 +164,7 @@ def filter_name_string(name):
         ret_name = re.sub(r"[/@#:]", " ", ret_name)
         ret_name = re.sub(r" \w$| \w |^\w ", r" ", ret_name)
         ret_name = ret_name.replace("::", " ")
-        ret_name = re.sub(r"  *", r" ", ret_name)
+        ret_name = re.sub(r" +", r" ", ret_name)
         ret_name = re.sub(r"^ ", r"", ret_name)
         ret_name = re.sub(r" $", r"", ret_name)
 
@@ -160,9 +176,11 @@ def filter_version_string(version):
         # Remove +git*
         # Remove -snapshot*
         # Replace / with space
-        ret_version = re.sub(r"\+git.*", r"", version, re.IGNORECASE)
-        ret_version = re.sub(r"-snapshot.*", r"", ret_version, re.IGNORECASE)
+        ret_version = re.sub(r"\+git.*", r"", version, flags=re.IGNORECASE)
+        ret_version = re.sub(r"-snapshot.*", r"", ret_version, flags=re.IGNORECASE)
         ret_version = re.sub(r"/", r" ", ret_version)
+        ret_version = re.sub(r"^v", r"", ret_version, flags=re.IGNORECASE)
+        ret_version = re.sub(r"\+*", r"", ret_version, flags=re.IGNORECASE)
         return ret_version
 
     def get_compid(self):
@@ -171,3 +189,40 @@ def get_compid(self):
             return compurl.split('/')[-1]
         except KeyError:
             return ''
+
+    def print_origins(self):
+        try:
+            for ori in self.data['origins']:
+                print(f"Comp '{self.name}/{self.version}' Origin '{ori['externalId']}' Name '{ori['name']}'")
+        except KeyError:
+            print(f"Comp '{self.name}/{self.version}' No Origin")
+
+    def get_origin_compnames(self):
+        compnames_arr = []
+        try:
+            for ori_entry in self.data['origins']:
+                ori = ori_entry['externalId']
+                ori_ver = ori_entry['name']
+                ori_string = ori.replace(f"{ori_ver}", '')
+                arr = re.split(r"[:/#]", ori_string)
+                new_name = arr[-2]
+                if new_name not in compnames_arr:
+                    logging.debug(
+                        f"Comp '{self.name}/{self.version}' Compname calculate from origin '{arr[-2]}' - origin='{ori}'")
+                    compnames_arr.append(arr[-2])
+            if self.filter_name.find(' ') == -1:
+                # Single word component name
+                if self.filter_name not in compnames_arr:
+                    compnames_arr.append(self.filter_name)
+        except (KeyError, IndexError):
+            logging.debug(f"Comp '{self.name}/{self.version}' Compname calculate from compname only '{self.name}'")
+            compnames_arr.append(self.filter_name)
+        return compnames_arr
+
+    def get_sigpaths(self):
+        data = ''
+        count = 0
+        for sigentry in self.sigentry_arr:
+            data += f"{sigentry.get_sigpath()}\n"
+            count += 1
+        return data

bd_sig_filter/ComponentListClass.py

@@ -87,11 +87,20 @@ def process(self):
         for comp in self.components:
             if comp.is_ignored():
                 continue
+            # DEBUG
+            # if comp.is_only_signature():
+            #     arr = comp.get_origin_compnames()
+            #     print(f"names '{arr}")
+            #     comp.print_sigpaths()
+            # continue
+            # END DEBUG
             if comp.is_dependency():
                 comp.set_reviewed()
                 comp.reason = "Mark REVIEWED - Dependency"
             elif comp.is_only_signature():
                 comp.process_signatures()
+            else:
+                comp.reason = 'No action - not Signature match'
 
         # look for duplicate components (same compid) and ignore
         logging.debug("\nDUPLICATE SIGNATURE MATCHES FILTER PHASE")
@@ -281,3 +290,11 @@ def get_component_report_data(self):
                          comp.ignore, comp.mark_reviewed, comp.reason])
         return data
 
+    def get_unmatched_list(self):
+        data = ''
+        for comp in self.components:
+            if comp.unmatched:
+                paths = comp.get_sigpaths()
+                orinames = ','.join(comp.oriname_arr)
+                data += f"Comp: {comp.name}/{comp.version} (Origin names={orinames}):\n{paths}"
+        return data

bd_sig_filter/SigEntryClass.py

@@ -16,9 +16,9 @@ def __init__(self, src_entry):
         except KeyError:
             return
 
-    def search_component(self, compname, compver):
-        logging.debug("")
-        logging.debug(f"search_component() Checking Comp '{compname}/{compver}' - {self.path}:")
+    def search_component(self, compname_arr, compver):
+        # logging.debug("")
+        # logging.debug(f"search_component() Checking Comp '{compname}/{compver}' - {self.path}:")
         # If component_version_reqd:
         # - folder matches compname and compver
         # - folder1 matches compname and folder2 matches compver
@@ -29,76 +29,103 @@ def search_component(self, compname, compver):
         # Bool2 - version found
         # Match_value - search result against both
 
-        compstring = f"{compname} {compver}"
-        element_in_compname = 0
-        compver_in_element = 0
 
-        # test of path search
-        newpath = self.path.replace(os.sep, " ")
-        newpath = re.sub(r"([a-zA-Z-]*)[0-9] ", "\1 ", newpath)
-        comp_in_path = fuzz.token_set_ratio(compstring, newpath)
-        logging.debug(f"search_component(): TEST comp_in_path is {comp_in_path}: path='{self.path}")
+        best_match_name = 0
+        best_match_ver = 0
+        # match_path = ''
+        for cname in compname_arr:
+            # compstring = f"{cname} {compver}"
 
-        found_compname_only = False
-        for element in self.elements:
-            pos = re.search(r"\.dll|\.obj|\.o|\.a|\.lib|\.iso|\.qcow2|\.vmdk|\.vdi|\.ova|\.nbi|\.vib|\.exe|\.img|"
-                            "\.bin|\.apk|\.aac|\.ipa|\.msi|\.zip|\.gz|\.tar|\.xz|\.lz|\.bz2|\.7z|\.rar|"
-                            "\.cpio|\.Z|\.lz4|\.lha|\.arj|\.jar|\.ear|\.war|\.rpm|\.deb|\.dmg|\.pki", element)
-            if pos is not None:
-                element = element[:pos.start()]
-            # How much of the element string is from the compname and version?
-            # - for example acl-1.3.0.jar
-            # - Value of 100 indicates either compname or version exists in element
-            element_in_compstring = fuzz.token_set_ratio(element, compstring)
-            element_in_compname = fuzz.token_set_ratio(element, compname)
-            compver_in_element = fuzz.token_set_ratio(compver, element)
+            # test of path search
+            newpath = self.path.replace(os.sep, " ")
+            # comp_in_path = fuzz.token_set_ratio(compstring, newpath)
+            compname_in_path = fuzz.token_set_ratio(cname, newpath)
+            compver_in_path = fuzz.token_set_ratio(compver, newpath)
+            if compname_in_path + compver_in_path > 90:
+                if compname_in_path + compver_in_path > best_match_name + best_match_ver:
+                    best_match_name = compname_in_path
+                    best_match_ver = compver_in_path
+                    # match_path = self.path
+                    logging.debug(f"search_component(): TEST '{cname}/{compver}' - {compname_in_path,compver_in_path}: path='{self.path}")
 
-            if element_in_compstring > 80:
-                if compver_in_element > 50:
-                    # element has both compname and version
-                    logging.debug(f"search_component() - MATCHED component name & version ({compstring}) in '{element}'")
-                    return True, True, element_in_compname + compver_in_element
-                elif element_in_compname > 50 and len(element) > 2:
-                    found_compname_only = True
-                    logging.debug(f"search_component() - FOUND component name ONLY ({compname}) in '{element}'")
-            elif found_compname_only:
-                if compver_in_element > 50:
-                    logging.debug(f"search_component() - MATCHED component version ({compver}) in '{element}'")
-                    return True, True, element_in_compname + compver_in_element
-                else:
-                    test = 1
+        name_bool = False
+        ver_bool = False
+        if best_match_name > 45:
+            name_bool = True
+        if best_match_ver > 45:
+            ver_bool = True
 
-        if found_compname_only:
-            logging.debug("search_component() - MATCHED Compname only")
-            return True, False, element_in_compname + compver_in_element
-
-        logging.debug(f"search_component() - NOT MATCHED")
-        return False, False, 0
+        return name_bool, ver_bool, best_match_name + best_match_ver
+        # compstring = f"{compname} {compver}"
+        # element_in_compname = 0
+        # compver_in_element = 0
+        # found_compname_only = False
+        # for element in self.elements:
+        #     pos = re.search(r"\.dll|\.obj|\.o|\.a|\.lib|\.iso|\.qcow2|\.vmdk|\.vdi|\.ova|\.nbi|\.vib|\.exe|\.img|"
+        #                     "\.bin|\.apk|\.aac|\.ipa|\.msi|\.zip|\.gz|\.tar|\.xz|\.lz|\.bz2|\.7z|\.rar|"
+        #                     "\.cpio|\.Z|\.lz4|\.lha|\.arj|\.jar|\.ear|\.war|\.rpm|\.deb|\.dmg|\.pki", element)
+        #     if pos is not None:
+        #         element = element[:pos.start()]
+        #     # How much of the element string is from the compname and version?
+        #     # - for example acl-1.3.0.jar
+        #     # - Value of 100 indicates either compname or version exists in element
+        #     element_in_compstring = fuzz.token_set_ratio(element, compstring)
+        #     element_in_compname = fuzz.token_set_ratio(element, compname)
+        #     compver_in_element = fuzz.token_set_ratio(compver, element)
+        #
+        #     if element_in_compstring > 80:
+        #         if compver_in_element > 50:
+        #             # element has both compname and version
+        #             logging.debug(f"search_component() - MATCHED component name & version ({compstring}) in '{element}'")
+        #             return True, True, element_in_compname + compver_in_element
+        #         elif element_in_compname > 50 and len(element) > 2:
+        #             found_compname_only = True
+        #             logging.debug(f"search_component() - FOUND component name ONLY ({compname}) in '{element}'")
+        #     elif found_compname_only:
+        #         if compver_in_element > 50:
+        #             logging.debug(f"search_component() - MATCHED component version ({compver}) in '{element}'")
+        #             return True, True, element_in_compname + compver_in_element
+        #         else:
+        #             test = 1
+        #
+        # if found_compname_only:
+        #     logging.debug("search_component() - MATCHED Compname only")
+        #     return True, False, element_in_compname + compver_in_element
+        #
+        # logging.debug(f"search_component() - NOT MATCHED")
 
 
     def filter_folders(self):
         # Return True if path should be ignored + reason
         if not global_values.no_ignore_synopsys:
-            syn_folders = ['.synopsys', 'synopsys-detect', '.coverity', 'synopsys-detect.jar',
-                           'scan.cli.impl-standalone.jar', 'seeker-agent.tgz', 'seeker-agent.zip',
-                           'Black_Duck_Scan_Installation']
-            for e in self.elements:
-                if e in syn_folders:
-                    return True, f"Found '{e}' in Signature match path '{self.path}'"
+            # syn_folders = ['.synopsys', 'synopsys-detect', '.coverity', 'synopsys-detect.jar',
+            #                'scan.cli.impl-standalone.jar', 'seeker-agent.tgz', 'seeker-agent.zip',
+            #                'Black_Duck_Scan_Installation']
+
+            syn_folders_re = (f"{os.sep}(\.synopsys|synopsys-detect|\.coverity|synopsys-detect.*\.jar|scan\.cli\.impl-standalone\.jar|"
+                              f"seeker-agent.*|Black_Duck_Scan_Installation){os.sep}")
+            res = re.search(syn_folders_re, self.path)
+            if res:
+                return True, f"Found {res.group()} folder in Signature match path '{self.path}'"
 
         if not global_values.no_ignore_defaults:
-            def_folders = ['.cache', '.m2', '.local', '.cache','.config', '.docker', '.npm', '.npmrc', '.pyenv',
-                           '.Trash', '.git', 'node_modules']
-            for e in self.elements:
-                if e in def_folders:
-                    return True, f"Found '{e}' in Signature match path '{self.path}'"
+            # def_folders = ['.cache', '.m2', '.local', '.cache','.config', '.docker', '.npm', '.npmrc', '.pyenv',
+            #                '.Trash', '.git', 'node_modules']
+            def_folders_re = (f"{os.sep}(\.cache|\.m2|\.local|\.config|\.docker|\.npm|\.npmrc|"
+                              f"\.pyenv|\.Trash|\.git|node_modules){os.sep}")
+            res = re.search(def_folders_re, os.sep + self.path + os.sep)
+            if res:
+                return True, f"Found {res.group()} folder in Signature match path '{self.path}'"
 
         if not global_values.no_ignore_test:
-            test_folders = r"^test$|^tests$|^testsuite$"
-            for e in self.elements:
-                if re.search(test_folders, e, flags=re.IGNORECASE) is not None:
-                    return True, f"Found '{e}' in Signature match path '{self.path}'"
+            test_folders = f"{os.sep}(test|tests|testsuite){os.sep}"
+            res = re.search(test_folders, os.sep + self.path + os.sep, flags=re.IGNORECASE)
+            if res:
+                return True, f"Found {res.group()} in Signature match path '{self.path}'"
                 # if e in test_folders:
                 #     return True, f"Found '{e}' in Signature match path '{self.path}'"
 
         return False, ''
+
+    def get_sigpath(self):
+        return(f"- {self.path}")

bd_sig_filter/config.py

@@ -26,6 +26,7 @@
 parser.add_argument("--no_ignore_synopsys", help="Do not ignore components in synopsys tool folders", action='store_true')
 parser.add_argument("--no_ignore_defaults", help="Do not ignore components in default folders", action='store_true')
 parser.add_argument("--ignore_no_path_matches", help="Also ignore components with no component/version match in signature path", action='store_true')
+parser.add_argument("--report_unmatched", help="Report unmatched (not reviewed or ignored) components", action='store_true')
 
 args = parser.parse_args()
 
@@ -114,6 +115,9 @@ def check_args():
             terminate = True
         global_values.report_file = args.report_file
 
+    if args.report_unmatched:
+        global_values.report_unmatched = True
+
 
     if terminate:
         sys.exit(2)