Merge pull request #4 from blackducksoftware/dev

matthewb66 · web-flow · commit a4a640e60b8d · 2024-06-17T14:12:14.000+01:00
v1.7 - Minor doc changes and code cleanup
diff --git a/README.md b/README.md
@@ -1,4 +1,4 @@
-# bd_sig_filter - v1.6
+# bd_sig_filter - v1.7
 BD Script to ignore components matched from Signature scan likely to be partial or invalid matches, and 
 mark components reviewed which are definitive matches (dependency or component name and version in matched path for
 signature matches).
@@ -137,41 +137,64 @@ The list of components shows the name, matchtypes and current ignore/review stat
 (after running the script with the `--ignore` and `--review` options) in the `To Be Ignored` and `To Be Reviewed` 
 columns with an explanation in the `Action` column.
 
-The following options can be specified:
-
-    --ignore:               Ignore components as shown in the `To Be Ignored` column
-    --review:               Mark components as reviewed as shown in the `To Be Reviewed` column
-    --no_ignore_test:       Do not ignore components with signature paths within test folders
-    --no_ignore_synopsys:   Do not ignore components with signature paths within Synopsys tools folders (for example '.synopsys')
-    --no_ignore_defaults:   Do not ignore components with signature paths in cache/config folders (for example '.git', '.m2', '.local')
-    --version_match_required:
-                            Enforce search for component version string in signature paths for marking reviewed
-                            (Paths containing only the component name will be used for matching otherwise)
-    --ignore_no_path_matches:
-                            Components with no match in the signature path are left unreviewed by default, allowing
-                            manual review. Use this option to ignore these components instead but use with caution
-                            as it may exclude components which are legitimate (the Signature match path does not
-                            have to include the component name or version).
+The `Match Score` value shows the result of fuzzy match searching for component name and version strings (note that
+origin component ID is used where available as opposed to the textual name of the component). A score of 200 shows
+an exact match of both component name and version in Signature paths; a lower value shows the possibility of less
+accurate matching.
+
+Options can be used to modify the behaviour of the script as follows:
+
+`--no_ignore_test`:
+        Stops components matched only by Signature scanning and containing test folders (test, tests,
+        testsuite or testsuites - case insensitive) being marked for ignore (which happens by default).
+
+`--no_ignore_synopsys`:
+        Stops components matched only by Signature scanning and containing Synopsys tools folders (.synopsys,
+        synopsys-detect, .coverity, synopsys-detect.\*.jar, scan.cli.impl-standalone.jar, seeker-agent.\*,
+        Black_Duck_Scan_Installation - case insensitive) being marked for ignore (which happens by default).
+
+`--no_ignore_defaults`:
+        Stops components matched only by Signature scanning and containing default folders (.cache, 
+        .m2, .local, .config, .docker, .npm, .npmrc, .pyenv, .Trash, .git, node_modules - case insensitive)
+        being marked for ignore (which happens by default).
+
+`--version_match_required`:
+        Enforce search for component version string in signature paths for marking components reviewed
+        (Paths containing only the component name will be used for matching otherwise)
+
+`--ignore_no_path_matches`:
+        Components with no match in the signature path are left unreviewed by default, allowing
+        manual review. Use this option to ignore these components instead but use with caution
+        as it may exclude components which are legitimate (the Signature match path does not
+        have to include the component name or version).
+
+`--report_unmatched`:
+        Create a list of Signature components which will be left UNreviewed 
 
 The options `--report_file` and `--logfile` can be used to output the tabular report and logging data to
 specified files.
 
 ## PROPOSED WORKFLOW
-The script can classify Signature scan results.
+The script can be used to classify Signature scan results.
 
-It can mark components as reviewed which are either Dependencies, or which have signature match paths containing
+It can mark components as reviewed which are either Dependencies, or which have Signature match paths containing
 the component name (and optionally component version) and which are therefore highly likely to be correctly identified
-by Signature matching. Fuzzy pattern matching is used so there is the possibility
-that components could be marked as reviewed where only a partial match exists, or components which should be matched
-are not identified meaning that some manual curation may still be required.
+by Signature matching.
 
-It will also ignore components only matched within extraneous folders (for example created by Synopsys tools, 
+It can also ignore components only Signature matched within extraneous folders (for example created by Synopsys tools, 
 config/cache folders or test folders).
 
 Components shown with `No action` are Signature matches where the component name or version 
 could not be identified in the signature paths, so they are potential false matches and require manual review.
-Specify the `--ignore_no_path_matches` option to ignore these components automatically,
-however this should be used with caution as these components may be valid and should be manually reviewed.
+
+After running the script and ignoring/reviewing components (using options `--ignore --review`), review the reported
+list of components from the script focussing on those marked with `No Action`. Optionally use the option `--report_unmatched`
+to list the `No Action` components with the full list of Signature match paths to enable assessment whether they should
+be included in the BOM.
+
+If, after inspection, all `No Action` components can be removed from the BOM, the `--ignore_no_path_matches` option can be used to
+ignore these components automatically, however this should be used with caution as these components may be valid 
+and should be manually reviewed.
 
 ## PROCESSING DUPLICATE COMPONENTS
 The script processes multiple versions of the same component in the BOM in several ways as described below:
diff --git a/bd_sig_filter/BOMClass.py b/bd_sig_filter/BOMClass.py
@@ -55,20 +55,20 @@ def report_summary(self):
                  ['After', self.complist.count(), self.complist.count_to_be_ignored(),
                   self.complist.count_to_be_reviewed(), self.complist.count_not_to_be_reviewed_ignored()]]
         print("SUMMARY:")
-        print(tabulate(table, headers=["", "Components", "Ignored", "Reviewed", "Neither"], tablefmt="simple"))
+        print(tabulate(table, headers=["", "Components", "Ignored", "Reviewed", "Neither (No Action)"], tablefmt="simple"))
         print()
 
         if global_values.report_file != '':
             with open(global_values.report_file, "w") as rfile:
                 # Writing data to a file
                 rfile.writelines("SUMMARY:")
-                rfile.writelines(tabulate(table, headers=["", "Components", "Ignored", "Reviewed", "Neither"],
+                rfile.writelines(tabulate(table, headers=["", "Components", "Ignored", "Reviewed", "Neither (No Action)"],
                                           tablefmt="Simple"))
                 rfile.writelines("")
 
     def report_full(self):
         table = self.complist.get_component_report_data()
-        print(tabulate(table, headers=["Component", "Match Type", "Ignored", "Reviewed", "To be Ignored",
+        print(tabulate(table, headers=["Component/Version", "Match Type", "Ignored", "Reviewed", "To be Ignored",
                                        "To be Reviewed", "Action"]))
         print()
         if global_values.report_file != '':
diff --git a/bd_sig_filter/SigEntryClass.py b/bd_sig_filter/SigEntryClass.py
@@ -10,24 +10,21 @@ def __init__(self, src_entry):
             self.src_entry = src_entry
             self.path = src_entry['commentPath']
             elements = re.split(r"!|#|" + os.sep, self.path)
-            # self.elements = self.path.replace("!", os.sep).replace("#", os.sep).split(os.sep)
             self.elements = list(filter(None, elements))
 
         except KeyError:
             return
 
     def search_component(self, compname_arr, compver):
-        # logging.debug("")
-        # logging.debug(f"search_component() Checking Comp '{compname}/{compver}' - {self.path}:")
         # If component_version_reqd:
         # - folder matches compname and compver
         # - folder1 matches compname and folder2 matches compver
         # Else:
         # - folder matches compname
         # Returns:
-        # Bool1 - compname found
-        # Bool2 - version found
-        # Match_value - search result against both
+        # - Bool1 - compname found
+        # - Bool2 - version found
+        # - Match_value - search result against both
 
 
         best_match_name = 0
@@ -39,96 +36,48 @@ def search_component(self, compname_arr, compver):
             # compstring = f"{cname} {compver}"
 
             # test of path search
-            # newpath = self.path.replace(os.sep, " ")
             rep = f"[{os.sep}!#]"
             newpath = re.sub(rep, ' ', self.path).lower()
-            # comp_in_path = fuzz.token_set_ratio(compstring, newpath)
             compname_setratio = fuzz.token_set_ratio(cname, newpath)
-            compname_sortratio = fuzz.token_sort_ratio(cname, newpath)
-            compname_partialratio = fuzz.partial_ratio(cname, newpath)
-            compver_setratio = fuzz.token_set_ratio(compver, newpath)
-            compver_sortratio = fuzz.token_sort_ratio(compver, newpath)
+            # compname_sortratio = fuzz.token_sort_ratio(cname, newpath)
+            # compname_partialratio = fuzz.partial_ratio(cname, newpath)
+            # compver_setratio = fuzz.token_set_ratio(compver, newpath)
+            # compver_sortratio = fuzz.token_sort_ratio(compver, newpath)
             compver_partialratio = fuzz.partial_ratio(compver, newpath)
 
             if compname_setratio + compver_partialratio > best_match_name + best_match_ver:
                 best_match_name = compname_setratio
                 best_match_ver = compver_partialratio
-                # match_path = self.path
                 logging.debug(f"search_component(): TEST '{cname}/{compver}' - {compname_setratio,compver_partialratio}: path='{self.path}")
 
         if best_match_name > 45:
             name_bool = True
         if best_match_ver > 80:
             ver_bool = True
         return name_bool, ver_bool, best_match_name + best_match_ver
-        # compstring = f"{compname} {compver}"
-        # element_in_compname = 0
-        # compver_in_element = 0
-        # found_compname_only = False
-        # for element in self.elements:
-        #     pos = re.search(r"\.dll|\.obj|\.o|\.a|\.lib|\.iso|\.qcow2|\.vmdk|\.vdi|\.ova|\.nbi|\.vib|\.exe|\.img|"
-        #                     "\.bin|\.apk|\.aac|\.ipa|\.msi|\.zip|\.gz|\.tar|\.xz|\.lz|\.bz2|\.7z|\.rar|"
-        #                     "\.cpio|\.Z|\.lz4|\.lha|\.arj|\.jar|\.ear|\.war|\.rpm|\.deb|\.dmg|\.pki", element)
-        #     if pos is not None:
-        #         element = element[:pos.start()]
-        #     # How much of the element string is from the compname and version?
-        #     # - for example acl-1.3.0.jar
-        #     # - Value of 100 indicates either compname or version exists in element
-        #     element_in_compstring = fuzz.token_set_ratio(element, compstring)
-        #     element_in_compname = fuzz.token_set_ratio(element, compname)
-        #     compver_in_element = fuzz.token_set_ratio(compver, element)
-        #
-        #     if element_in_compstring > 80:
-        #         if compver_in_element > 50:
-        #             # element has both compname and version
-        #             logging.debug(f"search_component() - MATCHED component name & version ({compstring}) in '{element}'")
-        #             return True, True, element_in_compname + compver_in_element
-        #         elif element_in_compname > 50 and len(element) > 2:
-        #             found_compname_only = True
-        #             logging.debug(f"search_component() - FOUND component name ONLY ({compname}) in '{element}'")
-        #     elif found_compname_only:
-        #         if compver_in_element > 50:
-        #             logging.debug(f"search_component() - MATCHED component version ({compver}) in '{element}'")
-        #             return True, True, element_in_compname + compver_in_element
-        #         else:
-        #             test = 1
-        #
-        # if found_compname_only:
-        #     logging.debug("search_component() - MATCHED Compname only")
-        #     return True, False, element_in_compname + compver_in_element
-        #
-        # logging.debug(f"search_component() - NOT MATCHED")
 
 
     def filter_folders(self):
         # Return True if path should be ignored + reason
         if not global_values.no_ignore_synopsys:
-            # syn_folders = ['.synopsys', 'synopsys-detect', '.coverity', 'synopsys-detect.jar',
-            #                'scan.cli.impl-standalone.jar', 'seeker-agent.tgz', 'seeker-agent.zip',
-            #                'Black_Duck_Scan_Installation']
-
             syn_folders_re = (f"\\{os.sep}(\.synopsys|synopsys-detect|\.coverity|synopsys-detect.*\.jar|scan\.cli\.impl-standalone\.jar|"
                               f"seeker-agent.*|Black_Duck_Scan_Installation)\\{os.sep}")
             res = re.search(syn_folders_re, os.sep + self.path + os.sep)
             if res:
                 return True, f"Found {res.group()} folder in Signature match path '{self.path}'"
 
         if not global_values.no_ignore_defaults:
-            # def_folders = ['.cache', '.m2', '.local', '.cache','.config', '.docker', '.npm', '.npmrc', '.pyenv',
-            #                '.Trash', '.git', 'node_modules']
             def_folders_re = (f"\\{os.sep}(\.cache|\.m2|\.local|\.config|\.docker|\.npm|\.npmrc|"
                               f"\.pyenv|\.Trash|\.git|node_modules)\\{os.sep}")
             res = re.search(def_folders_re, os.sep + self.path + os.sep)
             if res:
                 return True, f"Found {res.group()} folder in Signature match path '{self.path}'"
 
         if not global_values.no_ignore_test:
-            test_folders = f"\\{os.sep}(test|tests|testsuite)\\{os.sep}"
+            test_folders = f"\\{os.sep}(test|tests|testsuite|testsuites)\\{os.sep}"
             res = re.search(test_folders, os.sep + self.path + os.sep, flags=re.IGNORECASE)
             if res:
                 return True, f"Found {res.group()} in Signature match path '{self.path}'"
-                # if e in test_folders:
-                #     return True, f"Found '{e}' in Signature match path '{self.path}'"
 
         return False, ''
 
diff --git a/bd_sig_filter/config.py b/bd_sig_filter/config.py
@@ -118,7 +118,6 @@ def check_args():
     if args.report_unmatched:
         global_values.report_unmatched = True
 
-
     if terminate:
         sys.exit(2)
     return
diff --git a/pyproject.toml b/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "bd_sig_filter"
-version = "1.6"
+version = "1.7"
 authors = [
   { name="Matthew Brady", email="mbrad@synopsys.com" },
 ]

Original file line number	Diff line number	Diff line change
`@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"`
`4`	`4`
`5`	`5`	`[project]`
`6`	`6`	`name = "bd_sig_filter"`
`7`		`-version = "1.6"`
	`7`	`+version = "1.7"`
`8`	`8`	`authors = [`
`9`	`9`	`{ name="Matthew Brady", email="mbrad@synopsys.com" },`
`10`	`10`	`]`