From 053a6473f7f50a45ce5bdf8e3bf03dd61ceae6e5 Mon Sep 17 00:00:00 2001 From: Ian A <86781798+LuckySkyWalker@users.noreply.github.com> Date: Fri, 23 May 2025 09:29:07 +0100 Subject: [PATCH 1/3] Update refresh_project_copyrights.py Added dry run option --- examples/client/refresh_project_copyrights.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/examples/client/refresh_project_copyrights.py b/examples/client/refresh_project_copyrights.py index 366e848..fbbcf2a 100644 --- a/examples/client/refresh_project_copyrights.py +++ b/examples/client/refresh_project_copyrights.py @@ -48,7 +48,8 @@ def RepWarning(msg): parser.add_argument("--max-versions-per-project", dest='max_versions_per_project', type=int, help="Maximum versions per project to inspect else all") parser.add_argument("--max-components", dest='max_components', type=int, help="Maximum components to inspect in total else all") -parser.add_argument("--debug", dest='debug', type=int, default=0, help="Debug verbosity (0=none)") +parser.add_argument("--debug", dest='debug', type=int, default=0, help="Debug verbosity (0=none 'n'=level)") +parser.add_argument("--dryrun", dest='dry_run', type=int, default=0, help="Dry run test (0=no 1=yes)") parser.add_argument("--no-verify", dest='verify', action='store_false', help="Disable TLS certificate verification") parser.add_argument("-t", "--timeout", default=15, type=int, help="Adjust the (HTTP) session timeout value (default: 15s)") @@ -228,13 +229,17 @@ def RepWarning(msg): # refresh end point url += "/copyrights-refresh" - try: - response = bd.session.put(url, data=None, **refresh_kwargs) - RepDebug(5,'Refresh response %s' % response) - except urllib3.exceptions.ReadTimeoutError: - print('Failed to confirm copyrights refresh') + if args.dry_run != 0: + RepDebug(1, "DryRun: %s" % url) + else: + try: + response = bd.session.put(url, data=None, **refresh_kwargs) + RepDebug(5,'Refresh response %s' % response) + except ReadTimeoutError: + print('Failed to confirm copyrights refresh') + + my_statistics['_cntRefresh'] += 1 - my_statistics['_cntRefresh'] += 1 else: my_statistics['_cntNoOrigins'] += 1 url = 'n/a' From aea5f58173289a4d9af4e155e9b7b0287a1622ea Mon Sep 17 00:00:00 2001 From: Ian A <86781798+LuckySkyWalker@users.noreply.github.com> Date: Fri, 23 May 2025 14:35:05 +0100 Subject: [PATCH 2/3] Update refresh_project_copyrights.py Iterate through each origin per component and refresh each, increase timeout to 60 seconds, added dry run option --- examples/client/refresh_project_copyrights.py | 104 +++++++++++------- 1 file changed, 66 insertions(+), 38 deletions(-) diff --git a/examples/client/refresh_project_copyrights.py b/examples/client/refresh_project_copyrights.py index fbbcf2a..4244b4b 100644 --- a/examples/client/refresh_project_copyrights.py +++ b/examples/client/refresh_project_copyrights.py @@ -14,6 +14,8 @@ from pprint import pprint import array as arr +from urllib3.exceptions import ReadTimeoutError + http.client._MAXHEADERS = 1000 logging.basicConfig( @@ -52,8 +54,8 @@ def RepWarning(msg): parser.add_argument("--dryrun", dest='dry_run', type=int, default=0, help="Dry run test (0=no 1=yes)") parser.add_argument("--no-verify", dest='verify', action='store_false', help="Disable TLS certificate verification") -parser.add_argument("-t", "--timeout", default=15, type=int, help="Adjust the (HTTP) session timeout value (default: 15s)") -parser.add_argument("-r", "--retries", default=3, type=int, help="Adjust the number of retries on failure (default: 3)") +parser.add_argument("--timeout", default=60, type=int, help="Adjust the (HTTP) session timeout value (default: 60s)") +parser.add_argument("--retries", default=3, type=int, help="Adjust the number of retries on failure (default: 3)") args = parser.parse_args() @@ -64,8 +66,8 @@ def RepWarning(msg): # access the Black Duck platform bd = Client( base_url=args.base_url, - token=access_token, verify=args.verify, + token=access_token, timeout=args.timeout, retries=args.retries, ) @@ -205,8 +207,9 @@ def RepWarning(msg): break my_statistics['_cntComponents'] += 1 - RepDebug(4, ' Component: %s (%s)' % - (this_comp_data['componentName'], this_comp_data['componentVersionName'])) + comp_label = "{} ({})".format(this_comp_data['componentName'], this_comp_data['componentVersionName']) + + RepDebug(4, ' Component: %s' % comp_label) if this_comp_data['inputExternalIds'].__len__() > 0: inputExternalIds = this_comp_data['inputExternalIds'][0] @@ -218,46 +221,69 @@ def RepWarning(msg): # refresh the copyrights for this component if this_comp_data['origins'].__len__() > 0: - url = this_comp_data['origins'][0]['origin'] - else: - # no origins - RepWarning('No origin defined for [%s]' % this_comp_data['componentVersion']) -# url = this_comp_data['componentVersion'] - url = '' - - if len(url) > 0: - # refresh end point - url += "/copyrights-refresh" - - if args.dry_run != 0: - RepDebug(1, "DryRun: %s" % url) - else: - try: - response = bd.session.put(url, data=None, **refresh_kwargs) - RepDebug(5,'Refresh response %s' % response) - except ReadTimeoutError: - print('Failed to confirm copyrights refresh') - - my_statistics['_cntRefresh'] += 1 + + n_origin = 0 + + for this_origin in this_comp_data['origins']: + + n_origin += 1 + origin_id = this_origin['externalId'] + url = this_origin['origin'] + + # refresh with end point + url += "/copyrights-refresh" + + status = -1 + + if args.dry_run != 0: + RepDebug(1, "DryRun: no=%d origin=%s url=%s" % (n_origin, origin_id, url)) + else: + try: + response = bd.session.put(url, data=None, **refresh_kwargs) + RepDebug(5,'Refresh response: origin [%s] [%s]' % (this_origin, response)) + my_statistics['_cntRefresh'] += 1 + status= 0 + + except Exception: + print('Failed to confirm copyrights refresh') + status = 1 + + + # if recording the data - perhaps outputting to a CSV file + if args.dump_data: + my_data = {} + my_data['componentName'] = this_comp_data['componentName'] + my_data['componentVersion'] = this_comp_data['componentVersionName'] + my_data['status'] = status + my_data['url'] = url + + if hasattr(args, 'debug') and 5 <= args.debug: + pprint(my_data) + + # add to our list + all_my_comp_data.append(my_data) else: + # no origins defined + RepWarning('No origin(s) defined for [%s]' % comp_label) my_statistics['_cntNoOrigins'] += 1 + origin_id = '' + status = 3 url = 'n/a' + # if recording the data + if args.dump_data: + my_data = {} + my_data['componentName'] = this_comp_data['componentName'] + my_data['componentVersion'] = this_comp_data['componentVersionName'] + my_data['status'] = status + my_data['url'] = url - # if recording the data - perhaps outputting to a CSV file - if args.dump_data: - my_data = {} - my_data['componentName'] = this_comp_data['componentName'] - my_data['componentVersion'] = this_comp_data['componentVersionName'] - my_data['url'] = url - - if hasattr(args, 'debug') and 5 <= args.debug: - pprint(my_data) - - # add to our list - all_my_comp_data.append(my_data) + if hasattr(args, 'debug') and 5 <= args.debug: + pprint(my_data) + # add to our list + all_my_comp_data.append(my_data) # end of processing loop @@ -280,6 +306,7 @@ def RepWarning(msg): field_names = [ 'Component', 'Component Version', + 'Status', 'Url' ] @@ -290,6 +317,7 @@ def RepWarning(msg): row_data = { 'Component': my_comp_data['componentName'], 'Component Version': my_comp_data['componentVersion'], + 'Status': my_comp_data['status'], 'Url': my_comp_data['url'] } writer.writerow(row_data) From 90f1db34a13621dcee7458444e77f58ccbda6697 Mon Sep 17 00:00:00 2001 From: Ian A <86781798+LuckySkyWalker@users.noreply.github.com> Date: Tue, 27 May 2025 15:06:31 +0100 Subject: [PATCH 3/3] Update refresh_project_copyrights.py Counting origins and clearer debug output --- examples/client/refresh_project_copyrights.py | 176 ++++++++++++------ 1 file changed, 120 insertions(+), 56 deletions(-) diff --git a/examples/client/refresh_project_copyrights.py b/examples/client/refresh_project_copyrights.py index 4244b4b..a6e9d3b 100644 --- a/examples/client/refresh_project_copyrights.py +++ b/examples/client/refresh_project_copyrights.py @@ -4,6 +4,7 @@ # Ian Ashworth, May 2025 # import http.client +import signal from sys import api_version import sys import csv @@ -18,11 +19,18 @@ http.client._MAXHEADERS = 1000 +job_status = 0 + logging.basicConfig( level=logging.INFO, format="[%(asctime)s] {%(module)s:%(lineno)d} %(levelname)s - %(message)s" ) +# initialise +all_my_comp_data = [] +my_statistics = {} + + def RepDebug(level, msg): if hasattr(args, 'debug') and level <= args.debug: print("dbg{" + str(level) + "} " + msg) @@ -33,6 +41,59 @@ def RepWarning(msg): print("WARNING: " + msg) return True +def CompleteTask(job_status): + now = datetime.datetime.now() + my_statistics['_jobStatus'] = job_status + + print('Finished: %s' % now.strftime("%Y-%m-%d %H:%M:%S")) + print('Summary:') + pprint(my_statistics) + + # if dumping data + if args.dump_data: + # if outputting to a CSV file + if args.csv_file: + '''Note: See the BD API doc and in particular .../api-doc/public.html#_bom_vulnerability_endpoints + for a complete list of the fields available. The below code shows a subset of them just to + illustrate how to write out the data into a CSV format. + ''' + logging.info(f"Exporting {len(all_my_comp_data)} records to CSV file {args.csv_file}") + + with open(args.csv_file, 'w') as csv_f: + field_names = [ + 'Component', + 'Component Version', + 'Status', + 'Url' + ] + + writer = csv.DictWriter(csv_f, fieldnames=field_names) + writer.writeheader() + + for my_comp_data in all_my_comp_data: + row_data = { + 'Component': my_comp_data['componentName'], + 'Component Version': my_comp_data['componentVersion'], + 'Status': my_comp_data['status'], + 'Url': my_comp_data['url'] + } + writer.writerow(row_data) + else: + # print to screen + pprint(all_my_comp_data) + +def SignalHandler(sig, frame): + # Complete the work + print("Ctrl+C detected!") + + # tidy up and complete the job + CompleteTask(1) + sys.exit(job_status) + +# ------------------------------------------------------------------------------ +# register the signal handler +signal.signal(signal.SIGINT, SignalHandler) + # Parse command line arguments parser = argparse.ArgumentParser("Refresh copyrights for project/version components") @@ -46,10 +107,12 @@ def RepWarning(msg): parser.add_argument("--project", dest='project_name', help="Project name") parser.add_argument("--version", dest='version_name', help="Version name") -parser.add_argument("--max-projects", dest='max_projects', type=int, help="Maximum projects to inspect else all") +parser.add_argument("--max-projects", dest='max_projects', type=int, help="Maximum number of projects to inspect else all") parser.add_argument("--max-versions-per-project", dest='max_versions_per_project', type=int, help="Maximum versions per project to inspect else all") parser.add_argument("--max-components", dest='max_components', type=int, help="Maximum components to inspect in total else all") +parser.add_argument("--skip-projects", dest='skip_projects', type=int, help="Skip first 'n' projects to inspect") + parser.add_argument("--debug", dest='debug', type=int, default=0, help="Debug verbosity (0=none 'n'=level)") parser.add_argument("--dryrun", dest='dry_run', type=int, default=0, help="Dry run test (0=no 1=yes)") @@ -72,11 +135,10 @@ def RepWarning(msg): retries=args.retries, ) -# initialise -all_my_comp_data = [] -my_statistics = {} +str_unknown = "n/a" + # version of components API to call comp_api_version = 6 @@ -118,10 +180,13 @@ def RepWarning(msg): my_statistics['_cntProjects'] = 0 my_statistics['_cntVersions'] = 0 my_statistics['_cntComponents'] = 0 +my_statistics['_cntOrigins'] = 0 + my_statistics['_cntRefresh'] = 0 my_statistics['_cntNoOrigins'] = 0 my_statistics['_cntNoIDs'] = 0 - +my_statistics['_cntSkippedProjects'] = 0 +my_statistics['_jobStatus'] = 0 # record any control values if args.project_name: @@ -152,18 +217,33 @@ def RepWarning(msg): # all projects are in scope projects = bd.get_resource('projects') + +cnt_project = 0 +cnt_call = 0 + # loop through projects list for this_project in projects: + cnt_project += 1 + + # check if we are skipping over this project + if args.skip_projects and cnt_project <= args.skip_projects: + my_statistics['_cntSkippedProjects'] += 1 + RepDebug(1, 'Skipping project [%d] [%s]' % (cnt_project, this_project['name'])) + continue + # check if we have hit any limit if args.max_components and my_statistics['_cntComponents'] >= args.max_components: + RepDebug(1, 'Reached component limit [%d]' % args.max_components) break if args.max_projects and my_statistics['_cntProjects'] >= args.max_projects: + RepDebug(1, 'Reached project limit [%d]' % args.max_projects) break + # process this project my_statistics['_cntProjects'] += 1 - RepDebug(1, '## Project %d: %s' % (my_statistics['_cntProjects'], this_project['name'])) + RepDebug(1, '## Project: [%d] [%s]' % (cnt_project, this_project['name'])) if args.version_name: # note the specific project version of interest @@ -184,11 +264,11 @@ def RepWarning(msg): # check if we have hit any limit if args.max_components and my_statistics['_cntComponents'] >= args.max_components: - # exit component loop - at the limit + RepDebug(1, 'Reached component limit [%d]' % args.max_components) break if args.max_versions_per_project and nVersionsPerProject >= args.max_versions_per_project: - # exit loop - at the version per project limit + RepDebug(1, 'Reached versions per project limit [%d]' % args.max_versions_per_project) break nVersionsPerProject += 1 @@ -196,30 +276,40 @@ def RepWarning(msg): # Announce # logging.debug(f"Found {this_project['name']}:{this_version['versionName']}") - RepDebug(3, ' Version: %s' % this_version['versionName']) + RepDebug(3, ' Version: [%s]' % this_version['versionName']) # iterate through all components for this project version for this_comp_data in bd.get_resource('components', this_version, **comp_kwargs): if args.max_components and my_statistics['_cntComponents'] >= args.max_components: - # exit component loop - at the limit break my_statistics['_cntComponents'] += 1 - comp_label = "{} ({})".format(this_comp_data['componentName'], this_comp_data['componentVersionName']) - RepDebug(4, ' Component: %s' % comp_label) + if this_comp_data.get("componentName"): + comp_name = this_comp_data['componentName'] + else: + comp_name = str_unknown + + if this_comp_data.get("componentVersionName"): + comp_version_name = this_comp_data['componentVersionName'] + else: + comp_version_name = str_unknown + + comp_label = "{} ({})".format(comp_name, comp_version_name) + + RepDebug(4, ' Component: [%s]' % comp_label) if this_comp_data['inputExternalIds'].__len__() > 0: inputExternalIds = this_comp_data['inputExternalIds'][0] else: my_statistics['_cntNoIDs'] += 1 - inputExternalIds = "n/a" - RepDebug(2, ' ID: %s' % inputExternalIds) + inputExternalIds = str_unknown + RepDebug(2, ' ID: [%s]' % inputExternalIds) - # refresh the copyrights for this component + # refresh the copyrights for this component-origin if this_comp_data['origins'].__len__() > 0: n_origin = 0 @@ -227,17 +317,27 @@ def RepWarning(msg): for this_origin in this_comp_data['origins']: n_origin += 1 - origin_id = this_origin['externalId'] + my_statistics['_cntOrigins'] += 1 + + if this_origin.get('externalId'): + origin_id = this_origin['externalId'] + else: + origin_id = str_unknown + url = this_origin['origin'] # refresh with end point url += "/copyrights-refresh" status = -1 + cnt_call += 1 + call_id = "{}.{}".format(cnt_project, cnt_call) if args.dry_run != 0: - RepDebug(1, "DryRun: no=%d origin=%s url=%s" % (n_origin, origin_id, url)) + RepDebug(2, ' DryRun: %s - origin - no [%d] id [%s] url [%s]' % (call_id, n_origin, origin_id, url)) else: + RepDebug(3, + ' Origin: %s - origin - no [%d] id [%s] url [%s]' % (call_id, n_origin, origin_id, url)) try: response = bd.session.put(url, data=None, **refresh_kwargs) RepDebug(5,'Refresh response: origin [%s] [%s]' % (this_origin, response)) @@ -274,8 +374,8 @@ def RepWarning(msg): # if recording the data if args.dump_data: my_data = {} - my_data['componentName'] = this_comp_data['componentName'] - my_data['componentVersion'] = this_comp_data['componentVersionName'] + my_data['componentName'] = comp_name + my_data['componentVersion'] = comp_version_name my_data['status'] = status my_data['url'] = url @@ -287,42 +387,6 @@ def RepWarning(msg): # end of processing loop -now = datetime.datetime.now() -print('Finished: %s' % now.strftime("%Y-%m-%d %H:%M:%S")) -print('Summary:') -pprint(my_statistics) - -# if dumping data -if args.dump_data: - # if outputting to a CSV file - if args.csv_file: - '''Note: See the BD API doc and in particular .../api-doc/public.html#_bom_vulnerability_endpoints - for a complete list of the fields available. The below code shows a subset of them just to - illustrate how to write out the data into a CSV format. - ''' - logging.info(f"Exporting {len(all_my_comp_data)} records to CSV file {args.csv_file}") - - with open(args.csv_file, 'w') as csv_f: - field_names = [ - 'Component', - 'Component Version', - 'Status', - 'Url' - ] - - writer = csv.DictWriter(csv_f, fieldnames=field_names) - writer.writeheader() - - for my_comp_data in all_my_comp_data: - row_data = { - 'Component': my_comp_data['componentName'], - 'Component Version': my_comp_data['componentVersion'], - 'Status': my_comp_data['status'], - 'Url': my_comp_data['url'] - } - writer.writerow(row_data) - else: - # print to screen - pprint(all_my_comp_data) +CompleteTask(0) #end