Skip to content

Commit 90f1db3

Browse files
Update refresh_project_copyrights.py
Counting origins and clearer debug output
1 parent aea5f58 commit 90f1db3

File tree

1 file changed

+120
-56
lines changed

1 file changed

+120
-56
lines changed

examples/client/refresh_project_copyrights.py

Lines changed: 120 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
# Ian Ashworth, May 2025
55
#
66
import http.client
7+
import signal
78
from sys import api_version
89
import sys
910
import csv
@@ -18,11 +19,18 @@
1819

1920
http.client._MAXHEADERS = 1000
2021

22+
job_status = 0
23+
2124
logging.basicConfig(
2225
level=logging.INFO,
2326
format="[%(asctime)s] {%(module)s:%(lineno)d} %(levelname)s - %(message)s"
2427
)
2528

29+
# initialise
30+
all_my_comp_data = []
31+
my_statistics = {}
32+
33+
2634
def RepDebug(level, msg):
2735
if hasattr(args, 'debug') and level <= args.debug:
2836
print("dbg{" + str(level) + "} " + msg)
@@ -33,6 +41,59 @@ def RepWarning(msg):
3341
print("WARNING: " + msg)
3442
return True
3543

44+
def CompleteTask(job_status):
45+
now = datetime.datetime.now()
46+
my_statistics['_jobStatus'] = job_status
47+
48+
print('Finished: %s' % now.strftime("%Y-%m-%d %H:%M:%S"))
49+
print('Summary:')
50+
pprint(my_statistics)
51+
52+
# if dumping data
53+
if args.dump_data:
54+
# if outputting to a CSV file
55+
if args.csv_file:
56+
'''Note: See the BD API doc and in particular .../api-doc/public.html#_bom_vulnerability_endpoints
57+
for a complete list of the fields available. The below code shows a subset of them just to
58+
illustrate how to write out the data into a CSV format.
59+
'''
60+
logging.info(f"Exporting {len(all_my_comp_data)} records to CSV file {args.csv_file}")
61+
62+
with open(args.csv_file, 'w') as csv_f:
63+
field_names = [
64+
'Component',
65+
'Component Version',
66+
'Status',
67+
'Url'
68+
]
69+
70+
writer = csv.DictWriter(csv_f, fieldnames=field_names)
71+
writer.writeheader()
72+
73+
for my_comp_data in all_my_comp_data:
74+
row_data = {
75+
'Component': my_comp_data['componentName'],
76+
'Component Version': my_comp_data['componentVersion'],
77+
'Status': my_comp_data['status'],
78+
'Url': my_comp_data['url']
79+
}
80+
writer.writerow(row_data)
81+
else:
82+
# print to screen
83+
pprint(all_my_comp_data)
84+
85+
def SignalHandler(sig, frame):
86+
# Complete the work
87+
print("Ctrl+C detected!")
88+
89+
# tidy up and complete the job
90+
CompleteTask(1)
91+
sys.exit(job_status)
92+
93+
# ------------------------------------------------------------------------------
94+
# register the signal handler
95+
signal.signal(signal.SIGINT, SignalHandler)
96+
3697

3798
# Parse command line arguments
3899
parser = argparse.ArgumentParser("Refresh copyrights for project/version components")
@@ -46,10 +107,12 @@ def RepWarning(msg):
46107
parser.add_argument("--project", dest='project_name', help="Project name")
47108
parser.add_argument("--version", dest='version_name', help="Version name")
48109

49-
parser.add_argument("--max-projects", dest='max_projects', type=int, help="Maximum projects to inspect else all")
110+
parser.add_argument("--max-projects", dest='max_projects', type=int, help="Maximum number of projects to inspect else all")
50111
parser.add_argument("--max-versions-per-project", dest='max_versions_per_project', type=int, help="Maximum versions per project to inspect else all")
51112
parser.add_argument("--max-components", dest='max_components', type=int, help="Maximum components to inspect in total else all")
52113

114+
parser.add_argument("--skip-projects", dest='skip_projects', type=int, help="Skip first 'n' projects to inspect")
115+
53116
parser.add_argument("--debug", dest='debug', type=int, default=0, help="Debug verbosity (0=none 'n'=level)")
54117
parser.add_argument("--dryrun", dest='dry_run', type=int, default=0, help="Dry run test (0=no 1=yes)")
55118

@@ -72,11 +135,10 @@ def RepWarning(msg):
72135
retries=args.retries,
73136
)
74137

75-
# initialise
76-
all_my_comp_data = []
77-
my_statistics = {}
78138

79139

140+
str_unknown = "n/a"
141+
80142
# version of components API to call
81143
comp_api_version = 6
82144

@@ -118,10 +180,13 @@ def RepWarning(msg):
118180
my_statistics['_cntProjects'] = 0
119181
my_statistics['_cntVersions'] = 0
120182
my_statistics['_cntComponents'] = 0
183+
my_statistics['_cntOrigins'] = 0
184+
121185
my_statistics['_cntRefresh'] = 0
122186
my_statistics['_cntNoOrigins'] = 0
123187
my_statistics['_cntNoIDs'] = 0
124-
188+
my_statistics['_cntSkippedProjects'] = 0
189+
my_statistics['_jobStatus'] = 0
125190

126191
# record any control values
127192
if args.project_name:
@@ -152,18 +217,33 @@ def RepWarning(msg):
152217
# all projects are in scope
153218
projects = bd.get_resource('projects')
154219

220+
221+
cnt_project = 0
222+
cnt_call = 0
223+
155224
# loop through projects list
156225
for this_project in projects:
157226

227+
cnt_project += 1
228+
229+
# check if we are skipping over this project
230+
if args.skip_projects and cnt_project <= args.skip_projects:
231+
my_statistics['_cntSkippedProjects'] += 1
232+
RepDebug(1, 'Skipping project [%d] [%s]' % (cnt_project, this_project['name']))
233+
continue
234+
158235
# check if we have hit any limit
159236
if args.max_components and my_statistics['_cntComponents'] >= args.max_components:
237+
RepDebug(1, 'Reached component limit [%d]' % args.max_components)
160238
break
161239

162240
if args.max_projects and my_statistics['_cntProjects'] >= args.max_projects:
241+
RepDebug(1, 'Reached project limit [%d]' % args.max_projects)
163242
break
164243

244+
# process this project
165245
my_statistics['_cntProjects'] += 1
166-
RepDebug(1, '## Project %d: %s' % (my_statistics['_cntProjects'], this_project['name']))
246+
RepDebug(1, '## Project: [%d] [%s]' % (cnt_project, this_project['name']))
167247

168248
if args.version_name:
169249
# note the specific project version of interest
@@ -184,60 +264,80 @@ def RepWarning(msg):
184264

185265
# check if we have hit any limit
186266
if args.max_components and my_statistics['_cntComponents'] >= args.max_components:
187-
# exit component loop - at the limit
267+
RepDebug(1, 'Reached component limit [%d]' % args.max_components)
188268
break
189269

190270
if args.max_versions_per_project and nVersionsPerProject >= args.max_versions_per_project:
191-
# exit loop - at the version per project limit
271+
RepDebug(1, 'Reached versions per project limit [%d]' % args.max_versions_per_project)
192272
break
193273

194274
nVersionsPerProject += 1
195275
my_statistics['_cntVersions'] += 1
196276

197277
# Announce
198278
# logging.debug(f"Found {this_project['name']}:{this_version['versionName']}")
199-
RepDebug(3, ' Version: %s' % this_version['versionName'])
279+
RepDebug(3, ' Version: [%s]' % this_version['versionName'])
200280

201281

202282
# iterate through all components for this project version
203283
for this_comp_data in bd.get_resource('components', this_version, **comp_kwargs):
204284

205285
if args.max_components and my_statistics['_cntComponents'] >= args.max_components:
206-
# exit component loop - at the limit
207286
break
208287

209288
my_statistics['_cntComponents'] += 1
210-
comp_label = "{} ({})".format(this_comp_data['componentName'], this_comp_data['componentVersionName'])
211289

212-
RepDebug(4, ' Component: %s' % comp_label)
290+
if this_comp_data.get("componentName"):
291+
comp_name = this_comp_data['componentName']
292+
else:
293+
comp_name = str_unknown
294+
295+
if this_comp_data.get("componentVersionName"):
296+
comp_version_name = this_comp_data['componentVersionName']
297+
else:
298+
comp_version_name = str_unknown
299+
300+
comp_label = "{} ({})".format(comp_name, comp_version_name)
301+
302+
RepDebug(4, ' Component: [%s]' % comp_label)
213303

214304
if this_comp_data['inputExternalIds'].__len__() > 0:
215305
inputExternalIds = this_comp_data['inputExternalIds'][0]
216306
else:
217307
my_statistics['_cntNoIDs'] += 1
218-
inputExternalIds = "n/a"
219-
RepDebug(2, ' ID: %s' % inputExternalIds)
308+
inputExternalIds = str_unknown
309+
RepDebug(2, ' ID: [%s]' % inputExternalIds)
220310

221311

222-
# refresh the copyrights for this component
312+
# refresh the copyrights for this component-origin
223313
if this_comp_data['origins'].__len__() > 0:
224314

225315
n_origin = 0
226316

227317
for this_origin in this_comp_data['origins']:
228318

229319
n_origin += 1
230-
origin_id = this_origin['externalId']
320+
my_statistics['_cntOrigins'] += 1
321+
322+
if this_origin.get('externalId'):
323+
origin_id = this_origin['externalId']
324+
else:
325+
origin_id = str_unknown
326+
231327
url = this_origin['origin']
232328

233329
# refresh with end point
234330
url += "/copyrights-refresh"
235331

236332
status = -1
333+
cnt_call += 1
334+
call_id = "{}.{}".format(cnt_project, cnt_call)
237335

238336
if args.dry_run != 0:
239-
RepDebug(1, "DryRun: no=%d origin=%s url=%s" % (n_origin, origin_id, url))
337+
RepDebug(2, ' DryRun: %s - origin - no [%d] id [%s] url [%s]' % (call_id, n_origin, origin_id, url))
240338
else:
339+
RepDebug(3,
340+
' Origin: %s - origin - no [%d] id [%s] url [%s]' % (call_id, n_origin, origin_id, url))
241341
try:
242342
response = bd.session.put(url, data=None, **refresh_kwargs)
243343
RepDebug(5,'Refresh response: origin [%s] [%s]' % (this_origin, response))
@@ -274,8 +374,8 @@ def RepWarning(msg):
274374
# if recording the data
275375
if args.dump_data:
276376
my_data = {}
277-
my_data['componentName'] = this_comp_data['componentName']
278-
my_data['componentVersion'] = this_comp_data['componentVersionName']
377+
my_data['componentName'] = comp_name
378+
my_data['componentVersion'] = comp_version_name
279379
my_data['status'] = status
280380
my_data['url'] = url
281381

@@ -287,42 +387,6 @@ def RepWarning(msg):
287387

288388
# end of processing loop
289389

290-
now = datetime.datetime.now()
291-
print('Finished: %s' % now.strftime("%Y-%m-%d %H:%M:%S"))
292-
print('Summary:')
293-
pprint(my_statistics)
294-
295-
# if dumping data
296-
if args.dump_data:
297-
# if outputting to a CSV file
298-
if args.csv_file:
299-
'''Note: See the BD API doc and in particular .../api-doc/public.html#_bom_vulnerability_endpoints
300-
for a complete list of the fields available. The below code shows a subset of them just to
301-
illustrate how to write out the data into a CSV format.
302-
'''
303-
logging.info(f"Exporting {len(all_my_comp_data)} records to CSV file {args.csv_file}")
304-
305-
with open(args.csv_file, 'w') as csv_f:
306-
field_names = [
307-
'Component',
308-
'Component Version',
309-
'Status',
310-
'Url'
311-
]
312-
313-
writer = csv.DictWriter(csv_f, fieldnames=field_names)
314-
writer.writeheader()
315-
316-
for my_comp_data in all_my_comp_data:
317-
row_data = {
318-
'Component': my_comp_data['componentName'],
319-
'Component Version': my_comp_data['componentVersion'],
320-
'Status': my_comp_data['status'],
321-
'Url': my_comp_data['url']
322-
}
323-
writer.writerow(row_data)
324-
else:
325-
# print to screen
326-
pprint(all_my_comp_data)
390+
CompleteTask(0)
327391

328392
#end

0 commit comments

Comments
 (0)