Skip to content

Commit 4a81d4b

Browse files
author
Glenn Snyder
committed
2 parents 4ad5061 + c5b5428 commit 4a81d4b

File tree

4 files changed

+310
-13
lines changed

4 files changed

+310
-13
lines changed

‎README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,3 +110,4 @@ This package is available on PyPi:
110110
## Documentation ##
111111
Documentation for hub-rest-api-python can be found on the base project:
112112
[Hub REST API Python Wiki](https://github.com/blackducksoftware/hub-rest-api-python/wiki)
113+

‎examples/batch_policy_override.py

Lines changed: 50 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,17 @@
2222
specific language governing permissions and limitations
2323
under the License.
2424
25-
This scrit will override policies for components that are listed in a CSV file
25+
This scrit will override policies for components that are listed in a CSV or an EXCEL file
2626
Cmponents with status IN_VIOLATION will get policy status to IN_VIOLATION_OVERRIDEN
27-
identification of a project and cmponent wil be done based n following fields in the CSV file
27+
28+
Note: Override will happen only when the Override Date field is blank.
29+
Rows with blank Override Date and Ovveride Rationale will be ignored.
30+
31+
Note: EXCEL processing is rather simplistic, it will not process milti-sheet workbooks properly.
32+
It will skip all the header rows, until it finds "Name of Software Component" header.
33+
Rows after that will be processed according to abovementioned rules.
34+
35+
identification of a project and cmponent wil be done based n following fields in the input file
2836
2937
component_name = field 0 (Column A in Excel lingo)
3038
component_version = field 1 (Column B in Excel lingo)
@@ -41,7 +49,7 @@
4149
-h Show help
4250
-u BASE_URL URL of a Blackduck system
4351
-t TOKEN_FILE Authentication token file
44-
-i INPUT_FILE Input CSV file
52+
-i INPUT_FILE Input CSV or EXCEL file
4553
-nv Trust TLS certificate
4654
4755
@@ -78,6 +86,7 @@
7886
import json
7987
import logging
8088
import arrow
89+
import re
8190

8291
from itertools import islice
8392
from datetime import timedelta
@@ -124,28 +133,56 @@ def parse_command_args():
124133

125134
return parser.parse_args()
126135

127-
def main():
128-
args = parse_command_args()
129-
with open(args.token_file, 'r') as tf:
130-
access_token = tf.readline().strip()
131-
global bd
132-
bd = Client(base_url=args.base_url, token=access_token, verify=args.no_verify, timeout=60.0, retries=4)
133-
136+
def process_csv_file(filename):
134137
file = open(args.input_file)
135138
type(file)
136-
137139
csvreader = csv.reader(file)
138-
139140
for row in csvreader:
140141
component_name = row[0]
141142
component_version = row[1]
142143
policy_violation_status = row[8]
144+
override_date = row[10]
143145
override_rationale = row[11]
144146
project_name = row[13]
145147
project_version = row[14]
146-
if policy_violation_status == 'IN_VIOLATION':
148+
if policy_violation_status == 'IN_VIOLATION' and override_rationale and not override_date:
147149
logging.info(f"Attemting to override policy status for {component_name} {component_version} in {project_name} {project_version} with ''{override_rationale}''")
148150
override_policy_violaton(project_name, project_version, component_name, component_version, override_rationale)
149151

152+
def process_excel_file(filename):
153+
import openpyxl
154+
wb = openpyxl.load_workbook(filename)
155+
ws = wb.active
156+
process = False
157+
for row in ws.values:
158+
if process:
159+
component_name = row[0]
160+
component_version = row[1]
161+
policy_violation_status = row[8]
162+
override_date = row[10]
163+
override_rationale = row[11]
164+
project_name = row[13]
165+
project_version = row[14]
166+
if policy_violation_status == 'IN_VIOLATION' and override_rationale and not override_date:
167+
print ("overriding")
168+
logging.info(f"Attemting to override policy status for {component_name} {component_version} in {project_name} {project_version} with ''{override_rationale}''")
169+
override_policy_violaton(project_name, project_version, component_name, component_version, override_rationale)
170+
if not process:
171+
process = (row[0] == "Name of Software Component")
172+
173+
def main():
174+
args = parse_command_args()
175+
with open(args.token_file, 'r') as tf:
176+
access_token = tf.readline().strip()
177+
global bd
178+
bd = Client(base_url=args.base_url, token=access_token, verify=args.no_verify, timeout=60.0, retries=4)
179+
180+
if re.match(".+xlsx?$", args.input_file):
181+
print (f"Processing EXCEL file {args.input_file}")
182+
process_excel_file(args.input_file)
183+
else:
184+
print ("Processing as CSV")
185+
process_csv_file(args.input_file)
186+
150187
if __name__ == "__main__":
151188
sys.exit(main())

‎examples/download_all_scans.py

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
import argparse
2+
import json
3+
import logging
4+
import sys
5+
from datetime import timedelta
6+
from pprint import pprint
7+
8+
import arrow
9+
from blackduck import Client
10+
from blackduck.Utils import get_resource_name
11+
12+
logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', stream=sys.stderr, level=logging.DEBUG)
13+
logging.getLogger("requests").setLevel(logging.WARNING)
14+
logging.getLogger("urllib3").setLevel(logging.WARNING)
15+
logging.getLogger("blackduck").setLevel(logging.DEBUG)
16+
17+
def main():
18+
upstream_copyright_data_file='copyright_data.txt'
19+
20+
parser = argparse.ArgumentParser("Enumerate BOM componets without copyrigth statements. retrieve copyright statements form upstream channel and/or version")
21+
parser.add_argument("-u", "--base-url", required=True, help="Hub server URL e.g. https://your.blackduck.url")
22+
parser.add_argument("-t", "--token-file", dest='token_file', required=True, help="containing access token")
23+
parser.add_argument("-nv", "--no-verify", dest='verify', action='store_false', help="disable TLS certificate verification")
24+
parser.add_argument("-o", "--outputdir", dest='outputdir', default='outdir', help="disable TLS certificate verification")
25+
args = parser.parse_args()
26+
27+
with open(args.token_file, 'r') as tf:
28+
access_token = tf.readline().strip()
29+
30+
global bd
31+
bd = Client(base_url=args.base_url, token=access_token, verify=args.verify, timeout = 60.0)
32+
outdir=None
33+
if args.outputdir:
34+
outdir = args.outputdir
35+
import os
36+
if not os.path.exists(outdir):
37+
os.mkdir(outdir)
38+
projects = bd.get_resource('projects')
39+
for project in projects:
40+
versions = bd.get_resource('versions',project)
41+
for version in versions:
42+
codelocations = bd.get_resource('codelocations', version)
43+
for codelocation in codelocations:
44+
resources = bd.list_resources(codelocation)
45+
url = resources['scan-data']
46+
filename = url.split('/')[6]
47+
response = bd.session.get(url, stream=True, verify=False)
48+
print (response.status_code)
49+
print (response.ok)
50+
print (url)
51+
if response.ok:
52+
pathname = os.path.join(outdir, filename)
53+
with open(pathname, "wb") as f:
54+
for data in response.iter_content():
55+
f.write(data)
56+
print(f"{filename}, {pathname}")
57+
58+
if __name__ == "__main__":
59+
sys.exit(main())
60+

‎examples/generate-bom-report-with-cpe.py

Lines changed: 199 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,199 @@
1+
import csv
2+
import datetime
3+
import sys
4+
5+
from blackduck import Client
6+
import argparse
7+
import logging
8+
from pprint import pprint
9+
import http.client
10+
import requests
11+
http.client._MAXHEADERS = 1000
12+
13+
now = datetime.datetime.now()
14+
now = now.strftime("%d/%m/%Y %H:%M:%S")
15+
16+
17+
def get_cpe_from_nvd(cve_id, comp_name, comp_version):
18+
nvd_url = "https://services.nvd.nist.gov/rest/json/cve/1.0/"
19+
r = requests.get(nvd_url + cve_id)
20+
response = r.json()
21+
# cpe_id = response['result']['CVE_Items'][0]['configurations']['nodes'][0]['cpe_match'][0]['cpe23Uri']
22+
# return cpe_id
23+
cpe_list = []
24+
print("Processing Component:" + ' ' + comp_name + ' ' + comp_version + ' ' + "and CVE:" + ' ' + cve_id)
25+
# print("Processing CVE:" + ' ' + cve_id)
26+
for node_info in response['result']['CVE_Items'][0]['configurations']['nodes']:
27+
for item in node_info['cpe_match']:
28+
cpe_match = item['cpe23Uri']
29+
comp_name_split = comp_name.split()
30+
if ' ' in comp_name:
31+
comp_name_list = comp_name_split[1:]
32+
else:
33+
comp_name_list = comp_name_split
34+
for value in comp_name_list:
35+
if (cpe_match.find(value.lower()) != -1) and (len(value) > 2):
36+
print("Matched CPE =" + ' ' + cpe_match)
37+
cpe_list.append(cpe_match)
38+
break
39+
else:
40+
str_value = ''.join(map(str, value))
41+
n = 3
42+
# value_new = [str_value[index: index + n] for index in range(0, len(str_value), n)]
43+
value_new = [str_value[i:i + n] for i in range(0, len(str_value), n)]
44+
if (cpe_match.find(value_new[0].lower()) != -1) and (len(value_new[0]) > 2):
45+
print("Matched CPE =" + ' ' + cpe_match)
46+
cpe_list.append(cpe_match)
47+
# print(cve_id)
48+
break
49+
return ','.join(set(cpe_list))
50+
51+
52+
def csv_generator(bd, args):
53+
params = {
54+
'q': [f"name:{args.project_name}"]
55+
}
56+
projects = [p for p in bd.get_resource('projects', params=params) if p['name'] == args.project_name]
57+
assert len(
58+
projects) == 1, f"There should be one, and only one project named {args.project_name}. We found {len(projects)}"
59+
project = projects[0]
60+
61+
params = {
62+
'q': [f"versionName:{args.version_name}"]
63+
}
64+
versions = [v for v in bd.get_resource('versions', project, params=params) if v['versionName'] == args.version_name]
65+
assert len(
66+
versions) == 1, f"There should be one, and only one version named {args.version_name}. We found {len(versions)}"
67+
version = versions[0]
68+
69+
logging.debug(f"Found {project['name']}:{version['versionName']}")
70+
71+
all_bom_component_vulns = []
72+
73+
for bom_component_vuln in bd.get_resource('vulnerable-components', version):
74+
vuln_name = bom_component_vuln['vulnerabilityWithRemediation']['vulnerabilityName']
75+
vuln_source = bom_component_vuln['vulnerabilityWithRemediation']['source']
76+
upgrade_guidance = bd.get_json(f"{bom_component_vuln['componentVersion']}/upgrade-guidance")
77+
bom_component_vuln['upgrade_guidance'] = upgrade_guidance
78+
79+
vuln_detail_headers = {'Accept': 'application/vnd.blackducksoftware.vulnerability-4+json'}
80+
vuln_details = bd.get_json(f"/api/vulnerabilities/{vuln_name}", headers=vuln_detail_headers)
81+
bom_component_vuln['vulnerability_details'] = vuln_details
82+
83+
if 'related-vulnerability' in bd.list_resources(vuln_details):
84+
related_vuln = bd.get_resource("related-vulnerability", vuln_details, items=False)
85+
else:
86+
related_vuln = None
87+
bom_component_vuln['related_vulnerability'] = related_vuln
88+
all_bom_component_vulns.append(bom_component_vuln)
89+
90+
'''Note: See the BD API doc and in particular .../api-doc/public.html#_bom_vulnerability_endpoints
91+
for a complete list of the fields available. The below code shows a subset of them just to
92+
illustrate how to write out the data into a CSV format.
93+
'''
94+
logging.info(f"Exporting {len(all_bom_component_vulns)} records to CSV file {args.csv_file}")
95+
with open(args.csv_file, 'w') as csv_f:
96+
field_names = [
97+
'Vulnerability Name',
98+
'Vulnerability Description',
99+
'Remediation Status',
100+
'Component',
101+
'Component Version',
102+
'Exploit Available',
103+
'Workaround Available',
104+
'Solution Available',
105+
'Upgrade Guidance - short term',
106+
'Upgrade Guidance - long term',
107+
'Related Vuln',
108+
'CPE Data'
109+
]
110+
writer = csv.DictWriter(csv_f, fieldnames=field_names)
111+
writer.writeheader()
112+
for comp_vuln in all_bom_component_vulns:
113+
comp_name = comp_vuln['componentName']
114+
comp_version = comp_vuln['componentVersionName']
115+
if comp_vuln['related_vulnerability'] is not None:
116+
cve_id = comp_vuln['related_vulnerability'].get('name')
117+
# cpe_data = list(set(get_cpe_from_nvd(cve_id, comp_name)))
118+
cpe_data = get_cpe_from_nvd(cve_id, comp_name, comp_version)
119+
row_data = {
120+
'Vulnerability Name': comp_vuln['vulnerabilityWithRemediation']['vulnerabilityName'],
121+
'Vulnerability Description': comp_vuln['vulnerabilityWithRemediation']['description'],
122+
'Remediation Status': comp_vuln['vulnerabilityWithRemediation']['remediationStatus'],
123+
'Component': comp_vuln['componentName'],
124+
'Component Version': comp_vuln['componentVersionName'],
125+
'Exploit Available': comp_vuln['vulnerability_details'].get('exploitPublishDate', 'None available'),
126+
'Workaround Available': comp_vuln['vulnerability_details'].get('workaround', 'None available'),
127+
'Solution Available': comp_vuln['vulnerability_details'].get('solution', 'None available'),
128+
'Upgrade Guidance - short term': comp_vuln['upgrade_guidance'].get('shortTerm', 'None available'),
129+
'Upgrade Guidance - long term': comp_vuln['upgrade_guidance'].get('longTerm', 'None available'),
130+
'Related Vuln': comp_vuln['related_vulnerability'].get('name', 'None available'),
131+
'CPE Data': cpe_data
132+
}
133+
else:
134+
source = comp_vuln['vulnerabilityWithRemediation']['source']
135+
if source == 'NVD':
136+
cve_id = comp_vuln['vulnerabilityWithRemediation']['vulnerabilityName']
137+
cpe_data = get_cpe_from_nvd(cve_id, comp_name, comp_version)
138+
else:
139+
cpe_data = ''
140+
row_data = {
141+
'Vulnerability Name': comp_vuln['vulnerabilityWithRemediation']['vulnerabilityName'],
142+
'Vulnerability Description': comp_vuln['vulnerabilityWithRemediation']['description'],
143+
'Remediation Status': comp_vuln['vulnerabilityWithRemediation']['remediationStatus'],
144+
'Component': comp_vuln['componentName'],
145+
'Component Version': comp_vuln['componentVersionName'],
146+
'Exploit Available': comp_vuln['vulnerability_details'].get('exploitPublishDate', 'None available'),
147+
'Workaround Available': comp_vuln['vulnerability_details'].get('workaround', 'None available'),
148+
'Solution Available': comp_vuln['vulnerability_details'].get('solution', 'None available'),
149+
'Upgrade Guidance - short term': comp_vuln['upgrade_guidance'].get('shortTerm', 'None available'),
150+
'Upgrade Guidance - long term': comp_vuln['upgrade_guidance'].get('longTerm', 'None available'),
151+
'Related Vuln': 'None Available',
152+
'CPE Data': cpe_data
153+
}
154+
writer.writerow(row_data)
155+
156+
157+
def main(argv=None):
158+
logging.basicConfig(
159+
level=logging.INFO,
160+
format="[%(asctime)s] {%(module)s:%(lineno)d} %(levelname)s - %(message)s"
161+
)
162+
if argv is None:
163+
argv = sys.argv
164+
else:
165+
argv.extend(sys.argv)
166+
print("Today's Date:", now)
167+
print('############################################')
168+
print("Black Duck BoM to CPE Extraction Utility")
169+
print('############################################')
170+
parser = argparse.ArgumentParser("Extract CPE Data from NVD for Vulnerable BOM Components in a given Black Duck "
171+
"Project and Version")
172+
parser.add_argument("--base-url", required=True, help="Hub server URL e.g. https://your.blackduck.url")
173+
parser.add_argument("--token-file", dest='token_file', required=True, help="containing access token")
174+
parser.add_argument("--csv-file", dest='csv_file', required=True, help="Supply a CSV file name to get output "
175+
"formatted in CSV")
176+
parser.add_argument("--project", dest='project_name', required=True,
177+
help="Project that contains the BOM components")
178+
parser.add_argument("--version", dest='version_name', required=True,
179+
help="Version that contains the BOM components")
180+
parser.add_argument("--no-verify", dest='verify', action='store_false', help="disable TLS certificate verification")
181+
args = parser.parse_args()
182+
print(args)
183+
184+
# Initiate Black Duck Client Class
185+
print("Initiating Black Duck Client Class")
186+
with open(args.token_file, 'r') as tf:
187+
access_token = tf.readline().strip()
188+
189+
bd = Client(base_url=args.base_url, token=access_token, verify=args.verify)
190+
191+
if args.csv_file:
192+
print("Generating CSV Report for: " + args.project_name + ' ' + args.version_name)
193+
csv_generator(bd, args)
194+
else:
195+
parser.print_help()
196+
197+
198+
if __name__ == '__main__':
199+
main(sys.argv[1:])

0 commit comments

Comments
 (0)