Merge branch 'master' into LuckySkyWalker-refresh1

Author: LuckySkyWalker

examples/client/get_scan_missed_import_event.py

@@ -0,0 +1,96 @@
+"""
+Created on july 11, 2024
+@author: Dinesh Ravi
+Gather list of non matched components where blackduck could not able have match event with their kb for the bdio codelocation type
+"""
+from blackduck import Client
+
+import argparse
+
+import logging
+import json
+# py get_scan_missed_import_event.py --base-url=https://blackduck.company.com --token-file=.pt --project=ASTERIX2CLU3D_PR
+# OGRAM --version=AED2_ANDROID_S_2_2024-09-21_00-32 --company=company --no-verify > missing.txt
+logging.basicConfig(
+    level=logging.INFO,
+    format="[%(asctime)s] {%(module)s:%(lineno)d} %(levelname)s - %(message)s"
+)
+
+parser = argparse.ArgumentParser("Get the BOM components for a given project-version and the license details for each BOM component")
+parser.add_argument("--base-url", required=True, help="Hub server URL e.g. https://your.blackduck.url")
+parser.add_argument("--token-file", dest='token_file', required=True, help="containing access token")
+parser.add_argument("--project", dest='project_name', required=True, help="Project that contains the BOM components")
+parser.add_argument("--version", dest='version_name', required=True, help="Version that contains the BOM components")
+parser.add_argument("--company", dest='company_name', required=True, help="modules that contains the company name for separation")
+parser.add_argument("--no-verify", dest='verify', action='store_false', help="disable TLS certificate verification")
+args = parser.parse_args()
+
+with open(args.token_file, 'r') as tf:
+    access_token = tf.readline().strip()
+
+bd = Client(base_url=args.base_url, token=access_token, verify=args.verify)
+
+params = {
+    'q': [f"name:{args.project_name}"]
+}
+
+projects = [p for p in bd.get_resource('projects', params=params) if p['name'] == args.project_name]
+assert len(projects) == 1, f"There should be one, and only one project named {args.project_name}. We found {len(projects)}"
+project = projects[0]
+
+params = {
+    'q': [f"versionName:{args.version_name}"]
+}
+versions = [v for v in bd.get_resource('versions', project, params=params) if v['versionName'] == args.version_name]
+assert len(versions) == 1, f"There should be one, and only one version named {args.version_name}. We found {len(versions)}"
+version = versions[0]
+
+print(f"Found {project['name']}:{version['versionName']}")
+hashset_company = set()
+hashset_other = set()
+# ===================
+# print(version)
+# params = {
+#     'q': [f"name:*bdio*"]
+# }
+codelocations=[codelocation for codelocation in bd.get_resource('codelocations',version) if 'bdio' in codelocation['name']]
+# print(codelocations[0])
+# codelocation=codelocations[0]
+for codelocation in codelocations:
+    scans=bd.get_resource('scans',codelocation) 
+    for scan in scans:
+        events=[events for events in bd.get_resource('component-import-events',scan) if events['event']=="COMPONENT_MAPPING_FAILED"]
+        if len(events)>0:
+            print("============================")
+            print(f"codelocation_name: {codelocation['name']}")
+            print(f"matchCount: {scan['matchCount']}")
+            print(f"missing: {len(events)}")
+            
+            for i,event in enumerate(events,start=1):
+                print(f"--------------{i}")
+                externalId=event['externalId']
+                print(f"externalId: {externalId}")
+                print(f"importComponentName: {event['importComponentName']}")
+                print(f"importComponentVersionName: {event['importComponentVersionName']}")
+                if args.company_name in externalId:
+                    hashset_company.add(externalId)
+                else:
+                    hashset_other.add(externalId)
+
+                # {'event': 'COMPONENT_MAPPING_FAILED', 
+                #  'importComponentName': 'rsi-common-lib', 
+                #  'importComponentVersionName': '0.2.14', 
+                #  'externalId': 'com.company.aed2:rsi-common-lib:0.2.14', 
+                #  'failureReason': 'Unable to map scanned component version to Black Duck project version because no mapping is present for the given external identifier'}
+print("============================")
+sorted_company=sorted(hashset_company)
+sorted_other=sorted(hashset_other)
+
+print(f"========total missing Other components to get foss report: {len(hashset_other)}========")
+for i,missing in enumerate(sorted_other,start=1):
+    print(f"{i} {missing}")
+print(f"========total missing {args.company_name} components to manually add: {len(hashset_company)}========")
+for i,missing in enumerate(sorted_company,start=1):
+    print(f"{i} {missing}")
+total_missing=len(sorted_company)+len(sorted_other)
+print(f"========EB:{len(sorted_company)}+Other:{len(sorted_other)}={total_missing}========")

examples/client/refresh_project_copyrights.py

@@ -136,6 +136,7 @@ def SignalHandler(sig, frame):
 )
 
 
+str_unknown = "n/a"
 
 str_unknown = "n/a"
 
@@ -388,5 +389,4 @@ def SignalHandler(sig, frame):
 # end of processing loop
 
 CompleteTask(0)
-
 #end

examples/client/remap_codelocations.py

@@ -0,0 +1,91 @@
+"""
+Created on july 11, 2024
+
+@author: Dinesh Ravi
+
+Remap codelocations from a project version to another project version
+
+"""
+
+from blackduck import Client
+
+import argparse
+import json
+import logging
+import sys
+import time
+from pprint import pprint
+
+logging.basicConfig(
+    level=logging.DEBUG,
+    format="[%(asctime)s] {%(module)s:%(lineno)d} %(levelname)s - %(message)s",
+)
+
+parser = argparse.ArgumentParser(sys.argv[0])
+parser.add_argument(
+    "-u", "--bd_url", help="Hub server URL e.g. https://your.blackduck.url"
+)
+parser.add_argument(
+    "-t", "--token-file", help="File name of a file containing access token"
+)
+parser.add_argument(
+    "-nv",
+    "--no-verify",
+    dest="verify",
+    action="store_false",
+    help="disable TLS certificate verification",
+)
+parser.add_argument("project_name")
+parser.add_argument("version_name")
+parser.add_argument("update_pv_url")
+
+
+args = parser.parse_args()
+
+logging.basicConfig(
+    format="%(asctime)s:%(levelname)s:%(message)s",
+    stream=sys.stderr,
+    level=logging.DEBUG,
+)
+logging.getLogger("requests").setLevel(logging.WARNING)
+logging.getLogger("urllib3").setLevel(logging.WARNING)
+logging.getLogger("blackduck").setLevel(logging.WARNING)
+
+with open(args.token_file, "r") as tf:
+    access_token = tf.readline().strip()
+
+bd = Client(base_url=args.bd_url, token=access_token, verify=args.verify)
+
+params = {"q": [f"name:{args.project_name}"]}
+projects = [
+    p
+    for p in bd.get_resource("projects", params=params)
+    if p["name"] == args.project_name
+]
+assert (
+    len(projects) == 1
+), f"There should be one, and only one project named {args.project_name}. We found {len(projects)}"
+project = projects[0]
+
+params = {"q": [f"versionName:{args.version_name}"]}
+versions = [
+    v
+    for v in bd.get_resource("versions", project, params=params)
+    if v["versionName"] == args.version_name
+]
+assert (
+    len(versions) == 1
+), f"There should be one, and only one version named {args.version_name}. We found {len(versions)}"
+version = versions[0]
+
+logging.debug(f"Found {project['name']}:{version['versionName']}")
+
+
+codelocations = bd.get_resource("codelocations", version)
+# logging.info(f"Total Code locations '{len(list(codelocations))}'")
+for codelocation in codelocations:
+    logging.debug(f"Un-mapping code location {codelocation['name']}")
+    url = codelocation["_meta"]["href"]
+    codelocation["mappedProjectVersion"] = args.update_pv_url
+    result = bd.session.put(url, json=codelocation)
+    logging.info(f"Code location '{codelocation['name']}' unmap status {result}")

examples/client/upload_sbom.py

@@ -98,7 +98,7 @@ def find_or_create_project_group(group_name):
     else:
         return groups[0]['_meta']['href']
 
-def create_project_version(project_name,version_name,project_group, nickname = None):
+def create_project_version(project_name, version_name, project_group, nickname = None):
     version_data = {"distribution": "EXTERNAL", "phase": "DEVELOPMENT", "versionName": version_name}
     if nickname:
         version_data['nickname'] = nickname
@@ -120,7 +120,7 @@ def find_or_create_project_version(project_name, version_name, project_group):
         if version:
             pass
         else:
-            version = create_project_version(project_name, version_name)
+            version = create_project_version(project_name, version_name, project_group)
     else:
         version = create_project_version(project_name, version_name, project_group)
     project = find_project_by_name(project_name)