extracting version from the apposite section of the NVD json response and not from text (more accurate when available)

Author: sacca97

prospector/datamodel/advisory.py

@@ -61,7 +61,7 @@ def __init__(
         # references: List[str] = None,
         # references_content: List[str] = None,
         affected_products: List[str] = None,
-        versions: List[Tuple[str, str]] = None,
+        versions: Dict[str, List[str]] = None,
         files: Set[str] = None,
         keywords: Set[str] = None,
     ):
@@ -72,17 +72,17 @@ def __init__(
         self.references = references or dict()
         # self.references_content = references_content or list()
         self.affected_products = affected_products or list()
-        self.versions = versions or list()
+        self.versions = versions or dict()
         self.files = files or set()
         self.keywords = keywords or set()
 
     def analyze(
         self,
         fetch_references: bool = False,
     ):
-        self.versions = [
-            version for version in self.versions if version[0] != version[1]
-        ]
+        # self.versions = [
+        #     version for version in self.versions if version[0] != version[1]
+        # ]
         # self.versions.extend(extract_versions(self.description))
         # self.versions = list(set(self.versions))
 
@@ -92,7 +92,7 @@ def analyze(
         # TODO: this could be done on the words extracted from the description
         self.files.update(extract_affected_filenames(self.description))
 
-        self.keywords.update(extract_words_from_text(self.description))
+        self.keywords.update(set(extract_words_from_text(self.description)))
 
         logger.debug("References: " + str(self.references))
         # TODO: misses something because of subdomains not considered e.g. lists.apache.org
@@ -105,11 +105,6 @@ def analyze(
         # TODO: I should extract interesting stuff from the references immediately ad maintain them just for a fast lookup
         logger.debug(f"Relevant references: {len(self.references)}")
 
-        # if fetch_references:
-        #     self.references_content = [
-        #         " ".join(str(fetch_url(r)).split()) for r in self.references
-        #     ]
-
     def get_advisory(self):
         data = get_from_local(self.cve_id)
         if not data:
@@ -125,13 +120,28 @@ def parse_advisory(self, data):
         self.last_modified_timestamp = int(isoparse(data["lastModified"]).timestamp())
         self.description = data["descriptions"][0]["value"]
         self.references = [r["url"] for r in data.get("references", [])]
-        self.versions = [
-            (
-                item.get("versionStartIncluding", item.get("versionStartExcluding")),
-                item.get("versionEndExcluding", item.get("versionEndIncluding")),
-            )
-            for item in data["configurations"][0]["nodes"][0]["cpeMatch"]
+        self.versions = {
+            "affected": [
+                item.get("versionEndIncluding", item.get("versionStartIncluding"))
+                for item in data["configurations"][0]["nodes"][0]["cpeMatch"]
+            ],  # TODO: can return to tuples
+            "fixed": [
+                item.get("versionEndExcluding")
+                for item in data["configurations"][0]["nodes"][0]["cpeMatch"]
+            ],
+        }
+        self.versions["affected"] = [
+            v for v in self.versions["affected"] if v is not None
         ]
+        self.versions["fixed"] = [v for v in self.versions["fixed"] if v is not None]
+
+        # [
+        #     (
+        #         item.get("versionEndIncluding"),  # item.get("versionStartExcluding")
+        #         item.get("versionEndExcluding"),  # , item.get("versionEndIncluding")
+        #     )
+        #     for item in data["configurations"][0]["nodes"][0]["cpeMatch"]
+        # ]
 
 
 def get_from_nvd(cve_id: str):

prospector/datamodel/advisory_test.py

@@ -1,3 +1,5 @@
+import time
+
 from datamodel.advisory import AdvisoryRecord, build_advisory_record
 
 ADVISORY_TEXT = """Unspecified vulnerability in Uconnect before 15.26.1, as used
@@ -15,6 +17,42 @@
 but not further above (thus "limited" path traversal), if the calling code would
 use the result to construct a path value."""
 
+cve_list = """
+CVE-2020-13924
+CVE-2020-11987
+CVE-2020-17516
+CVE-2021-25640
+CVE-2020-11995
+CVE-2021-25646
+CVE-2020-13922
+CVE-2020-17514
+CVE-2020-11997
+CVE-2020-9492
+CVE-2020-1926
+CVE-2020-13950
+CVE-2020-35452
+CVE-2021-20190
+CVE-2020-27223
+CVE-2020-9493
+CVE-2020-17534
+CVE-2021-21295
+CVE-2021-25958
+CVE-2020-17532
+CVE-2020-17523
+CVE-2020-1946
+CVE-2020-17525
+CVE-2020-13949
+CVE-2021-25122
+CVE-2020-17522
+CVE-2020-13936
+CVE-2020-13959
+CVE-2021-23926
+CVE-2020-11988
+CVE-2019-10095
+CVE-2020-13929"""
+
+test = """The Apache Beam MongoDB connector in versions 2.10.0 to 2.16.0 has an option to disable SSL trust verification. However this configuration is not respected and the certificate verification disables trust verification in every case. This exclusion also gets registered globally which disables trust checking for any code running in the same JVM."""
+
 
 def test_advisory_basic():
     adv = AdvisoryRecord(
@@ -42,3 +80,11 @@ def test_build_advisory_record():
 
     assert advisory.cve_id == "CVE-2014-005"
     assert advisory.versions == []
+
+
+def test_private():
+    for i in cve_list.split()[:10]:
+        advisory = build_advisory_record(i, fetch_references=False)
+        print(advisory.cve_id, advisory.affected_products)
+        time.sleep(6)
+    raise NotImplementedError