Omegaconf implementation (#347)

Closes #344, #340

Author: sacca97

.github/workflows/python.yml

@@ -7,11 +7,9 @@ on:
   push:
     branches:
       - main
-      - test-fix-nlp
   pull_request:
     branches:
       - main
-      - text-fix-nlp
 
 jobs:
   build:
@@ -46,13 +44,11 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: 3.10.6
-      - name: Setup virtual environment
+      - name: Setup python environment
         run: |
           python -m pip install --upgrade pip
-          pip install virtualenv
           cd prospector
-          virtualenv ../venv
-          source ../venv/bin/activate
+          cp config-sample.yaml config.yaml
           pip install -r requirements.txt
           pip install -r requirements-dev.txt
           python -m spacy download en_core_web_sm
@@ -61,7 +57,7 @@ jobs:
         with:
           path: prospector
           max_line_length: 100
-          ignore: E203,E501,W503,F401,F403
+          ignore: E203,E501,W503,F401,F403,E999
       - name: Test with pytest
         env:
           GIT_CACHE: "/tmp/git-cache"
@@ -74,5 +70,4 @@ jobs:
         run: |
           cd prospector
           [ -d /tmp/git-cache ] || mkdir -p /tmp/git-cache
-          source ../venv/bin/activate
           pytest

.gitignore

@@ -43,6 +43,7 @@ prospector/data/nvd.csv
 .vscode/settings.json
 prospector/cov_html/*
 prospector/client/cli/cov_html/*
+prospector/config.yaml
 prospector/client/web/node-app/node_modules
 prospector/.coverage.*
 prospector/.coverage
@@ -53,6 +54,7 @@ prospector/prospector.code-workspace
 prospector/requests-cache.sqlite
 prospector/prospector-report.html
 prospector/test_report.html
+prospector/test_report.json
 prospector/.idea/*
 similarities.csv
 prospector/demo_ul.html

prospector/Makefile

@@ -7,31 +7,22 @@ END='\033[0m]'	# No Color
 DONE="$(GREEN)DONE$(END)"
 PROGRESS="$(YELLOW)....$(END)"
 
-test:
-	@echo "$(PROGRESS) TEST"
-	@echo "$(DONE) TEST"
 
 setup: requirements.txt
 	@echo "$(PROGRESS) Installing requirements"
 	@pip install -r requirements.txt
 	@echo "$(DONE) Installed requirements"
-	@echo "$(PROGRESS) Installing pre-commit and other modules"
+	@echo "$(PROGRESS) Installing pre-commit and spacy model"
 	@pre-commit install
 	@python -m spacy download en_core_web_sm
-	@echo "$(DONE) Installed pre-commit and other modules"
-	@mkdir -p $(GIT_CACHE)
-	@echo "$(DONE) Created directory $(GIT_CACHE)"
+	@echo "$(DONE) Installed pre-commit and spacy model"
 
 dev-setup: setup requirements-dev.txt
-	@mkdir -p $(CVE_DATA_PATH)
-	@echo "$(DONE) Created directory $(CVE_DATA_PATH)"
 	@echo "$(PROGRESS) Installing development requirements"
 	@pip install -r requirements-dev.txt
 	@echo "$(DONE) Installed development requirements"
 
 docker-setup:
-	mkdir -p $(GIT_CACHE)
-	mkdir -p $(CVE_DATA_PATH)
 	docker-compose up -d --build
 
 docker-clean:
@@ -47,20 +38,15 @@ docker-clean:
 	@docker system prune -a -f
 	@echo "$(DONE) Cleaned residue"
 
-run: client/cli/main.py
-#python client/cli/main.py CVE-2014-0050 --repository https://github.com/apache/commons-fileupload --use-nvd --tag-interval FILEUPLOAD_1_3_RC1:FILEUPLOAD_1_3_2_RC1
-	python client/cli/main.py CVE-2022-29599 --repository https://github.com/apache/maven-shared-utils --use-nvd
-#--tag-interval maven-shared-utils-0.1:maven-shared-utils-3.3.4
-
-select-run:
-	python client/cli/main.py $(cve) --repository $(repository) --use-nvd
 
 clean:
 	@rm -f prospector.log
-	@rm -rf $(GIT_CACHE)/*
 	@rm -rf __pycache__
 	@rm -rf */__pycache__
 	@rm -rf */*/__pycache__
-	@rm -rf *report.html
+	@rm -rf *.log
+	@rm -rf .pytest_cache
+	@rm -rf *.html
 	@rm -rf *.json
-	@rm -rf requests-cache.sqlite
+	@rm -rf *.sqlite
+#@rm -rf $(GIT_CACHE)/*

prospector/api/main.py

@@ -1,6 +1,5 @@
 import uvicorn
 from fastapi import FastAPI
-
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import HTMLResponse, RedirectResponse
 

prospector/api/routers/jobs.py

@@ -5,10 +5,9 @@
 from rq import Connection, Queue
 from rq.job import Job
 
-from log.logger import logger
 from api.routers.nvd_feed_update import main
 from git.git import do_clone
-
+from log.logger import logger
 
 redis_url = os.environ["REDIS_URL"]
 

prospector/api/routers/nvd.py

@@ -3,71 +3,98 @@
 import json
 import os
 
+import requests
 from fastapi import APIRouter, HTTPException
-from fastapi.responses import JSONResponse
 
 from log.logger import logger
 
-
 router = APIRouter(
     prefix="/nvd",
     tags=["nvd"],
     # dependencies=[Depends(oauth2_scheme)],
     responses={404: {"description": "Not found"}},
 )
 
-DATA_PATH = os.environ.get("CVE_DATA_PATH") or os.path.join(
-    os.path.realpath(os.path.dirname(__file__)), "..", "data"
-)
 
+NVD_REST_ENDPOINT = "https://services.nvd.nist.gov/rest/json/cves/2.0"
+NVD_API_KEY = os.getenv("NVD_API_KEY")
+DATA_PATH = os.getenv("CVE_DATA_PATH")
+
+
+def get_from_nvd(cve_id: str):
+    """
+    Get an advisory from the NVD dtabase
+    """
+    try:
+        if NVD_API_KEY is None:
+            logger.warning("No NVD API key provided, rate liting may apply")
+
+        headers = {"apiKey": NVD_API_KEY} if NVD_API_KEY else {}
+        params = {"cveId": cve_id}
 
-@router.get("/vulnerabilities/by-year/{year}")
-async def get_vuln_list_by_year(year: str):
-    logger.debug("Requested list of vulnerabilities for " + year)
+        response = requests.get(NVD_REST_ENDPOINT, headers=headers, params=params)
 
-    if len(year) != 4 or not year.isdigit():
-        return JSONResponse([])
+        if response.status_code != 200:
+            return False
 
-    data_dir = os.path.join(DATA_PATH, year)
-    if not os.path.isdir(data_dir):
-        logger.info("No data found for year " + year)
-        raise HTTPException(
-            status_code=404, detail="No vulnerabilities found for " + year
-        )
+        data = response.json()["vulnerabilities"]
+        if len(data) == 0:
+            return False
 
-    logger.debug("Serving data for year " + year)
-    vuln_ids = [vid.rstrip(".json") for vid in os.listdir(data_dir)]
-    results = {"count": len(vuln_ids), "data": vuln_ids}
-    return JSONResponse(results)
+        with open(f"{DATA_PATH}/{cve_id}.json", "w") as out:
+            json.dump(data[0]["cve"], out)
+
+        return data[0]["cve"]
+
+    except Exception:
+        return None
 
 
 @router.get("/vulnerabilities/{vuln_id}")
 async def get_vuln_data(vuln_id):
-    logger.debug("Requested data for vulnerability " + vuln_id)
 
-    year = vuln_id.split("-")[1]
-    json_file = os.path.join(DATA_PATH, year, vuln_id.upper() + ".json")
+    json_file = os.path.join(DATA_PATH, f"{vuln_id.upper()}.json")
     if not os.path.isfile(json_file):
-        logger.info("No file found: " + json_file)
-        raise HTTPException(
-            status_code=404, detail=json_file
-        )  # detail="Vulnerability data not found")
-
-    logger.debug("Serving file: " + json_file)
-    with open(json_file) as f:
-        data = json.loads(f.read())
-
+        logger.debug("Fallback to NVD")
+        data = get_from_nvd(vuln_id.upper())
+    else:
+        logger.debug("Vulnerability data found locally " + vuln_id)
+        with open(json_file) as f:
+            data = json.load(f)
+
+    if data is None:
+        raise HTTPException(status_code=404, detail="Vulnerability not found.")
+    # TODO: check what happens if there's some error here
     return data
 
 
-@router.get("/status")
-async def status():
-    logger.debug("Serving status page")
-    out = dict()
-    metadata_file = os.path.join(DATA_PATH, "metadata.json")
-    if os.path.isfile(metadata_file):
-        with open(metadata_file) as f:
-            metadata = json.loads(f.read())
-        out["metadata"] = metadata
-        return JSONResponse(out)
-    raise HTTPException(status_code=404, detail="Missing feed file")
+# @router.get("/status")
+# async def status():
+#     logger.debug("Serving status page")
+#     out = dict()
+#     metadata_file = os.path.join(DATA_PATH, "metadata.json")
+#     if os.path.isfile(metadata_file):
+#         with open(metadata_file) as f:
+#             metadata = json.loads(f.read())
+#         out["metadata"] = metadata
+#         return JSONResponse(out)
+#     raise HTTPException(status_code=404, detail="Missing feed file")
+
+# @router.get("/vulnerabilities/by-year/{year}")
+# async def get_vuln_list_by_year(year: str):
+#     logger.debug("Requested list of vulnerabilities for " + year)
+
+#     if len(year) != 4 or not year.isdigit():
+#         return JSONResponse([])
+
+#     data_dir = os.path.join(DATA_PATH, year)
+#     if not os.path.isdir(data_dir):
+#         logger.info("No data found for year " + year)
+#         raise HTTPException(
+#             status_code=404, detail="No vulnerabilities found for " + year
+#         )
+
+#     logger.debug("Serving data for year " + year)
+#     vuln_ids = [vid.rstrip(".json") for vid in os.listdir(data_dir)]
+#     results = {"count": len(vuln_ids), "data": vuln_ids}
+#     return JSONResponse(results)

prospector/api/routers/nvd_feed_update.py

@@ -26,7 +26,6 @@
 
 from log.logger import logger
 
-
 NVD_API_KEY = os.getenv("NVD_API_KEY", "")
 
 # note: The NVD has not data older than 2002

prospector/api/routers/preprocessed.py

@@ -1,6 +1,6 @@
 from typing import Any, Dict, List, Optional
 
-from fastapi import APIRouter
+from fastapi import APIRouter, HTTPException
 from fastapi.responses import JSONResponse
 
 from commitdb.postgres import PostgresCommitDB
@@ -23,7 +23,7 @@ async def get_commits(
     data = db.lookup(repository_url, commit_id)
 
     if len(data) == 0:
-        return JSONResponse(status_code=404, content={"message": "Not found"})
+        raise HTTPException(status_code=404, detail="Commit not found")
 
     return JSONResponse(data)
 

prospector/api/routers/users.py

@@ -1,9 +1,6 @@
 from fastapi import APIRouter, Depends, HTTPException
 from fastapi.security import OAuth2PasswordRequestForm
 
-# from http import HTTPStatus
-
-
 from ..dependencies import (
     User,
     UserInDB,
@@ -13,6 +10,9 @@
     oauth2_scheme,
 )
 
+# from http import HTTPStatus
+
+
 router = APIRouter(
     prefix="/users",
     tags=["users"],