Refactoring prospector code structure (#359)

## Main changes
- Reorganized code structure
- Fixed docker backend and worker containers
- Added first implementation to automate the process of running prospector
- Updated makefile 
- Updated backend to use config.yaml 
- Updated rules
- Prepared to use also MITRE API

---------

Co-authored-by: sacca97 <tommaso.sacchetti@gmail.com>
Co-authored-by: Antonino Sabetta <antonino.sabetta@sap.com>

Author: 3 people

.github/workflows/python.yml

@@ -39,7 +39,7 @@ jobs:
           # Maps tcp port 5432 on service container to the host
           - 5432:5432
         volumes:
-          - prospector/ddl:/docker-entrypoint-initdb.d
+          - ${{ github.workspace }}/prospector/ddl:/docker-entrypoint-initdb.d
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python 3.10

.gitignore

@@ -32,6 +32,7 @@ prospector/.env
 prospector/workspace.code-workspace
 prospector/disabled_tests/skip_test-commits.db
 prospector/disabled_tests/skip_test-vulnerabilities.db
+prospector/tracer_dataset_final_2
 prospector/results
 prospector/*.py
 prospector/.vscode/launch.json

prospector/.env-sample

@@ -17,5 +17,5 @@ POSTGRES_HOST=localhost
 POSTGRES_DBNAME=postgres
 POSTGRES_PASSWORD=example
 REDIS_URL=redis://localhost:6379/0
-NVD_API_KEY=yourNvdApiKey
+NVD_API_KEY=APIkey
 PYTHONPATH=.

prospector/.flake8

@@ -1,8 +1,9 @@
 [flake8]
-ignore = E203,E501,W503 #E203, E501, W503,F401,F403,W605
+ignore = E203,E501,W503,E501 E203, E501, W503,F401,F403,W605
 exclude =
     # No need to traverse our git directory
     .git,
+    empirical_study,
     # There's no value in checking cache directories
     __pycache__,
     # The conf file is mostly autogenerated, ignore it

prospector/.pre-commit-config.yaml

@@ -11,11 +11,11 @@ repos:
     hooks:
       - id: black
   - repo: https://github.com/pycqa/isort
-    rev: 5.10.1
+    rev: 5.12.0
     hooks:
       - id: isort
         args: ["--profile", "black", "--filter-files"]
-  - repo: https://gitlab.com/pycqa/flake8
+  - repo: https://github.com/pycqa/flake8
     rev: 5.0.4
     hooks:
       - id: flake8

prospector/api/static/index.html

@@ -14,11 +14,11 @@
     sys.path.append(path_root)
 
 
-import client.cli.report as report  # noqa: E402
-from client.cli.console import ConsoleWriter, MessageStatus  # noqa: E402
-from client.cli.prospector_client import TIME_LIMIT_AFTER  # noqa: E402
-from client.cli.prospector_client import TIME_LIMIT_BEFORE  # noqa: E402
-from client.cli.prospector_client import prospector  # noqa: E402; noqa: E402
+import core.report as report  # noqa: E402
+from cli.console import ConsoleWriter, MessageStatus  # noqa: E402
+from core.prospector import TIME_LIMIT_AFTER  # noqa: E402
+from core.prospector import TIME_LIMIT_BEFORE  # noqa: E402
+from core.prospector import prospector  # noqa: E402; noqa: E402
 
 # Load logger before doing anything else
 from log.logger import get_level, logger, pretty_log  # noqa: E402
@@ -30,7 +30,9 @@
 
 def main(argv):  # noqa: C901
     with ConsoleWriter("Initialization") as console:
+        print("before config: ", argv)
         config = get_configuration(argv)
+        print("after config: ", config.cve_id)
 
         if not config:
             logger.error("No configuration file found. Cannot proceed.")
@@ -69,16 +71,16 @@ def main(argv):  # noqa: C901
         repository_url=config.repository,
         publication_date=config.pub_date,
         vuln_descr=config.description,
-        # tag_interval=config.tag_interval,
         version_interval=config.version_interval,
         modified_files=config.modified_files,
         advisory_keywords=config.keywords,
         use_nvd=config.use_nvd,
-        fetch_references=config.fetch_references,
+        # fetch_references=config.fetch_references,
         backend_address=config.backend,
         use_backend=config.use_backend,
         git_cache=config.git_cache,
         limit_candidates=config.max_candidates,
+        # ignore_adv_refs=config.ignore_refs,
     )
 
     if config.preprocess_only: