new: [core] Added new heartbeat and report_error functions, and preparing a first trusted release on Pypi.

Author: cedricbonhomme

.github/workflows/release.yml

@@ -0,0 +1,27 @@
+on:
+  release:
+    types:
+      - published
+
+name: release
+
+jobs:
+  pypi-publish:
+    name: Upload release to PyPI
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/MISPSight
+
+    permissions:
+      id-token: write  # IMPORTANT: this permission is mandatory for trusted publishing
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Install Poetry
+        run: python -m pip install --upgrade pip poetry
+      - name: Build artifacts
+        run: poetry build
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

CHANGELOG.md

@@ -0,0 +1,23 @@
+# Changelog
+
+## Release 1.0.0 (2025-02-13)
+
+This release introduces the capability to report errors, warnings,
+and heartbeats to a Valkey datastore, facilitating centralized monitoring.
+
+
+## Release 0.4.1 (2024-12-09)
+
+Updated PyVulnerabilityLookup.
+
+
+## Release 0.4.0 (2024-11-19)
+
+- The client PyVulnerabilityLookup is now used in order to communicate with the API of the Vulnerability-Lookup instance.
+- When a duplicate sighting is detected a message is printed.
+- Improved management of UTC aware Datetime objects.
+
+
+##  Release 0.3.0 (2024-11-13)
+
+First working prototype - production ready.

mispsight/conf_sample.py

@@ -4,3 +4,10 @@
 
 vulnerability_lookup_base_url = "https://vulnerability.circl.lu/"
 vulnerability_auth_token = ""
+
+
+# Hearbeat mechanism
+heartbeat_enabled = True
+valkey_host = "127.0.0.1"
+valkey_port = 10002
+expiration_period = 18000

mispsight/config.py

@@ -37,3 +37,12 @@ def load_config(path):
 
 vulnerability_lookup_base_url = conf.vulnerability_lookup_base_url
 vulnerability_auth_token = conf.vulnerability_auth_token
+
+
+try:
+    heartbeat_enabled = True
+    valkey_host = conf.valkey_host
+    valkey_port = conf.valkey_port
+    expiration_period = conf.expiration_period
+except Exception:
+    heartbeat_enabled = False

mispsight/publish.py

@@ -5,6 +5,7 @@
 from pyvulnerabilitylookup import PyVulnerabilityLookup
 
 from mispsight import config
+from mispsight.utils import heartbeat, report_error
 
 
 def remove_case_insensitive_duplicates(input_list: list[str]) -> list[str]:
@@ -30,6 +31,9 @@ def push_sighting_to_vulnerability_lookup(attribute, vulnerability_ids):
         else:
             creation_timestamp = attribute.timestamp
         if not creation_timestamp:
+            report_error(
+                "warning", "push_sighting_to_vulnerability_lookup: no creation_stamp"
+            )
             continue
 
         # Create the sighting
@@ -45,15 +49,27 @@ def push_sighting_to_vulnerability_lookup(attribute, vulnerability_ids):
             r = vuln_lookup.create_sighting(sighting=sighting)
             if "message" in r:
                 print(r["message"])
+                if "duplicate" in r["message"]:
+                    level = "info"
+                else:
+                    level = "warning"
+                report_error(
+                    level, f"push_sighting_to_vulnerability_lookup: {r['message']}"
+                )
         except Exception as e:
             print(
                 f"Error when sending POST request to the Vulnerability-Lookup server:\n{e}"
             )
+            report_error(
+                "error",
+                f"Error when sending POST request to the Vulnerability-Lookup server: {e}",
+            )
 
 
 def main() -> None:
     parser = argparse.ArgumentParser(
-        prog="MISPSight", description="Allows access to the streaming API."
+        prog="MISPSight",
+        description="A client that retrieves vulnerability observations from a MISP server and pushes them to a Vulnerability-Lookup instance.",
     )
     parser.add_argument(
         "--since",
@@ -63,6 +79,9 @@ def main() -> None:
 
     arguments = parser.parse_args()
 
+    # Sends a heartbeat when the script launches
+    heartbeat()
+
     misp = PyMISP(config.misp_url, config.misp_key, config.misp_verifycert)
     print("Querying MISP…")
     attributes = misp.search(

mispsight/utils.py

@@ -0,0 +1,34 @@
+import time
+
+import valkey
+
+from mispsight import config
+
+if config.heartbeat_enabled:
+    valkey_client = valkey.Valkey(config.valkey_host, config.valkey_port)
+
+
+def heartbeat(key="process_heartbeat_MISPSight") -> None:
+    """Sends a heartbeat in the Valkey datastore."""
+    if not config.heartbeat_enabled:
+        return
+    try:
+        valkey_client.set(
+            key,
+            time.time(),
+            ex=config.expiration_period,
+        )
+    except Exception as e:
+        print(f"Heartbeat error: {e}")
+
+
+def report_error(level="warning", message="", key="process_logs_MISPSight") -> None:
+    """Reports an error or warning in the Valkey datastore."""
+    timestamp = time.time()
+    log_entry = {"timestamp": timestamp, "level": level, "message": message}
+    try:
+        # Add the log entry to a list, so multiple messages are preserved
+        valkey_client.rpush(key, str(log_entry))
+        valkey_client.expire(key, 86400)  # Expire after 24 hours
+    except Exception as e:
+        print(f"Error reporting failure: {e}")

poetry.lock

@@ -5,27 +5,29 @@ build-backend = "poetry.core.masonry.api"
 
 [project]
 name = "MISPSight"
-version = "0.4.2"
+version = "1.0.0"
 description = "A client that retrieves vulnerability observations from a MISP server and pushes them to a Vulnerability-Lookup instance."
 authors = [
     {name = "Cédric Bonhomme", email = "cedric.bonhomme@circl.lu"}
 ]
 license = "GPL-3.0-or-later"
 readme = "README.md"
-keywords = ["Vulnerability-Lookup", "vulnerability", "cve", "sighting", "observations", "MISP", "PyVulnerabilityLookup"]
-# classifieres is dynamic because we want to create Python classifiers automatically
+keywords = ["Vulnerability-Lookup", "vulnerability", "cve", "sighting", "observations", "MISP"]
+
 dynamic = ["classifiers"]
 
-requires-python = ">=3.10"
+requires-python = ">=3.10,<4.0"
 dependencies = [
     "pyvulnerabilitylookup (>=2.2.0)",
-    "pymisp (>=2.5.2)"
+    "pymisp (>=2.5.2)",
+    "valkey (>=6.1.0,<7.0.0)"
 ]
 
 
 [project.urls]
 Homepage = "https://github.com/vulnerability-lookup/MISPSight"
 Repository = "https://github.com/vulnerability-lookup/MISPSight"
+Changelog = "https://github.com/vulnerability-lookup/MISPSight/blob/main/CHANGELOG.md"
 
 
 [project.scripts]