Skip to content

Commit 17e44e3

Browse files
new: [core] Added new heartbeat and report_error functions, and preparing a first trusted release on Pypi.
1 parent 2ecf128 commit 17e44e3

File tree

8 files changed

+170
-17
lines changed

8 files changed

+170
-17
lines changed

.github/workflows/release.yml

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
on:
2+
release:
3+
types:
4+
- published
5+
6+
name: release
7+
8+
jobs:
9+
pypi-publish:
10+
name: Upload release to PyPI
11+
runs-on: ubuntu-latest
12+
environment:
13+
name: pypi
14+
url: https://pypi.org/p/MISPSight
15+
16+
permissions:
17+
id-token: write # IMPORTANT: this permission is mandatory for trusted publishing
18+
steps:
19+
- uses: actions/checkout@v4
20+
with:
21+
fetch-depth: 0
22+
- name: Install Poetry
23+
run: python -m pip install --upgrade pip poetry
24+
- name: Build artifacts
25+
run: poetry build
26+
- name: Publish package distributions to PyPI
27+
uses: pypa/gh-action-pypi-publish@release/v1

CHANGELOG.md

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
# Changelog
2+
3+
## Release 1.0.0 (2025-02-13)
4+
5+
This release introduces the capability to report errors, warnings,
6+
and heartbeats to a Valkey datastore, facilitating centralized monitoring.
7+
8+
9+
## Release 0.4.1 (2024-12-09)
10+
11+
Updated PyVulnerabilityLookup.
12+
13+
14+
## Release 0.4.0 (2024-11-19)
15+
16+
- The client PyVulnerabilityLookup is now used in order to communicate with the API of the Vulnerability-Lookup instance.
17+
- When a duplicate sighting is detected a message is printed.
18+
- Improved management of UTC aware Datetime objects.
19+
20+
21+
## Release 0.3.0 (2024-11-13)
22+
23+
First working prototype - production ready.

mispsight/conf_sample.py

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,3 +4,10 @@
44

55
vulnerability_lookup_base_url = "https://vulnerability.circl.lu/"
66
vulnerability_auth_token = ""
7+
8+
9+
# Hearbeat mechanism
10+
heartbeat_enabled = True
11+
valkey_host = "127.0.0.1"
12+
valkey_port = 10002
13+
expiration_period = 18000

mispsight/config.py

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,3 +37,12 @@ def load_config(path):
3737

3838
vulnerability_lookup_base_url = conf.vulnerability_lookup_base_url
3939
vulnerability_auth_token = conf.vulnerability_auth_token
40+
41+
42+
try:
43+
heartbeat_enabled = True
44+
valkey_host = conf.valkey_host
45+
valkey_port = conf.valkey_port
46+
expiration_period = conf.expiration_period
47+
except Exception:
48+
heartbeat_enabled = False

mispsight/publish.py

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
from pyvulnerabilitylookup import PyVulnerabilityLookup
66

77
from mispsight import config
8+
from mispsight.utils import heartbeat, report_error
89

910

1011
def remove_case_insensitive_duplicates(input_list: list[str]) -> list[str]:
@@ -30,6 +31,9 @@ def push_sighting_to_vulnerability_lookup(attribute, vulnerability_ids):
3031
else:
3132
creation_timestamp = attribute.timestamp
3233
if not creation_timestamp:
34+
report_error(
35+
"warning", "push_sighting_to_vulnerability_lookup: no creation_stamp"
36+
)
3337
continue
3438

3539
# Create the sighting
@@ -45,15 +49,27 @@ def push_sighting_to_vulnerability_lookup(attribute, vulnerability_ids):
4549
r = vuln_lookup.create_sighting(sighting=sighting)
4650
if "message" in r:
4751
print(r["message"])
52+
if "duplicate" in r["message"]:
53+
level = "info"
54+
else:
55+
level = "warning"
56+
report_error(
57+
level, f"push_sighting_to_vulnerability_lookup: {r['message']}"
58+
)
4859
except Exception as e:
4960
print(
5061
f"Error when sending POST request to the Vulnerability-Lookup server:\n{e}"
5162
)
63+
report_error(
64+
"error",
65+
f"Error when sending POST request to the Vulnerability-Lookup server: {e}",
66+
)
5267

5368

5469
def main() -> None:
5570
parser = argparse.ArgumentParser(
56-
prog="MISPSight", description="Allows access to the streaming API."
71+
prog="MISPSight",
72+
description="A client that retrieves vulnerability observations from a MISP server and pushes them to a Vulnerability-Lookup instance.",
5773
)
5874
parser.add_argument(
5975
"--since",
@@ -63,6 +79,9 @@ def main() -> None:
6379

6480
arguments = parser.parse_args()
6581

82+
# Sends a heartbeat when the script launches
83+
heartbeat()
84+
6685
misp = PyMISP(config.misp_url, config.misp_key, config.misp_verifycert)
6786
print("Querying MISP…")
6887
attributes = misp.search(

mispsight/utils.py

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
import time
2+
3+
import valkey
4+
5+
from mispsight import config
6+
7+
if config.heartbeat_enabled:
8+
valkey_client = valkey.Valkey(config.valkey_host, config.valkey_port)
9+
10+
11+
def heartbeat(key="process_heartbeat_MISPSight") -> None:
12+
"""Sends a heartbeat in the Valkey datastore."""
13+
if not config.heartbeat_enabled:
14+
return
15+
try:
16+
valkey_client.set(
17+
key,
18+
time.time(),
19+
ex=config.expiration_period,
20+
)
21+
except Exception as e:
22+
print(f"Heartbeat error: {e}")
23+
24+
25+
def report_error(level="warning", message="", key="process_logs_MISPSight") -> None:
26+
"""Reports an error or warning in the Valkey datastore."""
27+
timestamp = time.time()
28+
log_entry = {"timestamp": timestamp, "level": level, "message": message}
29+
try:
30+
# Add the log entry to a list, so multiple messages are preserved
31+
valkey_client.rpush(key, str(log_entry))
32+
valkey_client.expire(key, 86400) # Expire after 24 hours
33+
except Exception as e:
34+
print(f"Error reporting failure: {e}")

poetry.lock

Lines changed: 43 additions & 11 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

pyproject.toml

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,27 +5,29 @@ build-backend = "poetry.core.masonry.api"
55

66
[project]
77
name = "MISPSight"
8-
version = "0.4.2"
8+
version = "1.0.0"
99
description = "A client that retrieves vulnerability observations from a MISP server and pushes them to a Vulnerability-Lookup instance."
1010
authors = [
1111
{name = "Cédric Bonhomme", email = "cedric.bonhomme@circl.lu"}
1212
]
1313
license = "GPL-3.0-or-later"
1414
readme = "README.md"
15-
keywords = ["Vulnerability-Lookup", "vulnerability", "cve", "sighting", "observations", "MISP", "PyVulnerabilityLookup"]
16-
# classifieres is dynamic because we want to create Python classifiers automatically
15+
keywords = ["Vulnerability-Lookup", "vulnerability", "cve", "sighting", "observations", "MISP"]
16+
1717
dynamic = ["classifiers"]
1818

19-
requires-python = ">=3.10"
19+
requires-python = ">=3.10,<4.0"
2020
dependencies = [
2121
"pyvulnerabilitylookup (>=2.2.0)",
22-
"pymisp (>=2.5.2)"
22+
"pymisp (>=2.5.2)",
23+
"valkey (>=6.1.0,<7.0.0)"
2324
]
2425

2526

2627
[project.urls]
2728
Homepage = "https://github.com/vulnerability-lookup/MISPSight"
2829
Repository = "https://github.com/vulnerability-lookup/MISPSight"
30+
Changelog = "https://github.com/vulnerability-lookup/MISPSight/blob/main/CHANGELOG.md"
2931

3032

3133
[project.scripts]

0 commit comments

Comments
 (0)