From 3fe9311ae7f37648d1b89d7e90033e3ee8c886de Mon Sep 17 00:00:00 2001 From: Matthias Maderer Date: Mon, 31 Mar 2025 21:17:43 +0200 Subject: [PATCH 1/2] Namespace support --- agents/plugins/proxmox_bs | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/agents/plugins/proxmox_bs b/agents/plugins/proxmox_bs index e69991b..cdfe1c8 100755 --- a/agents/plugins/proxmox_bs +++ b/agents/plugins/proxmox_bs @@ -60,7 +60,7 @@ command_section() { source /etc/check_mk/proxmox_bs.env PBS_REPO="${PBS_USERNAME}@localhost" -proxmox-backup-client login +#proxmox-backup-client login #Not needed? Causes long delay. OUTPUT_FORMAT="--output-format json-pretty" @@ -80,11 +80,29 @@ jq -r '.[].name' "$TMP_DATASTORES" | while read -r name; do command_section -t "$TMP_GC" -p "$name" \ "proxmox-backup-manager garbage-collection status" "$name" $OUTPUT_FORMAT jq -r '.upid' "$TMP_GC" >> "$TMP_UPIDS" - command_section -p "$name" \ - "proxmox-backup-client list" --repository "$PBS_REPO:$name" $OUTPUT_FORMAT - command_section -p "$name" \ - "proxmox-backup-client snapshot list" --repository "$PBS_REPO:$name" \ - $OUTPUT_FORMAT + + #for each namespace in repository + TMP_NAMESPACE=$( mktemp -p /tmp/ ) + ns=$(/bin/env proxmox-backup-client namespace list --repository "$PBS_REPO:$name" --output-format text) + + #concat all jsons from each namespace + currlist="[]" + echo "===proxmox-backup-client list===$name" + while IFS= read -r line; do + l=$(/bin/env proxmox-backup-client list --repository "$PBS_REPO:$name" --ns $line $OUTPUT_FORMAT) + currlist=$(jq -s 'add' <(echo $currjson) <(echo $l)) + done <<< "$ns" + echo $currlist + + #concat all jsons from each namespace + currsnap="[]" + echo "===proxmox-backup-client snapshot list===$name" + while IFS= read -r line; do + j=$(/bin/env proxmox-backup-client snapshot list --repository "$PBS_REPO:$name" --ns $line $OUTPUT_FORMAT) + currsnap=$(jq -s 'add' <(echo $currsnap) <(echo $j)) + done <<< "$ns" + echo $currsnap + command_section -p "$name" \ "proxmox-backup-client status" --repository "$PBS_REPO:$name" \ $OUTPUT_FORMAT From ab08be3362550679421b4172cf56f68ff44817f2 Mon Sep 17 00:00:00 2001 From: Matthias Maderer Date: Mon, 31 Mar 2025 21:42:56 +0200 Subject: [PATCH 2/2] One Check_MK service for each client --- agent_based/proxmox_bs.py | 245 +++++++++++++++++++++++++++++++++ web/plugins/wato/proxmox_bs.py | 49 ++++++- 2 files changed, 293 insertions(+), 1 deletion(-) diff --git a/agent_based/proxmox_bs.py b/agent_based/proxmox_bs.py index 6821fa1..b39e49e 100644 --- a/agent_based/proxmox_bs.py +++ b/agent_based/proxmox_bs.py @@ -14,11 +14,15 @@ ServiceLabel, State, Result, + render ) from .utils import df import re import json +import time +from datetime import datetime + proxmox_bs_subsection_start = re.compile("^===") proxmox_bs_subsection_int = re.compile("===.*$") proxmox_bs_subsection_end = re.compile("^=") @@ -228,3 +232,244 @@ def proxmox_bs_checks(item, params, section): check_ruleset_name="filesystem", ) + + + + + + + + +# Check for each single client +def proxmox_bs_gen_clientname(client_json): + if "comment" in client_json and "backup-id" in client_json: + return str(client_json["backup-id"]) + "-" + str(client_json["comment"]) + +def proxmox_bs_clients_discovery(section): + for n, k, c in proxmox_bs_subsections_discovery(section): + if n == "proxmox-backup-client snapshot list": + clients = [] + + for client_section in json.loads(c): + if "comment" in client_section and "backup-id" in client_section: + cn = proxmox_bs_gen_clientname(client_section) + + if not cn in clients: + clients.append(cn) + + for client_name in clients: + yield Service( + item=client_name, + #labels=[ServiceLabel('pbs/datastore', 'yes')] + ) + + return + + +# Example JSON output from check +#[ +#{ +# "backup-id":"103", +# "backup-time":1742890730, +# "backup-type":"vm", +# "comment":"pfsense01", +# "files":[ +# { +# "crypt-mode":"none", +# "filename":"qemu-server.conf.blob", +# "size":487 +# }, +# { +# "crypt-mode":"none", +# "filename":"drive-scsi0.img.fidx", +# "size":34359738368 +# }, +# { +# "crypt-mode":"none", +# "filename":"index.json.blob", +# "size":414 +# }, +# { +# "filename":"client.log.blob" +# } +# ], +# "owner":"user", +# "protected":false, +# "size":34359739269 +#}, +#{ +# "backup-id":"103", +# "backup-time":1742550846, +# "backup-type":"vm", +# "comment":"pfsense01", +# "files":[ +# { +# "crypt-mode":"none", +# "filename":"qemu-server.conf.blob", +# "size":487 +# }, +# { +# "crypt-mode":"none", +# "filename":"drive-scsi0.img.fidx", +# "size":34359738368 +# }, +# { +# "crypt-mode":"none", +# "filename":"index.json.blob", +# "size":514 +# }, +# { +# "filename":"client.log.blob" +# } +# ], +# "owner":"user", +# "protected":false, +# "size":34359739369, +# "verification":{ +# "state":"ok", +# "upid":"UPID:pbs:000002C0:000007BA:00000001:67DE4FC4:verificationjob:fs01\\x3av\\x2dee54fa7e\\x2d61f0:root@pam:" +# } +#} +#] + + + + +def proxmox_bs_clients_checks(item, params, section): + clients = {} + + #structure results from check output + for n, k, c in proxmox_bs_subsections_checks(section): + if (n == "proxmox-backup-client snapshot list"): + for e in json.loads(c): + #Get clientname + cn = proxmox_bs_gen_clientname(e) + + #Only process do fruther processing for current item + if cn != item: + continue + + if not cn in clients: + clients[cn] = {} + + #Verification states + if not "verification" in clients[cn]: + clients[cn]["verification"] = {} + + clients[cn]["verification"]["ok"] = {} + clients[cn]["verification"]["ok"]["newest_date"] = None + clients[cn]["verification"]["ok"]["count"] = 0 + + clients[cn]["verification"]["failed"] = {} + clients[cn]["verification"]["failed"]["newest_date"] = None + clients[cn]["verification"]["failed"]["count"] = 0 + + clients[cn]["verification"]["notdone"] = {} + clients[cn]["verification"]["notdone"]["newest_date"] = None + clients[cn]["verification"]["notdone"]["count"] = 0 + + #Backup age + dt = int(e["backup-time"]) + + if "verification" in e: + verify_state = e.get("verification", {}).get("state", "na") + if verify_state == "ok": + clients[cn]["verification"]["ok"]["count"] += 1 + if clients[cn]["verification"]["ok"]["newest_date"] == None: + clients[cn]["verification"]["ok"]["newest_date"] = dt + elif clients[cn]["verification"]["ok"]["newest_date"] < dt: + clients[cn]["verification"]["ok"]["newest_date"] = dt + + else: + clients[cn]["verification"]["failed"]["count"] += 1 + if clients[cn]["verification"]["failed"]["newest_date"] == None: + clients[cn]["verification"]["failed"]["newest_date"] = dt + elif clients[cn]["verification"]["failed"]["newest_date"] < dt: + clients[cn]["verification"]["failed"]["newest_date"] = dt + else: + clients[cn]["verification"]["notdone"]["count"] += 1 + if clients[cn]["verification"]["notdone"]["newest_date"] == None: + clients[cn]["verification"]["notdone"]["newest_date"] = dt + elif clients[cn]["verification"]["notdone"]["newest_date"] < dt: + clients[cn]["verification"]["notdone"]["newest_date"] = dt + + + #Process client result and yield results (in the clients array should only be the client matching the item) + for cn in clients: + if cn != item: #useless, because filtering for the right item is done above. But leave it there for safty. + continue + + #OK + dpt = "" + if clients[cn]["verification"]["ok"]["count"] < params["snapshot_min_ok"]: + s=State.WARN + dpt= " (minimum of %s backups not reached)" % params["snapshot_min_ok"] + elif clients[cn]["verification"]["ok"]["count"] >= params["snapshot_min_ok"]: + s=State.OK + dpt= "" + + yield Result(state=s, summary=( + 'Snapshots verify OK: %d%s' % (clients[cn]["verification"]["ok"]["count"],dpt) + )) + + #Age Check OK + if clients[cn]["verification"]["ok"]["newest_date"] != None: + age = int(time.time() - clients[cn]["verification"]["ok"]["newest_date"]) + + warn_age, critical_age = params['bkp_age'] + + if age >= critical_age: + s = State.CRIT + elif age >= warn_age: + s = State.WARN + else: + s = State.OK + + yield Result(state=s, summary=( + 'Timestamp latest verify OK: %s, Age: %s' % (render.datetime(clients[cn]["verification"]["ok"]["newest_date"]), render.timespan(age)) + )) + else: + s = State.WARN + yield Result(state=s, summary=( + 'Timestamp latest verify OK: No verified snapshot found' + )) + + #Not verified + yield Result(state=State.OK, summary=( + 'Snapshots verify notdone: %d' % clients[cn]["verification"]["notdone"]["count"] + )) + + if clients[cn]["verification"]["notdone"]["newest_date"] != None: + age = int(time.time() - clients[cn]["verification"]["notdone"]["newest_date"]) + + yield Result(state=State.OK, summary=( + 'Timestamp latest unverified: %s, Age: %s' % (render.datetime(clients[cn]["verification"]["notdone"]["newest_date"]), render.timespan(age)) + )) + else: + s = State.WARN + yield Result(state=State.OK, summary=( + 'Timestamp latest unverified: No unverified snapshot found' + )) + + + #Failed + if clients[cn]["verification"]["failed"]["count"] > 0: + s=State.CRIT + else: + s=State.OK + + yield Result(state=s, summary=( + 'Snapshots verify failed: %d' % clients[cn]["verification"]["failed"]["count"] + )) + + +default_proxmox_bs_clients_params={'bkp_age': (172800, 259200), 'snapshot_min_ok': 1} +register.check_plugin( + name="proxmox_bs_clients", + service_name="PBS Client %s", + sections=["proxmox_bs"], + discovery_function=proxmox_bs_clients_discovery, + check_function=proxmox_bs_clients_checks, + check_default_parameters=default_proxmox_bs_clients_params, + check_ruleset_name="proxmox_bs_clients" +) diff --git a/web/plugins/wato/proxmox_bs.py b/web/plugins/wato/proxmox_bs.py index ee48ab7..b57990d 100644 --- a/web/plugins/wato/proxmox_bs.py +++ b/web/plugins/wato/proxmox_bs.py @@ -10,6 +10,8 @@ from cmk.gui.plugins.wato import ( HostRulespec, rulespec_registry, + CheckParameterRulespecWithItem, + RulespecGroupCheckParametersStorage, ) from cmk.gui.valuespec import ( Alternative, @@ -17,6 +19,10 @@ FixedValue, Password, TextInput, + Tuple, + Age, + Integer, + TextAscii, ) @@ -58,5 +64,46 @@ def _valuespec_agent_config_proxmox_bs(): name="agent_config:proxmox_bs", valuespec=_valuespec_agent_config_proxmox_bs, )) -except ImportError: +except: pass + + + + +def _parameter_proxmox_bs_clients(): + return Dictionary( + required_keys=[], + elements = [ + ('bkp_age', + Tuple( + title = "Age of last Snapshot with state verified OK before changing to warn or critical", + elements = [ + Age(title=_("Warning at or above backup age"), + default_value = 604800, #warn after 7*24*60*60=604800 one week + help=_("If the backup is older than the specified time, the check changes to warn.") + ), + Age(title=_("Critical at or above backup age"), + default_value = 864000, #critical after 10*24*60*60=604800 10 days + help=_("If the backup is older than the specified time, the check changes to critical.") + ), + ] + ) + ), + ('snapshot_min_ok', + Integer(title=_("Minium Snapshots with state verified OK"), + default_value = 1, + help=_("Change to warn, if not enough snapshots stored on the PBS Server") + ), + ) + ] + ) + +rulespec_registry.register( + CheckParameterRulespecWithItem( + check_group_name="proxmox_bs_clients", + group=RulespecGroupCheckParametersStorage, + item_spec=lambda: TextAscii(title=_('PBS Client ID'), ), #The text before the item Filter Box in a specific rule + match_type='dict', + parameter_valuespec=_parameter_proxmox_bs_clients, + title=lambda: _("Proxmox Backup Server (PBS) Clients"), + ))