Skip to content

Add support for namespaces and service for each client #10

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Add support for namespaces and service for each client #10

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3fe9311
Namespace support
Mar 31, 2025
ab08be3
One Check_MK service for each client
Mar 31, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
245 changes: 245 additions & 0 deletions agent_based/proxmox_bs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,15 @@
ServiceLabel,
State,
Result,
render
)
from .utils import df
import re
import json

import time
from datetime import datetime

proxmox_bs_subsection_start = re.compile("^===")
proxmox_bs_subsection_int = re.compile("===.*$")
proxmox_bs_subsection_end = re.compile("^=")
Expand Down Expand Up @@ -228,3 +232,244 @@ def proxmox_bs_checks(item, params, section):
check_ruleset_name="filesystem",
)









# Check for each single client
def proxmox_bs_gen_clientname(client_json):
if "comment" in client_json and "backup-id" in client_json:
return str(client_json["backup-id"]) + "-" + str(client_json["comment"])

def proxmox_bs_clients_discovery(section):
for n, k, c in proxmox_bs_subsections_discovery(section):
if n == "proxmox-backup-client snapshot list":
clients = []

for client_section in json.loads(c):
if "comment" in client_section and "backup-id" in client_section:
cn = proxmox_bs_gen_clientname(client_section)

if not cn in clients:
clients.append(cn)

for client_name in clients:
yield Service(
item=client_name,
#labels=[ServiceLabel('pbs/datastore', 'yes')]
)

return


# Example JSON output from check
#[
#{
# "backup-id":"103",
# "backup-time":1742890730,
# "backup-type":"vm",
# "comment":"pfsense01",
# "files":[
# {
# "crypt-mode":"none",
# "filename":"qemu-server.conf.blob",
# "size":487
# },
# {
# "crypt-mode":"none",
# "filename":"drive-scsi0.img.fidx",
# "size":34359738368
# },
# {
# "crypt-mode":"none",
# "filename":"index.json.blob",
# "size":414
# },
# {
# "filename":"client.log.blob"
# }
# ],
# "owner":"user",
# "protected":false,
# "size":34359739269
#},
#{
# "backup-id":"103",
# "backup-time":1742550846,
# "backup-type":"vm",
# "comment":"pfsense01",
# "files":[
# {
# "crypt-mode":"none",
# "filename":"qemu-server.conf.blob",
# "size":487
# },
# {
# "crypt-mode":"none",
# "filename":"drive-scsi0.img.fidx",
# "size":34359738368
# },
# {
# "crypt-mode":"none",
# "filename":"index.json.blob",
# "size":514
# },
# {
# "filename":"client.log.blob"
# }
# ],
# "owner":"user",
# "protected":false,
# "size":34359739369,
# "verification":{
# "state":"ok",
# "upid":"UPID:pbs:000002C0:000007BA:00000001:67DE4FC4:verificationjob:fs01\\x3av\\x2dee54fa7e\\x2d61f0:root@pam:"
# }
#}
#]




def proxmox_bs_clients_checks(item, params, section):
clients = {}

#structure results from check output
for n, k, c in proxmox_bs_subsections_checks(section):
if (n == "proxmox-backup-client snapshot list"):
for e in json.loads(c):
#Get clientname
cn = proxmox_bs_gen_clientname(e)

#Only process do fruther processing for current item
if cn != item:
continue

if not cn in clients:
clients[cn] = {}

#Verification states
if not "verification" in clients[cn]:
clients[cn]["verification"] = {}

clients[cn]["verification"]["ok"] = {}
clients[cn]["verification"]["ok"]["newest_date"] = None
clients[cn]["verification"]["ok"]["count"] = 0

clients[cn]["verification"]["failed"] = {}
clients[cn]["verification"]["failed"]["newest_date"] = None
clients[cn]["verification"]["failed"]["count"] = 0

clients[cn]["verification"]["notdone"] = {}
clients[cn]["verification"]["notdone"]["newest_date"] = None
clients[cn]["verification"]["notdone"]["count"] = 0

#Backup age
dt = int(e["backup-time"])

if "verification" in e:
verify_state = e.get("verification", {}).get("state", "na")
if verify_state == "ok":
clients[cn]["verification"]["ok"]["count"] += 1
if clients[cn]["verification"]["ok"]["newest_date"] == None:
clients[cn]["verification"]["ok"]["newest_date"] = dt
elif clients[cn]["verification"]["ok"]["newest_date"] < dt:
clients[cn]["verification"]["ok"]["newest_date"] = dt

else:
clients[cn]["verification"]["failed"]["count"] += 1
if clients[cn]["verification"]["failed"]["newest_date"] == None:
clients[cn]["verification"]["failed"]["newest_date"] = dt
elif clients[cn]["verification"]["failed"]["newest_date"] < dt:
clients[cn]["verification"]["failed"]["newest_date"] = dt
else:
clients[cn]["verification"]["notdone"]["count"] += 1
if clients[cn]["verification"]["notdone"]["newest_date"] == None:
clients[cn]["verification"]["notdone"]["newest_date"] = dt
elif clients[cn]["verification"]["notdone"]["newest_date"] < dt:
clients[cn]["verification"]["notdone"]["newest_date"] = dt


#Process client result and yield results (in the clients array should only be the client matching the item)
for cn in clients:
if cn != item: #useless, because filtering for the right item is done above. But leave it there for safty.
continue

#OK
dpt = ""
if clients[cn]["verification"]["ok"]["count"] < params["snapshot_min_ok"]:
s=State.WARN
dpt= " (minimum of %s backups not reached)" % params["snapshot_min_ok"]
elif clients[cn]["verification"]["ok"]["count"] >= params["snapshot_min_ok"]:
s=State.OK
dpt= ""

yield Result(state=s, summary=(
'Snapshots verify OK: %d%s' % (clients[cn]["verification"]["ok"]["count"],dpt)
))

#Age Check OK
if clients[cn]["verification"]["ok"]["newest_date"] != None:
age = int(time.time() - clients[cn]["verification"]["ok"]["newest_date"])

warn_age, critical_age = params['bkp_age']

if age >= critical_age:
s = State.CRIT
elif age >= warn_age:
s = State.WARN
else:
s = State.OK

yield Result(state=s, summary=(
'Timestamp latest verify OK: %s, Age: %s' % (render.datetime(clients[cn]["verification"]["ok"]["newest_date"]), render.timespan(age))
))
else:
s = State.WARN
yield Result(state=s, summary=(
'Timestamp latest verify OK: No verified snapshot found'
))

#Not verified
yield Result(state=State.OK, summary=(
'Snapshots verify notdone: %d' % clients[cn]["verification"]["notdone"]["count"]
))

if clients[cn]["verification"]["notdone"]["newest_date"] != None:
age = int(time.time() - clients[cn]["verification"]["notdone"]["newest_date"])

yield Result(state=State.OK, summary=(
'Timestamp latest unverified: %s, Age: %s' % (render.datetime(clients[cn]["verification"]["notdone"]["newest_date"]), render.timespan(age))
))
else:
s = State.WARN
yield Result(state=State.OK, summary=(
'Timestamp latest unverified: No unverified snapshot found'
))


#Failed
if clients[cn]["verification"]["failed"]["count"] > 0:
s=State.CRIT
else:
s=State.OK

yield Result(state=s, summary=(
'Snapshots verify failed: %d' % clients[cn]["verification"]["failed"]["count"]
))


default_proxmox_bs_clients_params={'bkp_age': (172800, 259200), 'snapshot_min_ok': 1}
register.check_plugin(
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We've updated to use API v2, as such this would not work with the most recent release version. This would be the new syntax:

check_plugin_proxmox_bs_clients = CheckPlugin(
    name="proxmox_bs_clients",
    service_name="PBS Client %s",
    sections=["proxmox_bs"],
    discovery_function=proxmox_bs_clients_discovery,
    check_function=proxmox_bs_clients_checks,
    check_default_parameters=default_proxmox_bs_clients_params,
    check_ruleset_name="proxmox_bs_clients",
)

Copy link
Contributor

@fdriessler fdriessler Apr 28, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This will likely also have to be changed:
$OUTPUT_FORMAT was adjusted to be json instead of json-pretty, as the newlines were increasing the file size unnecessarily, even quadrupling its size in one of our tests; the updated plugin reads the line in its entirety as a json to further process it instead of reading multiple lines and using key words to create the Section. As such,the discovery and check will also have to be adjusted.
The Section, as it is created by the parse function has the following structure:

{
  "tasks": [
    <"UPID:*">
  ],
  "data_stores": [
    "store1",
    "store2",
    ...
  ],
  "versions":{...},
  "datastore_list": [
    {...},
    {...},
    ...
  ],
  "task_list": [
    {...},
    {...},
    ...
  ]
}

name="proxmox_bs_clients",
service_name="PBS Client %s",
sections=["proxmox_bs"],
discovery_function=proxmox_bs_clients_discovery,
check_function=proxmox_bs_clients_checks,
check_default_parameters=default_proxmox_bs_clients_params,
check_ruleset_name="proxmox_bs_clients"
)
30 changes: 24 additions & 6 deletions agents/plugins/proxmox_bs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ command_section() {

source /etc/check_mk/proxmox_bs.env
PBS_REPO="${PBS_USERNAME}@localhost"
proxmox-backup-client login
#proxmox-backup-client login #Not needed? Causes long delay.

OUTPUT_FORMAT="--output-format json-pretty"

Expand All @@ -80,11 +80,29 @@ jq -r '.[].name' "$TMP_DATASTORES" | while read -r name; do
command_section -t "$TMP_GC" -p "$name" \
"proxmox-backup-manager garbage-collection status" "$name" $OUTPUT_FORMAT
jq -r '.upid' "$TMP_GC" >> "$TMP_UPIDS"
command_section -p "$name" \
"proxmox-backup-client list" --repository "$PBS_REPO:$name" $OUTPUT_FORMAT
command_section -p "$name" \
"proxmox-backup-client snapshot list" --repository "$PBS_REPO:$name" \
$OUTPUT_FORMAT

#for each namespace in repository
TMP_NAMESPACE=$( mktemp -p /tmp/ )
ns=$(/bin/env proxmox-backup-client namespace list --repository "$PBS_REPO:$name" --output-format text)

#concat all jsons from each namespace
currlist="[]"
echo "===proxmox-backup-client list===$name"
while IFS= read -r line; do
l=$(/bin/env proxmox-backup-client list --repository "$PBS_REPO:$name" --ns $line $OUTPUT_FORMAT)
currlist=$(jq -s 'add' <(echo $currjson) <(echo $l))
done <<< "$ns"
echo $currlist

#concat all jsons from each namespace
currsnap="[]"
echo "===proxmox-backup-client snapshot list===$name"
while IFS= read -r line; do
j=$(/bin/env proxmox-backup-client snapshot list --repository "$PBS_REPO:$name" --ns $line $OUTPUT_FORMAT)
currsnap=$(jq -s 'add' <(echo $currsnap) <(echo $j))
done <<< "$ns"
echo $currsnap

command_section -p "$name" \
"proxmox-backup-client status" --repository "$PBS_REPO:$name" \
$OUTPUT_FORMAT
Expand Down
49 changes: 48 additions & 1 deletion web/plugins/wato/proxmox_bs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,19 @@
from cmk.gui.plugins.wato import (
HostRulespec,
rulespec_registry,
CheckParameterRulespecWithItem,
RulespecGroupCheckParametersStorage,
)
from cmk.gui.valuespec import (
Alternative,
Dictionary,
FixedValue,
Password,
TextInput,
Tuple,
Age,
Integer,
TextAscii,
)


Expand Down Expand Up @@ -58,5 +64,46 @@ def _valuespec_agent_config_proxmox_bs():
name="agent_config:proxmox_bs",
valuespec=_valuespec_agent_config_proxmox_bs,
))
except ImportError:
except:
pass




def _parameter_proxmox_bs_clients():
return Dictionary(
required_keys=[],
elements = [
('bkp_age',
Tuple(
title = "Age of last Snapshot with state verified OK before changing to warn or critical",
elements = [
Age(title=_("Warning at or above backup age"),
default_value = 604800, #warn after 7*24*60*60=604800 one week
help=_("If the backup is older than the specified time, the check changes to warn.")
),
Age(title=_("Critical at or above backup age"),
default_value = 864000, #critical after 10*24*60*60=604800 10 days
help=_("If the backup is older than the specified time, the check changes to critical.")
),
]
)
),
('snapshot_min_ok',
Integer(title=_("Minium Snapshots with state verified OK"),
default_value = 1,
help=_("Change to warn, if not enough snapshots stored on the PBS Server")
),
)
]
)

rulespec_registry.register(
CheckParameterRulespecWithItem(
check_group_name="proxmox_bs_clients",
group=RulespecGroupCheckParametersStorage,
item_spec=lambda: TextAscii(title=_('PBS Client ID'), ), #The text before the item Filter Box in a specific rule
match_type='dict',
parameter_valuespec=_parameter_proxmox_bs_clients,
title=lambda: _("Proxmox Backup Server (PBS) Clients"),
))