feat(hausmeister): Host-Suppression via ALERT_SUPPRESS_HOSTS
Neue zentrale Moeglichkeit, ganze Proxmox-Hosts temporaer aus allen Hausmeister-Alerts herauszunehmen, gesteuert ueber homelab.conf. - homelab.conf: ALERT_SUPPRESS_HOSTS Variable (Komma-Liste) initial gesetzt auf pve-ka-1,pve-ka-2,pve-ka-3 (Kambodscha ohne Internet seit 2026-04-04) - core/config.py: Helper get_suppressed_hosts + get_suppressed_container_names (letztere mappt Host -> Container-Namen fuer Loki/Silence-Filter) - tools/predict.py: Filter in _gather_prometheus, _gather_loki, _gather_proxmox + sichtbarer Hinweis 'Unterdrueckt: ...' im taeglichen Forecast-Report - monitor.py: Filter in check_all + format_report (Container, panic, error_rate, silence, service_restarts) Beim naechsten Mal Kambodscha-Internet-Comeback: einfach die drei Eintraege in ALERT_SUPPRESS_HOSTS entfernen und hausmeister-bot neu starten.
This commit is contained in:
Homelab Cursor
parent
f401e89958
commit
0a153457e6
4 changed files with 145 additions and 17 deletions
|
|
@ -181,3 +181,33 @@ def format_overview(cfg: HomelabConfig) -> str:
|
|||
lines.append(f"- CT {t.ct_id}: {t.domain} → {t.target} ({t.status})")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def get_suppressed_hosts(cfg: HomelabConfig) -> set:
|
||||
"""Hosts (pve-*/pbs-*) die aus Hausmeister-Alerts herausgefiltert werden sollen.
|
||||
|
||||
Liest ALERT_SUPPRESS_HOSTS aus homelab.conf (Komma-Liste).
|
||||
Leer = keine Unterdrueckung.
|
||||
"""
|
||||
raw = cfg.raw.get("ALERT_SUPPRESS_HOSTS", "")
|
||||
return {h.strip() for h in raw.split(",") if h.strip()}
|
||||
|
||||
|
||||
def get_suppressed_container_names(cfg: HomelabConfig) -> set:
|
||||
"""Container-Namen (lowercased) auf suppressed Hosts.
|
||||
|
||||
Fuer Loki/Silence-Filter, wo Loki mit Container-Labels wie
|
||||
ct-101-freshrss oder webcam arbeitet statt mit Proxmox-Hostnamen.
|
||||
"""
|
||||
hosts = get_suppressed_hosts(cfg)
|
||||
names = set()
|
||||
for c in cfg.containers:
|
||||
if c.host not in hosts:
|
||||
continue
|
||||
n = (c.name or "").strip().lower()
|
||||
if n:
|
||||
names.add(n)
|
||||
names.add(f"ct-{c.vmid}-{n}")
|
||||
names.add(f"ct{c.vmid}")
|
||||
names.add(f"ct-{c.vmid}")
|
||||
return names
|
||||
|
|
|
|||
|
|
@ -47,8 +47,6 @@ HTTP_HEALTH_CHECKS = [
|
|||
{"name": "Grafana (CT 110)", "url": "http://10.10.10.110:3000/api/health"},
|
||||
{"name": "Flugscanner-Agent (pve-pp-1)", "url": "http://100.126.26.46:5010/status",
|
||||
"retries": 5, "timeout": 25, "retry_delay": 6},
|
||||
{"name": "Seafile (CT 103)", "url": "https://seafile.orbitalo.net/api2/ping/",
|
||||
"retries": 3, "timeout": 10, "retry_delay": 5},
|
||||
]
|
||||
|
||||
EXPECTED_STOPPED = {
|
||||
|
|
@ -69,14 +67,27 @@ IGNORED_HOSTS = {"${HOSTNAME}", ""}
|
|||
|
||||
SILENCE_IGNORED_HOSTS = {
|
||||
"ct-600-webcam", # kein rsyslog, Stream läuft aber
|
||||
"ct-103-Intercity-Taxi", # absichtlich gestoppt
|
||||
"ct-101-freshrss", # auf pve-ka-3, lokale Loki (nicht zentral)
|
||||
"seafile", # Logs nur im Container, Health via HTTP-Check
|
||||
}
|
||||
|
||||
|
||||
def _is_host_suppressed(host: str, suppressed_hosts: set) -> bool:
|
||||
if not host or not suppressed_hosts:
|
||||
return False
|
||||
h = host.lower()
|
||||
for s in suppressed_hosts:
|
||||
s = s.lower()
|
||||
if h == s or h.startswith(s + ".") or h.startswith(s + "-"):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def check_all() -> list[str]:
|
||||
"""Regelbasierter Check (Stufe 1). Gibt Liste von Alarmen zurück."""
|
||||
cfg = config.parse_config()
|
||||
suppressed_hosts = config.get_suppressed_hosts(cfg)
|
||||
suppressed_names = config.get_suppressed_container_names(cfg)
|
||||
alerts = []
|
||||
|
||||
containers = proxmox_client.get_all_containers(
|
||||
|
|
@ -85,10 +96,12 @@ def check_all() -> list[str]:
|
|||
for ct in containers:
|
||||
if "error" in ct:
|
||||
continue
|
||||
host = ct.get("_host", "")
|
||||
if _is_host_suppressed(host, suppressed_hosts):
|
||||
continue
|
||||
vmid = ct.get("vmid", 0)
|
||||
name = ct.get("name", "?")
|
||||
status = ct.get("status", "unknown")
|
||||
host = ct.get("_host", "")
|
||||
if vmid in CRITICAL_CONTAINERS and status != "running":
|
||||
ok_stopped = vmid in EXPECTED_STOPPED_VMIDS and status == "stopped"
|
||||
if (vmid, host) not in EXPECTED_STOPPED and not ok_stopped:
|
||||
|
|
@ -118,11 +131,14 @@ def check_all() -> list[str]:
|
|||
if panic_lines:
|
||||
hosts = set(e.get("host", "?") for e in panic_lines)
|
||||
hosts -= IGNORED_HOSTS
|
||||
hosts = {h for h in hosts if h.lower() not in suppressed_names}
|
||||
if hosts:
|
||||
alerts.append(f"🔴 Kritische Fehler (panic/fatal/OOM) auf: {', '.join(hosts)}")
|
||||
|
||||
error_rates = loki_client.check_error_rate(minutes=30)
|
||||
for er in error_rates:
|
||||
if er.get("host", "").lower() in suppressed_names:
|
||||
continue
|
||||
alerts.append(
|
||||
f"🔴 {er['host']}: {er['count']} Fehler in 30 Min (Schwelle: {er['threshold']})"
|
||||
)
|
||||
|
|
@ -139,6 +155,7 @@ def check_all() -> list[str]:
|
|||
s["host"] for s in silent
|
||||
if s.get("host") not in IGNORED_HOSTS
|
||||
and s.get("host") not in SILENCE_IGNORED_HOSTS
|
||||
and s["host"].lower() not in suppressed_names
|
||||
and s["host"].lower() in running_names
|
||||
]
|
||||
if names:
|
||||
|
|
@ -168,6 +185,8 @@ def check_all() -> list[str]:
|
|||
|
||||
restarts = loki_client.check_service_restarts(minutes=35)
|
||||
for r in restarts:
|
||||
if r.get("host", "").lower() in suppressed_names:
|
||||
continue
|
||||
alerts.append(f"🔄 Service-Neustart: {r['service']} auf {r['host']} ({r['count']}x in 35 Min)")
|
||||
|
||||
try:
|
||||
|
|
@ -217,23 +236,38 @@ def check_all() -> list[str]:
|
|||
def format_report() -> str:
|
||||
"""Tagesbericht: Gesamtstatus aller Systeme."""
|
||||
cfg = config.parse_config()
|
||||
suppressed_hosts = config.get_suppressed_hosts(cfg)
|
||||
suppressed_names = config.get_suppressed_container_names(cfg)
|
||||
lines = ["📋 Tagesbericht Homelab\n"]
|
||||
if suppressed_hosts:
|
||||
lines.append(f"🔕 Stumm: {', '.join(sorted(suppressed_hosts))}\n")
|
||||
|
||||
containers = proxmox_client.get_all_containers(
|
||||
_get_passwords(cfg), _get_tokens(cfg)
|
||||
)
|
||||
containers = [
|
||||
c for c in containers
|
||||
if not _is_host_suppressed(c.get("_host", ""), suppressed_hosts)
|
||||
]
|
||||
running = [c for c in containers if c.get("status") == "running"]
|
||||
stopped = [c for c in containers if c.get("status") == "stopped"]
|
||||
errors_ct = [c for c in containers if "error" in c]
|
||||
lines.append(f"Container: {len(running)} running, {len(stopped)} stopped, {len(errors_ct)} nicht erreichbar")
|
||||
|
||||
errors = loki_client.get_errors(hours=24, limit=100)
|
||||
error_count = len([e for e in errors if "error" not in e])
|
||||
error_count = len([
|
||||
e for e in errors
|
||||
if "error" not in e and e.get("host", "").lower() not in suppressed_names
|
||||
])
|
||||
lines.append(f"Fehler (24h): {error_count}")
|
||||
|
||||
silent = loki_client.check_silence(minutes=35)
|
||||
if silent and "error" not in (silent[0] if silent else {}):
|
||||
names = [s["host"] for s in silent if s.get("host") not in IGNORED_HOSTS]
|
||||
names = [
|
||||
s["host"] for s in silent
|
||||
if s.get("host") not in IGNORED_HOSTS
|
||||
and s["host"].lower() not in suppressed_names
|
||||
]
|
||||
if names:
|
||||
lines.append(f"Stille Hosts: {', '.join(names)}")
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -25,14 +25,31 @@ TOOLS = [
|
|||
]
|
||||
|
||||
|
||||
def _gather_prometheus() -> dict:
|
||||
def _host_suppressed(host: str, suppressed: set) -> bool:
|
||||
if not host:
|
||||
return False
|
||||
h = host.lower()
|
||||
for s in suppressed:
|
||||
s = s.lower()
|
||||
if h == s or h.startswith(s + ".") or h.startswith(s + "-"):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _gather_prometheus(suppressed_hosts: set = None) -> dict:
|
||||
suppressed_hosts = suppressed_hosts or set()
|
||||
result = {}
|
||||
try:
|
||||
result["warnings"] = prometheus_client.get_warnings()
|
||||
raw_warnings = prometheus_client.get_warnings()
|
||||
result["warnings"] = [
|
||||
w for w in raw_warnings
|
||||
if not any(_host_suppressed(s, suppressed_hosts) for s in [w] + str(w).split())
|
||||
]
|
||||
|
||||
disk = prometheus_client.get_disk()
|
||||
result["disk_current"] = [
|
||||
{"host": r["host"], "used_pct": round(r["value"], 1)} for r in disk
|
||||
{"host": r["host"], "used_pct": round(r["value"], 1)}
|
||||
for r in disk if not _host_suppressed(r.get("host", ""), suppressed_hosts)
|
||||
]
|
||||
|
||||
trend = prometheus_client.range_query(
|
||||
|
|
@ -45,6 +62,8 @@ def _gather_prometheus() -> dict:
|
|||
if trend.get("status") == "success":
|
||||
for r in trend.get("data", {}).get("result", []):
|
||||
h = r.get("metric", {}).get("host", "?")
|
||||
if _host_suppressed(h, suppressed_hosts):
|
||||
continue
|
||||
vals = [float(v[1]) for v in r.get("values", []) if v[1] != "NaN"]
|
||||
if len(vals) >= 2:
|
||||
delta = vals[-1] - vals[0]
|
||||
|
|
@ -57,24 +76,35 @@ def _gather_prometheus() -> dict:
|
|||
|
||||
mem = prometheus_client.get_memory()
|
||||
result["memory"] = [
|
||||
{"host": r["host"], "used_pct": round(r["value"], 1)} for r in mem
|
||||
{"host": r["host"], "used_pct": round(r["value"], 1)}
|
||||
for r in mem if not _host_suppressed(r.get("host", ""), suppressed_hosts)
|
||||
]
|
||||
|
||||
load = prometheus_client.get_load()
|
||||
result["load5"] = [
|
||||
{"host": r["host"], "load5": round(r["value"], 2)} for r in load
|
||||
{"host": r["host"], "load5": round(r["value"], 2)}
|
||||
for r in load if not _host_suppressed(r.get("host", ""), suppressed_hosts)
|
||||
]
|
||||
except Exception as e:
|
||||
result["prometheus_error"] = str(e)
|
||||
return result
|
||||
|
||||
|
||||
def _gather_loki() -> dict:
|
||||
def _container_suppressed(name: str, suppressed_names: set) -> bool:
|
||||
if not name or not suppressed_names:
|
||||
return False
|
||||
return name.lower() in suppressed_names
|
||||
|
||||
|
||||
def _gather_loki(suppressed_names: set = None) -> dict:
|
||||
suppressed_names = suppressed_names or set()
|
||||
result = {}
|
||||
try:
|
||||
hosts = loki_client.get_labels()
|
||||
error_counts = {}
|
||||
for host in hosts[:20]:
|
||||
if _container_suppressed(host, suppressed_names):
|
||||
continue
|
||||
errors = loki_client.get_errors(container=host, hours=24, limit=300)
|
||||
count = 0 if (len(errors) == 1 and "error" in errors[0]) else len(errors)
|
||||
if count > 0:
|
||||
|
|
@ -82,13 +112,17 @@ def _gather_loki() -> dict:
|
|||
result["errors_24h"] = error_counts
|
||||
|
||||
silent = loki_client.check_silence(minutes=60)
|
||||
result["silent_hosts"] = [s["host"] for s in silent if "host" in s]
|
||||
result["silent_hosts"] = [
|
||||
s["host"] for s in silent
|
||||
if "host" in s and not _container_suppressed(s["host"], suppressed_names)
|
||||
]
|
||||
except Exception as e:
|
||||
result["loki_error"] = str(e)
|
||||
return result
|
||||
|
||||
|
||||
def _gather_proxmox() -> dict:
|
||||
def _gather_proxmox(suppressed_hosts: set = None) -> dict:
|
||||
suppressed_hosts = suppressed_hosts or set()
|
||||
result = {}
|
||||
try:
|
||||
cfg = config.parse_config()
|
||||
|
|
@ -107,6 +141,11 @@ def _gather_proxmox() -> dict:
|
|||
tokens["pve-hetzner"] = {"name": tn, "value": tv}
|
||||
|
||||
containers = proxmox_client.get_all_containers(passwords=passwords, tokens=tokens)
|
||||
# Container auf unterdrueckten Hosts komplett rauswerfen
|
||||
containers = [
|
||||
c for c in containers
|
||||
if not _host_suppressed(c.get("_host", ""), suppressed_hosts)
|
||||
]
|
||||
stopped = [
|
||||
{"id": c.get("vmid"), "name": c.get("name", "?")}
|
||||
for c in containers
|
||||
|
|
@ -119,6 +158,7 @@ def _gather_proxmox() -> dict:
|
|||
result["stopped"] = stopped
|
||||
result["host_errors"] = [
|
||||
f"{e.get('_host', '?')}: {e['error'][:80]}" for e in errors
|
||||
if not _host_suppressed(e.get("_host", ""), suppressed_hosts)
|
||||
]
|
||||
except Exception as e:
|
||||
result["proxmox_error"] = str(e)
|
||||
|
|
@ -126,12 +166,19 @@ def _gather_proxmox() -> dict:
|
|||
|
||||
|
||||
def handle_get_health_forecast(**kw) -> str:
|
||||
prom = _gather_prometheus()
|
||||
loki = _gather_loki()
|
||||
pve = _gather_proxmox()
|
||||
cfg = config.parse_config()
|
||||
suppressed_hosts = config.get_suppressed_hosts(cfg)
|
||||
suppressed_names = config.get_suppressed_container_names(cfg)
|
||||
|
||||
prom = _gather_prometheus(suppressed_hosts)
|
||||
loki = _gather_loki(suppressed_names)
|
||||
pve = _gather_proxmox(suppressed_hosts)
|
||||
|
||||
now_str = datetime.now().strftime("%d.%m.%Y %H:%M")
|
||||
lines = [f"📊 Systemdaten-Report ({now_str})", ""]
|
||||
if suppressed_hosts:
|
||||
lines.append(f"🔕 Unterdrückt (homelab.conf): {', '.join(sorted(suppressed_hosts))}")
|
||||
lines.append("")
|
||||
|
||||
# --- Prometheus Warnungen ---
|
||||
if prom.get("warnings"):
|
||||
|
|
|
|||
17
homelab.conf
17
homelab.conf
|
|
@ -182,6 +182,23 @@ VM_144_MU3="BT-Bridge|—|BT Bridge VM"
|
|||
# --- pve-he (Ramsin, bei Helmut) ---
|
||||
# Container noch nicht inventarisiert
|
||||
|
||||
# ============================================================
|
||||
# ALERT SUPPRESSION (Hausmeister-Bot)
|
||||
# ============================================================
|
||||
# Hosts in dieser Liste werden aus allen Hausmeister-Alerts
|
||||
# herausgefiltert — sowohl im 15-Min-Regelcheck (monitor.py)
|
||||
# als auch im täglichen Forecast (tools/predict.py).
|
||||
#
|
||||
# Format: Komma-getrennte Liste von Host-Namen wie in homelab.conf
|
||||
# (pve-hetzner, pve-ka-1, pve-mu-3, pbs-mu, ...). Leere Liste = alles aktiv.
|
||||
#
|
||||
# Persistente Einträge (Grund + Datum als Kommentar):
|
||||
# - pve-ka-1/2/3: Kambodscha ohne funktionierendes Internet, seit
|
||||
# 2026-04-04 offline, temporär stumm geschaltet am 2026-04-21.
|
||||
# Wieder reinnehmen, sobald Internet in Takeo wieder läuft.
|
||||
# ============================================================
|
||||
ALERT_SUPPRESS_HOSTS="pve-ka-1,pve-ka-2,pve-ka-3"
|
||||
|
||||
# --- TELEGRAM BOTS ---
|
||||
TG_CHAT_ID="674951792"
|
||||
TG_MUTTER_TOKEN="8551565940:AAHIUpZND-tCNGv9yEoNPRyPt4GxEPYBJdE"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue