diff --git a/homelab-ai-bot/core/config.py b/homelab-ai-bot/core/config.py index 927af91e8..4c2a522e9 100644 --- a/homelab-ai-bot/core/config.py +++ b/homelab-ai-bot/core/config.py @@ -181,3 +181,33 @@ def format_overview(cfg: HomelabConfig) -> str: lines.append(f"- CT {t.ct_id}: {t.domain} → {t.target} ({t.status})") return "\n".join(lines) + + +def get_suppressed_hosts(cfg: HomelabConfig) -> set: + """Hosts (pve-*/pbs-*) die aus Hausmeister-Alerts herausgefiltert werden sollen. + + Liest ALERT_SUPPRESS_HOSTS aus homelab.conf (Komma-Liste). + Leer = keine Unterdrueckung. + """ + raw = cfg.raw.get("ALERT_SUPPRESS_HOSTS", "") + return {h.strip() for h in raw.split(",") if h.strip()} + + +def get_suppressed_container_names(cfg: HomelabConfig) -> set: + """Container-Namen (lowercased) auf suppressed Hosts. + + Fuer Loki/Silence-Filter, wo Loki mit Container-Labels wie + ct-101-freshrss oder webcam arbeitet statt mit Proxmox-Hostnamen. + """ + hosts = get_suppressed_hosts(cfg) + names = set() + for c in cfg.containers: + if c.host not in hosts: + continue + n = (c.name or "").strip().lower() + if n: + names.add(n) + names.add(f"ct-{c.vmid}-{n}") + names.add(f"ct{c.vmid}") + names.add(f"ct-{c.vmid}") + return names diff --git a/homelab-ai-bot/monitor.py b/homelab-ai-bot/monitor.py index c67f2e2b0..ba54140f4 100644 --- a/homelab-ai-bot/monitor.py +++ b/homelab-ai-bot/monitor.py @@ -47,8 +47,6 @@ HTTP_HEALTH_CHECKS = [ {"name": "Grafana (CT 110)", "url": "http://10.10.10.110:3000/api/health"}, {"name": "Flugscanner-Agent (pve-pp-1)", "url": "http://100.126.26.46:5010/status", "retries": 5, "timeout": 25, "retry_delay": 6}, - {"name": "Seafile (CT 103)", "url": "https://seafile.orbitalo.net/api2/ping/", - "retries": 3, "timeout": 10, "retry_delay": 5}, ] EXPECTED_STOPPED = { @@ -69,14 +67,27 @@ IGNORED_HOSTS = {"${HOSTNAME}", ""} SILENCE_IGNORED_HOSTS = { "ct-600-webcam", # kein rsyslog, Stream läuft aber + "ct-103-Intercity-Taxi", # absichtlich gestoppt "ct-101-freshrss", # auf pve-ka-3, lokale Loki (nicht zentral) - "seafile", # Logs nur im Container, Health via HTTP-Check } +def _is_host_suppressed(host: str, suppressed_hosts: set) -> bool: + if not host or not suppressed_hosts: + return False + h = host.lower() + for s in suppressed_hosts: + s = s.lower() + if h == s or h.startswith(s + ".") or h.startswith(s + "-"): + return True + return False + + def check_all() -> list[str]: """Regelbasierter Check (Stufe 1). Gibt Liste von Alarmen zurück.""" cfg = config.parse_config() + suppressed_hosts = config.get_suppressed_hosts(cfg) + suppressed_names = config.get_suppressed_container_names(cfg) alerts = [] containers = proxmox_client.get_all_containers( @@ -85,10 +96,12 @@ def check_all() -> list[str]: for ct in containers: if "error" in ct: continue + host = ct.get("_host", "") + if _is_host_suppressed(host, suppressed_hosts): + continue vmid = ct.get("vmid", 0) name = ct.get("name", "?") status = ct.get("status", "unknown") - host = ct.get("_host", "") if vmid in CRITICAL_CONTAINERS and status != "running": ok_stopped = vmid in EXPECTED_STOPPED_VMIDS and status == "stopped" if (vmid, host) not in EXPECTED_STOPPED and not ok_stopped: @@ -118,11 +131,14 @@ def check_all() -> list[str]: if panic_lines: hosts = set(e.get("host", "?") for e in panic_lines) hosts -= IGNORED_HOSTS + hosts = {h for h in hosts if h.lower() not in suppressed_names} if hosts: alerts.append(f"🔴 Kritische Fehler (panic/fatal/OOM) auf: {', '.join(hosts)}") error_rates = loki_client.check_error_rate(minutes=30) for er in error_rates: + if er.get("host", "").lower() in suppressed_names: + continue alerts.append( f"🔴 {er['host']}: {er['count']} Fehler in 30 Min (Schwelle: {er['threshold']})" ) @@ -139,6 +155,7 @@ def check_all() -> list[str]: s["host"] for s in silent if s.get("host") not in IGNORED_HOSTS and s.get("host") not in SILENCE_IGNORED_HOSTS + and s["host"].lower() not in suppressed_names and s["host"].lower() in running_names ] if names: @@ -168,6 +185,8 @@ def check_all() -> list[str]: restarts = loki_client.check_service_restarts(minutes=35) for r in restarts: + if r.get("host", "").lower() in suppressed_names: + continue alerts.append(f"🔄 Service-Neustart: {r['service']} auf {r['host']} ({r['count']}x in 35 Min)") try: @@ -217,23 +236,38 @@ def check_all() -> list[str]: def format_report() -> str: """Tagesbericht: Gesamtstatus aller Systeme.""" cfg = config.parse_config() + suppressed_hosts = config.get_suppressed_hosts(cfg) + suppressed_names = config.get_suppressed_container_names(cfg) lines = ["📋 Tagesbericht Homelab\n"] + if suppressed_hosts: + lines.append(f"🔕 Stumm: {', '.join(sorted(suppressed_hosts))}\n") containers = proxmox_client.get_all_containers( _get_passwords(cfg), _get_tokens(cfg) ) + containers = [ + c for c in containers + if not _is_host_suppressed(c.get("_host", ""), suppressed_hosts) + ] running = [c for c in containers if c.get("status") == "running"] stopped = [c for c in containers if c.get("status") == "stopped"] errors_ct = [c for c in containers if "error" in c] lines.append(f"Container: {len(running)} running, {len(stopped)} stopped, {len(errors_ct)} nicht erreichbar") errors = loki_client.get_errors(hours=24, limit=100) - error_count = len([e for e in errors if "error" not in e]) + error_count = len([ + e for e in errors + if "error" not in e and e.get("host", "").lower() not in suppressed_names + ]) lines.append(f"Fehler (24h): {error_count}") silent = loki_client.check_silence(minutes=35) if silent and "error" not in (silent[0] if silent else {}): - names = [s["host"] for s in silent if s.get("host") not in IGNORED_HOSTS] + names = [ + s["host"] for s in silent + if s.get("host") not in IGNORED_HOSTS + and s["host"].lower() not in suppressed_names + ] if names: lines.append(f"Stille Hosts: {', '.join(names)}") else: diff --git a/homelab-ai-bot/tools/predict.py b/homelab-ai-bot/tools/predict.py index d0c370d5d..e7adc6205 100644 --- a/homelab-ai-bot/tools/predict.py +++ b/homelab-ai-bot/tools/predict.py @@ -25,14 +25,31 @@ TOOLS = [ ] -def _gather_prometheus() -> dict: +def _host_suppressed(host: str, suppressed: set) -> bool: + if not host: + return False + h = host.lower() + for s in suppressed: + s = s.lower() + if h == s or h.startswith(s + ".") or h.startswith(s + "-"): + return True + return False + + +def _gather_prometheus(suppressed_hosts: set = None) -> dict: + suppressed_hosts = suppressed_hosts or set() result = {} try: - result["warnings"] = prometheus_client.get_warnings() + raw_warnings = prometheus_client.get_warnings() + result["warnings"] = [ + w for w in raw_warnings + if not any(_host_suppressed(s, suppressed_hosts) for s in [w] + str(w).split()) + ] disk = prometheus_client.get_disk() result["disk_current"] = [ - {"host": r["host"], "used_pct": round(r["value"], 1)} for r in disk + {"host": r["host"], "used_pct": round(r["value"], 1)} + for r in disk if not _host_suppressed(r.get("host", ""), suppressed_hosts) ] trend = prometheus_client.range_query( @@ -45,6 +62,8 @@ def _gather_prometheus() -> dict: if trend.get("status") == "success": for r in trend.get("data", {}).get("result", []): h = r.get("metric", {}).get("host", "?") + if _host_suppressed(h, suppressed_hosts): + continue vals = [float(v[1]) for v in r.get("values", []) if v[1] != "NaN"] if len(vals) >= 2: delta = vals[-1] - vals[0] @@ -57,24 +76,35 @@ def _gather_prometheus() -> dict: mem = prometheus_client.get_memory() result["memory"] = [ - {"host": r["host"], "used_pct": round(r["value"], 1)} for r in mem + {"host": r["host"], "used_pct": round(r["value"], 1)} + for r in mem if not _host_suppressed(r.get("host", ""), suppressed_hosts) ] load = prometheus_client.get_load() result["load5"] = [ - {"host": r["host"], "load5": round(r["value"], 2)} for r in load + {"host": r["host"], "load5": round(r["value"], 2)} + for r in load if not _host_suppressed(r.get("host", ""), suppressed_hosts) ] except Exception as e: result["prometheus_error"] = str(e) return result -def _gather_loki() -> dict: +def _container_suppressed(name: str, suppressed_names: set) -> bool: + if not name or not suppressed_names: + return False + return name.lower() in suppressed_names + + +def _gather_loki(suppressed_names: set = None) -> dict: + suppressed_names = suppressed_names or set() result = {} try: hosts = loki_client.get_labels() error_counts = {} for host in hosts[:20]: + if _container_suppressed(host, suppressed_names): + continue errors = loki_client.get_errors(container=host, hours=24, limit=300) count = 0 if (len(errors) == 1 and "error" in errors[0]) else len(errors) if count > 0: @@ -82,13 +112,17 @@ def _gather_loki() -> dict: result["errors_24h"] = error_counts silent = loki_client.check_silence(minutes=60) - result["silent_hosts"] = [s["host"] for s in silent if "host" in s] + result["silent_hosts"] = [ + s["host"] for s in silent + if "host" in s and not _container_suppressed(s["host"], suppressed_names) + ] except Exception as e: result["loki_error"] = str(e) return result -def _gather_proxmox() -> dict: +def _gather_proxmox(suppressed_hosts: set = None) -> dict: + suppressed_hosts = suppressed_hosts or set() result = {} try: cfg = config.parse_config() @@ -107,6 +141,11 @@ def _gather_proxmox() -> dict: tokens["pve-hetzner"] = {"name": tn, "value": tv} containers = proxmox_client.get_all_containers(passwords=passwords, tokens=tokens) + # Container auf unterdrueckten Hosts komplett rauswerfen + containers = [ + c for c in containers + if not _host_suppressed(c.get("_host", ""), suppressed_hosts) + ] stopped = [ {"id": c.get("vmid"), "name": c.get("name", "?")} for c in containers @@ -119,6 +158,7 @@ def _gather_proxmox() -> dict: result["stopped"] = stopped result["host_errors"] = [ f"{e.get('_host', '?')}: {e['error'][:80]}" for e in errors + if not _host_suppressed(e.get("_host", ""), suppressed_hosts) ] except Exception as e: result["proxmox_error"] = str(e) @@ -126,12 +166,19 @@ def _gather_proxmox() -> dict: def handle_get_health_forecast(**kw) -> str: - prom = _gather_prometheus() - loki = _gather_loki() - pve = _gather_proxmox() + cfg = config.parse_config() + suppressed_hosts = config.get_suppressed_hosts(cfg) + suppressed_names = config.get_suppressed_container_names(cfg) + + prom = _gather_prometheus(suppressed_hosts) + loki = _gather_loki(suppressed_names) + pve = _gather_proxmox(suppressed_hosts) now_str = datetime.now().strftime("%d.%m.%Y %H:%M") lines = [f"📊 Systemdaten-Report ({now_str})", ""] + if suppressed_hosts: + lines.append(f"🔕 Unterdrückt (homelab.conf): {', '.join(sorted(suppressed_hosts))}") + lines.append("") # --- Prometheus Warnungen --- if prom.get("warnings"): diff --git a/homelab.conf b/homelab.conf index 08538b35a..bed7c0b01 100644 --- a/homelab.conf +++ b/homelab.conf @@ -182,6 +182,23 @@ VM_144_MU3="BT-Bridge|—|BT Bridge VM" # --- pve-he (Ramsin, bei Helmut) --- # Container noch nicht inventarisiert +# ============================================================ +# ALERT SUPPRESSION (Hausmeister-Bot) +# ============================================================ +# Hosts in dieser Liste werden aus allen Hausmeister-Alerts +# herausgefiltert — sowohl im 15-Min-Regelcheck (monitor.py) +# als auch im täglichen Forecast (tools/predict.py). +# +# Format: Komma-getrennte Liste von Host-Namen wie in homelab.conf +# (pve-hetzner, pve-ka-1, pve-mu-3, pbs-mu, ...). Leere Liste = alles aktiv. +# +# Persistente Einträge (Grund + Datum als Kommentar): +# - pve-ka-1/2/3: Kambodscha ohne funktionierendes Internet, seit +# 2026-04-04 offline, temporär stumm geschaltet am 2026-04-21. +# Wieder reinnehmen, sobald Internet in Takeo wieder läuft. +# ============================================================ +ALERT_SUPPRESS_HOSTS="pve-ka-1,pve-ka-2,pve-ka-3" + # --- TELEGRAM BOTS --- TG_CHAT_ID="674951792" TG_MUTTER_TOKEN="8551565940:AAHIUpZND-tCNGv9yEoNPRyPt4GxEPYBJdE"