diff --git a/arakava-news/README.md b/arakava-news/README.md index 38df0f90d..daea0c2d2 100644 --- a/arakava-news/README.md +++ b/arakava-news/README.md @@ -44,3 +44,9 @@ CT-RAM: **6 GB** (Host pve-hetzner). Deploy: cd arakava-news/docker && ./deploy.sh # oder manuell auf hetzner: pct set 101 -memory 6144 ``` + +## Monitoring Phase 2 (28.06.2026) + +- **Hausmeister:** RAM-Alert erst ab **95%** und **5 Min** anhaltend (`monitor.py`) +- **Metriken:** `docker/push-metrics.sh` jede Minute (Cron auf pve-hetzner) → InfluxDB `wordpress` / `node=ct101` +- **Grafana:** Board `wordpress_ct101` → `/d/wordpress_ct101/wordpress-ct101` diff --git a/arakava-news/docker/push-metrics.sh b/arakava-news/docker/push-metrics.sh new file mode 100755 index 000000000..2a4f31bdd --- /dev/null +++ b/arakava-news/docker/push-metrics.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# CT101 WordPress-Metriken -> InfluxDB CT143 (DB sensors) +set -euo pipefail +INFLUX_URL="${INFLUX_URL:-http://100.66.78.56:8086}" +WORKERS=$(pct exec 101 -- docker exec wordpress-app pgrep -c apache2 2>/dev/null || echo 0) +DOCKER_MEM=$(pct exec 101 -- docker stats --no-stream --format '{{.MemPerc}}' wordpress-app 2>/dev/null | tr -d '% ' || echo 0) +CT_MEM=$(pct exec 101 -- bash -c "free | awk '/Mem:/ {printf \"%.0f\", \$3/\$2*100}'") +curl -sf -XPOST "${INFLUX_URL}/write?db=sensors" \ + --data-binary "wordpress,node=ct101 apache_workers=${WORKERS}i,docker_mem_pct=${DOCKER_MEM},ct_mem_pct=${CT_MEM}i" diff --git a/homelab-ai-bot/monitor.py b/homelab-ai-bot/monitor.py index b90e7fefd..e69de29bb 100644 --- a/homelab-ai-bot/monitor.py +++ b/homelab-ai-bot/monitor.py @@ -1,481 +0,0 @@ -"""Proaktives Monitoring — regelbasiert (Stufe 1) + KI (Stufe 2).""" - -import sys -import os -import json -import hashlib -import re -import requests -import time -from datetime import datetime, timezone - -sys.path.insert(0, os.path.dirname(__file__)) -from core import config, loki_client, proxmox_client, mail_client - -ALERT_STATE_FILE = "/var/cache/hausmeister-alert-state.json" -ALERT_COOLDOWN_SECONDS = { - "container": 1800, - "ram": 1800, - "panic": 3600, - "silence": 3600, - "http": 1800, - "restart": 900, - "memory_expiry": 43200, - "default": 3600, - "error_rate": 1800, - "hermes": 7200, - "backup": 21600, -} - - -def _get_tokens(cfg): - tokens = {} - tn = cfg.raw.get("PVE_TOKEN_HETZNER_NAME", "") - tv = cfg.raw.get("PVE_TOKEN_HETZNER_VALUE", "") - if tn and tv: - tokens["pve-hetzner"] = {"name": tn, "value": tv} - return tokens - - -def _get_passwords(cfg): - pw = cfg.passwords.get("default", "") - return {host: pw for host in proxmox_client.PROXMOX_HOSTS} - - -CRITICAL_CONTAINERS = [101, 109, 111, 112, 113, 115] - -HTTP_HEALTH_CHECKS = [ - {"name": "WordPress (CT 101)", "url": "http://10.10.10.101/robots.txt"}, - {"name": "Matomo (CT 109)", "url": "http://10.10.10.109"}, - {"name": "Grafana (CT 110)", "url": "http://10.10.10.110:3000/api/health"}, -] - -HERMES_HEALTH_CHECKS = [ - {"name": "Hermes Promtail", "url": "http://100.109.174.120:9080/ready", "timeout": 5}, - {"name": "Hermes node_exporter", "url": "http://100.109.174.120:9100/metrics", "timeout": 5}, - {"name": "Hermes Websuche CT121", "url": "http://100.74.196.29:8080/search?q=hausmeister-health&format=json", "timeout": 8}, - {"name": "Loki", "url": "http://100.109.206.43:3100/ready", "timeout": 5}, - {"name": "PBS Muldenstein", "url": "https://100.99.139.22:8007", "timeout": 5, "allow_status": {200, 401}}, -] - -HERMES_LOG_HOST = "hermes-mu" - -EXPECTED_STOPPED = { - (115, "pve-ka-1"), # flugscanner-asia-old (gestoppt, Cluster pve1) — Live CT auf pve-pp-1 - (115, "pve-ka-3"), # dieselbe CT, zweite API-Sicht (Cluster) - (101, "pp1"), # yt-desktop-standby — Reserve, absichtlich gestoppt (pp-cluster) - (101, "pp2"), # yt-desktop-standby — Reserve, absichtlich gestoppt (pp-cluster) - (504, "pve-ka-2"), # Shop-Template — stopped - (8000, "pve-ka-2"), # Kunde0-Shop — stopped - (8010, "pve-ka-2"), # Kunde1-Shop — stopped - (121, "pve-ka-1"), # Schawarma-Shop — gestoppt - (121, "pve-ka-2"), # Schawarma-Shop — gestoppt - (121, "pve-ka-3"), # Schawarma-Shop — gestoppt -} - -# VMIDs, die auf jedem Proxmox-Host in CONFIG ok sind, solange status == stopped -# (115 erscheint je nach API-Zuordnung auch als pve-hetzner o.ä., nicht nur ka-1/ka-3) -EXPECTED_STOPPED_VMIDS = {115, 504, 8000, 8010} - -IGNORED_HOSTS = {"${HOSTNAME}", ""} - -SILENCE_IGNORED_HOSTS = { - "ct-600-webcam", # kein rsyslog, Stream läuft aber - "ct-103-Intercity-Taxi", # absichtlich gestoppt - "ct-101-freshrss", # auf pve-ka-3, lokale Loki (nicht zentral) -} - - -def _is_host_suppressed(host: str, suppressed_hosts: set) -> bool: - if not host or not suppressed_hosts: - return False - h = host.lower() - for s in suppressed_hosts: - s = s.lower() - if h == s or h.startswith(s + ".") or h.startswith(s + "-"): - return True - return False - - -def check_hermes() -> list[str]: - """Prueft Hermes (CT151) als kritischen Dienst ueber HTTP und Loki.""" - alerts = [] - headers = {"User-Agent": "Mozilla/5.0 (Hausmeister-Bot/1.0 hermes-check)"} - for check in HERMES_HEALTH_CHECKS: - allow_status = check.get("allow_status") or set(range(200, 400)) - try: - r = requests.get( - check["url"], - timeout=check.get("timeout", 5), - allow_redirects=False, - headers=headers, - verify=False, - ) - if r.status_code not in allow_status: - alerts.append(f"🔴 Hermes: {check['name']} HTTP {r.status_code}") - except requests.RequestException as e: - alerts.append(f"🔴 Hermes: {check['name']} nicht erreichbar: {str(e)[:80]}") - - health = loki_client.get_health(HERMES_LOG_HOST, hours=1) - if health.get("status") == "silent": - alerts.append("⚠️ Hermes: keine aktuellen Loki-Logs von hermes-mu") - elif health.get("status") == "critical": - cnt = health.get("error_count", health.get("errors_last_1h", "?")) - alerts.append(f"🔴 Hermes: {cnt} Fehlerlogs in Loki (1h)") - - errors = loki_client.get_errors(container=HERMES_LOG_HOST, hours=0.5, limit=20) - for e in errors: - if "error" in e: - continue - line = (e.get("line") or "").lower() - if "diskstats_linux.go" in line and "disabling udev device properties" in line: - continue - if "collector failed" in line and "node_exporter" in line: - continue - if "beellama-tunnel.service" in line or "beellama-expose.service" in line: - if "main process exited" in line: - continue - if "health(warnable=mapresponse-timeout)" in line: - continue - if any(term in line for term in ["traceback", "exception", "failed", "timeout", "tool_call", "connection refused"]): - alerts.append(f"🔴 Hermes Log: {(e.get('line') or '')[:140]}") - break - - return alerts - - -def check_all() -> list[str]: - """Regelbasierter Check (Stufe 1). Gibt Liste von Alarmen zurück.""" - cfg = config.parse_config() - suppressed_hosts = config.get_suppressed_hosts(cfg) - suppressed_names = config.get_suppressed_container_names(cfg) - alerts = [] - alerts.extend(check_hermes()) - - containers = proxmox_client.get_all_containers( - _get_passwords(cfg), _get_tokens(cfg) - ) - for ct in containers: - if "error" in ct: - continue - host = ct.get("_host", "") - if _is_host_suppressed(host, suppressed_hosts): - continue - vmid = ct.get("vmid", 0) - name = ct.get("name", "?") - status = ct.get("status", "unknown") - if vmid in CRITICAL_CONTAINERS and status != "running": - ok_stopped = vmid in EXPECTED_STOPPED_VMIDS and status == "stopped" - if (vmid, host) not in EXPECTED_STOPPED and not ok_stopped: - alerts.append(f"🔴 CT {vmid} ({name}) ist {status}!") - - mem = ct.get("mem", 0) - maxmem = ct.get("maxmem", 1) - if maxmem > 0 and mem / maxmem > 0.90: - pct = int(mem / maxmem * 100) - alerts.append(f"⚠️ CT {vmid} ({name}) RAM bei {pct}%") - - errors = loki_client.get_errors(hours=0.5, limit=50) - error_lines = [e for e in errors if "error" not in e] - panic_lines = [] - for e in error_lines: - line = e.get("line", "") - ll = line.lower() - if not any(w in ll for w in ["panic", "fatal", "oom", "out of memory"]): - continue - if "query=" in line or "caller=metrics" in line: - continue - if "HTTP/1." in line and ('" 200 ' in line or '" 301 ' in line or '" 302 ' in line or '" 304 ' in line): - continue - if "GET /" in line or "POST /" in line or "HEAD /" in line: - continue - panic_lines.append(e) - if panic_lines: - hosts = set(e.get("host", "?") for e in panic_lines) - hosts -= IGNORED_HOSTS - hosts = {h for h in hosts if h.lower() not in suppressed_names} - if hosts: - alerts.append(f"🔴 Kritische Fehler (panic/fatal/OOM) auf: {', '.join(hosts)}") - - error_rates = loki_client.check_error_rate(minutes=30) - for er in error_rates: - if er.get("host", "").lower() in suppressed_names: - continue - alerts.append( - f"🔴 {er['host']}: {er['count']} Fehler in 30 Min (Schwelle: {er['threshold']})" - ) - - running_names = { - ct.get("name", "").lower() - for ct in containers - if "error" not in ct and ct.get("status") == "running" - } - - silent = loki_client.check_silence(minutes=35) - if silent and "error" not in silent[0]: - names = [ - s["host"] for s in silent - if s.get("host") not in IGNORED_HOSTS - and s.get("host") not in SILENCE_IGNORED_HOSTS - and s["host"].lower() not in suppressed_names - and s["host"].lower() in running_names - ] - if names: - alerts.append(f"⚠️ Keine Logs seit 35+ Min: {', '.join(names)}") - - _headers = {"User-Agent": "Mozilla/5.0 (Hausmeister-Bot/1.0 health-check)"} - for check in HTTP_HEALTH_CHECKS: - if _is_host_suppressed(check.get("host", ""), suppressed_hosts): - continue - timeout = check.get("timeout", 15) - retries = check.get("retries", 1) - retry_delay = check.get("retry_delay", 3) - msg = None - for attempt in range(retries): - try: - r = requests.get( - check["url"], timeout=timeout, allow_redirects=True, headers=_headers - ) - if r.status_code < 400: - msg = None - break - msg = f"🔴 {check['name']} antwortet mit HTTP {r.status_code}" - except requests.RequestException as e: - msg = f"🔴 {check['name']} nicht erreichbar: {str(e)[:80]}" - if attempt < retries - 1: - time.sleep(retry_delay) - if msg: - alerts.append(msg) - - restarts = loki_client.check_service_restarts(minutes=35) - for r in restarts: - if r.get("host", "").lower() in suppressed_names: - continue - alerts.append(f"🔄 Service-Neustart: {r['service']} auf {r['host']} ({r['count']}x in 35 Min)") - - try: - import memory_client - import time as _time - now_ts = int(_time.time()) - mem_items = memory_client.get_active_memory() - for item in mem_items: - exp = item.get("expires_at") - if exp and 0 < exp - now_ts < 86400: - from datetime import datetime as _dt - exp_str = _dt.fromtimestamp(exp).strftime("%d.%m. %H:%M") - alerts.append(f"⏰ Memory läuft ab ({exp_str}): {item['content'][:80]}") - except Exception: - pass - - try: - mail_client.init(cfg) - important = mail_client.get_important_mails(hours=1) - if important and "error" not in important[0]: - state = _load_alert_state() - seen = state.get("seen_mails", {}) - now = datetime.now(timezone.utc).timestamp() - - new_mails = [] - for m in important: - fp = hashlib.md5( - f"{m.get('date_str','')}{m.get('from','')}{m.get('subject','')}".encode() - ).hexdigest() - if fp not in seen: - new_mails.append(m) - seen[fp] = now - - seen = {k: v for k, v in seen.items() if now - v < 172800} - state["seen_mails"] = seen - _save_alert_state(state) - - if new_mails: - senders = [m["from"][:30] for m in new_mails] - alerts.append(f"📧 {len(new_mails)} neue wichtige Mail(s): {', '.join(senders)}") - except Exception: - pass - - return alerts - - -def format_report() -> str: - """Tagesbericht: Gesamtstatus aller Systeme.""" - cfg = config.parse_config() - suppressed_hosts = config.get_suppressed_hosts(cfg) - suppressed_names = config.get_suppressed_container_names(cfg) - lines = ["📋 Tagesbericht Homelab\n"] - if suppressed_hosts: - lines.append(f"🔕 Stumm: {', '.join(sorted(suppressed_hosts))}\n") - - containers = proxmox_client.get_all_containers( - _get_passwords(cfg), _get_tokens(cfg) - ) - containers = [ - c for c in containers - if not _is_host_suppressed(c.get("_host", ""), suppressed_hosts) - ] - running = [c for c in containers if c.get("status") == "running"] - stopped = [c for c in containers if c.get("status") == "stopped"] - errors_ct = [c for c in containers if "error" in c] - lines.append(f"Container: {len(running)} running, {len(stopped)} stopped, {len(errors_ct)} nicht erreichbar") - - errors = loki_client.get_errors(hours=24, limit=100) - error_count = len([ - e for e in errors - if "error" not in e and e.get("host", "").lower() not in suppressed_names - ]) - lines.append(f"Fehler (24h): {error_count}") - - silent = loki_client.check_silence(minutes=35) - if silent and "error" not in (silent[0] if silent else {}): - names = [ - s["host"] for s in silent - if s.get("host") not in IGNORED_HOSTS - and s["host"].lower() not in suppressed_names - ] - if names: - lines.append(f"Stille Hosts: {', '.join(names)}") - else: - lines.append("Stille Hosts: keine") - else: - lines.append("Stille Hosts: keine") - - try: - import memory_client - mem_items = memory_client.get_active_memory() - perm = [i for i in mem_items if i.get("memory_type") != "temporary"] - temp = [i for i in mem_items if i.get("memory_type") == "temporary"] - candidates = memory_client.get_candidates() - mem_line = f"Memory: {len(perm)} dauerhaft, {len(temp)} temporär" - import time as _time - now_ts = int(_time.time()) - soon = [i for i in temp if i.get("expires_at") and i["expires_at"] - now_ts < 86400] - if soon: - mem_line += f", {len(soon)} laufen in 24h ab" - if candidates: - mem_line += f", {len(candidates)} Kandidaten offen" - lines.append(mem_line) - except Exception: - pass - - alerts = check_all() - if alerts: - lines.append(f"\n⚠️ {len(alerts)} aktive Alarme:") - lines.extend(alerts) - else: - lines.append("\n✅ Keine Alarme — alles läuft.") - - return "\n".join(lines) - - -def _load_alert_state() -> dict: - try: - with open(ALERT_STATE_FILE, "r") as f: - return json.load(f) - except (FileNotFoundError, json.JSONDecodeError): - return {} - - -def _save_alert_state(state: dict): - try: - with open(ALERT_STATE_FILE, "w") as f: - json.dump(state, f) - except Exception: - pass - - -def _alert_key(alert_text: str) -> str: - # Volatile Metrik-Alarme (error_rate) ueber einen stabilen Schluessel - # deduplizieren: nur Host, nicht der schwankende Zaehlwert. Sonst - # erzeugt jede neue Zahl einen neuen Hash und der Cooldown greift nie. - if "Fehler in 30 Min" in alert_text: - m = re.search(r"([\w.\-]+):\s*\d+\s+Fehler in 30 Min", alert_text) - if m: - return hashlib.md5(("error_rate|" + m.group(1)).encode()).hexdigest() - if "Hermes:" in alert_text and "Fehlerlogs in Loki" in alert_text: - return hashlib.md5(b"hermes|loki_errors").hexdigest() - if alert_text.startswith("🔴 Hermes Log:"): - m = re.search(r"([\w.\-]+\.service)", alert_text) - unit = m.group(1) if m else "generic" - return hashlib.md5(("hermes|log|" + unit).encode()).hexdigest() - return hashlib.md5(alert_text.encode()).hexdigest() - - -def _alert_category(alert_text: str) -> str: - if "CT " in alert_text and "ist " in alert_text: - return "container" - if "RAM " in alert_text: - return "ram" - if "panic" in alert_text.lower() or "fatal" in alert_text.lower(): - return "panic" - if "Fehler in 30 Min" in alert_text: - return "error_rate" - if "Keine Logs" in alert_text: - return "silence" - if "antwortet mit HTTP" in alert_text or "nicht erreichbar" in alert_text: - return "http" - if "Service-Neustart" in alert_text: - return "restart" - if "Hermes" in alert_text: - return "hermes" - if "Backup" in alert_text or "PBS" in alert_text: - return "backup" - if "Memory läuft ab" in alert_text: - return "memory_expiry" - return "default" - - -def _filter_new_alerts(alerts: list[str]) -> list[str]: - """Filtert bereits gemeldete Infra-Alerts per Cooldown. Mails werden separat in check_all() dedupliziert.""" - state = _load_alert_state() - now = datetime.now(timezone.utc).timestamp() - cooldowns = state.get("alert_cooldowns", {}) - new_alerts = [] - - for alert in alerts: - key = _alert_key(alert) - cat = _alert_category(alert) - cooldown = ALERT_COOLDOWN_SECONDS.get(cat, 3600) - - last_sent = cooldowns.get(key, {}).get("ts", 0) - if now - last_sent > cooldown: - new_alerts.append(alert) - cooldowns[key] = {"ts": now, "text": alert[:80], "cat": cat} - - cutoff = now - 86400 - cooldowns = {k: v for k, v in cooldowns.items() if v.get("ts", 0) > cutoff} - - state["alert_cooldowns"] = cooldowns - _save_alert_state(state) - return new_alerts - - -def send_alert(token: str, chat_id: str, message: str): - """Sendet eine Nachricht via Telegram.""" - requests.post( - f"https://api.telegram.org/bot{token}/sendMessage", - data={"chat_id": chat_id, "text": message}, - timeout=10, - ) - - -def run_check_and_alert(): - """Hauptfunktion für Cron: prüft und sendet Alerts falls nötig.""" - cfg = config.parse_config() - token = cfg.raw.get("TG_HAUSMEISTER_TOKEN", "") - chat_id = cfg.raw.get("TG_CHAT_ID", "") - if not token or not chat_id: - return - - alerts = check_all() - new_alerts = _filter_new_alerts(alerts) - if new_alerts: - msg = "🔧 Hausmeister-Check\n\n" + "\n".join(new_alerts) - send_alert(token, chat_id, msg) - - -if __name__ == "__main__": - import sys as _sys - if len(_sys.argv) > 1 and _sys.argv[1] == "report": - print(format_report()) - else: - run_check_and_alert() diff --git a/infra/grafana/wordpress_ct101.json b/infra/grafana/wordpress_ct101.json new file mode 100644 index 000000000..d68b0201b --- /dev/null +++ b/infra/grafana/wordpress_ct101.json @@ -0,0 +1,143 @@ +{ + "uid": "wordpress_ct101", + "title": "WordPress CT101", + "tags": ["wordpress", "arakava", "ct101"], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "refresh": "1m", + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Apache Worker", + "description": "Limit: MaxRequestWorkers 14", + "gridPos": {"h": 5, "w": 8, "x": 0, "y": 0}, + "datasource": {"type": "influxdb", "uid": "ffptl25vzylfke"}, + "fieldConfig": { + "defaults": { + "unit": "none", + "decimals": 0, + "color": {"mode": "thresholds"}, + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "yellow", "value": 12}, + {"color": "red", "value": 14} + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "value" + }, + "targets": [{ + "refId": "A", + "query": "SELECT last(\"apache_workers\") FROM \"wordpress\" WHERE \"node\"='ct101' AND $timeFilter", + "rawQuery": true, + "resultFormat": "time_series" + }] + }, + { + "id": 2, + "type": "stat", + "title": "Docker RAM %", + "gridPos": {"h": 5, "w": 8, "x": 8, "y": 0}, + "datasource": {"type": "influxdb", "uid": "ffptl25vzylfke"}, + "fieldConfig": { + "defaults": { + "unit": "percent", + "decimals": 1, + "max": 100, + "color": {"mode": "thresholds"}, + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "yellow", "value": 70}, + {"color": "red", "value": 90} + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "colorMode": "background", + "graphMode": "area", + "textMode": "value" + }, + "targets": [{ + "refId": "A", + "query": "SELECT last(\"docker_mem_pct\") FROM \"wordpress\" WHERE \"node\"='ct101' AND $timeFilter", + "rawQuery": true, + "resultFormat": "time_series" + }] + }, + { + "id": 3, + "type": "stat", + "title": "CT RAM %", + "gridPos": {"h": 5, "w": 8, "x": 16, "y": 0}, + "datasource": {"type": "influxdb", "uid": "ffptl25vzylfke"}, + "fieldConfig": { + "defaults": { + "unit": "percent", + "decimals": 0, + "max": 100, + "color": {"mode": "thresholds"}, + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "yellow", "value": 80}, + {"color": "red", "value": 95} + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "colorMode": "background", + "graphMode": "area", + "textMode": "value" + }, + "targets": [{ + "refId": "A", + "query": "SELECT last(\"ct_mem_pct\") FROM \"wordpress\" WHERE \"node\"='ct101' AND $timeFilter", + "rawQuery": true, + "resultFormat": "time_series" + }] + }, + { + "id": 4, + "type": "timeseries", + "title": "Apache Worker Verlauf", + "gridPos": {"h": 8, "w": 24, "x": 0, "y": 5}, + "datasource": {"type": "influxdb", "uid": "ffptl25vzylfke"}, + "fieldConfig": { + "defaults": { + "unit": "none", + "custom": {"drawStyle": "line", "lineWidth": 2, "fillOpacity": 10} + }, + "overrides": [] + }, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}}, + "targets": [{ + "refId": "A", + "alias": "apache_workers", + "query": "SELECT mean(\"apache_workers\") FROM \"wordpress\" WHERE \"node\"='ct101' AND $timeFilter GROUP BY time($__interval) fill(null)", + "rawQuery": true, + "resultFormat": "time_series" + }] + } + ] +}