514 lines
19 KiB
Python
514 lines
19 KiB
Python
"""Proaktives Monitoring — regelbasiert (Stufe 1) + KI (Stufe 2)."""
|
|
|
|
import sys
|
|
import os
|
|
import json
|
|
import hashlib
|
|
import re
|
|
import requests
|
|
import time
|
|
from datetime import datetime, timezone
|
|
|
|
sys.path.insert(0, os.path.dirname(__file__))
|
|
from core import config, loki_client, proxmox_client, mail_client
|
|
|
|
ALERT_STATE_FILE = "/var/cache/hausmeister-alert-state.json"
|
|
ALERT_COOLDOWN_SECONDS = {
|
|
"container": 1800,
|
|
"ram": 1800,
|
|
"panic": 3600,
|
|
"silence": 3600,
|
|
"http": 1800,
|
|
"restart": 900,
|
|
"memory_expiry": 43200,
|
|
"default": 3600,
|
|
"error_rate": 1800,
|
|
"hermes": 7200,
|
|
"backup": 21600,
|
|
}
|
|
|
|
|
|
def _get_tokens(cfg):
|
|
tokens = {}
|
|
tn = cfg.raw.get("PVE_TOKEN_HETZNER_NAME", "")
|
|
tv = cfg.raw.get("PVE_TOKEN_HETZNER_VALUE", "")
|
|
if tn and tv:
|
|
tokens["pve-hetzner"] = {"name": tn, "value": tv}
|
|
return tokens
|
|
|
|
|
|
def _get_passwords(cfg):
|
|
pw = cfg.passwords.get("default", "")
|
|
return {host: pw for host in proxmox_client.PROXMOX_HOSTS}
|
|
|
|
|
|
RAM_ALERT_THRESHOLD = 0.95
|
|
RAM_ALERT_PERSIST_SECONDS = 300
|
|
|
|
CRITICAL_CONTAINERS = [101, 109, 111, 112, 113, 115]
|
|
|
|
HTTP_HEALTH_CHECKS = [
|
|
{"name": "WordPress (CT 101)", "url": "http://10.10.10.101/robots.txt"},
|
|
{"name": "Matomo (CT 109)", "url": "http://10.10.10.109"},
|
|
{"name": "Grafana (CT 110)", "url": "http://10.10.10.110:3000/api/health"},
|
|
]
|
|
|
|
HERMES_HEALTH_CHECKS = [
|
|
{"name": "Hermes Promtail", "url": "http://100.109.174.120:9080/ready", "timeout": 5},
|
|
{"name": "Hermes node_exporter", "url": "http://100.109.174.120:9100/metrics", "timeout": 5},
|
|
{"name": "Hermes Websuche CT121", "url": "http://100.74.196.29:8080/search?q=hausmeister-health&format=json", "timeout": 8},
|
|
{"name": "Loki", "url": "http://100.109.206.43:3100/ready", "timeout": 5},
|
|
{"name": "PBS Muldenstein", "url": "https://100.99.139.22:8007", "timeout": 5, "allow_status": {200, 401}},
|
|
]
|
|
|
|
HERMES_LOG_HOST = "hermes-mu"
|
|
|
|
EXPECTED_STOPPED = {
|
|
(115, "pve-ka-1"), # flugscanner-asia-old (gestoppt, Cluster pve1) — Live CT auf pve-pp-1
|
|
(115, "pve-ka-3"), # dieselbe CT, zweite API-Sicht (Cluster)
|
|
(101, "pp1"), # yt-desktop-standby — Reserve, absichtlich gestoppt (pp-cluster)
|
|
(101, "pp2"), # yt-desktop-standby — Reserve, absichtlich gestoppt (pp-cluster)
|
|
(504, "pve-ka-2"), # Shop-Template — stopped
|
|
(8000, "pve-ka-2"), # Kunde0-Shop — stopped
|
|
(8010, "pve-ka-2"), # Kunde1-Shop — stopped
|
|
(121, "pve-ka-1"), # Schawarma-Shop — gestoppt
|
|
(121, "pve-ka-2"), # Schawarma-Shop — gestoppt
|
|
(121, "pve-ka-3"), # Schawarma-Shop — gestoppt
|
|
}
|
|
|
|
# VMIDs, die auf jedem Proxmox-Host in CONFIG ok sind, solange status == stopped
|
|
# (115 erscheint je nach API-Zuordnung auch als pve-hetzner o.ä., nicht nur ka-1/ka-3)
|
|
EXPECTED_STOPPED_VMIDS = {115, 504, 8000, 8010}
|
|
|
|
IGNORED_HOSTS = {"${HOSTNAME}", ""}
|
|
|
|
SILENCE_IGNORED_HOSTS = {
|
|
"ct-600-webcam", # kein rsyslog, Stream läuft aber
|
|
"ct-103-Intercity-Taxi", # absichtlich gestoppt
|
|
"ct-101-freshrss", # auf pve-ka-3, lokale Loki (nicht zentral)
|
|
}
|
|
|
|
|
|
def _is_host_suppressed(host: str, suppressed_hosts: set) -> bool:
|
|
if not host or not suppressed_hosts:
|
|
return False
|
|
h = host.lower()
|
|
for s in suppressed_hosts:
|
|
s = s.lower()
|
|
if h == s or h.startswith(s + ".") or h.startswith(s + "-"):
|
|
return True
|
|
return False
|
|
|
|
|
|
def check_hermes() -> list[str]:
|
|
"""Prueft Hermes (CT151) als kritischen Dienst ueber HTTP und Loki."""
|
|
alerts = []
|
|
headers = {"User-Agent": "Mozilla/5.0 (Hausmeister-Bot/1.0 hermes-check)"}
|
|
for check in HERMES_HEALTH_CHECKS:
|
|
allow_status = check.get("allow_status") or set(range(200, 400))
|
|
try:
|
|
r = requests.get(
|
|
check["url"],
|
|
timeout=check.get("timeout", 5),
|
|
allow_redirects=False,
|
|
headers=headers,
|
|
verify=False,
|
|
)
|
|
if r.status_code not in allow_status:
|
|
alerts.append(f"🔴 Hermes: {check['name']} HTTP {r.status_code}")
|
|
except requests.RequestException as e:
|
|
alerts.append(f"🔴 Hermes: {check['name']} nicht erreichbar: {str(e)[:80]}")
|
|
|
|
health = loki_client.get_health(HERMES_LOG_HOST, hours=1)
|
|
if health.get("status") == "silent":
|
|
alerts.append("⚠️ Hermes: keine aktuellen Loki-Logs von hermes-mu")
|
|
elif health.get("status") == "critical":
|
|
cnt = health.get("error_count", health.get("errors_last_1h", "?"))
|
|
alerts.append(f"🔴 Hermes: {cnt} Fehlerlogs in Loki (1h)")
|
|
|
|
errors = loki_client.get_errors(container=HERMES_LOG_HOST, hours=0.5, limit=20)
|
|
for e in errors:
|
|
if "error" in e:
|
|
continue
|
|
line = (e.get("line") or "").lower()
|
|
if "diskstats_linux.go" in line and "disabling udev device properties" in line:
|
|
continue
|
|
if "collector failed" in line and "node_exporter" in line:
|
|
continue
|
|
if "beellama-tunnel.service" in line or "beellama-expose.service" in line:
|
|
if "main process exited" in line:
|
|
continue
|
|
if "health(warnable=mapresponse-timeout)" in line:
|
|
continue
|
|
if any(term in line for term in ["traceback", "exception", "failed", "timeout", "tool_call", "connection refused"]):
|
|
alerts.append(f"🔴 Hermes Log: {(e.get('line') or '')[:140]}")
|
|
break
|
|
|
|
return alerts
|
|
|
|
|
|
|
|
def _check_ram_alert(vmid: int, name: str, mem: int, maxmem: int, ram_high_since: dict, alerts: list):
|
|
"""RAM >95% erst melden, wenn der Zustand RAM_ALERT_PERSIST_SECONDS anhaelt."""
|
|
ratio = mem / maxmem
|
|
key = str(vmid)
|
|
now_ts = time.time()
|
|
if ratio > RAM_ALERT_THRESHOLD:
|
|
if key not in ram_high_since:
|
|
ram_high_since[key] = now_ts
|
|
return
|
|
if now_ts - ram_high_since[key] >= RAM_ALERT_PERSIST_SECONDS:
|
|
pct = int(ratio * 100)
|
|
alerts.append(f"⚠️ CT {vmid} ({name}) RAM bei {pct}%")
|
|
else:
|
|
ram_high_since.pop(key, None)
|
|
|
|
|
|
def check_all(state: dict | None = None) -> list[str]:
|
|
"""Regelbasierter Check (Stufe 1). Gibt Liste von Alarmen zurück."""
|
|
if state is None:
|
|
state = _load_alert_state()
|
|
ram_high_since = state.setdefault("ram_high_since", {})
|
|
cfg = config.parse_config()
|
|
suppressed_hosts = config.get_suppressed_hosts(cfg)
|
|
suppressed_names = config.get_suppressed_container_names(cfg)
|
|
alerts = []
|
|
alerts.extend(check_hermes())
|
|
|
|
containers = proxmox_client.get_all_containers(
|
|
_get_passwords(cfg), _get_tokens(cfg)
|
|
)
|
|
for ct in containers:
|
|
if "error" in ct:
|
|
continue
|
|
host = ct.get("_host", "")
|
|
if _is_host_suppressed(host, suppressed_hosts):
|
|
continue
|
|
vmid = ct.get("vmid", 0)
|
|
name = ct.get("name", "?")
|
|
status = ct.get("status", "unknown")
|
|
if vmid in CRITICAL_CONTAINERS and status != "running":
|
|
ok_stopped = vmid in EXPECTED_STOPPED_VMIDS and status == "stopped"
|
|
if (vmid, host) not in EXPECTED_STOPPED and not ok_stopped:
|
|
alerts.append(f"🔴 CT {vmid} ({name}) ist {status}!")
|
|
|
|
mem = ct.get("mem", 0)
|
|
maxmem = ct.get("maxmem", 1)
|
|
if maxmem > 0:
|
|
_check_ram_alert(vmid, name, mem, maxmem, ram_high_since, alerts)
|
|
|
|
errors = loki_client.get_errors(hours=0.5, limit=50)
|
|
error_lines = [e for e in errors if "error" not in e]
|
|
panic_lines = []
|
|
for e in error_lines:
|
|
line = e.get("line", "")
|
|
ll = line.lower()
|
|
if not any(w in ll for w in ["panic", "fatal", "oom", "out of memory"]):
|
|
continue
|
|
if "query=" in line or "caller=metrics" in line:
|
|
continue
|
|
if "HTTP/1." in line and ('" 200 ' in line or '" 301 ' in line or '" 302 ' in line or '" 304 ' in line):
|
|
continue
|
|
if "GET /" in line or "POST /" in line or "HEAD /" in line:
|
|
continue
|
|
panic_lines.append(e)
|
|
if panic_lines:
|
|
hosts = set(e.get("host", "?") for e in panic_lines)
|
|
hosts -= IGNORED_HOSTS
|
|
hosts = {h for h in hosts if h.lower() not in suppressed_names}
|
|
if hosts:
|
|
alerts.append(f"🔴 Kritische Fehler (panic/fatal/OOM) auf: {', '.join(hosts)}")
|
|
|
|
error_rates = loki_client.check_error_rate(minutes=30)
|
|
for er in error_rates:
|
|
if er.get("host", "").lower() in suppressed_names:
|
|
continue
|
|
alerts.append(
|
|
f"🔴 {er['host']}: {er['count']} Fehler in 30 Min (Schwelle: {er['threshold']})"
|
|
)
|
|
|
|
running_names = {
|
|
ct.get("name", "").lower()
|
|
for ct in containers
|
|
if "error" not in ct and ct.get("status") == "running"
|
|
}
|
|
|
|
silent = loki_client.check_silence(minutes=35)
|
|
if silent and "error" not in silent[0]:
|
|
names = [
|
|
s["host"] for s in silent
|
|
if s.get("host") not in IGNORED_HOSTS
|
|
and s.get("host") not in SILENCE_IGNORED_HOSTS
|
|
and s["host"].lower() not in suppressed_names
|
|
and s["host"].lower() in running_names
|
|
]
|
|
if names:
|
|
alerts.append(f"⚠️ Keine Logs seit 35+ Min: {', '.join(names)}")
|
|
|
|
_headers = {"User-Agent": "Mozilla/5.0 (Hausmeister-Bot/1.0 health-check)"}
|
|
for check in HTTP_HEALTH_CHECKS:
|
|
if _is_host_suppressed(check.get("host", ""), suppressed_hosts):
|
|
continue
|
|
timeout = check.get("timeout", 15)
|
|
retries = check.get("retries", 1)
|
|
retry_delay = check.get("retry_delay", 3)
|
|
msg = None
|
|
for attempt in range(retries):
|
|
try:
|
|
r = requests.get(
|
|
check["url"], timeout=timeout, allow_redirects=True, headers=_headers
|
|
)
|
|
if r.status_code < 400:
|
|
msg = None
|
|
break
|
|
msg = f"🔴 {check['name']} antwortet mit HTTP {r.status_code}"
|
|
except requests.RequestException as e:
|
|
msg = f"🔴 {check['name']} nicht erreichbar: {str(e)[:80]}"
|
|
if attempt < retries - 1:
|
|
time.sleep(retry_delay)
|
|
if msg:
|
|
alerts.append(msg)
|
|
|
|
restarts = loki_client.check_service_restarts(minutes=35)
|
|
for r in restarts:
|
|
if r.get("host", "").lower() in suppressed_names:
|
|
continue
|
|
alerts.append(f"🔄 Service-Neustart: {r['service']} auf {r['host']} ({r['count']}x in 35 Min)")
|
|
|
|
try:
|
|
import memory_client
|
|
import time as _time
|
|
now_ts = int(_time.time())
|
|
mem_items = memory_client.get_active_memory()
|
|
for item in mem_items:
|
|
exp = item.get("expires_at")
|
|
if exp and 0 < exp - now_ts < 86400:
|
|
from datetime import datetime as _dt
|
|
exp_str = _dt.fromtimestamp(exp).strftime("%d.%m. %H:%M")
|
|
alerts.append(f"⏰ Memory läuft ab ({exp_str}): {item['content'][:80]}")
|
|
except Exception:
|
|
pass
|
|
|
|
try:
|
|
mail_client.init(cfg)
|
|
important = mail_client.get_important_mails(hours=1)
|
|
if important and "error" not in important[0]:
|
|
state = _load_alert_state()
|
|
seen = state.get("seen_mails", {})
|
|
now = datetime.now(timezone.utc).timestamp()
|
|
|
|
new_mails = []
|
|
for m in important:
|
|
fp = hashlib.md5(
|
|
f"{m.get('date_str','')}{m.get('from','')}{m.get('subject','')}".encode()
|
|
).hexdigest()
|
|
if fp not in seen:
|
|
new_mails.append(m)
|
|
seen[fp] = now
|
|
|
|
seen = {k: v for k, v in seen.items() if now - v < 172800}
|
|
state["seen_mails"] = seen
|
|
_save_alert_state(state)
|
|
|
|
if new_mails:
|
|
senders = [m["from"][:30] for m in new_mails]
|
|
alerts.append(f"📧 {len(new_mails)} neue wichtige Mail(s): {', '.join(senders)}")
|
|
except Exception:
|
|
pass
|
|
|
|
return alerts
|
|
|
|
|
|
def format_report() -> str:
|
|
"""Tagesbericht: Gesamtstatus aller Systeme."""
|
|
cfg = config.parse_config()
|
|
suppressed_hosts = config.get_suppressed_hosts(cfg)
|
|
suppressed_names = config.get_suppressed_container_names(cfg)
|
|
lines = ["📋 Tagesbericht Homelab\n"]
|
|
if suppressed_hosts:
|
|
lines.append(f"🔕 Stumm: {', '.join(sorted(suppressed_hosts))}\n")
|
|
|
|
containers = proxmox_client.get_all_containers(
|
|
_get_passwords(cfg), _get_tokens(cfg)
|
|
)
|
|
containers = [
|
|
c for c in containers
|
|
if not _is_host_suppressed(c.get("_host", ""), suppressed_hosts)
|
|
]
|
|
running = [c for c in containers if c.get("status") == "running"]
|
|
stopped = [c for c in containers if c.get("status") == "stopped"]
|
|
errors_ct = [c for c in containers if "error" in c]
|
|
lines.append(f"Container: {len(running)} running, {len(stopped)} stopped, {len(errors_ct)} nicht erreichbar")
|
|
|
|
errors = loki_client.get_errors(hours=24, limit=100)
|
|
error_count = len([
|
|
e for e in errors
|
|
if "error" not in e and e.get("host", "").lower() not in suppressed_names
|
|
])
|
|
lines.append(f"Fehler (24h): {error_count}")
|
|
|
|
silent = loki_client.check_silence(minutes=35)
|
|
if silent and "error" not in (silent[0] if silent else {}):
|
|
names = [
|
|
s["host"] for s in silent
|
|
if s.get("host") not in IGNORED_HOSTS
|
|
and s["host"].lower() not in suppressed_names
|
|
]
|
|
if names:
|
|
lines.append(f"Stille Hosts: {', '.join(names)}")
|
|
else:
|
|
lines.append("Stille Hosts: keine")
|
|
else:
|
|
lines.append("Stille Hosts: keine")
|
|
|
|
try:
|
|
import memory_client
|
|
mem_items = memory_client.get_active_memory()
|
|
perm = [i for i in mem_items if i.get("memory_type") != "temporary"]
|
|
temp = [i for i in mem_items if i.get("memory_type") == "temporary"]
|
|
candidates = memory_client.get_candidates()
|
|
mem_line = f"Memory: {len(perm)} dauerhaft, {len(temp)} temporär"
|
|
import time as _time
|
|
now_ts = int(_time.time())
|
|
soon = [i for i in temp if i.get("expires_at") and i["expires_at"] - now_ts < 86400]
|
|
if soon:
|
|
mem_line += f", {len(soon)} laufen in 24h ab"
|
|
if candidates:
|
|
mem_line += f", {len(candidates)} Kandidaten offen"
|
|
lines.append(mem_line)
|
|
except Exception:
|
|
pass
|
|
|
|
alerts = check_all()
|
|
if alerts:
|
|
lines.append(f"\n⚠️ {len(alerts)} aktive Alarme:")
|
|
lines.extend(alerts)
|
|
else:
|
|
lines.append("\n✅ Keine Alarme — alles läuft.")
|
|
|
|
return "\n".join(lines)
|
|
|
|
|
|
def _load_alert_state() -> dict:
|
|
try:
|
|
with open(ALERT_STATE_FILE, "r") as f:
|
|
return json.load(f)
|
|
except (FileNotFoundError, json.JSONDecodeError):
|
|
return {}
|
|
|
|
|
|
def _save_alert_state(state: dict):
|
|
try:
|
|
with open(ALERT_STATE_FILE, "w") as f:
|
|
json.dump(state, f)
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def _alert_key(alert_text: str) -> str:
|
|
# Volatile Metrik-Alarme (error_rate) ueber einen stabilen Schluessel
|
|
# deduplizieren: nur Host, nicht der schwankende Zaehlwert. Sonst
|
|
# erzeugt jede neue Zahl einen neuen Hash und der Cooldown greift nie.
|
|
if "RAM bei" in alert_text:
|
|
m = re.search(r"CT (\d+)", alert_text)
|
|
if m:
|
|
return hashlib.md5(("ram|" + m.group(1)).encode()).hexdigest()
|
|
if "Fehler in 30 Min" in alert_text:
|
|
m = re.search(r"([\w.\-]+):\s*\d+\s+Fehler in 30 Min", alert_text)
|
|
if m:
|
|
return hashlib.md5(("error_rate|" + m.group(1)).encode()).hexdigest()
|
|
if "Hermes:" in alert_text and "Fehlerlogs in Loki" in alert_text:
|
|
return hashlib.md5(b"hermes|loki_errors").hexdigest()
|
|
if alert_text.startswith("🔴 Hermes Log:"):
|
|
m = re.search(r"([\w.\-]+\.service)", alert_text)
|
|
unit = m.group(1) if m else "generic"
|
|
return hashlib.md5(("hermes|log|" + unit).encode()).hexdigest()
|
|
return hashlib.md5(alert_text.encode()).hexdigest()
|
|
|
|
|
|
def _alert_category(alert_text: str) -> str:
|
|
if "CT " in alert_text and "ist " in alert_text:
|
|
return "container"
|
|
if "RAM " in alert_text:
|
|
return "ram"
|
|
if "panic" in alert_text.lower() or "fatal" in alert_text.lower():
|
|
return "panic"
|
|
if "RAM bei" in alert_text:
|
|
m = re.search(r"CT (\d+)", alert_text)
|
|
if m:
|
|
return hashlib.md5(("ram|" + m.group(1)).encode()).hexdigest()
|
|
if "Fehler in 30 Min" in alert_text:
|
|
return "error_rate"
|
|
if "Keine Logs" in alert_text:
|
|
return "silence"
|
|
if "antwortet mit HTTP" in alert_text or "nicht erreichbar" in alert_text:
|
|
return "http"
|
|
if "Service-Neustart" in alert_text:
|
|
return "restart"
|
|
if "Hermes" in alert_text:
|
|
return "hermes"
|
|
if "Backup" in alert_text or "PBS" in alert_text:
|
|
return "backup"
|
|
if "Memory läuft ab" in alert_text:
|
|
return "memory_expiry"
|
|
return "default"
|
|
|
|
|
|
def _filter_new_alerts(alerts: list[str], state: dict | None = None) -> list[str]:
|
|
"""Filtert bereits gemeldete Infra-Alerts per Cooldown. Mails werden separat in check_all() dedupliziert."""
|
|
if state is None:
|
|
state = _load_alert_state()
|
|
now = datetime.now(timezone.utc).timestamp()
|
|
cooldowns = state.get("alert_cooldowns", {})
|
|
new_alerts = []
|
|
|
|
for alert in alerts:
|
|
key = _alert_key(alert)
|
|
cat = _alert_category(alert)
|
|
cooldown = ALERT_COOLDOWN_SECONDS.get(cat, 3600)
|
|
|
|
last_sent = cooldowns.get(key, {}).get("ts", 0)
|
|
if now - last_sent > cooldown:
|
|
new_alerts.append(alert)
|
|
cooldowns[key] = {"ts": now, "text": alert[:80], "cat": cat}
|
|
|
|
cutoff = now - 86400
|
|
cooldowns = {k: v for k, v in cooldowns.items() if v.get("ts", 0) > cutoff}
|
|
|
|
state["alert_cooldowns"] = cooldowns
|
|
_save_alert_state(state)
|
|
return new_alerts
|
|
|
|
|
|
def send_alert(token: str, chat_id: str, message: str):
|
|
"""Sendet eine Nachricht via Telegram."""
|
|
requests.post(
|
|
f"https://api.telegram.org/bot{token}/sendMessage",
|
|
data={"chat_id": chat_id, "text": message},
|
|
timeout=10,
|
|
)
|
|
|
|
|
|
def run_check_and_alert():
|
|
"""Hauptfunktion für Cron: prüft und sendet Alerts falls nötig."""
|
|
cfg = config.parse_config()
|
|
token = cfg.raw.get("TG_HAUSMEISTER_TOKEN", "")
|
|
chat_id = cfg.raw.get("TG_CHAT_ID", "")
|
|
if not token or not chat_id:
|
|
return
|
|
|
|
state = _load_alert_state()
|
|
alerts = check_all(state)
|
|
new_alerts = _filter_new_alerts(alerts, state)
|
|
_save_alert_state(state)
|
|
if new_alerts:
|
|
msg = "🔧 Hausmeister-Check\n\n" + "\n".join(new_alerts)
|
|
send_alert(token, chat_id, msg)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import sys as _sys
|
|
if len(_sys.argv) > 1 and _sys.argv[1] == "report":
|
|
print(format_report())
|
|
else:
|
|
run_check_and_alert()
|