Add Watchdog + Telegram alerts, Cron auf 15 Min

scripts/sync-state.sh

@@ -1,26 +1,81 @@
 #!/bin/bash
 # ============================================================
 # homelab-brain Auto-Sync Script
-# Läuft täglich 03:00 Uhr auf pve-hetzner
-# Aktualisiert STATE.md Dateien und pushed nach GitHub
+# Läuft alle 15 Min auf pve-hetzner
+# Aktualisiert STATE.md Dateien, pushed nach GitHub
+# und sendet Telegram-Alerts bei Service-Ausfällen
 #
 # Setup (einmalig auf pve-hetzner):
 #   git clone https://<TOKEN>@github.com/Orbitalo/homelab-brain.git /opt/homelab-brain
 #   chmod +x /opt/homelab-brain/scripts/sync-state.sh
-#   echo "0 3 * * * root /opt/homelab-brain/scripts/sync-state.sh >> /var/log/homelab-sync.log 2>&1" >> /etc/crontab
+#   echo "*/15 * * * * root /opt/homelab-brain/scripts/sync-state.sh >> /var/log/homelab-sync.log 2>&1" >> /etc/crontab
 # ============================================================
 
 set -euo pipefail
 REPO="/opt/homelab-brain"
 GH_TOKEN="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"
+TG_TOKEN="8551565940:AAHIUpZND-tCNGv9yEoNPRyPt4GxEPYBJdE"
+TG_CHAT="674951792"
+DEBOUNCE_DIR="/tmp/homelab_watchdog"
 DATE=$(date '+%Y-%m-%d %H:%M')
 CHANGED=0
 
+mkdir -p "$DEBOUNCE_DIR"
 log() { echo "[$(date '+%H:%M:%S')] $1"; }
 
+# Telegram Alert mit Debounce (10 Min pro Alert-Typ)
+tg_alert() {
+    local key="$1" msg="$2"
+    local lockfile="$DEBOUNCE_DIR/${key}.lock"
+    local now=$(date +%s)
+    if [ -f "$lockfile" ]; then
+        local last=$(cat "$lockfile")
+        if [ $((now - last)) -lt 600 ]; then
+            return  # Debounce aktiv
+        fi
+    fi
+    curl -s -X POST "https://api.telegram.org/bot${TG_TOKEN}/sendMessage" \
+        -d "chat_id=${TG_CHAT}" \
+        -d "text=🚨 Homelab Watchdog%0A%0A${msg}" \
+        -d "parse_mode=Markdown" > /dev/null 2>&1
+    echo "$now" > "$lockfile"
+    log "Alert gesendet: $key"
+}
+
+# Service-Watchdog: prüft ob ein CT-Service läuft
+check_service() {
+    local ct="$1" service="$2" name="$3"
+    local status
+    status=$(pct exec "$ct" -- systemctl is-active "$service" 2>/dev/null || echo "unknown")
+    if [ "$status" != "active" ]; then
+        tg_alert "service_${service}" "⚠️ *${name}* ist DOWN%0AService: ${service}%0ACT: ${ct}%0AStatus: ${status}"
+        echo "DOWN"
+    else
+        # Wenn vorheriger Alert vorhanden → Recovery-Meldung
+        if [ -f "$DEBOUNCE_DIR/service_${service}.lock" ]; then
+            curl -s -X POST "https://api.telegram.org/bot${TG_TOKEN}/sendMessage" \
+                -d "chat_id=${TG_CHAT}" \
+                -d "text=✅ *${name}* wieder online" \
+                -d "parse_mode=Markdown" > /dev/null 2>&1
+            rm -f "$DEBOUNCE_DIR/service_${service}.lock"
+        fi
+        echo "active"
+    fi
+}
+
 cd "$REPO"
 git pull --quiet
 
+# ─────────────────────────────────────────────────────
+# 0. SERVICE WATCHDOG
+# ─────────────────────────────────────────────────────
+log "Watchdog läuft..."
+RSS_LIVE=$(check_service 109 rss-manager "RSS Manager")
+WP_LIVE=$(pct exec 101 -- docker inspect --format='{{.State.Status}}' wordpress-app 2>/dev/null || echo "unknown")
+if [ "$WP_LIVE" != "running" ]; then
+    tg_alert "wordpress" "⚠️ *WordPress Docker* ist DOWN%0AStatus: ${WP_LIVE}%0ACT: 101"
+fi
+
 # ─────────────────────────────────────────────────────
 # 1. ARAKAVA NEWS STATE
 # ─────────────────────────────────────────────────────