292 lines
11 KiB
Bash
Executable file
292 lines
11 KiB
Bash
Executable file
#!/bin/bash
|
|
# ============================================================
|
|
# homelab-brain Auto-Sync Script
|
|
# Läuft alle 15 Min auf pve-hetzner
|
|
# Aktualisiert STATE.md Dateien, pushed nach GitHub
|
|
# und sendet Telegram-Alerts bei Service-Ausfällen
|
|
#
|
|
# Setup (einmalig auf pve-hetzner):
|
|
# git clone https://<TOKEN>@github.com/Orbitalo/homelab-brain.git /opt/homelab-brain
|
|
# chmod +x /opt/homelab-brain/scripts/sync-state.sh
|
|
# echo "*/15 * * * * root /opt/homelab-brain/scripts/sync-state.sh >> /var/log/homelab-sync.log 2>&1" >> /etc/crontab
|
|
# ============================================================
|
|
|
|
set -euo pipefail
|
|
REPO="/opt/homelab-brain"
|
|
GH_TOKEN="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"
|
|
FORGEJO_TOKEN="04f9ca5b5db295ececd2656d7b113c37d90c38fd"
|
|
TG_TOKEN="8551565940:AAHIUpZND-tCNGv9yEoNPRyPt4GxEPYBJdE"
|
|
TG_CHAT="674951792"
|
|
DEBOUNCE_DIR="/tmp/homelab_watchdog"
|
|
DATE=$(date '+%Y-%m-%d %H:%M')
|
|
CHANGED=0
|
|
|
|
mkdir -p "$DEBOUNCE_DIR"
|
|
log() { echo "[$(date '+%H:%M:%S')] $1"; }
|
|
|
|
# Telegram Alert mit Debounce (10 Min pro Alert-Typ)
|
|
tg_alert() {
|
|
local key="$1" msg="$2"
|
|
local lockfile="$DEBOUNCE_DIR/${key}.lock"
|
|
local now=$(date +%s)
|
|
if [ -f "$lockfile" ]; then
|
|
local last=$(cat "$lockfile")
|
|
if [ $((now - last)) -lt 600 ]; then
|
|
return # Debounce aktiv
|
|
fi
|
|
fi
|
|
curl -s -X POST "https://api.telegram.org/bot${TG_TOKEN}/sendMessage" \
|
|
-d "chat_id=${TG_CHAT}" \
|
|
-d "text=🚨 Homelab Watchdog%0A%0A${msg}" \
|
|
-d "parse_mode=Markdown" > /dev/null 2>&1
|
|
echo "$now" > "$lockfile"
|
|
log "Alert gesendet: $key"
|
|
}
|
|
|
|
# Service-Watchdog: prüft ob ein CT-Service läuft
|
|
check_service() {
|
|
local ct="$1" service="$2" name="$3"
|
|
local status
|
|
status=$(pct exec "$ct" -- systemctl is-active "$service" 2>/dev/null || echo "unknown")
|
|
if [ "$status" != "active" ]; then
|
|
tg_alert "service_${service}" "⚠️ *${name}* ist DOWN%0AService: ${service}%0ACT: ${ct}%0AStatus: ${status}"
|
|
echo "DOWN"
|
|
else
|
|
# Wenn vorheriger Alert vorhanden → Recovery-Meldung
|
|
if [ -f "$DEBOUNCE_DIR/service_${service}.lock" ]; then
|
|
curl -s -X POST "https://api.telegram.org/bot${TG_TOKEN}/sendMessage" \
|
|
-d "chat_id=${TG_CHAT}" \
|
|
-d "text=✅ *${name}* wieder online" \
|
|
-d "parse_mode=Markdown" > /dev/null 2>&1
|
|
rm -f "$DEBOUNCE_DIR/service_${service}.lock"
|
|
fi
|
|
echo "active"
|
|
fi
|
|
}
|
|
|
|
cd "$REPO"
|
|
git pull --quiet
|
|
|
|
# ─────────────────────────────────────────────────────
|
|
# 0. SERVICE WATCHDOG
|
|
# ─────────────────────────────────────────────────────
|
|
log "Watchdog läuft..."
|
|
RSS_LIVE=$(check_service 109 rss-manager "RSS Manager")
|
|
WP_LIVE=$(pct exec 101 -- docker inspect --format='{{.State.Status}}' wordpress-app 2>/dev/null || echo "unknown")
|
|
if [ "$WP_LIVE" != "running" ]; then
|
|
tg_alert "wordpress" "⚠️ *WordPress Docker* ist DOWN%0AStatus: ${WP_LIVE}%0ACT: 101"
|
|
fi
|
|
|
|
# ─────────────────────────────────────────────────────
|
|
# 1. ARAKAVA NEWS STATE
|
|
# ─────────────────────────────────────────────────────
|
|
log "Sammle Arakava News Status..."
|
|
|
|
RSS_STATUS=$(pct exec 109 -- systemctl is-active rss-manager 2>/dev/null || echo "unknown")
|
|
WP_STATUS=$(pct exec 101 -- docker inspect --format='{{.State.Status}}' wordpress-app 2>/dev/null || echo "unknown")
|
|
|
|
# Letzte Feed-Aktivität aus SQLite
|
|
FEED_ACTIVITY=$(pct exec 109 -- python3 -c "
|
|
import sqlite3, json
|
|
db = sqlite3.connect('/opt/rss-manager/rss_manager.db')
|
|
rows = db.execute(\"SELECT name, last_run FROM feeds WHERE enabled=1 ORDER BY last_run DESC LIMIT 5\").fetchall()
|
|
for r in rows: print(f' {r[0]}: {r[1] or \"nie\"}')
|
|
" 2>/dev/null || echo " (nicht abrufbar)")
|
|
|
|
# Letzte Fehler (letzte 24h)
|
|
ERRORS=$(pct exec 109 -- bash -c "grep -c 'ERROR' /opt/rss-manager/logs/service.log 2>/dev/null || echo 0")
|
|
LAST_ERROR=$(pct exec 109 -- bash -c "grep 'ERROR' /opt/rss-manager/logs/service.log 2>/dev/null | tail -1 || echo 'keine'")
|
|
|
|
# OpenRouter Balance
|
|
OR_BALANCE=$(pct exec 109 -- python3 -c "
|
|
import requests, sys
|
|
try:
|
|
r = requests.get('https://openrouter.ai/api/v1/auth/key',
|
|
headers={'Authorization': 'Bearer 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'},
|
|
timeout=5)
|
|
d = r.json().get('data', {})
|
|
remaining = float(d.get('limit', 20)) - float(d.get('usage', 0))
|
|
print(f'\${remaining:.2f} verbleibend')
|
|
except Exception as e:
|
|
print(f'(nicht abrufbar: {e})')
|
|
" 2>/dev/null || echo "(nicht abrufbar)")
|
|
|
|
cat > "$REPO/arakava-news/STATE.md" << EOF
|
|
# Arakava News — Live State
|
|
> Auto-generiert: $DATE | Manueller Abschnitt am Ende.
|
|
|
|
## Service Status
|
|
| Service | Status |
|
|
|---|---|
|
|
| rss-manager (CT 109) | $RSS_STATUS |
|
|
| WordPress Docker (CT 101) | $WP_STATUS |
|
|
| n8n Workflows | ⛔ deaktiviert |
|
|
|
|
## Letzte Feed-Aktivität (Top 5)
|
|
$FEED_ACTIVITY
|
|
|
|
## Fehler (letzte 24h)
|
|
- Fehler gesamt: $ERRORS
|
|
- Letzter Fehler: $LAST_ERROR
|
|
|
|
## OpenRouter Guthaben
|
|
$OR_BALANCE
|
|
|
|
## Credentials
|
|
- WordPress: https://arakava-news-2.orbitalo.net | admin / eJIyhW0p5PFacjvvKGufKeXS
|
|
- RSS Manager: http://<CT109-IP>:8080
|
|
- OpenRouter: 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
|
|
|
|
## Feeds (17 aktiv)
|
|
| ID | Name | Kategorie | Schedule |
|
|
|---|---|---|---|
|
|
| 1 | Dr. Bines Substack | 13 | 08/14/20 Uhr |
|
|
| 3 | NachDenkSeiten | 5 | 07/13/19 Uhr |
|
|
| 4 | Tichys Einblick | 6 | 07:30/13:30/19:30 |
|
|
| 5 | Junge Freiheit | 7 | 08/14/20 Uhr |
|
|
| 6 | PAZ | 8 | 08:30/14:30/20:30 |
|
|
| 7 | Apollo News | 9 | 09/15/21 Uhr |
|
|
| 8 | Apolut | 10 | 09:30/15:30/21:30 |
|
|
| 9 | Achgut.com | 15 | 10/16/22 Uhr |
|
|
| 10 | Heise Security | 11 | alle 4h |
|
|
| 11 | Golem.de | 12 | alle 2h |
|
|
| 12 | Heise Online | 3 | alle 3h |
|
|
| 13 | Rubikon.news | 17 | alle 3h |
|
|
| 14 | Corona-Transition | 18 | alle 4h |
|
|
| 15 | Photon.info (KI) | 3 | alle 6h |
|
|
| 16 | Antispiegel | 20 | 08:30/14:30/20:30 |
|
|
| 17 | Riehle News | 21 | 09:00 Uhr |
|
|
|
|
## Code (CT 109: /opt/rss-manager/)
|
|
- poster.py, scheduler.py, app.py, db.py, github_researcher.py
|
|
- Vollcode: github.com/Orbitalo/Wordpress-V3-MCP-Projekt
|
|
|
|
## Offene Aufgaben v3
|
|
- [ ] CT 112 anlegen, Docker Compose v3
|
|
- [ ] Retry-Logik in poster.py
|
|
- [ ] Telegram Alerting bei Feed-Fehlern
|
|
|
|
## Notizen (manuell)
|
|
EOF
|
|
CHANGED=1
|
|
log "Arakava News STATE.md aktualisiert"
|
|
|
|
# ─────────────────────────────────────────────────────
|
|
# 2. INFRASTRUKTUR STATE
|
|
# ─────────────────────────────────────────────────────
|
|
log "Sammle Infrastruktur Status..."
|
|
|
|
# Container-Status auf pve-hetzner
|
|
CT_STATUS=$(pvesh get /nodes/pve-hetzner/lxc --output-format=text 2>/dev/null | \
|
|
awk '{print $1, $2}' | grep -v "^vmid" || echo "(nicht abrufbar)")
|
|
|
|
# Disk-Auslastung
|
|
DISK_ROOT=$(df -h / | awk 'NR==2{print $5 " von " $2}')
|
|
DISK_DATA=$(df -h /var/lib/vz | awk 'NR==2{print $5 " von " $2}' 2>/dev/null || echo "n/a")
|
|
|
|
cat > "$REPO/infrastructure/STATE.md" << EOF
|
|
# Infrastruktur — Live State
|
|
> Auto-generiert: $DATE
|
|
|
|
## pve-hetzner Disk
|
|
| Mount | Belegt |
|
|
|---|---|
|
|
| / (root) | $DISK_ROOT |
|
|
| /var/lib/vz (VMs) | $DISK_DATA |
|
|
|
|
## Container auf pve-hetzner
|
|
| CT | Name | Tailscale IP | Dienste |
|
|
|---|---|---|---|
|
|
| 100 | traefik | 100.78.77.115 | Traefik, Pangolin, Uptime-Kuma |
|
|
| 101 | moltbot | 100.91.212.19 | @MutterbotAI_bot |
|
|
| 102 | dify | 100.113.136.30 | Dify RAG + @DifyRagBot |
|
|
| 103 | seafile | 100.75.247.60 | Seafile (seafile.orbitalo.net) |
|
|
| 104 | n8n | 100.125.102.93 | n8n (Workflows deaktiviert) |
|
|
| 107 | ragflow | 100.116.125.12 | RAGFlow (in Einrichtung) |
|
|
| 109 | rss-manager | — | RSS Manager + KI |
|
|
| 110 | portainer | 100.109.206.43 | Portainer UI |
|
|
| 144 | muldenstein-backup | — | Backup-Archiv |
|
|
| 999 | cluster-docu | 100.79.8.49 | Dokumentation |
|
|
|
|
## Container auf pve1 Kambodscha
|
|
| CT | Name | IP | Dienste |
|
|
|---|---|---|---|
|
|
| 135 | edelmetall | 192.168.0.219 | Streamlit Gold/Silber |
|
|
| 888 | MCP-Proxmox | 192.168.0.116 | Proxmox MCP |
|
|
| 999 | cluster-docu | 192.168.0.209 | Doku-Mirror |
|
|
|
|
## Container auf pve3 Muldenstein
|
|
| CT | Name | IP | Dienste |
|
|
|---|---|---|---|
|
|
| 134 | gold-silber-de | 100.69.161.128 | Dashboard DE (blei.orbitalo.info) |
|
|
| 143 | raspi-broker | 192.168.178.36 | InfluxDB, Grafana, ioBroker |
|
|
|
|
## Zugangsdaten
|
|
- pve-hetzner: root / Astral-Proxmox!2026
|
|
- pve1: root / astral66
|
|
- Alle lokalen CTs: root / astral66
|
|
- Seafile: admin@orbitalo.net / astral66
|
|
- n8n: wuttig@gmx.de / Astral66
|
|
- Dify: admin@orbitalo.net / astral66
|
|
|
|
## Notizen (manuell)
|
|
EOF
|
|
CHANGED=1
|
|
log "Infrastruktur STATE.md aktualisiert"
|
|
|
|
# ─────────────────────────────────────────────────────
|
|
# 3. SMART HOME STATE
|
|
# ─────────────────────────────────────────────────────
|
|
log "Sammle Smart Home Status..."
|
|
|
|
# Backup-Status
|
|
LAST_BACKUP=$(ls -t /home/backup-muldenstein/backups/*.tar.gz 2>/dev/null | head -1 | xargs ls -lh 2>/dev/null | awk '{print $5, $6, $7, $8}' || echo "nicht abrufbar")
|
|
BACKUP_COUNT=$(ls /home/backup-muldenstein/backups/*.tar.gz 2>/dev/null | wc -l || echo "0")
|
|
|
|
cat > "$REPO/smart-home/STATE.md" << EOF
|
|
# Smart Home Muldenstein — Live State
|
|
> Auto-generiert: $DATE
|
|
|
|
## Backup-Status
|
|
- Letztes Backup: $LAST_BACKUP
|
|
- Backups gesamt: $BACKUP_COUNT
|
|
- Ziel: /home/backup-muldenstein/backups/ (CT 144)
|
|
|
|
## Services (CT 143: 192.168.178.36)
|
|
| Dienst | URL |
|
|
|---|---|
|
|
| Grafana | https://grafana.orbitalo.net |
|
|
| ioBroker | http://192.168.178.36:8081 |
|
|
| InfluxDB | http://192.168.178.36:8086 |
|
|
|
|
## Grafana Alerts → Telegram 674951792
|
|
- Promtail DOWN (> 5 Min keine Daten)
|
|
- CPU > 70%
|
|
- Memory > 80%
|
|
- Disk > 90%
|
|
|
|
## Backup-Zeitplan
|
|
- täglich 04:00 → Script: /root/backup-to-hetzner.sh (auf pve3)
|
|
- Retention: 30d tägl, 90d wöchl, unbegrenzt monatl
|
|
|
|
## Notizen (manuell)
|
|
EOF
|
|
CHANGED=1
|
|
log "Smart Home STATE.md aktualisiert"
|
|
|
|
# ─────────────────────────────────────────────────────
|
|
# 4. GIT COMMIT & PUSH
|
|
# ─────────────────────────────────────────────────────
|
|
if [ "$CHANGED" -eq 1 ]; then
|
|
log "Committe Änderungen..."
|
|
git -C "$REPO" add -A
|
|
git -C "$REPO" -c user.email="sync@homelab" -c user.name="Auto-Sync" \
|
|
commit -m "Auto-Sync: $DATE" --quiet || true
|
|
git -C "$REPO" push \
|
|
"https://orbitalo:${FORGEJO_TOKEN}@git.orbitalo.net/orbitalo/homelab-brain.git" main --quiet
|
|
log "Push erfolgreich"
|
|
else
|
|
log "Keine Änderungen"
|
|
fi
|
|
|
|
log "Sync abgeschlossen"
|