318 lines
12 KiB
Bash
Executable file
318 lines
12 KiB
Bash
Executable file
#!/bin/bash
|
|
# ============================================================
|
|
# homelab-brain Auto-Sync Script
|
|
# Läuft alle 15 Min auf pve-hetzner via Cron
|
|
# Aktualisiert STATE.md Dateien, pushed nach Forgejo
|
|
# Telegram-Alerts bei Service-Ausfällen und Push-Fehlern
|
|
# Stand: 08.03.2026
|
|
# ============================================================
|
|
|
|
set -euo pipefail
|
|
REPO="/opt/homelab-brain"
|
|
FORGEJO_TOKEN="5402da0447b0eb6aede721a8748a08974ddc5c42"
|
|
TG_TOKEN="8551565940:AAHIUpZND-tCNGv9yEoNPRyPt4GxEPYBJdE"
|
|
TG_CHAT="674951792"
|
|
DEBOUNCE_DIR="/tmp/homelab_watchdog"
|
|
DATE=$(date '+%Y-%m-%d %H:%M')
|
|
CHANGED=0
|
|
|
|
mkdir -p "$DEBOUNCE_DIR"
|
|
log() { echo "[$(date '+%H:%M:%S')] $1"; }
|
|
|
|
# Telegram Alert mit Debounce (10 Min pro Alert-Typ)
|
|
tg_alert() {
|
|
local key="$1" msg="$2"
|
|
local lockfile="$DEBOUNCE_DIR/${key}.lock"
|
|
local now=$(date +%s)
|
|
if [ -f "$lockfile" ]; then
|
|
local last=$(cat "$lockfile")
|
|
if [ $((now - last)) -lt 600 ]; then
|
|
return # Debounce aktiv
|
|
fi
|
|
fi
|
|
curl -s -X POST "https://api.telegram.org/bot${TG_TOKEN}/sendMessage" \
|
|
-d "chat_id=${TG_CHAT}" \
|
|
-d "text=Homelab Watchdog%0A%0A${msg}" \
|
|
-d "parse_mode=Markdown" > /dev/null 2>&1
|
|
echo "$now" > "$lockfile"
|
|
log "Alert gesendet: $key"
|
|
}
|
|
|
|
# Service-Watchdog: prüft ob ein CT-Service läuft
|
|
check_service() {
|
|
local ct="$1" service="$2" name="$3"
|
|
local status
|
|
status=$(pct exec "$ct" -- systemctl is-active "$service" 2>/dev/null || echo "unknown")
|
|
if [ "$status" != "active" ]; then
|
|
tg_alert "service_${service}" "*${name}* ist DOWN%0AService: ${service}%0ACT: ${ct}%0AStatus: ${status}"
|
|
echo "DOWN"
|
|
else
|
|
if [ -f "$DEBOUNCE_DIR/service_${service}.lock" ]; then
|
|
curl -s -X POST "https://api.telegram.org/bot${TG_TOKEN}/sendMessage" \
|
|
-d "chat_id=${TG_CHAT}" \
|
|
-d "text=*${name}* wieder online" \
|
|
-d "parse_mode=Markdown" > /dev/null 2>&1
|
|
rm -f "$DEBOUNCE_DIR/service_${service}.lock"
|
|
fi
|
|
echo "active"
|
|
fi
|
|
}
|
|
|
|
cd "$REPO"
|
|
git pull --quiet 2>/dev/null || true
|
|
|
|
# ─────────────────────────────────────────────────────
|
|
# 0. SERVICE WATCHDOG
|
|
# ─────────────────────────────────────────────────────
|
|
log "Watchdog läuft..."
|
|
RSS_LIVE=$(check_service 109 rss-manager "RSS Manager")
|
|
WP_LIVE=$(pct exec 101 -- docker inspect --format='{{.State.Status}}' wordpress-app 2>/dev/null || echo "unknown")
|
|
if [ "$WP_LIVE" != "running" ]; then
|
|
tg_alert "wordpress" "*WordPress Docker* ist DOWN%0AStatus: ${WP_LIVE}%0ACT: 101"
|
|
fi
|
|
|
|
# ─────────────────────────────────────────────────────
|
|
# 1. ARAKAVA NEWS STATE
|
|
# ─────────────────────────────────────────────────────
|
|
log "Sammle Arakava News Status..."
|
|
|
|
RSS_STATUS=$(pct exec 109 -- systemctl is-active rss-manager 2>/dev/null || echo "unknown")
|
|
WP_STATUS=$(pct exec 101 -- docker inspect --format='{{.State.Status}}' wordpress-app 2>/dev/null || echo "unknown")
|
|
|
|
# Letzte Feed-Aktivität aus SQLite
|
|
FEED_ACTIVITY=$(pct exec 109 -- python3 -c "
|
|
import sqlite3
|
|
db = sqlite3.connect('/opt/rss-manager/rss_manager.db')
|
|
rows = db.execute(\"SELECT name, last_run FROM feeds WHERE enabled=1 ORDER BY last_run DESC LIMIT 5\").fetchall()
|
|
for r in rows: print(f' {r[0]}: {r[1] or \"nie\"}')
|
|
" 2>/dev/null || echo " (nicht abrufbar)")
|
|
|
|
# Fehler letzte 24h
|
|
ERRORS=$(pct exec 109 -- bash -c "grep -c 'ERROR' /opt/rss-manager/logs/service.log 2>/dev/null || echo 0")
|
|
LAST_ERROR=$(pct exec 109 -- bash -c "grep 'ERROR' /opt/rss-manager/logs/service.log 2>/dev/null | tail -1 || echo 'keine'")
|
|
|
|
# OpenRouter Balance
|
|
OR_BALANCE=$(pct exec 109 -- python3 -c "
|
|
import requests
|
|
try:
|
|
r = requests.get('https://openrouter.ai/api/v1/auth/key',
|
|
headers={'Authorization': 'Bearer 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'},
|
|
timeout=5)
|
|
d = r.json().get('data', {})
|
|
remaining = float(d.get('limit', 20)) - float(d.get('usage', 0))
|
|
print(f'\${remaining:.2f} verbleibend')
|
|
except Exception as e:
|
|
print(f'(nicht abrufbar: {e})')
|
|
" 2>/dev/null || echo "(nicht abrufbar)")
|
|
|
|
cat > "$REPO/arakava-news/STATE.md" << EOF
|
|
# Arakava News — Live State
|
|
> Auto-generiert: $DATE
|
|
|
|
## Service Status
|
|
| Service | CT | Status |
|
|
|---|---|---|
|
|
| rss-manager | 109 | $RSS_STATUS |
|
|
| WordPress Docker | 101 | $WP_STATUS |
|
|
|
|
## Letzte Feed-Aktivität (Top 5)
|
|
$FEED_ACTIVITY
|
|
|
|
## Fehler (letzte 24h)
|
|
- Fehler gesamt: $ERRORS
|
|
- Letzter Fehler: $LAST_ERROR
|
|
|
|
## OpenRouter Guthaben
|
|
$OR_BALANCE
|
|
|
|
## URLs
|
|
- Blog: https://arakavanews.com
|
|
- Admin: https://arakavanews.com/wp-admin (admin / eJIyhW0p5PFacjvvKGufKeXS)
|
|
- RSS Manager: http://100.113.244.101:8080 (admin / astral66)
|
|
- Matomo: https://matomo.orbitalo.net (admin / astral66)
|
|
|
|
## Container (Primary — pve-hetzner)
|
|
| CT | Dienst | Tailscale |
|
|
|---|---|---|
|
|
| 101 | WordPress + MySQL (Docker) | 100.91.212.19 |
|
|
| 109 | RSS Manager + Matomo | 100.113.244.101 |
|
|
|
|
## Container (Mirror — pve3 Muldenstein)
|
|
| CT | Dienst | Tailscale |
|
|
|---|---|---|
|
|
| 600 | WordPress Mirror (Docker) | 100.92.205.101 |
|
|
| 601 | RSS Manager Mirror + n8n | — (kein Tailscale) |
|
|
|
|
## Aktive Feeds (17)
|
|
| ID | Name | Schedule |
|
|
|---|---|---|
|
|
| 1 | Dr. Bines Substack | 08/14/20 Uhr |
|
|
| 3 | NachDenkSeiten | 07/13/19 Uhr |
|
|
| 4 | Tichys Einblick | 07:30/13:30/19:30 |
|
|
| 5 | Junge Freiheit | 08/14/20 Uhr |
|
|
| 6 | PAZ | 08:30/14:30/20:30 |
|
|
| 7 | Apollo News | 09/15/21 Uhr |
|
|
| 8 | Apolut | 09:30/15:30/21:30 |
|
|
| 9 | Achgut.com | 10/16/22 Uhr |
|
|
| 10 | Heise Security | alle 4h |
|
|
| 11 | Golem.de | alle 2h |
|
|
| 12 | Heise Online | alle 3h |
|
|
| 13 | Rubikon.news | alle 3h |
|
|
| 14 | Corona-Transition | alle 4h |
|
|
| 15 | Photon.info (KI-Analyse) | alle 6h |
|
|
| 16 | Antispiegel | 08:30/14:30/20:30 |
|
|
| 17 | Riehle News | 09:00 Uhr |
|
|
|
|
## Code (CT 109: /opt/rss-manager/)
|
|
poster.py, scheduler.py, app.py, db.py
|
|
|
|
## Änderungshistorie
|
|
- 08.03.2026: Domain arakavanews.com live, Mirror CT 600/601 auf pve3
|
|
- 24.02.2026: Scheduler Lock gegen Doppelstarts
|
|
- 24.02.2026: Telegram auf HTML-Modus (Sonderzeichen-Fix)
|
|
- 24.02.2026: Werbeartikel-Blacklist (Anzeige:, Sponsored, etc.)
|
|
- 23.02.2026: Matomo von CT 113 → CT 109 migriert
|
|
- 23.02.2026: CT 100/102/104/105/106/113 gelöscht
|
|
EOF
|
|
CHANGED=1
|
|
log "Arakava News STATE.md aktualisiert"
|
|
|
|
# ─────────────────────────────────────────────────────
|
|
# 2. INFRASTRUKTUR STATE
|
|
# ─────────────────────────────────────────────────────
|
|
log "Sammle Infrastruktur Status..."
|
|
|
|
DISK_ROOT=$(df -h / | awk 'NR==2{print $5 " von " $2}')
|
|
DISK_DATA=$(df -h /var/lib/vz | awk 'NR==2{print $5 " von " $2}' 2>/dev/null || echo "n/a")
|
|
|
|
cat > "$REPO/infrastructure/STATE.md" << EOF
|
|
# Infrastruktur — Live State
|
|
> Auto-generiert: $DATE
|
|
|
|
## pve-hetzner Disk
|
|
| Mount | Belegt |
|
|
|---|---|
|
|
| / (root) | $DISK_ROOT |
|
|
| /var/lib/vz (VMs/CTs) | $DISK_DATA |
|
|
|
|
## Aktive Container auf pve-hetzner
|
|
| CT | Name | Tailscale IP | Dienste |
|
|
|---|---|---|---|
|
|
| 101 | wordpress-v2 | 100.91.212.19 | WordPress + MySQL (Docker) |
|
|
| 103 | seafile | 100.75.247.60 | Seafile (seafile.orbitalo.net) |
|
|
| 109 | rss-manager | 100.113.244.101 | RSS Manager + Matomo |
|
|
| 110 | portainer | 100.109.206.43 | Portainer Docker UI |
|
|
| 111 | forgejo | 100.89.246.60 | Forgejo Git (http://100.89.246.60:3000) |
|
|
| 144 | muldenstein-backup | — | Backup-Archiv |
|
|
| 999 | cluster-docu | 100.79.8.49 | Dokumentation (http://100.79.8.49:8080) |
|
|
|
|
## Gelöschte Container (24.02.2026)
|
|
| CT | Name | Grund |
|
|
|---|---|---|
|
|
| 100 | traefik | Abgelöst durch Cloudflare Tunnel |
|
|
| 102 | dify | Experiment fehlgeschlagen |
|
|
| 104 | n8n | Nicht aktiv genutzt |
|
|
| 105 | debian-12 | Nicht genutzt |
|
|
| 106 | wordpress-news | Abgelöst durch CT 101 |
|
|
| 113 | matomo | Integriert in CT 109 |
|
|
|
|
## Container auf pve1 (Kambodscha)
|
|
| CT | Name | Dienste |
|
|
|---|---|---|
|
|
| 136 | gold-silber-v3 | Edelmetall-Bot (Tailscale: 100.72.230.87) |
|
|
| 143 | smart-home | ioBroker + Grafana + InfluxDB |
|
|
|
|
## Container auf pve3 (Muldenstein)
|
|
| CT | Name | Tailscale IP | Dienste |
|
|
|---|---|---|---|
|
|
| 139 | Syncthing-Muldenstein | — | Syncthing |
|
|
| 141 | syncthing | — | Syncthing |
|
|
| 142 | WG-easy | — | WireGuard |
|
|
| 143 | Raspi-Broker | — | MQTT Broker |
|
|
| 145 | flugscanner-mu | — | Flugpreisscanner Node |
|
|
| 504 | projektscan-template | — | Projektscan |
|
|
| 600 | wp-mirror | 100.92.205.101 | WordPress Mirror (Redundanz CT 101) |
|
|
| 601 | rss-mirror | — | RSS Manager Mirror (Redundanz CT 109) |
|
|
|
|
## Routing
|
|
- Cloudflare Tunnel CT 101: arakavanews.com → :80
|
|
- Cloudflare Tunnel CT 101: arakava-news-2.orbitalo.net → 301 → arakavanews.com
|
|
- Cloudflare Tunnel CT 109: matomo.orbitalo.net → :80
|
|
- Cloudflare Tunnel CT 600: Standby (WordPress Mirror)
|
|
- Cloudflare Tunnel CT 601: Standby (RSS Manager Mirror)
|
|
- Kein Traefik, kein PBS-Gateway mehr
|
|
|
|
## Zugangsdaten
|
|
- pve-hetzner: root / Astral-Proxmox!2026
|
|
- pve1: root / astral66
|
|
- Alle CTs: root / astral66
|
|
- Seafile: admin@orbitalo.net / astral66
|
|
- Forgejo: orbitalo / astral66
|
|
|
|
## Telegram Bots
|
|
| Bot | Token (Auszug) | Chat-ID |
|
|
|---|---|---|
|
|
| Mutter (@MutterbotAI_bot) | 8551565940:... | 674951792 |
|
|
EOF
|
|
CHANGED=1
|
|
log "Infrastruktur STATE.md aktualisiert"
|
|
|
|
# ─────────────────────────────────────────────────────
|
|
# 3. SMART HOME STATE
|
|
# ─────────────────────────────────────────────────────
|
|
log "Sammle Smart Home Status..."
|
|
|
|
LAST_BACKUP=$(ls -t /home/backup-muldenstein/backups/*.tar.gz 2>/dev/null | head -1 | xargs ls -lh 2>/dev/null | awk '{print $5, $6, $7, $8}' || echo "nicht abrufbar")
|
|
BACKUP_COUNT=$(ls /home/backup-muldenstein/backups/*.tar.gz 2>/dev/null | wc -l || echo "0")
|
|
|
|
cat > "$REPO/smart-home/STATE.md" << EOF
|
|
# Smart Home Muldenstein — Live State
|
|
> Auto-generiert: $DATE
|
|
|
|
## Backup-Status
|
|
- Letztes Backup: $LAST_BACKUP
|
|
- Backups gesamt: $BACKUP_COUNT
|
|
- Ziel: /home/backup-muldenstein/backups/ (CT 144)
|
|
|
|
## Services (CT 143)
|
|
| Dienst | URL |
|
|
|---|---|
|
|
| Grafana | https://grafana.orbitalo.net |
|
|
| ioBroker | http://192.168.178.36:8081 |
|
|
| InfluxDB | http://192.168.178.36:8086 |
|
|
|
|
## Grafana Alerts → Telegram 674951792
|
|
- Promtail DOWN (> 5 Min keine Daten)
|
|
- CPU > 70%
|
|
- Memory > 80%
|
|
- Disk > 90%
|
|
|
|
## Backup-Zeitplan
|
|
- täglich 04:00 → /root/backup-to-hetzner.sh (auf pve3)
|
|
- Retention: 30d tägl, 90d wöchl, unbegrenzt monatl
|
|
EOF
|
|
CHANGED=1
|
|
log "Smart Home STATE.md aktualisiert"
|
|
|
|
# ─────────────────────────────────────────────────────
|
|
# 4. GIT COMMIT & PUSH
|
|
# ─────────────────────────────────────────────────────
|
|
if [ "$CHANGED" -eq 1 ]; then
|
|
log "Committe Änderungen..."
|
|
git -C "$REPO" add -A
|
|
git -C "$REPO" -c user.email="sync@homelab" -c user.name="Auto-Sync" \
|
|
commit -m "Auto-Sync: $DATE" --quiet || true
|
|
if git -C "$REPO" push \
|
|
"http://orbitalo:${FORGEJO_TOKEN}@100.89.246.60:3000/orbitalo/homelab-brain.git" main --quiet 2>/tmp/git-push-err; then
|
|
log "Push erfolgreich"
|
|
rm -f "$DEBOUNCE_DIR/git_push.lock"
|
|
else
|
|
ERR=$(cat /tmp/git-push-err | head -1)
|
|
log "Push FEHLER: $ERR"
|
|
tg_alert "git_push" "*Homelab Git-Sync fehlgeschlagen*%0A%0AFehler: ${ERR}%0AZeit: ${DATE}"
|
|
fi
|
|
else
|
|
log "Keine Änderungen"
|
|
fi
|
|
|
|
log "Sync abgeschlossen"
|