diff --git a/arakava-news/scripts/install-promtail-ct600.sh b/arakava-news/scripts/install-promtail-ct600.sh new file mode 100755 index 000000000..d53a31821 --- /dev/null +++ b/arakava-news/scripts/install-promtail-ct600.sh @@ -0,0 +1,91 @@ +#!/bin/bash +# Promtail auf CT600 wp-mirror (pve-mu-3) — wie CT101, Loki via Tailscale +set -euo pipefail +apt-get update && apt-get install -y wget unzip curl systemd-journal-remote 2>/dev/null || apt-get install -y wget unzip curl +cd /tmp +wget -q https://github.com/grafana/loki/releases/download/v3.0.0/promtail-linux-amd64.zip +unzip -o promtail-linux-amd64.zip +chmod +x promtail-linux-amd64 +mv promtail-linux-amd64 /usr/local/bin/promtail +cat > /etc/promtail-config.yml << 'EOF' +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +positions: + filename: /tmp/positions.yaml + +clients: + - url: http://100.109.206.43:3100/loki/api/v1/push + +scrape_configs: + - job_name: journal + journal: + max_age: 12h + labels: + job: systemd + host: wp-mirror + relabel_configs: + - source_labels: [__journal__systemd_unit] + target_label: unit + + - job_name: system + static_configs: + - targets: + - localhost + labels: + job: varlogs + host: wp-mirror + __path__: /var/log/*log + + - job_name: docker + static_configs: + - targets: + - localhost + labels: + job: dockerlogs + host: wp-mirror + __path__: /var/lib/docker/containers/*/*.log + pipeline_stages: + - json: + expressions: + output: log + stream: stream + attrs: + - json: + expressions: + tag: + source: attrs + - regex: + expression: (?P(?:[^|]*[^|])) + source: tag + - timestamp: + format: RFC3339Nano + source: time + - labels: + tag: + stream: + container_name: + - output: + source: output +EOF +cat > /etc/systemd/system/promtail.service << 'EOF' +[Unit] +Description=Promtail Log Agent +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=root +ExecStart=/usr/local/bin/promtail -config.file=/etc/promtail-config.yml +Restart=always +RestartSec=10 + +[Install] +WantedBy=multi-user.target +EOF +systemctl daemon-reload +systemctl enable promtail +systemctl restart promtail +systemctl is-active promtail diff --git a/arakava-news/scripts/wp-mirror-sync.sh b/arakava-news/scripts/wp-mirror-sync.sh index 8c7e1027b..f08d87614 100755 --- a/arakava-news/scripts/wp-mirror-sync.sh +++ b/arakava-news/scripts/wp-mirror-sync.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash # Pull WordPress CT101 (pve-hetzner) → CT600 wp-mirror (pve-mu-3) # CT101 impact: read-only mysqldump (--single-transaction) + read-only tar. No restarts. +# Logs: Host /var/log/wp-mirror-sync.log + CT600 /var/log/wp-mirror-sync.log → Loki (Promtail) set -euo pipefail HETZNER_HOST="${HETZNER_HOST:-root@100.88.230.59}" @@ -8,6 +9,7 @@ HETZNER_CT="${HETZNER_CT:-101}" MIRROR_CT="${MIRROR_CT:-600}" STAGING="${STAGING:-/opt/wp-mirror-sync/staging}" LOG="${LOG:-/var/log/wp-mirror-sync.log}" +LOG_CT="/var/log/wp-mirror-sync.log" LOCK_FILE="/var/run/wp-mirror-sync.lock" FAILOVER_FLAG="/opt/wp-mirror-sync/FAILOVER_ACTIVE" MIRROR_SITEURL="http://192.168.178.142" @@ -15,9 +17,29 @@ MYSQL_USER="wordpress" MYSQL_PASS="astral66" MYSQL_DB="wordpress" WP_VOL="/var/lib/docker/volumes/wordpress_wp_data/_data" +MIRROR_RUNNING=0 log() { - echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "$LOG" + local msg="[$(date '+%Y-%m-%d %H:%M:%S')] $*" + echo "$msg" | tee -a "$LOG" + if [[ "$MIRROR_RUNNING" == "1" ]]; then + echo "$msg" | pct exec "$MIRROR_CT" -- tee -a "$LOG_CT" >/dev/null 2>&1 || true + fi +} + +ensure_mirror_ct() { + if [[ "$(pct status "$MIRROR_CT" 2>/dev/null | awk '{print $2}')" != "running" ]]; then + pct start "$MIRROR_CT" + fi + for _ in $(seq 1 30); do + if pct exec "$MIRROR_CT" -- test -d /var/log 2>/dev/null; then + MIRROR_RUNNING=1 + pct exec "$MIRROR_CT" -- touch "$LOG_CT" 2>/dev/null || true + return 0 + fi + sleep 2 + done + return 1 } ct101_http_ok() { @@ -62,6 +84,12 @@ if ! ct101_http_ok; then exit 1 fi +log "Starting CT600 (Promtail/Loki)..." +if ! ensure_mirror_ct; then + log "ABORT: CT600 did not become ready" + exit 1 +fi + log "START sync CT101 → CT600" DB_FILE="${STAGING}/wordpress.sql.gz" @@ -82,10 +110,6 @@ log "File sync via tar (read-only, ~5–8 min)..." tar_sync_files log "Files staged: $(du -sh "${STAGING}/wpfiles" | awk '{print $1}')" -log "Starting CT600 for import..." -if [[ "$(pct status "$MIRROR_CT" | awk '{print $2}')" != "running" ]]; then - pct start "$MIRROR_CT" -fi for _ in $(seq 1 45); do if pct exec "$MIRROR_CT" -- docker info >/dev/null 2>&1; then break @@ -122,4 +146,5 @@ fi log "Stopping CT600 (standby)..." pct stop "$MIRROR_CT" +MIRROR_RUNNING=0 log "DONE" diff --git a/homelab-ai-bot/core/loki_client.py b/homelab-ai-bot/core/loki_client.py index 144ed7c27..21771b6e0 100644 --- a/homelab-ai-bot/core/loki_client.py +++ b/homelab-ai-bot/core/loki_client.py @@ -208,6 +208,34 @@ def check_error_rate(minutes: int = 30) -> list[dict]: return alerts +def check_wp_mirror_sync(hours: float = 26) -> list[str]: + """Prüft CT600 wp-mirror Sync-Logs in Loki (host=wp-mirror).""" + alerts = [] + aborts = query_logs( + '{host="wp-mirror"} |~ "ABORT|WARN: CT101"', + hours=hours, + limit=10, + ) + for e in aborts: + if "error" in e: + continue + line = e.get("line", "").strip() + if line: + alerts.append(f"⚠️ wp-mirror Sync: {line[:160]}") + break + + try: + from zoneinfo import ZoneInfo + now = datetime.now(ZoneInfo("Europe/Berlin")) + except Exception: + now = datetime.now(timezone.utc) + if now.hour >= 6: + done = query_logs('{host="wp-mirror"} |= "DONE"', hours=hours, limit=5) + if not [e for e in done if "error" not in e]: + alerts.append("⚠️ wp-mirror: kein erfolgreicher Sync (DONE) in den letzten 26h") + return alerts + + def format_logs(entries: list[dict], max_lines: int = 30) -> str: """Format log entries for human/LLM consumption.""" if not entries: diff --git a/homelab-ai-bot/monitor.py b/homelab-ai-bot/monitor.py index 9872cba1a..af3888787 100644 --- a/homelab-ai-bot/monitor.py +++ b/homelab-ai-bot/monitor.py @@ -25,6 +25,7 @@ ALERT_COOLDOWN_SECONDS = { "error_rate": 1800, "hermes": 7200, "backup": 21600, + "mirror_sync": 21600, } @@ -86,6 +87,7 @@ SILENCE_IGNORED_HOSTS = { "ct-600-webcam", # kein rsyslog, Stream läuft aber "ct-103-Intercity-Taxi", # absichtlich gestoppt "ct-101-freshrss", # auf pve-ka-3, lokale Loki (nicht zentral) + "wp-mirror", # CT600 Reserve — gestoppt, Logs nur während Sync } @@ -174,6 +176,7 @@ def check_all(state: dict | None = None) -> list[str]: suppressed_names = config.get_suppressed_container_names(cfg) alerts = [] alerts.extend(check_hermes()) + alerts.extend(loki_client.check_wp_mirror_sync()) containers = proxmox_client.get_all_containers( _get_passwords(cfg), _get_tokens(cfg) @@ -451,6 +454,8 @@ def _alert_category(alert_text: str) -> str: return "backup" if "Memory läuft ab" in alert_text: return "memory_expiry" + if "wp-mirror" in alert_text: + return "mirror_sync" return "default"