Fix: Edelmetall Scraper Pro Aurum CSS Modules + Preisreihenfolge (24.02.2026)

edelmetall/code/prices.py

@@ -0,0 +1,150 @@
+"""Einheitliche Preis-API — eine Quelle der Wahrheit für Spot + Händler."""
+import requests
+import sqlite3
+from typing import Optional
+from .db import get_db
+
+OZ = 31.1035  # Gramm pro Unze
+
+
+def get_exchange_rate() -> float:
+    """EUR/USD Kurs aus DB, Fallback 0.92."""
+    try:
+        conn = get_db()
+        row = conn.execute(
+            "SELECT rate FROM exchange_rates ORDER BY timestamp DESC LIMIT 1"
+        ).fetchone()
+        conn.close()
+        return row["rate"] if row else 0.92
+    except Exception:
+        return 0.92
+
+
+def fetch_spot() -> Optional[dict]:
+    """
+    Holt Gold/Silber Spotpreise.
+    Gibt EUR und USD zurück — immer beide Einheiten klar benannt.
+    """
+    try:
+        r_gold   = requests.get("https://api.gold-api.com/price/XAU", timeout=10)
+        r_silver = requests.get("https://api.gold-api.com/price/XAG", timeout=10)
+
+        gold_usd   = r_gold.json().get("price", 0)
+        silver_usd = r_silver.json().get("price", 0)
+        eur_usd    = get_exchange_rate()
+
+        gold_eur   = gold_usd   * eur_usd
+        silver_eur = silver_usd * eur_usd
+
+        return {
+            # USD pro Unze
+            "gold_usd_oz":    gold_usd,
+            "silver_usd_oz":  silver_usd,
+            # EUR pro Unze
+            "gold_eur_oz":    gold_eur,
+            "silver_eur_oz":  silver_eur,
+            # EUR pro Gramm (für Validierung + Portfolio)
+            "gold_eur_g":     gold_eur   / OZ,
+            "silver_eur_g":   silver_eur / OZ,
+            # Meta
+            "eur_usd":        eur_usd,
+            "ratio":          gold_usd / silver_usd if silver_usd > 0 else 0,
+        }
+    except Exception as e:
+        return None
+
+
+def save_spot(spot: dict):
+    """Spotpreis in DB speichern."""
+    conn = get_db()
+    conn.execute("""
+        INSERT INTO spot_prices (gold_usd, silver_usd, gold_eur, silver_eur, eur_usd)
+        VALUES (?, ?, ?, ?, ?)
+    """, (
+        spot["gold_usd_oz"], spot["silver_usd_oz"],
+        spot["gold_eur_oz"], spot["silver_eur_oz"],
+        spot["eur_usd"],
+    ))
+    conn.commit()
+    conn.close()
+
+
+def get_latest_spot() -> Optional[dict]:
+    """Letzten gespeicherten Spotpreis aus DB."""
+    conn = get_db()
+    row = conn.execute("""
+        SELECT gold_usd, silver_usd, gold_eur, silver_eur, eur_usd, timestamp
+        FROM spot_prices ORDER BY timestamp DESC LIMIT 1
+    """).fetchone()
+    conn.close()
+    if not row:
+        return None
+    eur_usd = row["eur_usd"] or 0.92
+    gold_eur_oz   = row["gold_eur"]   or (row["gold_usd"]   * eur_usd)
+    silver_eur_oz = row["silver_eur"] or (row["silver_usd"] * eur_usd)
+    return {
+        "gold_usd_oz":   row["gold_usd"],
+        "silver_usd_oz": row["silver_usd"],
+        "gold_eur_oz":   gold_eur_oz,
+        "silver_eur_oz": silver_eur_oz,
+        "gold_eur_g":    gold_eur_oz   / OZ,
+        "silver_eur_g":  silver_eur_oz / OZ,
+        "eur_usd":       eur_usd,
+        "ratio":         row["gold_usd"] / row["silver_usd"] if row["silver_usd"] else 0,
+        "timestamp":     row["timestamp"],
+    }
+
+
+def get_best_dealer_prices(hours: int = 6) -> dict:
+    """
+    Bester Ankaufpreis für 1oz Münzen der letzten N Stunden.
+    Einfach, direkt, keine komplexe Filterlogik.
+    """
+    conn = get_db()
+
+    # Gold: Krügerrand bevorzugen, dann andere Standardmünzen (Median statt Max)
+    gold_rows = conn.execute("""
+        SELECT product, buy_price, sell_price, weight_g,
+               COALESCE(buy_price, sell_price) / weight_g AS ppg
+        FROM gold_prices
+        WHERE weight_g BETWEEN 30 AND 32
+          AND COALESCE(buy_price, sell_price) > 0
+          AND timestamp >= datetime('now', ?)
+        ORDER BY ppg
+    """, (f"-{hours} hours",)).fetchall()
+
+    # Krügerrand bevorzugen, sonst Median
+    gold = next(
+        (r for r in gold_rows if "krügerrand" in r["product"].lower() or "kruger" in r["product"].lower()),
+        gold_rows[len(gold_rows)//2] if gold_rows else None
+    )
+
+    # Silber: Standardmünzen (Maple, Philharmoniker etc.), Median
+    silver_rows = conn.execute("""
+        SELECT product, buy_price, sell_price, weight_g,
+               COALESCE(buy_price, sell_price) / weight_g AS ppg
+        FROM silver_prices
+        WHERE weight_g BETWEEN 30 AND 32
+          AND COALESCE(buy_price, sell_price) > 0
+          AND timestamp >= datetime('now', ?)
+        ORDER BY ppg
+    """, (f"-{hours} hours",)).fetchall()
+
+    silver = next(
+        (r for r in silver_rows if any(k in r["product"].lower()
+         for k in ["maple", "philharmoniker", "känguru", "britannia", "kruger"])),
+        silver_rows[len(silver_rows)//2] if silver_rows else None
+    )
+
+    conn.close()
+    def _price(row):
+        if not row: return 0
+        return row["buy_price"] if row["buy_price"] else (row["sell_price"] if row["sell_price"] else round(row["ppg"] * 31.1035, 2))
+    return {
+        "gold_ppg":     gold["ppg"]    if gold   else 0,
+        "gold_oz":      _price(gold),
+        "gold_product": gold["product"]   if gold   else "—",
+        "silver_ppg":     silver["ppg"]    if silver else 0,
+        "silver_oz":      _price(silver),
+        "silver_product": silver["product"]   if silver else "—",
+    }

edelmetall/code/proaurum.py

@@ -0,0 +1,135 @@
+"""Pro Aurum Scraper — Selenium, Stealth Mode."""
+import re
+import random
+import time
+import logging
+from datetime import datetime
+
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.chrome.service import Service
+
+logger = logging.getLogger(__name__)
+
+OZ = 31.1035
+
+GOLD_URL   = "https://www.proaurum.de/shop/gold/goldmuenzen-zur-kapitalanlage/"
+SILVER_URL = "https://www.proaurum.de/shop/silber/silbermuenzen/"
+
+USER_AGENTS = [
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/121.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/121.0.0.0 Safari/537.36",
+]
+
+
+def _make_driver() -> webdriver.Chrome:
+    opts = Options()
+    opts.add_argument(f"--user-agent={random.choice(USER_AGENTS)}")
+    opts.add_argument("--headless=new")
+    opts.add_argument("--no-sandbox")
+    opts.add_argument("--disable-dev-shm-usage")
+    opts.add_argument("--disable-blink-features=AutomationControlled")
+    opts.add_experimental_option("excludeSwitches", ["enable-automation"])
+    opts.add_experimental_option("useAutomationExtension", False)
+    driver = webdriver.Chrome(service=Service("/bin/chromedriver"), options=opts)
+    driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
+        "source": "Object.defineProperty(navigator, 'webdriver', {get: () => undefined});"
+    })
+    return driver
+
+
+def _parse_price(text: str) -> float | None:
+    cleaned = text.replace(".", "").replace(",", ".").replace("€", "").strip()
+    try:
+        return float(cleaned)
+    except ValueError:
+        return None
+
+
+def _weight_from_name(name: str) -> float:
+    n = name.lower()
+    if "10 kg" in n or "10 kilogramm" in n: return 10000.0
+    if "1 kg" in n or "kilogramm" in n:     return 1000.0
+    if "10 unzen" in n or "10 oz" in n:     return OZ * 10
+    if "1/2" in n:  return OZ / 2
+    if "1/4" in n:  return OZ / 4
+    if "1/10" in n: return OZ / 10
+    return OZ  # Standard 1oz
+
+
+def _scrape_page(driver: webdriver.Chrome, url: str) -> list[dict]:
+    logger.info(f"Lade {url}")
+    driver.get(url)
+    time.sleep(random.uniform(4, 7))
+
+    products = []
+    # Pro Aurum CSS Modules (Stand 2026-02)
+    # productButton-buy  = "Kaufen"     = Kaufpreis (sell_price, was Kunde zahlt)
+    # productButton-sell = "Verkaufen"  = Ankaufspreis (buy_price, was Händler zahlt)
+    # Preise stehen in buySellSection-price: erst Kauf-, dann Ankaufspreis
+    cards = driver.find_elements("css selector", "[class*='product-root']")
+
+    for card in cards:
+        try:
+            # Name: Text ohne € Zeichen
+            name = ""
+            for sel in ["a", "h2", "h3", "[class*='name']"]:
+                els = card.find_elements("css selector", sel)
+                for el in els:
+                    t = el.text.strip()
+                    if t and len(t) > 5 and "€" not in t and "Kaufen" not in t and "Verkaufen" not in t:
+                        name = t.split("\n")[0].strip()
+                        break
+                if name:
+                    break
+
+            # Beide Preise in Reihenfolge holen
+            # [0] = Kaufpreis (sell_price), [1] = Ankaufspreis (buy_price)
+            price_els = card.find_elements("css selector", "[class*='buySellSection-price']")
+            sell = _parse_price(price_els[0].text) if len(price_els) > 0 else None
+            buy  = _parse_price(price_els[1].text) if len(price_els) > 1 else None
+
+            # Fallback: alle €-Preise wenn buySellSection nicht gefunden
+            if not sell:
+                for el in card.find_elements("css selector", "[class*='price'], [class*='Price']"):
+                    v = _parse_price(el.text)
+                    if v and v > 100:
+                        sell = v
+                        break
+
+            weight = _weight_from_name(name)
+
+            if sell and sell > 0:
+                products.append({
+                    "product":    name,
+                    "sell_price": sell,
+                    "buy_price":  buy,
+                    "weight_g":   weight,
+                })
+        except Exception:
+            continue
+
+    return products
+
+
+def scrape() -> dict:
+    """
+    Scrapt Pro Aurum. Gibt {'gold': [...], 'silver': [...]} zurück.
+    Wirft Exception wenn es scheitert — Fallback wird vom Aufrufer gehandelt.
+    """
+    t0     = datetime.now()
+    driver = _make_driver()
+    try:
+        # Zufällige Wartezeit (Anti-Bot)
+        wait = random.randint(60, 150)
+        logger.info(f"Warte {wait}s vor dem Scraping...")
+        time.sleep(wait)
+
+        gold   = _scrape_page(driver, GOLD_URL)
+        silver = _scrape_page(driver, SILVER_URL)
+
+        duration = (datetime.now() - t0).total_seconds()
+        logger.info(f"Pro Aurum: {len(gold)} Gold, {len(silver)} Silber in {duration:.0f}s")
+        return {"gold": gold, "silver": silver, "source": "proaurum"}
+    finally:
+        driver.quit()

edelmetall/code/scrape.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+"""
+Haupt-Scraper: Pro Aurum primär, Degussa als Fallback.
+Wird per Cron aufgerufen.
+"""
+import sys
+import logging
+from datetime import datetime
+sys.path.insert(0, "/opt/edelmetall")
+
+from core.db import get_db, init_db
+from core.prices import fetch_spot, save_spot
+from scrapers import proaurum, degussa
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(message)s",
+    handlers=[
+        logging.FileHandler("/opt/edelmetall/logs/scraper.log"),
+        logging.StreamHandler(),
+    ]
+)
+logger = logging.getLogger(__name__)
+
+
+def save_prices(data: dict):
+    """Speichert Scraper-Ergebnis in DB."""
+    conn  = get_db()
+    c     = conn.cursor()
+    source = data["source"]
+
+    for p in data["gold"]:
+        c.execute("""
+            INSERT INTO gold_prices (product, buy_price, sell_price, weight_g, source)
+            VALUES (?, ?, ?, ?, ?)
+        """, (p["product"], p.get("buy_price"), p.get("sell_price"), p["weight_g"], source))
+
+    for p in data["silver"]:
+        c.execute("""
+            INSERT INTO silver_prices (product, buy_price, sell_price, weight_g, source)
+            VALUES (?, ?, ?, ?, ?)
+        """, (p["product"], p.get("buy_price"), p.get("sell_price"), p["weight_g"], source))
+
+    conn.commit()
+    conn.close()
+
+
+def log_run(source: str, success: bool, gold: int, silver: int, duration: float, error: str = None):
+    conn = get_db()
+    conn.execute("""
+        INSERT INTO scraper_log (source, success, gold_count, silver_count, duration_s, error)
+        VALUES (?, ?, ?, ?, ?, ?)
+    """, (source, int(success), gold, silver, duration, error))
+    conn.commit()
+    conn.close()
+
+
+def main():
+    init_db()
+    t0 = datetime.now()
+
+    # 1. Spot-Preise holen
+    spot = fetch_spot()
+    if spot:
+        save_spot(spot)
+        logger.info(f"Spot: Gold {spot['gold_eur_oz']:.2f} EUR/oz | Silber {spot['silver_eur_oz']:.2f} EUR/oz")
+    else:
+        logger.warning("Spot-Preise nicht abrufbar")
+
+    # 2. Händlerpreise: Pro Aurum → Degussa Fallback
+    data     = None
+    source   = None
+    error    = None
+
+    try:
+        logger.info("Versuche Pro Aurum (Selenium)...")
+        data   = proaurum.scrape()
+        source = "proaurum"
+        logger.info(f"Pro Aurum: {len(data['gold'])} Gold, {len(data['silver'])} Silber")
+        if not data["gold"] and not data["silver"]:
+            raise ValueError("Pro Aurum: 0 Produkte — Seite geaendert, Fallback zu Degussa")
+    except Exception as e:
+        logger.warning(f"Pro Aurum gescheitert: {e} — wechsle zu Degussa")
+        error = str(e)
+        try:
+            data   = degussa.scrape()
+            source = "degussa"
+            error  = None
+            logger.info(f"Degussa Fallback: {len(data['gold'])} Gold, {len(data['silver'])} Silber")
+        except Exception as e2:
+            logger.error(f"Degussa auch gescheitert: {e2}")
+            log_run("both_failed", False, 0, 0,
+                    (datetime.now() - t0).total_seconds(), f"PA: {e} | DG: {e2}")
+            sys.exit(1)
+
+    save_prices(data)
+    duration = (datetime.now() - t0).total_seconds()
+    log_run(source, True, len(data["gold"]), len(data["silver"]), duration, error)
+
+    logger.info(f"=== Fertig in {duration:.0f}s: {len(data['gold'])} Gold, {len(data['silver'])} Silber ===")
+
+
+if __name__ == "__main__":
+    main()