From c3c5c8a9cc9b198adce779bbd08ff768b0cb3c95 Mon Sep 17 00:00:00 2001 From: Auto-Sync Date: Tue, 24 Feb 2026 13:09:21 +0100 Subject: [PATCH] Fix: Edelmetall Scraper Pro Aurum CSS Modules + Preisreihenfolge (24.02.2026) --- edelmetall/code/prices.py | 150 ++++++++++++++++++++++++++++++++++++ edelmetall/code/proaurum.py | 135 ++++++++++++++++++++++++++++++++ edelmetall/code/scrape.py | 104 +++++++++++++++++++++++++ 3 files changed, 389 insertions(+) create mode 100644 edelmetall/code/prices.py create mode 100644 edelmetall/code/proaurum.py create mode 100644 edelmetall/code/scrape.py diff --git a/edelmetall/code/prices.py b/edelmetall/code/prices.py new file mode 100644 index 00000000..fbaa807a --- /dev/null +++ b/edelmetall/code/prices.py @@ -0,0 +1,150 @@ +"""Einheitliche Preis-API — eine Quelle der Wahrheit für Spot + Händler.""" +import requests +import sqlite3 +from typing import Optional +from .db import get_db + +OZ = 31.1035 # Gramm pro Unze + + +def get_exchange_rate() -> float: + """EUR/USD Kurs aus DB, Fallback 0.92.""" + try: + conn = get_db() + row = conn.execute( + "SELECT rate FROM exchange_rates ORDER BY timestamp DESC LIMIT 1" + ).fetchone() + conn.close() + return row["rate"] if row else 0.92 + except Exception: + return 0.92 + + +def fetch_spot() -> Optional[dict]: + """ + Holt Gold/Silber Spotpreise. + Gibt EUR und USD zurück — immer beide Einheiten klar benannt. + """ + try: + r_gold = requests.get("https://api.gold-api.com/price/XAU", timeout=10) + r_silver = requests.get("https://api.gold-api.com/price/XAG", timeout=10) + + gold_usd = r_gold.json().get("price", 0) + silver_usd = r_silver.json().get("price", 0) + eur_usd = get_exchange_rate() + + gold_eur = gold_usd * eur_usd + silver_eur = silver_usd * eur_usd + + return { + # USD pro Unze + "gold_usd_oz": gold_usd, + "silver_usd_oz": silver_usd, + # EUR pro Unze + "gold_eur_oz": gold_eur, + "silver_eur_oz": silver_eur, + # EUR pro Gramm (für Validierung + Portfolio) + "gold_eur_g": gold_eur / OZ, + "silver_eur_g": silver_eur / OZ, + # Meta + "eur_usd": eur_usd, + "ratio": gold_usd / silver_usd if silver_usd > 0 else 0, + } + except Exception as e: + return None + + +def save_spot(spot: dict): + """Spotpreis in DB speichern.""" + conn = get_db() + conn.execute(""" + INSERT INTO spot_prices (gold_usd, silver_usd, gold_eur, silver_eur, eur_usd) + VALUES (?, ?, ?, ?, ?) + """, ( + spot["gold_usd_oz"], spot["silver_usd_oz"], + spot["gold_eur_oz"], spot["silver_eur_oz"], + spot["eur_usd"], + )) + conn.commit() + conn.close() + + +def get_latest_spot() -> Optional[dict]: + """Letzten gespeicherten Spotpreis aus DB.""" + conn = get_db() + row = conn.execute(""" + SELECT gold_usd, silver_usd, gold_eur, silver_eur, eur_usd, timestamp + FROM spot_prices ORDER BY timestamp DESC LIMIT 1 + """).fetchone() + conn.close() + if not row: + return None + eur_usd = row["eur_usd"] or 0.92 + gold_eur_oz = row["gold_eur"] or (row["gold_usd"] * eur_usd) + silver_eur_oz = row["silver_eur"] or (row["silver_usd"] * eur_usd) + return { + "gold_usd_oz": row["gold_usd"], + "silver_usd_oz": row["silver_usd"], + "gold_eur_oz": gold_eur_oz, + "silver_eur_oz": silver_eur_oz, + "gold_eur_g": gold_eur_oz / OZ, + "silver_eur_g": silver_eur_oz / OZ, + "eur_usd": eur_usd, + "ratio": row["gold_usd"] / row["silver_usd"] if row["silver_usd"] else 0, + "timestamp": row["timestamp"], + } + + +def get_best_dealer_prices(hours: int = 6) -> dict: + """ + Bester Ankaufpreis für 1oz Münzen der letzten N Stunden. + Einfach, direkt, keine komplexe Filterlogik. + """ + conn = get_db() + + # Gold: Krügerrand bevorzugen, dann andere Standardmünzen (Median statt Max) + gold_rows = conn.execute(""" + SELECT product, buy_price, sell_price, weight_g, + COALESCE(buy_price, sell_price) / weight_g AS ppg + FROM gold_prices + WHERE weight_g BETWEEN 30 AND 32 + AND COALESCE(buy_price, sell_price) > 0 + AND timestamp >= datetime('now', ?) + ORDER BY ppg + """, (f"-{hours} hours",)).fetchall() + + # Krügerrand bevorzugen, sonst Median + gold = next( + (r for r in gold_rows if "krügerrand" in r["product"].lower() or "kruger" in r["product"].lower()), + gold_rows[len(gold_rows)//2] if gold_rows else None + ) + + # Silber: Standardmünzen (Maple, Philharmoniker etc.), Median + silver_rows = conn.execute(""" + SELECT product, buy_price, sell_price, weight_g, + COALESCE(buy_price, sell_price) / weight_g AS ppg + FROM silver_prices + WHERE weight_g BETWEEN 30 AND 32 + AND COALESCE(buy_price, sell_price) > 0 + AND timestamp >= datetime('now', ?) + ORDER BY ppg + """, (f"-{hours} hours",)).fetchall() + + silver = next( + (r for r in silver_rows if any(k in r["product"].lower() + for k in ["maple", "philharmoniker", "känguru", "britannia", "kruger"])), + silver_rows[len(silver_rows)//2] if silver_rows else None + ) + + conn.close() + def _price(row): + if not row: return 0 + return row["buy_price"] if row["buy_price"] else (row["sell_price"] if row["sell_price"] else round(row["ppg"] * 31.1035, 2)) + return { + "gold_ppg": gold["ppg"] if gold else 0, + "gold_oz": _price(gold), + "gold_product": gold["product"] if gold else "—", + "silver_ppg": silver["ppg"] if silver else 0, + "silver_oz": _price(silver), + "silver_product": silver["product"] if silver else "—", + } diff --git a/edelmetall/code/proaurum.py b/edelmetall/code/proaurum.py new file mode 100644 index 00000000..7767558b --- /dev/null +++ b/edelmetall/code/proaurum.py @@ -0,0 +1,135 @@ +"""Pro Aurum Scraper — Selenium, Stealth Mode.""" +import re +import random +import time +import logging +from datetime import datetime + +from selenium import webdriver +from selenium.webdriver.chrome.options import Options +from selenium.webdriver.chrome.service import Service + +logger = logging.getLogger(__name__) + +OZ = 31.1035 + +GOLD_URL = "https://www.proaurum.de/shop/gold/goldmuenzen-zur-kapitalanlage/" +SILVER_URL = "https://www.proaurum.de/shop/silber/silbermuenzen/" + +USER_AGENTS = [ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/121.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/121.0.0.0 Safari/537.36", +] + + +def _make_driver() -> webdriver.Chrome: + opts = Options() + opts.add_argument(f"--user-agent={random.choice(USER_AGENTS)}") + opts.add_argument("--headless=new") + opts.add_argument("--no-sandbox") + opts.add_argument("--disable-dev-shm-usage") + opts.add_argument("--disable-blink-features=AutomationControlled") + opts.add_experimental_option("excludeSwitches", ["enable-automation"]) + opts.add_experimental_option("useAutomationExtension", False) + driver = webdriver.Chrome(service=Service("/bin/chromedriver"), options=opts) + driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", { + "source": "Object.defineProperty(navigator, 'webdriver', {get: () => undefined});" + }) + return driver + + +def _parse_price(text: str) -> float | None: + cleaned = text.replace(".", "").replace(",", ".").replace("€", "").strip() + try: + return float(cleaned) + except ValueError: + return None + + +def _weight_from_name(name: str) -> float: + n = name.lower() + if "10 kg" in n or "10 kilogramm" in n: return 10000.0 + if "1 kg" in n or "kilogramm" in n: return 1000.0 + if "10 unzen" in n or "10 oz" in n: return OZ * 10 + if "1/2" in n: return OZ / 2 + if "1/4" in n: return OZ / 4 + if "1/10" in n: return OZ / 10 + return OZ # Standard 1oz + + +def _scrape_page(driver: webdriver.Chrome, url: str) -> list[dict]: + logger.info(f"Lade {url}") + driver.get(url) + time.sleep(random.uniform(4, 7)) + + products = [] + # Pro Aurum CSS Modules (Stand 2026-02) + # productButton-buy = "Kaufen" = Kaufpreis (sell_price, was Kunde zahlt) + # productButton-sell = "Verkaufen" = Ankaufspreis (buy_price, was Händler zahlt) + # Preise stehen in buySellSection-price: erst Kauf-, dann Ankaufspreis + cards = driver.find_elements("css selector", "[class*='product-root']") + + for card in cards: + try: + # Name: Text ohne € Zeichen + name = "" + for sel in ["a", "h2", "h3", "[class*='name']"]: + els = card.find_elements("css selector", sel) + for el in els: + t = el.text.strip() + if t and len(t) > 5 and "€" not in t and "Kaufen" not in t and "Verkaufen" not in t: + name = t.split("\n")[0].strip() + break + if name: + break + + # Beide Preise in Reihenfolge holen + # [0] = Kaufpreis (sell_price), [1] = Ankaufspreis (buy_price) + price_els = card.find_elements("css selector", "[class*='buySellSection-price']") + sell = _parse_price(price_els[0].text) if len(price_els) > 0 else None + buy = _parse_price(price_els[1].text) if len(price_els) > 1 else None + + # Fallback: alle €-Preise wenn buySellSection nicht gefunden + if not sell: + for el in card.find_elements("css selector", "[class*='price'], [class*='Price']"): + v = _parse_price(el.text) + if v and v > 100: + sell = v + break + + weight = _weight_from_name(name) + + if sell and sell > 0: + products.append({ + "product": name, + "sell_price": sell, + "buy_price": buy, + "weight_g": weight, + }) + except Exception: + continue + + return products + + +def scrape() -> dict: + """ + Scrapt Pro Aurum. Gibt {'gold': [...], 'silver': [...]} zurück. + Wirft Exception wenn es scheitert — Fallback wird vom Aufrufer gehandelt. + """ + t0 = datetime.now() + driver = _make_driver() + try: + # Zufällige Wartezeit (Anti-Bot) + wait = random.randint(60, 150) + logger.info(f"Warte {wait}s vor dem Scraping...") + time.sleep(wait) + + gold = _scrape_page(driver, GOLD_URL) + silver = _scrape_page(driver, SILVER_URL) + + duration = (datetime.now() - t0).total_seconds() + logger.info(f"Pro Aurum: {len(gold)} Gold, {len(silver)} Silber in {duration:.0f}s") + return {"gold": gold, "silver": silver, "source": "proaurum"} + finally: + driver.quit() diff --git a/edelmetall/code/scrape.py b/edelmetall/code/scrape.py new file mode 100644 index 00000000..60ebd3c6 --- /dev/null +++ b/edelmetall/code/scrape.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +""" +Haupt-Scraper: Pro Aurum primär, Degussa als Fallback. +Wird per Cron aufgerufen. +""" +import sys +import logging +from datetime import datetime +sys.path.insert(0, "/opt/edelmetall") + +from core.db import get_db, init_db +from core.prices import fetch_spot, save_spot +from scrapers import proaurum, degussa + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s", + handlers=[ + logging.FileHandler("/opt/edelmetall/logs/scraper.log"), + logging.StreamHandler(), + ] +) +logger = logging.getLogger(__name__) + + +def save_prices(data: dict): + """Speichert Scraper-Ergebnis in DB.""" + conn = get_db() + c = conn.cursor() + source = data["source"] + + for p in data["gold"]: + c.execute(""" + INSERT INTO gold_prices (product, buy_price, sell_price, weight_g, source) + VALUES (?, ?, ?, ?, ?) + """, (p["product"], p.get("buy_price"), p.get("sell_price"), p["weight_g"], source)) + + for p in data["silver"]: + c.execute(""" + INSERT INTO silver_prices (product, buy_price, sell_price, weight_g, source) + VALUES (?, ?, ?, ?, ?) + """, (p["product"], p.get("buy_price"), p.get("sell_price"), p["weight_g"], source)) + + conn.commit() + conn.close() + + +def log_run(source: str, success: bool, gold: int, silver: int, duration: float, error: str = None): + conn = get_db() + conn.execute(""" + INSERT INTO scraper_log (source, success, gold_count, silver_count, duration_s, error) + VALUES (?, ?, ?, ?, ?, ?) + """, (source, int(success), gold, silver, duration, error)) + conn.commit() + conn.close() + + +def main(): + init_db() + t0 = datetime.now() + + # 1. Spot-Preise holen + spot = fetch_spot() + if spot: + save_spot(spot) + logger.info(f"Spot: Gold {spot['gold_eur_oz']:.2f} EUR/oz | Silber {spot['silver_eur_oz']:.2f} EUR/oz") + else: + logger.warning("Spot-Preise nicht abrufbar") + + # 2. Händlerpreise: Pro Aurum → Degussa Fallback + data = None + source = None + error = None + + try: + logger.info("Versuche Pro Aurum (Selenium)...") + data = proaurum.scrape() + source = "proaurum" + logger.info(f"Pro Aurum: {len(data['gold'])} Gold, {len(data['silver'])} Silber") + if not data["gold"] and not data["silver"]: + raise ValueError("Pro Aurum: 0 Produkte — Seite geaendert, Fallback zu Degussa") + except Exception as e: + logger.warning(f"Pro Aurum gescheitert: {e} — wechsle zu Degussa") + error = str(e) + try: + data = degussa.scrape() + source = "degussa" + error = None + logger.info(f"Degussa Fallback: {len(data['gold'])} Gold, {len(data['silver'])} Silber") + except Exception as e2: + logger.error(f"Degussa auch gescheitert: {e2}") + log_run("both_failed", False, 0, 0, + (datetime.now() - t0).total_seconds(), f"PA: {e} | DG: {e2}") + sys.exit(1) + + save_prices(data) + duration = (datetime.now() - t0).total_seconds() + log_run(source, True, len(data["gold"]), len(data["silver"]), duration, error) + + logger.info(f"=== Fertig in {duration:.0f}s: {len(data['gold'])} Gold, {len(data['silver'])} Silber ===") + + +if __name__ == "__main__": + main()