Fix: Edelmetall Scraper Pro Aurum CSS Modules + Preisreihenfolge (24.02.2026)
This commit is contained in:
Auto-Sync
parent
3c85cc81d2
commit
c3c5c8a9cc
3 changed files with 389 additions and 0 deletions
150
edelmetall/code/prices.py
Normal file
150
edelmetall/code/prices.py
Normal file
|
|
@ -0,0 +1,150 @@
|
|||
"""Einheitliche Preis-API — eine Quelle der Wahrheit für Spot + Händler."""
|
||||
import requests
|
||||
import sqlite3
|
||||
from typing import Optional
|
||||
from .db import get_db
|
||||
|
||||
OZ = 31.1035 # Gramm pro Unze
|
||||
|
||||
|
||||
def get_exchange_rate() -> float:
|
||||
"""EUR/USD Kurs aus DB, Fallback 0.92."""
|
||||
try:
|
||||
conn = get_db()
|
||||
row = conn.execute(
|
||||
"SELECT rate FROM exchange_rates ORDER BY timestamp DESC LIMIT 1"
|
||||
).fetchone()
|
||||
conn.close()
|
||||
return row["rate"] if row else 0.92
|
||||
except Exception:
|
||||
return 0.92
|
||||
|
||||
|
||||
def fetch_spot() -> Optional[dict]:
|
||||
"""
|
||||
Holt Gold/Silber Spotpreise.
|
||||
Gibt EUR und USD zurück — immer beide Einheiten klar benannt.
|
||||
"""
|
||||
try:
|
||||
r_gold = requests.get("https://api.gold-api.com/price/XAU", timeout=10)
|
||||
r_silver = requests.get("https://api.gold-api.com/price/XAG", timeout=10)
|
||||
|
||||
gold_usd = r_gold.json().get("price", 0)
|
||||
silver_usd = r_silver.json().get("price", 0)
|
||||
eur_usd = get_exchange_rate()
|
||||
|
||||
gold_eur = gold_usd * eur_usd
|
||||
silver_eur = silver_usd * eur_usd
|
||||
|
||||
return {
|
||||
# USD pro Unze
|
||||
"gold_usd_oz": gold_usd,
|
||||
"silver_usd_oz": silver_usd,
|
||||
# EUR pro Unze
|
||||
"gold_eur_oz": gold_eur,
|
||||
"silver_eur_oz": silver_eur,
|
||||
# EUR pro Gramm (für Validierung + Portfolio)
|
||||
"gold_eur_g": gold_eur / OZ,
|
||||
"silver_eur_g": silver_eur / OZ,
|
||||
# Meta
|
||||
"eur_usd": eur_usd,
|
||||
"ratio": gold_usd / silver_usd if silver_usd > 0 else 0,
|
||||
}
|
||||
except Exception as e:
|
||||
return None
|
||||
|
||||
|
||||
def save_spot(spot: dict):
|
||||
"""Spotpreis in DB speichern."""
|
||||
conn = get_db()
|
||||
conn.execute("""
|
||||
INSERT INTO spot_prices (gold_usd, silver_usd, gold_eur, silver_eur, eur_usd)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
""", (
|
||||
spot["gold_usd_oz"], spot["silver_usd_oz"],
|
||||
spot["gold_eur_oz"], spot["silver_eur_oz"],
|
||||
spot["eur_usd"],
|
||||
))
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
|
||||
def get_latest_spot() -> Optional[dict]:
|
||||
"""Letzten gespeicherten Spotpreis aus DB."""
|
||||
conn = get_db()
|
||||
row = conn.execute("""
|
||||
SELECT gold_usd, silver_usd, gold_eur, silver_eur, eur_usd, timestamp
|
||||
FROM spot_prices ORDER BY timestamp DESC LIMIT 1
|
||||
""").fetchone()
|
||||
conn.close()
|
||||
if not row:
|
||||
return None
|
||||
eur_usd = row["eur_usd"] or 0.92
|
||||
gold_eur_oz = row["gold_eur"] or (row["gold_usd"] * eur_usd)
|
||||
silver_eur_oz = row["silver_eur"] or (row["silver_usd"] * eur_usd)
|
||||
return {
|
||||
"gold_usd_oz": row["gold_usd"],
|
||||
"silver_usd_oz": row["silver_usd"],
|
||||
"gold_eur_oz": gold_eur_oz,
|
||||
"silver_eur_oz": silver_eur_oz,
|
||||
"gold_eur_g": gold_eur_oz / OZ,
|
||||
"silver_eur_g": silver_eur_oz / OZ,
|
||||
"eur_usd": eur_usd,
|
||||
"ratio": row["gold_usd"] / row["silver_usd"] if row["silver_usd"] else 0,
|
||||
"timestamp": row["timestamp"],
|
||||
}
|
||||
|
||||
|
||||
def get_best_dealer_prices(hours: int = 6) -> dict:
|
||||
"""
|
||||
Bester Ankaufpreis für 1oz Münzen der letzten N Stunden.
|
||||
Einfach, direkt, keine komplexe Filterlogik.
|
||||
"""
|
||||
conn = get_db()
|
||||
|
||||
# Gold: Krügerrand bevorzugen, dann andere Standardmünzen (Median statt Max)
|
||||
gold_rows = conn.execute("""
|
||||
SELECT product, buy_price, sell_price, weight_g,
|
||||
COALESCE(buy_price, sell_price) / weight_g AS ppg
|
||||
FROM gold_prices
|
||||
WHERE weight_g BETWEEN 30 AND 32
|
||||
AND COALESCE(buy_price, sell_price) > 0
|
||||
AND timestamp >= datetime('now', ?)
|
||||
ORDER BY ppg
|
||||
""", (f"-{hours} hours",)).fetchall()
|
||||
|
||||
# Krügerrand bevorzugen, sonst Median
|
||||
gold = next(
|
||||
(r for r in gold_rows if "krügerrand" in r["product"].lower() or "kruger" in r["product"].lower()),
|
||||
gold_rows[len(gold_rows)//2] if gold_rows else None
|
||||
)
|
||||
|
||||
# Silber: Standardmünzen (Maple, Philharmoniker etc.), Median
|
||||
silver_rows = conn.execute("""
|
||||
SELECT product, buy_price, sell_price, weight_g,
|
||||
COALESCE(buy_price, sell_price) / weight_g AS ppg
|
||||
FROM silver_prices
|
||||
WHERE weight_g BETWEEN 30 AND 32
|
||||
AND COALESCE(buy_price, sell_price) > 0
|
||||
AND timestamp >= datetime('now', ?)
|
||||
ORDER BY ppg
|
||||
""", (f"-{hours} hours",)).fetchall()
|
||||
|
||||
silver = next(
|
||||
(r for r in silver_rows if any(k in r["product"].lower()
|
||||
for k in ["maple", "philharmoniker", "känguru", "britannia", "kruger"])),
|
||||
silver_rows[len(silver_rows)//2] if silver_rows else None
|
||||
)
|
||||
|
||||
conn.close()
|
||||
def _price(row):
|
||||
if not row: return 0
|
||||
return row["buy_price"] if row["buy_price"] else (row["sell_price"] if row["sell_price"] else round(row["ppg"] * 31.1035, 2))
|
||||
return {
|
||||
"gold_ppg": gold["ppg"] if gold else 0,
|
||||
"gold_oz": _price(gold),
|
||||
"gold_product": gold["product"] if gold else "—",
|
||||
"silver_ppg": silver["ppg"] if silver else 0,
|
||||
"silver_oz": _price(silver),
|
||||
"silver_product": silver["product"] if silver else "—",
|
||||
}
|
||||
135
edelmetall/code/proaurum.py
Normal file
135
edelmetall/code/proaurum.py
Normal file
|
|
@ -0,0 +1,135 @@
|
|||
"""Pro Aurum Scraper — Selenium, Stealth Mode."""
|
||||
import re
|
||||
import random
|
||||
import time
|
||||
import logging
|
||||
from datetime import datetime
|
||||
|
||||
from selenium import webdriver
|
||||
from selenium.webdriver.chrome.options import Options
|
||||
from selenium.webdriver.chrome.service import Service
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
OZ = 31.1035
|
||||
|
||||
GOLD_URL = "https://www.proaurum.de/shop/gold/goldmuenzen-zur-kapitalanlage/"
|
||||
SILVER_URL = "https://www.proaurum.de/shop/silber/silbermuenzen/"
|
||||
|
||||
USER_AGENTS = [
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/121.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/121.0.0.0 Safari/537.36",
|
||||
]
|
||||
|
||||
|
||||
def _make_driver() -> webdriver.Chrome:
|
||||
opts = Options()
|
||||
opts.add_argument(f"--user-agent={random.choice(USER_AGENTS)}")
|
||||
opts.add_argument("--headless=new")
|
||||
opts.add_argument("--no-sandbox")
|
||||
opts.add_argument("--disable-dev-shm-usage")
|
||||
opts.add_argument("--disable-blink-features=AutomationControlled")
|
||||
opts.add_experimental_option("excludeSwitches", ["enable-automation"])
|
||||
opts.add_experimental_option("useAutomationExtension", False)
|
||||
driver = webdriver.Chrome(service=Service("/bin/chromedriver"), options=opts)
|
||||
driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
|
||||
"source": "Object.defineProperty(navigator, 'webdriver', {get: () => undefined});"
|
||||
})
|
||||
return driver
|
||||
|
||||
|
||||
def _parse_price(text: str) -> float | None:
|
||||
cleaned = text.replace(".", "").replace(",", ".").replace("€", "").strip()
|
||||
try:
|
||||
return float(cleaned)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _weight_from_name(name: str) -> float:
|
||||
n = name.lower()
|
||||
if "10 kg" in n or "10 kilogramm" in n: return 10000.0
|
||||
if "1 kg" in n or "kilogramm" in n: return 1000.0
|
||||
if "10 unzen" in n or "10 oz" in n: return OZ * 10
|
||||
if "1/2" in n: return OZ / 2
|
||||
if "1/4" in n: return OZ / 4
|
||||
if "1/10" in n: return OZ / 10
|
||||
return OZ # Standard 1oz
|
||||
|
||||
|
||||
def _scrape_page(driver: webdriver.Chrome, url: str) -> list[dict]:
|
||||
logger.info(f"Lade {url}")
|
||||
driver.get(url)
|
||||
time.sleep(random.uniform(4, 7))
|
||||
|
||||
products = []
|
||||
# Pro Aurum CSS Modules (Stand 2026-02)
|
||||
# productButton-buy = "Kaufen" = Kaufpreis (sell_price, was Kunde zahlt)
|
||||
# productButton-sell = "Verkaufen" = Ankaufspreis (buy_price, was Händler zahlt)
|
||||
# Preise stehen in buySellSection-price: erst Kauf-, dann Ankaufspreis
|
||||
cards = driver.find_elements("css selector", "[class*='product-root']")
|
||||
|
||||
for card in cards:
|
||||
try:
|
||||
# Name: Text ohne € Zeichen
|
||||
name = ""
|
||||
for sel in ["a", "h2", "h3", "[class*='name']"]:
|
||||
els = card.find_elements("css selector", sel)
|
||||
for el in els:
|
||||
t = el.text.strip()
|
||||
if t and len(t) > 5 and "€" not in t and "Kaufen" not in t and "Verkaufen" not in t:
|
||||
name = t.split("\n")[0].strip()
|
||||
break
|
||||
if name:
|
||||
break
|
||||
|
||||
# Beide Preise in Reihenfolge holen
|
||||
# [0] = Kaufpreis (sell_price), [1] = Ankaufspreis (buy_price)
|
||||
price_els = card.find_elements("css selector", "[class*='buySellSection-price']")
|
||||
sell = _parse_price(price_els[0].text) if len(price_els) > 0 else None
|
||||
buy = _parse_price(price_els[1].text) if len(price_els) > 1 else None
|
||||
|
||||
# Fallback: alle €-Preise wenn buySellSection nicht gefunden
|
||||
if not sell:
|
||||
for el in card.find_elements("css selector", "[class*='price'], [class*='Price']"):
|
||||
v = _parse_price(el.text)
|
||||
if v and v > 100:
|
||||
sell = v
|
||||
break
|
||||
|
||||
weight = _weight_from_name(name)
|
||||
|
||||
if sell and sell > 0:
|
||||
products.append({
|
||||
"product": name,
|
||||
"sell_price": sell,
|
||||
"buy_price": buy,
|
||||
"weight_g": weight,
|
||||
})
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
return products
|
||||
|
||||
|
||||
def scrape() -> dict:
|
||||
"""
|
||||
Scrapt Pro Aurum. Gibt {'gold': [...], 'silver': [...]} zurück.
|
||||
Wirft Exception wenn es scheitert — Fallback wird vom Aufrufer gehandelt.
|
||||
"""
|
||||
t0 = datetime.now()
|
||||
driver = _make_driver()
|
||||
try:
|
||||
# Zufällige Wartezeit (Anti-Bot)
|
||||
wait = random.randint(60, 150)
|
||||
logger.info(f"Warte {wait}s vor dem Scraping...")
|
||||
time.sleep(wait)
|
||||
|
||||
gold = _scrape_page(driver, GOLD_URL)
|
||||
silver = _scrape_page(driver, SILVER_URL)
|
||||
|
||||
duration = (datetime.now() - t0).total_seconds()
|
||||
logger.info(f"Pro Aurum: {len(gold)} Gold, {len(silver)} Silber in {duration:.0f}s")
|
||||
return {"gold": gold, "silver": silver, "source": "proaurum"}
|
||||
finally:
|
||||
driver.quit()
|
||||
104
edelmetall/code/scrape.py
Normal file
104
edelmetall/code/scrape.py
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
Haupt-Scraper: Pro Aurum primär, Degussa als Fallback.
|
||||
Wird per Cron aufgerufen.
|
||||
"""
|
||||
import sys
|
||||
import logging
|
||||
from datetime import datetime
|
||||
sys.path.insert(0, "/opt/edelmetall")
|
||||
|
||||
from core.db import get_db, init_db
|
||||
from core.prices import fetch_spot, save_spot
|
||||
from scrapers import proaurum, degussa
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s [%(levelname)s] %(message)s",
|
||||
handlers=[
|
||||
logging.FileHandler("/opt/edelmetall/logs/scraper.log"),
|
||||
logging.StreamHandler(),
|
||||
]
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def save_prices(data: dict):
|
||||
"""Speichert Scraper-Ergebnis in DB."""
|
||||
conn = get_db()
|
||||
c = conn.cursor()
|
||||
source = data["source"]
|
||||
|
||||
for p in data["gold"]:
|
||||
c.execute("""
|
||||
INSERT INTO gold_prices (product, buy_price, sell_price, weight_g, source)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
""", (p["product"], p.get("buy_price"), p.get("sell_price"), p["weight_g"], source))
|
||||
|
||||
for p in data["silver"]:
|
||||
c.execute("""
|
||||
INSERT INTO silver_prices (product, buy_price, sell_price, weight_g, source)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
""", (p["product"], p.get("buy_price"), p.get("sell_price"), p["weight_g"], source))
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
|
||||
def log_run(source: str, success: bool, gold: int, silver: int, duration: float, error: str = None):
|
||||
conn = get_db()
|
||||
conn.execute("""
|
||||
INSERT INTO scraper_log (source, success, gold_count, silver_count, duration_s, error)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
""", (source, int(success), gold, silver, duration, error))
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
|
||||
def main():
|
||||
init_db()
|
||||
t0 = datetime.now()
|
||||
|
||||
# 1. Spot-Preise holen
|
||||
spot = fetch_spot()
|
||||
if spot:
|
||||
save_spot(spot)
|
||||
logger.info(f"Spot: Gold {spot['gold_eur_oz']:.2f} EUR/oz | Silber {spot['silver_eur_oz']:.2f} EUR/oz")
|
||||
else:
|
||||
logger.warning("Spot-Preise nicht abrufbar")
|
||||
|
||||
# 2. Händlerpreise: Pro Aurum → Degussa Fallback
|
||||
data = None
|
||||
source = None
|
||||
error = None
|
||||
|
||||
try:
|
||||
logger.info("Versuche Pro Aurum (Selenium)...")
|
||||
data = proaurum.scrape()
|
||||
source = "proaurum"
|
||||
logger.info(f"Pro Aurum: {len(data['gold'])} Gold, {len(data['silver'])} Silber")
|
||||
if not data["gold"] and not data["silver"]:
|
||||
raise ValueError("Pro Aurum: 0 Produkte — Seite geaendert, Fallback zu Degussa")
|
||||
except Exception as e:
|
||||
logger.warning(f"Pro Aurum gescheitert: {e} — wechsle zu Degussa")
|
||||
error = str(e)
|
||||
try:
|
||||
data = degussa.scrape()
|
||||
source = "degussa"
|
||||
error = None
|
||||
logger.info(f"Degussa Fallback: {len(data['gold'])} Gold, {len(data['silver'])} Silber")
|
||||
except Exception as e2:
|
||||
logger.error(f"Degussa auch gescheitert: {e2}")
|
||||
log_run("both_failed", False, 0, 0,
|
||||
(datetime.now() - t0).total_seconds(), f"PA: {e} | DG: {e2}")
|
||||
sys.exit(1)
|
||||
|
||||
save_prices(data)
|
||||
duration = (datetime.now() - t0).total_seconds()
|
||||
log_run(source, True, len(data["gold"]), len(data["silver"]), duration, error)
|
||||
|
||||
logger.info(f"=== Fertig in {duration:.0f}s: {len(data['gold'])} Gold, {len(data['silver'])} Silber ===")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Add table
Reference in a new issue