104 lines
3.3 KiB
Python
104 lines
3.3 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Haupt-Scraper: Pro Aurum primär, Degussa als Fallback.
|
|
Wird per Cron aufgerufen.
|
|
"""
|
|
import sys
|
|
import logging
|
|
from datetime import datetime
|
|
sys.path.insert(0, "/opt/edelmetall")
|
|
|
|
from core.db import get_db, init_db
|
|
from core.prices import fetch_spot, save_spot
|
|
from scrapers import proaurum, degussa
|
|
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format="%(asctime)s [%(levelname)s] %(message)s",
|
|
handlers=[
|
|
logging.FileHandler("/opt/edelmetall/logs/scraper.log"),
|
|
logging.StreamHandler(),
|
|
]
|
|
)
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def save_prices(data: dict):
|
|
"""Speichert Scraper-Ergebnis in DB."""
|
|
conn = get_db()
|
|
c = conn.cursor()
|
|
source = data["source"]
|
|
|
|
for p in data["gold"]:
|
|
c.execute("""
|
|
INSERT INTO gold_prices (product, buy_price, sell_price, weight_g, source)
|
|
VALUES (?, ?, ?, ?, ?)
|
|
""", (p["product"], p.get("buy_price"), p.get("sell_price"), p["weight_g"], source))
|
|
|
|
for p in data["silver"]:
|
|
c.execute("""
|
|
INSERT INTO silver_prices (product, buy_price, sell_price, weight_g, source)
|
|
VALUES (?, ?, ?, ?, ?)
|
|
""", (p["product"], p.get("buy_price"), p.get("sell_price"), p["weight_g"], source))
|
|
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
|
|
def log_run(source: str, success: bool, gold: int, silver: int, duration: float, error: str = None):
|
|
conn = get_db()
|
|
conn.execute("""
|
|
INSERT INTO scraper_log (source, success, gold_count, silver_count, duration_s, error)
|
|
VALUES (?, ?, ?, ?, ?, ?)
|
|
""", (source, int(success), gold, silver, duration, error))
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
|
|
def main():
|
|
init_db()
|
|
t0 = datetime.now()
|
|
|
|
# 1. Spot-Preise holen
|
|
spot = fetch_spot()
|
|
if spot:
|
|
save_spot(spot)
|
|
logger.info(f"Spot: Gold {spot['gold_eur_oz']:.2f} EUR/oz | Silber {spot['silver_eur_oz']:.2f} EUR/oz")
|
|
else:
|
|
logger.warning("Spot-Preise nicht abrufbar")
|
|
|
|
# 2. Händlerpreise: Pro Aurum → Degussa Fallback
|
|
data = None
|
|
source = None
|
|
error = None
|
|
|
|
try:
|
|
logger.info("Versuche Pro Aurum (Selenium)...")
|
|
data = proaurum.scrape()
|
|
source = "proaurum"
|
|
logger.info(f"Pro Aurum: {len(data['gold'])} Gold, {len(data['silver'])} Silber")
|
|
if not data["gold"] and not data["silver"]:
|
|
raise ValueError("Pro Aurum: 0 Produkte — Seite geaendert, Fallback zu Degussa")
|
|
except Exception as e:
|
|
logger.warning(f"Pro Aurum gescheitert: {e} — wechsle zu Degussa")
|
|
error = str(e)
|
|
try:
|
|
data = degussa.scrape()
|
|
source = "degussa"
|
|
error = None
|
|
logger.info(f"Degussa Fallback: {len(data['gold'])} Gold, {len(data['silver'])} Silber")
|
|
except Exception as e2:
|
|
logger.error(f"Degussa auch gescheitert: {e2}")
|
|
log_run("both_failed", False, 0, 0,
|
|
(datetime.now() - t0).total_seconds(), f"PA: {e} | DG: {e2}")
|
|
sys.exit(1)
|
|
|
|
save_prices(data)
|
|
duration = (datetime.now() - t0).total_seconds()
|
|
log_run(source, True, len(data["gold"]), len(data["silver"]), duration, error)
|
|
|
|
logger.info(f"=== Fertig in {duration:.0f}s: {len(data['gold'])} Gold, {len(data['silver'])} Silber ===")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|