fix: vision_preis_lokal von Ollama (qwen3-vl:32b) auf OpenRouter (gpt-4o-mini) umgestellt
GPU auf KI-Server wird fuer Coding-Agent (Qwen3-Coder-Next) benoetigt. Roundtrip-Erkennung und Sidebar-Artefakt-Filter bleiben funktional. Inkl. bisher uncommittete Aenderungen (Vision-Pipeline, Flex-Scans, Worker-Updates). Ref: Issue #75 Phase 1
This commit is contained in:
parent
510ef99691
commit
b091649e6a
4 changed files with 376 additions and 28 deletions
|
|
@ -6,6 +6,7 @@ import threading
|
|||
import requests
|
||||
import schedule
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Optional
|
||||
from db import (init_db, get_conn, log, source_health_update,
|
||||
source_health_ist_pausiert, source_health_reset_daily,
|
||||
source_health_get_all, scan_result_save)
|
||||
|
|
@ -17,6 +18,8 @@ _vision_client = OpenAI(
|
|||
base_url="https://openrouter.ai/api/v1",
|
||||
api_key=os.environ.get("OPENROUTER_API_KEY")
|
||||
)
|
||||
OLLAMA_VISION_URL = "http://100.84.255.83:11434"
|
||||
|
||||
|
||||
# ── Telegram ──────────────────────────────────────────────────────────────────
|
||||
TELEGRAM_TOKEN = os.environ.get("TELEGRAM_BOT_TOKEN", "")
|
||||
|
|
@ -194,6 +197,129 @@ def klassifiziere_screenshot(screenshot_b64: str) -> str:
|
|||
return "Unbekannt"
|
||||
|
||||
|
||||
|
||||
def vision_preis_lokal(screenshot_b64: str) -> float | None:
|
||||
"""Vision-KI (gpt-4o-mini via OpenRouter) liest guenstigsten Roundtrip-Preis aus Screenshot.
|
||||
Frueher lokal (qwen3-vl:32b), jetzt Cloud — GPU frei fuer Coding-Agent."""
|
||||
if not screenshot_b64:
|
||||
return None
|
||||
try:
|
||||
prompt = (
|
||||
"Look at this flight search screenshot. "
|
||||
"I need the cheapest ROUNDTRIP (Hin- und Rueckflug) price in EUR from the search results. "
|
||||
"IMPORTANT: Ignore one-way (Hinflug only) prices. Ignore sidebar filters. Ignore ads. "
|
||||
"If the page shows roundtrip results: answer with the cheapest roundtrip price as a number only, e.g.: 872 "
|
||||
"If the page shows only one-way results or no roundtrip prices: answer 0"
|
||||
)
|
||||
response = _vision_client.chat.completions.create(
|
||||
model="openai/gpt-4o-mini",
|
||||
max_tokens=30,
|
||||
temperature=0,
|
||||
messages=[{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": prompt},
|
||||
{"type": "image_url", "image_url": {
|
||||
"url": f"data:image/jpeg;base64,{screenshot_b64}"
|
||||
}}
|
||||
]
|
||||
}]
|
||||
)
|
||||
txt = response.choices[0].message.content.strip()
|
||||
m = re.search(r'\d{3,5}', txt)
|
||||
if m:
|
||||
v = float(m.group(0))
|
||||
if 600 <= v <= 2500:
|
||||
log(f"Vision-Preis: {v:.0f}\u20ac erkannt (OpenRouter)")
|
||||
return v
|
||||
if v == 0:
|
||||
return None
|
||||
except Exception as e:
|
||||
log(f"Vision-Preis Fehler: {e}", "WARN")
|
||||
return None
|
||||
|
||||
def vision_verifiziere_preise(screenshot_b64: str, screenshot_id: int) -> Optional[float]:
|
||||
"""Verifiziert gespeicherte Preise via lokaler Vision-KI (qwen3-vl:32b).
|
||||
|
||||
Ablauf:
|
||||
- KI liest guenstigsten Preis aus Screenshot
|
||||
- Stimmt mit gespeichertem Preis ueberein (<=20% Abweichung): ki_verified=1 setzen
|
||||
- Weicht >20% ab (Sidebar-Artefakt): original plausibel=0 UND neuer Eintrag mit
|
||||
scanner='[scanner]_ki' und ki-Preis angelegt → erscheint als eigener Eintrag im UI
|
||||
"""
|
||||
if not screenshot_b64 or not screenshot_id:
|
||||
return None
|
||||
ki_preis = vision_preis_lokal(screenshot_b64)
|
||||
if ki_preis is None:
|
||||
return None
|
||||
try:
|
||||
from datetime import datetime as _dt
|
||||
conn = get_conn()
|
||||
# Alle Preise dieses Screenshots holen (mit Metadaten fuer neuen Eintrag)
|
||||
rows = conn.execute("""
|
||||
SELECT id, preis, scanner, node, job_id, waehrung, airline, abflug, ankunft,
|
||||
von, nach, booking_url, kabine_erkannt, preis_korrigiert, korrektur_grund
|
||||
FROM prices WHERE screenshot_id=?
|
||||
""", (screenshot_id,)).fetchall()
|
||||
|
||||
if not rows:
|
||||
conn.close()
|
||||
log(f"Vision-Check screenshot {screenshot_id}: keine Preiszeilen zum Abgleich")
|
||||
return None
|
||||
|
||||
now_str = _dt.now().isoformat()
|
||||
korrigiert = 0
|
||||
bestaetigt = 0
|
||||
|
||||
for row in rows:
|
||||
(price_id, raw_preis, scanner, node, job_id, waehrung, airline,
|
||||
abflug, ankunft, von, nach, booking_url, kabine, preis_korr, korr_grund) = row
|
||||
|
||||
diff_pct = abs(raw_preis - ki_preis) / ki_preis if ki_preis > 0 else 1
|
||||
|
||||
if diff_pct > 0.15:
|
||||
# Original als Artefakt markieren
|
||||
conn.execute("""
|
||||
UPDATE prices SET ki_preis_visual=?, ki_verified=0, ki_verified_at=NULL,
|
||||
plausibel=0,
|
||||
plausi_grund='Vision-KI: ' || ROUND(?) || 'EUR vs scraped ' || ROUND(?) || 'EUR (Artefakt — nicht verifiziert)'
|
||||
WHERE id=?
|
||||
""", (ki_preis, ki_preis, raw_preis, price_id))
|
||||
|
||||
# Neuer verifizierter Eintrag (scanner + '_ki')
|
||||
conn.execute("""
|
||||
INSERT OR IGNORE INTO prices
|
||||
(job_id, scanner, node, preis, waehrung, airline, abflug, ankunft,
|
||||
von, nach, booking_url, screenshot_id, kabine_erkannt,
|
||||
plausibel, plausi_grund, preis_korrigiert, korrektur_grund,
|
||||
ki_preis_visual, ki_verified, ki_verified_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1,
|
||||
'Vision-KI verifiziert', ?, ?, ?, 1, ?)
|
||||
""", (
|
||||
job_id, scanner + '_ki', node, ki_preis, waehrung or 'EUR',
|
||||
airline, abflug, ankunft, von, nach, booking_url, screenshot_id,
|
||||
kabine, preis_korr, korr_grund, ki_preis, now_str,
|
||||
))
|
||||
conn.commit()
|
||||
log(f"Vision-Check: {raw_preis:.0f}€ → Artefakt; neuer Eintrag {ki_preis:.0f}€ ({scanner}_ki)")
|
||||
korrigiert += 1
|
||||
else:
|
||||
# Bestaetigt: nur Felder updaten, kein neuer Eintrag noetig
|
||||
conn.execute("""
|
||||
UPDATE prices SET ki_preis_visual=?, ki_verified=1, ki_verified_at=?
|
||||
WHERE id=?
|
||||
""", (ki_preis, now_str, price_id))
|
||||
conn.commit()
|
||||
bestaetigt += 1
|
||||
|
||||
conn.close()
|
||||
log(f"Vision-Check screenshot {screenshot_id}: {bestaetigt} bestaetigt, {korrigiert} korrigiert (KI: {ki_preis:.0f}€)")
|
||||
return ki_preis
|
||||
except Exception as e:
|
||||
log(f"Vision-Verifizierung Fehler: {e}", "WARN")
|
||||
return None
|
||||
|
||||
|
||||
# ── Cleanup ───────────────────────────────────────────────────────────────────
|
||||
def cleanup_alte_screenshots(tage=30):
|
||||
"""Löscht Screenshots die älter als `tage` Tage sind."""
|
||||
|
|
@ -375,9 +501,29 @@ def dispatch_job(node, job, tage_override=None):
|
|||
log(f"👁 KI-Fallback: {dropped} Preise verworfen (außerhalb {KI_FALLBACK_MIN}-{KI_FALLBACK_MAX}€ — vermutlich One-Way)")
|
||||
|
||||
try:
|
||||
pruefe_preis_alert(results, job)
|
||||
pruefe_preisanstieg(results, job)
|
||||
speichere_preise(results, node["name"], job, screenshot_id, kabine_erkannt)
|
||||
ki_vis = vision_verifiziere_preise(screenshot_b64, screenshot_id)
|
||||
# Telegram erst NACH Vision-Abgleich — Zahl = was die KI im Screenshot liest
|
||||
if screenshot_id and screenshot_b64:
|
||||
alert_rows = _alert_results_aus_db(screenshot_id, job_id)
|
||||
if alert_rows and _vision_hat_preise_verifiziert(screenshot_id):
|
||||
pruefe_preis_alert(alert_rows, job)
|
||||
pruefe_preisanstieg(alert_rows, job)
|
||||
elif alert_rows and ki_vis is None:
|
||||
log(
|
||||
f"📵 Preis-Alert übersprungen — Vision lieferte keinen Preis "
|
||||
f"({node['name']}/{job['scanner']})",
|
||||
"WARN",
|
||||
)
|
||||
elif alert_rows and not _vision_hat_preise_verifiziert(screenshot_id):
|
||||
log(
|
||||
f"📵 Preis-Alert übersprungen — keine ki_verified-Zeilen "
|
||||
f"(Screenshot {screenshot_id})",
|
||||
"WARN",
|
||||
)
|
||||
else:
|
||||
pruefe_preis_alert(results, job)
|
||||
pruefe_preisanstieg(results, job)
|
||||
except Exception as e:
|
||||
log(f"Speicher-Fehler {node['name']}/{job['scanner']}: {e}", "ERROR")
|
||||
return True
|
||||
|
|
@ -408,6 +554,49 @@ def speichere_screenshot(screenshot_b64, node_name, job):
|
|||
return None
|
||||
|
||||
|
||||
|
||||
def _vision_hat_preise_verifiziert(screenshot_id: int) -> bool:
|
||||
"""Mind. eine Zeile dieses Screenshots wurde mit Vision abgeglichen (ki_verified=1)."""
|
||||
conn = get_conn()
|
||||
row = conn.execute(
|
||||
"SELECT COUNT(*) AS c FROM prices WHERE screenshot_id=? AND ki_verified=1",
|
||||
(screenshot_id,),
|
||||
).fetchone()
|
||||
conn.close()
|
||||
return bool(row and row["c"] and row["c"] > 0)
|
||||
|
||||
|
||||
def _alert_results_aus_db(screenshot_id: int, job_id: int) -> list:
|
||||
"""Preise fuer Telegram-Alert nach Vision: nur plausibel (NULL/1), mit Korrigierung."""
|
||||
conn = get_conn()
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT preis, preis_korrigiert, abflug, ankunft, booking_url, scanner
|
||||
FROM prices
|
||||
WHERE screenshot_id = ? AND job_id = ?
|
||||
AND (plausibel IS NULL OR plausibel = 1)
|
||||
ORDER BY COALESCE(preis_korrigiert, preis) ASC
|
||||
""",
|
||||
(screenshot_id, job_id),
|
||||
).fetchall()
|
||||
conn.close()
|
||||
out = []
|
||||
for r in rows:
|
||||
raw = float(r["preis"] or 0)
|
||||
pk = r["preis_korrigiert"]
|
||||
eff = float(pk) if pk is not None else raw
|
||||
if eff <= 0:
|
||||
continue
|
||||
out.append({
|
||||
"preis": eff,
|
||||
"abflug": r["abflug"] or "",
|
||||
"ankunft": r["ankunft"] or "",
|
||||
"booking_url": r["booking_url"] or "",
|
||||
"scanner": r["scanner"] or "",
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
ALERT_SCHWELLE_EUR = 900 # Telegram-Alert wenn CX unter diesen Preis fällt
|
||||
|
||||
def pruefe_preis_alert(results, job):
|
||||
|
|
@ -427,6 +616,23 @@ def pruefe_preis_alert(results, job):
|
|||
f"⚠️ Sofort auf Buchungsseite prüfen — Preise ändern sich schnell."
|
||||
)
|
||||
log(f"💰 PREIS-ALERT: {preis:.0f}EUR {scanner} — Telegram gesendet")
|
||||
# memory_service_event
|
||||
try:
|
||||
import requests as _rq, json as _json
|
||||
_rq.post("http://100.121.192.94:8400/events", json={
|
||||
"source": "flugscanner",
|
||||
"event_type": "price_alert",
|
||||
"object_key": f"{scanner}_{abflug}_{preis:.0f}",
|
||||
"payload_json": _json.dumps({
|
||||
"scanner": scanner,
|
||||
"preis_eur": preis,
|
||||
"abflug": abflug,
|
||||
"booking_url": url,
|
||||
"schwelle": ALERT_SCHWELLE_EUR,
|
||||
}, ensure_ascii=False),
|
||||
}, headers={"Authorization": "Bearer Ai8eeQibV6Z1RWc7oNPim4PXB4vILU1nRW2-XgRcX2M"}, timeout=3)
|
||||
except Exception:
|
||||
pass
|
||||
break
|
||||
|
||||
|
||||
|
|
@ -495,7 +701,7 @@ def speichere_preise(results, node_name, job, screenshot_id=None, kabine_erkannt
|
|||
continue
|
||||
|
||||
conn.execute("""
|
||||
INSERT INTO prices
|
||||
INSERT OR IGNORE INTO prices
|
||||
(job_id, scanner, node, preis, waehrung, airline, abflug, ankunft,
|
||||
von, nach, booking_url, screenshot_id, kabine_erkannt,
|
||||
plausibel, plausi_grund, preis_korrigiert, korrektur_grund)
|
||||
|
|
|
|||
|
|
@ -338,14 +338,28 @@ async function ladeUebersicht() {
|
|||
const buchBtn = p.booking_url
|
||||
? `<a href="${p.booking_url}" target="_blank" class="btn btn-sm" style="text-decoration:none">Öffnen ↗</a>`
|
||||
: '—';
|
||||
// KI-Verifizierung: entweder _ki-Eintrag (korrigiert) oder ki_verified=1 (bestaetigt)
|
||||
const isKiKorrigiert = p.scanner && p.scanner.endsWith('_ki');
|
||||
const isKiBestaetigt = !isKiKorrigiert && p.ki_verified == 1;
|
||||
const isKiVerified = isKiKorrigiert || isKiBestaetigt;
|
||||
const scannerBase = isKiKorrigiert ? p.scanner.slice(0, -3) : p.scanner;
|
||||
const kiLabel = isKiKorrigiert
|
||||
? `<br><span title="Preis durch Vision-KI korrigiert (war Sidebar-Artefakt)" style="background:#1e3a5f;color:#60a5fa;padding:0.1rem 0.4rem;border-radius:3px;font-size:0.7rem;cursor:help">👁 KI-korrigiert</span>`
|
||||
: isKiBestaetigt
|
||||
? `<br><span title="Preis durch lokale Vision-KI (qwen3-vl) bestätigt${p.ki_preis_visual ? ': KI sieht ' + p.ki_preis_visual + '€' : ''}" style="background:#052e16;color:#4ade80;padding:0.1rem 0.4rem;border-radius:3px;font-size:0.7rem;cursor:help">👁 KI ✓</span>`
|
||||
: '';
|
||||
const scannerLabel = isMulticity
|
||||
? `<strong style="color:#818cf8">🇭🇰 HKG Stopover</strong><br><span style="font-size:0.72rem;color:#64748b">+~${HOTEL_HKG}€ Hotel</span>`
|
||||
: p.scanner;
|
||||
: (scannerBase + kiLabel);
|
||||
const verdaechtig = (ps === 0);
|
||||
const preisFarbe = verdaechtig ? '#ef4444' : (isMulticity ? '#a78bfa' : '#34d399');
|
||||
const preisFarbe = verdaechtig ? '#ef4444' : (isMulticity ? '#a78bfa' : isKiKorrigiert ? '#60a5fa' : '#34d399');
|
||||
// Bei bestaetigten Eintraegen: ki_preis_visual anzeigen wenn vorhanden und abweichend
|
||||
const kiPreisHinweis = (isKiBestaetigt && p.ki_preis_visual && Math.abs(p.ki_preis_visual - p.preis) > 5)
|
||||
? `<br><span style="font-size:0.7rem;color:#94a3b8">KI sieht: ${p.ki_preis_visual}€</span>`
|
||||
: '';
|
||||
const gesamtHtml = isMulticity
|
||||
? `<strong style="color:${preisFarbe}">${p.preis} €</strong><br><span style="font-size:0.75rem;color:#64748b">∑ ~${Math.round(p.preis)+HOTEL_HKG} € inkl. Hotel</span>`
|
||||
: `<strong style="color:${preisFarbe}">${p.preis} €</strong>`;
|
||||
: `<strong style="color:${preisFarbe}">${p.preis} €</strong>${kiPreisHinweis}`;
|
||||
const ssBtn = p.screenshot_id
|
||||
? `<button onclick="zeigeScreenshot(${p.screenshot_id},'${p.scanner} · ${p.node} · ${p.abflug||''}')"
|
||||
style="background:#1e3a5f;border:1px solid #2563eb;color:#93c5fd;padding:0.2rem 0.5rem;border-radius:5px;cursor:pointer;font-size:0.8rem">
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ def job():
|
|||
tage = data.get("tage", 30)
|
||||
aufenthalt = data.get("aufenthalt_tage", 60)
|
||||
trip_type = data.get("trip_type", "roundtrip")
|
||||
kabine = data.get("kabine", "premium_economy")
|
||||
kabine = data.get("kabine", "economy")
|
||||
gepaeck = data.get("gepaeck", "1koffer+handgepaeck")
|
||||
airline_filter = data.get("airline_filter", "")
|
||||
layover_min = data.get("layover_min", 120)
|
||||
|
|
|
|||
|
|
@ -36,8 +36,8 @@ def _validate_results(results, scanner_name, kabine="economy"):
|
|||
return results
|
||||
|
||||
|
||||
def _check_cabin_on_page(body, title, kabine="premium_economy"):
|
||||
"""Prüft ob die Seite die gewünschte Kabinenklasse bestätigt."""
|
||||
def _check_cabin_on_page(body, title, kabine="economy"):
|
||||
"""Prüft ob die Seite die gewünschte Kabinenklasse grob bestätigt."""
|
||||
text = (title + " " + body[:3000]).lower()
|
||||
if kabine == "premium_economy":
|
||||
pe_keywords = ["premium economy", "premium eco", "premiumeconomy",
|
||||
|
|
@ -49,6 +49,9 @@ def _check_cabin_on_page(body, title, kabine="premium_economy"):
|
|||
if eco_only[0]:
|
||||
print("[QC] WARNUNG: Seite zeigt 'Economy' OHNE 'Premium' — möglicherweise falsche Kabine!")
|
||||
return False
|
||||
elif kabine == "economy":
|
||||
if "business" in text and "economy" not in text[:800]:
|
||||
print("[QC] WARNUNG: Seite evtl. nur Business sichtbar — prüfen")
|
||||
return True
|
||||
|
||||
|
||||
|
|
@ -79,7 +82,7 @@ def _filter_roundtrip_only(results):
|
|||
|
||||
|
||||
def scrape(scanner, von, nach, tage=30, aufenthalt_tage=60,
|
||||
trip_type="roundtrip", kabine="premium_economy",
|
||||
trip_type="roundtrip", kabine="economy",
|
||||
gepaeck="1koffer+handgepaeck", airline_filter="",
|
||||
layover_min=120, layover_max=300,
|
||||
max_flugzeit_h=22, max_stops=2,
|
||||
|
|
@ -134,6 +137,68 @@ def _dismiss_cookie_banner(sb):
|
|||
return False
|
||||
|
||||
|
||||
|
||||
def _dismiss_comparison_popup(sb):
|
||||
"""Vergleichs-Popups (Opodo, Skyscanner etc.) wegklicken bevor Screenshot gemacht wird."""
|
||||
# Erst Escape versuchen (funktioniert bei den meisten Modals)
|
||||
try:
|
||||
sb.driver.execute_script("document.dispatchEvent(new KeyboardEvent('keydown', {key: 'Escape', keyCode: 27, bubbles: true}));")
|
||||
sb.sleep(0.5)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Dann gezielt Close-Buttons suchen
|
||||
for sel in [
|
||||
'button[aria-label*="lose"]',
|
||||
'button[aria-label*="chließen"]',
|
||||
'button[aria-label*="Schließen"]',
|
||||
'[class*="modal"] button[class*="close"]',
|
||||
'[class*="dialog"] button[class*="close"]',
|
||||
'[class*="overlay"] button[class*="close"]',
|
||||
'[class*="popup"] button[class*="close"]',
|
||||
'button[class*="dismiss"]',
|
||||
'[data-testid*="close"]',
|
||||
'//button[contains(@aria-label, "lose")]',
|
||||
'//button[contains(., "Schließen")]',
|
||||
'//button[contains(., "Nein")]',
|
||||
'//button[contains(., "Nicht jetzt")]',
|
||||
'//button[contains(., "Vielleicht später")]',
|
||||
]:
|
||||
try:
|
||||
sb.click(sel, timeout=1)
|
||||
print(f"[Popup] Geschlossen: {sel[:60]}")
|
||||
sb.sleep(0.8)
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# JavaScript-Fallback: alle sichtbaren Modals/Overlays entfernen
|
||||
try:
|
||||
removed = sb.driver.execute_script("""
|
||||
var removed = 0;
|
||||
var selectors = ['[class*="modal"]', '[class*="overlay"]', '[class*="dialog"]',
|
||||
'[class*="popup"]', '[role="dialog"]'];
|
||||
selectors.forEach(function(sel) {
|
||||
document.querySelectorAll(sel).forEach(function(el) {
|
||||
var style = window.getComputedStyle(el);
|
||||
if (style.display !== 'none' && style.visibility !== 'hidden'
|
||||
&& el.offsetHeight > 100) {
|
||||
el.remove();
|
||||
removed++;
|
||||
}
|
||||
});
|
||||
});
|
||||
return removed;
|
||||
""")
|
||||
if removed:
|
||||
print(f"[Popup] JS: {removed} Elemente entfernt")
|
||||
sb.sleep(0.5)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def _take_screenshot(sb):
|
||||
"""Full-Page Screenshot via CDP (JPEG 55%, max 3000px). Gibt base64-String zurück."""
|
||||
try:
|
||||
|
|
@ -216,7 +281,9 @@ def _booking_url_momondo(von, nach, abflug, rueck, kc, bags=1,
|
|||
|
||||
|
||||
def _booking_url_trip(von, nach, abflug_fmt, rueck_fmt, kc, von_name, nach_name, airline=""):
|
||||
params = f"DDate1={abflug_fmt}&class={kc}&curr=EUR"
|
||||
# flightType=RT erzwingt Roundtrip-Suche auf Trip.com
|
||||
flight_type = "RT" if rueck_fmt else "OW"
|
||||
params = f"DDate1={abflug_fmt}&class={kc}&curr=EUR&flightType={flight_type}"
|
||||
if rueck_fmt:
|
||||
params += f"&DDate2={rueck_fmt}"
|
||||
if airline:
|
||||
|
|
@ -267,6 +334,54 @@ def _preise_aus_body(body, scanner, abflug):
|
|||
return results[:10]
|
||||
|
||||
|
||||
|
||||
def _kayak_header_preis(sb) -> float | None:
|
||||
"""Liest den 'Günstigste Option' Preis aus dem KAYAK-Summary-Header.
|
||||
Dieser Wert ist der zuverlässigste Anker — kommt direkt aus den Suchergebnissen."""
|
||||
try:
|
||||
# JavaScript: suche die summary-bar Elemente
|
||||
price = sb.driver.execute_script("""
|
||||
// KAYAK zeigt "Günstigste Option" + Preis in einem summary-container
|
||||
var containers = document.querySelectorAll('[class*="rec-col"], [class*="recommended"], [class*="summary"], [class*="option-header"]');
|
||||
for (var c of containers) {
|
||||
var txt = c.innerText || '';
|
||||
var m = txt.match(/(\d[\d.]{1,6})\s?€|€\s?(\d[\d.]{1,6})/);
|
||||
if (m) {
|
||||
var raw = (m[1] || m[2]).replace('.','').replace(',','.');
|
||||
var v = parseFloat(raw);
|
||||
if (v > 300 && v < 5000) return v;
|
||||
}
|
||||
}
|
||||
// Fallback: suche im Seitentitel / h1
|
||||
var h = document.querySelector('h1, [class*="title"]');
|
||||
if (h) {
|
||||
var m2 = (h.innerText||'').match(/(\d[\d.]{2,6})\s?€/);
|
||||
if (m2) return parseFloat(m2[1].replace('.',''));
|
||||
}
|
||||
return null;
|
||||
""")
|
||||
if price:
|
||||
print(f"[KY] Header-Preis: {price} EUR")
|
||||
return float(price)
|
||||
except Exception as e:
|
||||
print(f"[KY] Header-Preis Fehler: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def _filter_sidebar_preise(results: list, anker: float | None, scanner: str) -> list:
|
||||
"""Filtert Sidebar-Preise (Airline-Filter, Preisslider) heraus.
|
||||
Behalte nur Preise die >= 80% des Anker-Preises sind (Sidebar-Preise sind viel günstiger)."""
|
||||
if not anker or not results:
|
||||
return results
|
||||
min_valid = anker * 0.80
|
||||
filtered = [r for r in results if r["preis"] >= min_valid]
|
||||
removed = len(results) - len(filtered)
|
||||
if removed:
|
||||
print(f"[{scanner}] {removed} Sidebar-Preise entfernt (unter {min_valid:.0f} EUR)")
|
||||
return filtered if filtered else results # Fallback: alle behalten wenn alle rausgefiltert
|
||||
|
||||
|
||||
|
||||
def _consent_google(sb):
|
||||
"""Google Consent-Seite (DSGVO) behandeln."""
|
||||
if "consent" in sb.get_current_url() or "Bevor Sie" in sb.get_title():
|
||||
|
|
@ -318,7 +433,7 @@ def _gf_fill_field(sb, selectors, text, field_name):
|
|||
|
||||
|
||||
def scrape_google_flights(von, nach, tage=30, aufenthalt_tage=60,
|
||||
trip_type="roundtrip", kabine="premium_economy",
|
||||
trip_type="roundtrip", kabine="economy",
|
||||
gepaeck="1koffer+handgepaeck", airline_filter="",
|
||||
layover_min=120, layover_max=300,
|
||||
max_flugzeit_h=22, max_stops=2):
|
||||
|
|
@ -326,7 +441,7 @@ def scrape_google_flights(von, nach, tage=30, aufenthalt_tage=60,
|
|||
abflug_de = (datetime.now() + timedelta(days=tage)).strftime("%d.%m.%Y")
|
||||
rueck = (datetime.now() + timedelta(days=tage + aufenthalt_tage)).strftime("%Y-%m-%d") \
|
||||
if trip_type == "roundtrip" else ""
|
||||
kc = KABINE_GOOGLE.get(kabine, "w")
|
||||
kc = KABINE_GOOGLE.get(kabine, "e")
|
||||
booking_url = _booking_url_google(von, nach, abflug, rueck, kc)
|
||||
|
||||
stadtname = {"FRA": "Frankfurt", "HAN": "Hanoi", "KTI": "Phnom Penh",
|
||||
|
|
@ -554,18 +669,19 @@ def scrape_google_flights(von, nach, tage=30, aufenthalt_tage=60,
|
|||
results = dedup
|
||||
|
||||
print(f"[GF] Ergebnis: {[r['preis'] for r in results[:5]]}")
|
||||
_dismiss_comparison_popup(sb)
|
||||
screenshot_b64 = _take_screenshot(sb)
|
||||
return results[:10], screenshot_b64
|
||||
|
||||
|
||||
def scrape_kayak(von, nach, tage=30, aufenthalt_tage=60,
|
||||
trip_type="roundtrip", kabine="premium_economy",
|
||||
trip_type="roundtrip", kabine="economy",
|
||||
gepaeck="1koffer+handgepaeck", airline_filter="",
|
||||
layover_min=120, layover_max=300,
|
||||
max_flugzeit_h=22, max_stops=2):
|
||||
abflug = (datetime.now() + timedelta(days=tage)).strftime("%Y-%m-%d")
|
||||
rueck = (datetime.now() + timedelta(days=tage + aufenthalt_tage)).strftime("%Y-%m-%d") if trip_type == "roundtrip" else ""
|
||||
kc = KABINE_KAYAK.get(kabine, "w")
|
||||
kc = KABINE_KAYAK.get(kabine, "e")
|
||||
bags = 1 if "koffer" in gepaeck else 0
|
||||
booking_url = _booking_url_kayak(von, nach, abflug, rueck, kc, bags,
|
||||
layover_min, layover_max, airline_filter,
|
||||
|
|
@ -611,20 +727,24 @@ def scrape_kayak(von, nach, tage=30, aufenthalt_tage=60,
|
|||
results.append(r)
|
||||
|
||||
# Kabinen-Verifikation: prüfe ob "Premium Economy" in der Seite steht
|
||||
pe_confirmed = _check_cabin_on_page(body, title, "premium_economy")
|
||||
pe_confirmed = _check_cabin_on_page(body, title, kabine)
|
||||
if not pe_confirmed:
|
||||
print(f"[KY{airline_label}] WARNUNG: Premium Economy nicht auf Seite bestätigt!")
|
||||
|
||||
# Sidebar-Preise herausfiltern: Header-Preis als Ankerwert holen
|
||||
anker = _kayak_header_preis(sb)
|
||||
results = _filter_sidebar_preise(results, anker, f"kayak{airline_label}")
|
||||
results = _validate_results(results, f"kayak{airline_label}", kabine)
|
||||
print(f"[KY{airline_label}] Ergebnis: {[r['preis'] for r in results[:5]]}")
|
||||
_dismiss_cookie_banner(sb)
|
||||
sb.sleep(3)
|
||||
_dismiss_comparison_popup(sb)
|
||||
screenshot_b64 = _take_screenshot(sb)
|
||||
return results[:10], screenshot_b64
|
||||
|
||||
|
||||
def scrape_trip(von, nach, tage=30, aufenthalt_tage=60,
|
||||
trip_type="roundtrip", kabine="premium_economy",
|
||||
trip_type="roundtrip", kabine="economy",
|
||||
gepaeck="1koffer+handgepaeck", airline_filter="",
|
||||
layover_min=120, layover_max=300,
|
||||
max_flugzeit_h=22, max_stops=2):
|
||||
|
|
@ -632,7 +752,7 @@ def scrape_trip(von, nach, tage=30, aufenthalt_tage=60,
|
|||
rueck_fmt = (datetime.now() + timedelta(days=tage + aufenthalt_tage)).strftime("%Y%m%d") if trip_type == "roundtrip" else ""
|
||||
abflug_iso = (datetime.now() + timedelta(days=tage)).strftime("%Y-%m-%d")
|
||||
rueck_iso = (datetime.now() + timedelta(days=tage + aufenthalt_tage)).strftime("%Y-%m-%d") if trip_type == "roundtrip" else ""
|
||||
kc = KABINE_TRIP.get(kabine, "W")
|
||||
kc = KABINE_TRIP.get(kabine, "Y")
|
||||
|
||||
stadtname = {"FRA": "frankfurt", "HAN": "hanoi", "KTI": "phnom-penh",
|
||||
"PNH": "phnom-penh", "BKK": "bangkok", "SGN": "ho-chi-minh-city"}
|
||||
|
|
@ -684,7 +804,7 @@ def scrape_trip(von, nach, tage=30, aufenthalt_tage=60,
|
|||
r["booking_url"] = booking_url
|
||||
results.append(r)
|
||||
|
||||
pe_confirmed = _check_cabin_on_page(body, title, "premium_economy")
|
||||
pe_confirmed = _check_cabin_on_page(body, title, kabine)
|
||||
if not pe_confirmed:
|
||||
print("[TR] WARNUNG: Premium Economy nicht auf Seite bestätigt!")
|
||||
|
||||
|
|
@ -692,6 +812,7 @@ def scrape_trip(von, nach, tage=30, aufenthalt_tage=60,
|
|||
print(f"[TR] Ergebnis: {[r['preis'] for r in results[:5]]}")
|
||||
_dismiss_cookie_banner(sb)
|
||||
sb.sleep(2)
|
||||
_dismiss_comparison_popup(sb)
|
||||
screenshot_b64 = _take_screenshot(sb)
|
||||
return results[:10], screenshot_b64
|
||||
|
||||
|
|
@ -716,7 +837,7 @@ def _booking_url_kayak_multicity(von, nach, via, abflug, via_datum, rueck, kc, b
|
|||
|
||||
|
||||
def scrape_kayak_multicity(von, nach, tage=30, aufenthalt_tage=60,
|
||||
kabine="premium_economy",
|
||||
kabine="economy",
|
||||
gepaeck="1koffer+handgepaeck",
|
||||
airline_filter="",
|
||||
via="HKG", stopover_min_h=20, stopover_max_h=30):
|
||||
|
|
@ -727,7 +848,7 @@ def scrape_kayak_multicity(von, nach, tage=30, aufenthalt_tage=60,
|
|||
abflug = (datetime.now() + timedelta(days=tage)).strftime("%Y-%m-%d")
|
||||
via_datum = (datetime.now() + timedelta(days=tage + 1)).strftime("%Y-%m-%d")
|
||||
rueck = (datetime.now() + timedelta(days=tage + 1 + aufenthalt_tage)).strftime("%Y-%m-%d")
|
||||
kc = KABINE_KAYAK.get(kabine, "w")
|
||||
kc = KABINE_KAYAK.get(kabine, "e")
|
||||
bags = 1 if "koffer" in gepaeck else 0
|
||||
airline_label = f" [{airline_filter}]" if airline_filter else ""
|
||||
|
||||
|
|
@ -783,12 +904,13 @@ def scrape_kayak_multicity(von, nach, tage=30, aufenthalt_tage=60,
|
|||
print(f"[MC{airline_label}] Ergebnis: {[r['preis'] for r in results[:5]]}")
|
||||
_dismiss_cookie_banner(sb)
|
||||
sb.sleep(3)
|
||||
_dismiss_comparison_popup(sb)
|
||||
screenshot_b64 = _take_screenshot(sb)
|
||||
return results[:10], screenshot_b64
|
||||
|
||||
|
||||
def scrape_momondo(von, nach, tage=30, aufenthalt_tage=60,
|
||||
trip_type="roundtrip", kabine="premium_economy",
|
||||
trip_type="roundtrip", kabine="economy",
|
||||
gepaeck="1koffer+handgepaeck", airline_filter="",
|
||||
layover_min=120, layover_max=300,
|
||||
max_flugzeit_h=22, max_stops=2):
|
||||
|
|
@ -796,7 +918,7 @@ def scrape_momondo(von, nach, tage=30, aufenthalt_tage=60,
|
|||
abflug = (datetime.now() + timedelta(days=tage)).strftime("%Y-%m-%d")
|
||||
rueck = (datetime.now() + timedelta(days=tage + aufenthalt_tage)).strftime("%Y-%m-%d") \
|
||||
if trip_type == "roundtrip" else ""
|
||||
kc = KABINE_KAYAK.get(kabine, "w")
|
||||
kc = KABINE_KAYAK.get(kabine, "e")
|
||||
bags = 1 if "koffer" in gepaeck else 0
|
||||
booking_url = _booking_url_momondo(von, nach, abflug, rueck, kc, bags,
|
||||
layover_min, layover_max, airline_filter,
|
||||
|
|
@ -855,20 +977,24 @@ def scrape_momondo(von, nach, tage=30, aufenthalt_tage=60,
|
|||
r["airline"] = airline_filter or ""
|
||||
results.append(r)
|
||||
|
||||
pe_confirmed = _check_cabin_on_page(body, title, "premium_economy")
|
||||
pe_confirmed = _check_cabin_on_page(body, title, kabine)
|
||||
if not pe_confirmed:
|
||||
print(f"[MO{airline_label}] WARNUNG: Premium Economy nicht auf Seite bestätigt!")
|
||||
|
||||
# Sidebar-Preise herausfiltern
|
||||
anker_mo = _kayak_header_preis(sb) # Momondo hat gleiches Layout wie Kayak
|
||||
results = _filter_sidebar_preise(results, anker_mo, f"momondo{airline_label}")
|
||||
results = _validate_results(results, f"momondo{airline_label}", kabine)
|
||||
print(f"[MO{airline_label}] Ergebnis: {[r['preis'] for r in results[:5]]}")
|
||||
_dismiss_cookie_banner(sb)
|
||||
sb.sleep(2)
|
||||
_dismiss_comparison_popup(sb)
|
||||
screenshot_b64 = _take_screenshot(sb)
|
||||
return results[:10], screenshot_b64
|
||||
|
||||
|
||||
def scrape_wego(von, nach, tage=30, aufenthalt_tage=60,
|
||||
trip_type="roundtrip", kabine="premium_economy",
|
||||
trip_type="roundtrip", kabine="economy",
|
||||
gepaeck="1koffer+handgepaeck", airline_filter="",
|
||||
layover_min=120, layover_max=300,
|
||||
max_flugzeit_h=22, max_stops=2):
|
||||
|
|
@ -879,7 +1005,7 @@ def scrape_wego(von, nach, tage=30, aufenthalt_tage=60,
|
|||
|
||||
KABINE_WEGO = {"economy": "economy", "premium_economy": "premiumEconomy",
|
||||
"business": "business", "first": "first"}
|
||||
kc = KABINE_WEGO.get(kabine, "premiumEconomy")
|
||||
kc = KABINE_WEGO.get(kabine, "economy")
|
||||
|
||||
stadtname_wego = {"FRA": "frankfurt", "KTI": "phnom-penh", "HAN": "hanoi",
|
||||
"BKK": "bangkok", "SGN": "ho-chi-minh-city", "HKG": "hong-kong"}
|
||||
|
|
@ -931,6 +1057,7 @@ def scrape_wego(von, nach, tage=30, aufenthalt_tage=60,
|
|||
results.append(r)
|
||||
|
||||
print(f"[WG] Ergebnis: {[r['preis'] for r in results[:5]]}")
|
||||
_dismiss_comparison_popup(sb)
|
||||
screenshot_b64 = _take_screenshot(sb)
|
||||
return results[:10], screenshot_b64
|
||||
|
||||
|
|
@ -954,7 +1081,7 @@ def _parse_preis_usd(text):
|
|||
|
||||
|
||||
def scrape_traveloka(von, nach, tage=30, aufenthalt_tage=60,
|
||||
trip_type="roundtrip", kabine="premium_economy",
|
||||
trip_type="roundtrip", kabine="economy",
|
||||
gepaeck="1koffer+handgepaeck", airline_filter="",
|
||||
layover_min=120, layover_max=300,
|
||||
max_flugzeit_h=22, max_stops=2):
|
||||
|
|
@ -1009,12 +1136,13 @@ def scrape_traveloka(von, nach, tage=30, aufenthalt_tage=60,
|
|||
results.sort(key=lambda x: x["preis"])
|
||||
results = _validate_results(results, "traveloka", "premium_economy")
|
||||
print(f"[TV] Ergebnis: {[r['preis'] for r in results[:5]]}")
|
||||
_dismiss_comparison_popup(sb)
|
||||
screenshot_b64 = _take_screenshot(sb)
|
||||
return results[:10], screenshot_b64
|
||||
|
||||
|
||||
def scrape_skyscanner(von, nach, tage=30, aufenthalt_tage=60,
|
||||
trip_type="roundtrip", kabine="premium_economy",
|
||||
trip_type="roundtrip", kabine="economy",
|
||||
gepaeck="1koffer+handgepaeck", airline_filter="",
|
||||
layover_min=120, layover_max=300,
|
||||
max_flugzeit_h=22, max_stops=2):
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue