fix: Google Flights scraper — find_elements ohne timeout, robustere Formular-Logik
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Cursor
parent
3e6c1011dd
commit
c8dcce6c17
1 changed files with 109 additions and 79 deletions
|
|
@ -158,110 +158,116 @@ def _consent_google(sb):
|
|||
return False
|
||||
|
||||
|
||||
def _gf_fill_field(sb, selectors, text, field_name):
|
||||
"""Hilfsfunktion: Textfeld in Google Flights füllen + ersten Vorschlag klicken."""
|
||||
for sel in selectors:
|
||||
try:
|
||||
field = sb.find_element(sel, timeout=3)
|
||||
field.clear()
|
||||
sb.sleep(0.3)
|
||||
field.send_keys(text)
|
||||
sb.sleep(2)
|
||||
# Ersten Autocomplete-Vorschlag klicken
|
||||
try:
|
||||
sb.find_element('[role="option"]', timeout=3).click()
|
||||
sb.sleep(0.8)
|
||||
except Exception:
|
||||
pass
|
||||
print(f"[GF] {field_name} gesetzt: {text}")
|
||||
return True
|
||||
except Exception:
|
||||
continue
|
||||
print(f"[GF] {field_name} fehlgeschlagen — kein Feld gefunden")
|
||||
return False
|
||||
|
||||
|
||||
def scrape_google_flights(von, nach, tage=30, aufenthalt_tage=60,
|
||||
trip_type="roundtrip", kabine="premium_economy",
|
||||
gepaeck="1koffer+handgepaeck", airline_filter="",
|
||||
layover_min=120, layover_max=300,
|
||||
max_flugzeit_h=22, max_stops=2):
|
||||
abflug = (datetime.now() + timedelta(days=tage)).strftime("%Y-%m-%d")
|
||||
abflug_de = (datetime.now() + timedelta(days=tage)).strftime("%d.%m.%Y")
|
||||
rueck = (datetime.now() + timedelta(days=tage + aufenthalt_tage)).strftime("%Y-%m-%d") if trip_type == "roundtrip" else ""
|
||||
rueck_de = (datetime.now() + timedelta(days=tage + aufenthalt_tage)).strftime("%d.%m.%Y") if trip_type == "roundtrip" else ""
|
||||
abflug = (datetime.now() + timedelta(days=tage)).strftime("%Y-%m-%d")
|
||||
abflug_de = (datetime.now() + timedelta(days=tage)).strftime("%d.%m.%Y")
|
||||
rueck = (datetime.now() + timedelta(days=tage + aufenthalt_tage)).strftime("%Y-%m-%d") \
|
||||
if trip_type == "roundtrip" else ""
|
||||
kc = KABINE_GOOGLE.get(kabine, "w")
|
||||
booking_url = _booking_url_google(von, nach, abflug, rueck, kc)
|
||||
|
||||
# Stadtname-Mapping für die Suchfelder
|
||||
stadtname = {"FRA": "Frankfurt", "HAN": "Hanoi", "KTI": "Phnom Penh",
|
||||
"PNH": "Phnom Penh", "BKK": "Bangkok", "SGN": "Ho Chi Minh"}
|
||||
"PNH": "Phnom Penh", "BKK": "Bangkok", "SGN": "Ho Chi Minh City"}
|
||||
von_name = stadtname.get(von, von)
|
||||
nach_name = stadtname.get(nach, nach)
|
||||
results = []
|
||||
screenshot_b64 = ""
|
||||
|
||||
print(f"[GF] Formular-Suche: {von_name}→{nach_name} {abflug_de}")
|
||||
results = []
|
||||
print(f"[GF] Suche: {von_name}→{nach_name} {abflug_de}")
|
||||
|
||||
with SB(uc=True, headless=True, chromium_arg="--no-sandbox --disable-dev-shm-usage") as sb:
|
||||
# 1. Startseite laden
|
||||
sb.open("https://www.google.com/travel/flights?hl=de&curr=EUR")
|
||||
sb.sleep(6)
|
||||
sb.sleep(5)
|
||||
_consent_google(sb)
|
||||
sb.sleep(2)
|
||||
|
||||
if _consent_google(sb):
|
||||
sb.sleep(4)
|
||||
|
||||
# 2. Kabinen-Typ setzen (Roundtrip / Economy dropdown)
|
||||
# ── 1. Kabine auf "Premium Economy" setzen ──────────────────────────
|
||||
try:
|
||||
# Kabinen-Auswahl: nach "Premium Economy" suchen
|
||||
kab_label = {"e": "Economy", "w": "Premium Economy", "b": "Business", "f": "First"}
|
||||
print(f"[GF] Versuche Kabine zu setzen: {kab_label.get(kc, 'Premium Economy')}")
|
||||
# Kabinen-Dropdown öffnen (2. Dropdown in der Suchleiste)
|
||||
kab_btns = sb.find_elements('[class*="cabin"] button, [aria-label*="abine"], [data-value*="cabin"]', timeout=3)
|
||||
if not kab_btns:
|
||||
kab_btns = sb.find_elements('button[class*="VfPpkd"]', timeout=2)
|
||||
if kab_btns:
|
||||
kab_btns[1].click()
|
||||
# VfPpkd-Buttons: [0]=Hin+Rück [1]=Economy(Klasse)
|
||||
btns = sb.find_elements('button[class*="VfPpkd"]')
|
||||
if len(btns) >= 2:
|
||||
btns[1].click()
|
||||
sb.sleep(1)
|
||||
# Option "Premium Economy" im Dropdown auswählen
|
||||
for opt_sel in ['[data-value="2"]',
|
||||
'li[class*="premium"]',
|
||||
'[role="option"]:nth-child(3)']:
|
||||
try:
|
||||
sb.find_element(opt_sel, timeout=2).click()
|
||||
sb.sleep(0.5)
|
||||
print(f"[GF] Kabine gesetzt via {opt_sel}")
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as e:
|
||||
print(f"[GF] Kabinen-Klick fehlgeschlagen: {e}")
|
||||
print(f"[GF] Kabine: {e}")
|
||||
|
||||
# 3. Abflugort eingeben
|
||||
try:
|
||||
von_field = None
|
||||
for sel in ['input[placeholder*="Von"], input[aria-label*="Von"], input[aria-label*="Abflug"], '
|
||||
'input[placeholder*="Where from"]']:
|
||||
elems = sb.find_elements(sel, timeout=2)
|
||||
if elems:
|
||||
von_field = elems[0]
|
||||
break
|
||||
if not von_field:
|
||||
# Fallback: erster Input im Suchformular
|
||||
von_field = sb.find_element('div[class*="target"] input', timeout=3)
|
||||
# ── 2. Von-Feld befüllen ────────────────────────────────────────────
|
||||
_gf_fill_field(sb, [
|
||||
'input[aria-label*="Von"]',
|
||||
'input[aria-label*="Abflugort"]',
|
||||
'input[placeholder*="Von"]',
|
||||
'input[aria-label*="Where from"]',
|
||||
], von_name, "Von")
|
||||
|
||||
von_field.clear()
|
||||
von_field.send_keys(von_name)
|
||||
sb.sleep(2)
|
||||
# Ersten Vorschlag auswählen
|
||||
sb.find_element('[role="option"]', timeout=3).click()
|
||||
sb.sleep(1)
|
||||
print(f"[GF] Abflugort gesetzt: {von_name}")
|
||||
except Exception as e:
|
||||
print(f"[GF] Abflugort-Eingabe fehlgeschlagen: {e}")
|
||||
# ── 3. Nach-Feld befüllen ───────────────────────────────────────────
|
||||
_gf_fill_field(sb, [
|
||||
'input[aria-label*="Wohin"]',
|
||||
'input[aria-label*="Zielort"]',
|
||||
'input[placeholder*="Wohin"]',
|
||||
'input[aria-label*="Where to"]',
|
||||
], nach_name, "Nach")
|
||||
|
||||
# 4. Zielort eingeben
|
||||
try:
|
||||
nach_field = sb.find_element(
|
||||
'input[placeholder*="Wohin"], input[aria-label*="Ziel"], input[aria-label*="Where to"]',
|
||||
timeout=3)
|
||||
nach_field.clear()
|
||||
nach_field.send_keys(nach_name)
|
||||
sb.sleep(2)
|
||||
sb.find_element('[role="option"]', timeout=3).click()
|
||||
sb.sleep(1)
|
||||
print(f"[GF] Zielort gesetzt: {nach_name}")
|
||||
except Exception as e:
|
||||
print(f"[GF] Zielort-Eingabe fehlgeschlagen: {e}")
|
||||
# ── 4. Suchen-Button klicken ────────────────────────────────────────
|
||||
gesucht = False
|
||||
for sel in ['button[aria-label*="Suchen"]', 'button[aria-label*="Search"]',
|
||||
'button[jsname="vLv7Lb"]', 'button[type="submit"]']:
|
||||
try:
|
||||
sb.find_element(sel, timeout=2).click()
|
||||
print(f"[GF] Suche gestartet via {sel}")
|
||||
gesucht = True
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
# 5. Suchen-Button klicken
|
||||
try:
|
||||
for sel in ['button[aria-label*="Suchen"], button[aria-label*="Search"]',
|
||||
'button[jsname="vLv7Lb"]', 'button[class*="search"]']:
|
||||
btns = sb.find_elements(sel, timeout=2)
|
||||
if btns:
|
||||
btns[0].click()
|
||||
print(f"[GF] Suche gestartet via {sel}")
|
||||
break
|
||||
except Exception as e:
|
||||
print(f"[GF] Such-Button fehlgeschlagen: {e}")
|
||||
|
||||
sb.sleep(12)
|
||||
sb.sleep(14)
|
||||
title = sb.get_title()
|
||||
body = sb.get_text("body")
|
||||
print(f"[GF] Title: {title[:80]} | Body: {len(body)} chars")
|
||||
print(f"[GF] Title: {title[:80]} | Body: {len(body)} chars | Suche-OK: {gesucht}")
|
||||
|
||||
# 6. Preise extrahieren
|
||||
# ── 5. Preise extrahieren ───────────────────────────────────────────
|
||||
# a) aria-label Elemente
|
||||
try:
|
||||
elems = sb.find_elements('[aria-label*="€"], [aria-label*="EUR"]', timeout=3)
|
||||
for elem in elems[:20]:
|
||||
label = elem.get_attribute('aria-label') or elem.text
|
||||
p = _parse_preis(label)
|
||||
for elem in sb.find_elements('[aria-label*="€"], [aria-label*="EUR"]')[:30]:
|
||||
lbl = elem.get_attribute("aria-label") or elem.text
|
||||
p = _parse_preis(lbl)
|
||||
if p and p > 400:
|
||||
results.append({"scanner": "google_flights", "preis": p,
|
||||
"waehrung": "EUR", "airline": "",
|
||||
|
|
@ -270,15 +276,39 @@ def scrape_google_flights(von, nach, tage=30, aufenthalt_tage=60,
|
|||
except Exception:
|
||||
pass
|
||||
|
||||
# Fallback: Regex über Body (nur plausible Preise > 400 EUR)
|
||||
# b) sichtbare Preistexte in Ergebnisliste
|
||||
if not results:
|
||||
for sel in ['.YMlIz', '.FpEdX', '[class*="price"]', 'span[class*="preis"]']:
|
||||
try:
|
||||
for elem in sb.find_elements(sel)[:20]:
|
||||
p = _parse_preis(elem.text)
|
||||
if p and p > 400:
|
||||
results.append({"scanner": "google_flights", "preis": p,
|
||||
"waehrung": "EUR", "airline": "",
|
||||
"abflug": abflug, "ankunft": rueck,
|
||||
"booking_url": booking_url})
|
||||
if results:
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# c) Body-Regex Fallback
|
||||
if not results:
|
||||
for r in _preise_aus_body(body, "google_flights", abflug):
|
||||
if r["preis"] > 400:
|
||||
r["ankunft"] = rueck
|
||||
r["ankunft"] = rueck
|
||||
r["booking_url"] = booking_url
|
||||
results.append(r)
|
||||
|
||||
results = [r for r in results if r["preis"] > 400]
|
||||
seen = set()
|
||||
dedup = []
|
||||
for r in results:
|
||||
if r["preis"] not in seen:
|
||||
seen.add(r["preis"])
|
||||
dedup.append(r)
|
||||
results = dedup
|
||||
|
||||
print(f"[GF] Ergebnis: {[r['preis'] for r in results[:5]]}")
|
||||
screenshot_b64 = _take_screenshot(sb)
|
||||
return results[:10], screenshot_b64
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue