fix: robusteres Scraping mit mehreren Selektoren + Regex-Fallback

Co-authored-by: Cursor <cursoragent@cursor.com>

node/src/worker.py

@@ -15,49 +15,104 @@ def scrape(scanner, von, nach, tage=30):
     return fn(von, nach, tage)
 
 
-def parse_preis(text):
+def parse_preise_aus_text(text, scanner, abflug):
-    """Zahl aus einem Preis-String extrahieren."""
+    """EUR-Preise aus Seitentext per Regex extrahieren."""
-    matches = re.findall(r'[\d.,]+', text.replace(',', '.'))
+    results = []
-    for m in matches:
+    # Muster: 578 € oder €578 oder EUR 578 oder 578 EUR oder 1.234 €
+    patterns = [
+        r'(\d{1,2}[.,]\d{3})\s*[€]',   # 1.234 €
+        r'[€]\s*(\d{3,4})',              # €578
+        r'(\d{3,4})\s*[€]',             # 578 €
+        r'EUR\s*(\d{3,4})',              # EUR 578
+        r'(\d{3,4})\s*EUR',             # 578 EUR
+    ]
+    seen = set()
+    for pattern in patterns:
+        for m in re.findall(pattern, text):
+            clean = m.replace('.', '').replace(',', '')
             try:
-            v = float(m.replace(',', ''))
+                preis = float(clean)
-            if 50 < v < 10000:
+                if 100 < preis < 5000 and preis not in seen:
-                return round(v, 2)
+                    seen.add(preis)
-        except Exception:
+                    results.append({
+                        "scanner": scanner,
+                        "preis": preis,
+                        "waehrung": "EUR",
+                        "airline": "",
+                        "abflug": abflug,
+                        "ankunft": ""
+                    })
+            except ValueError:
                 continue
-    return None
+    # Maximal 10 günstigste
+    results.sort(key=lambda x: x["preis"])
+    return results[:10]
 
 
 def scrape_google_flights(von, nach, tage=30):
-    results = []
     abflug = (datetime.now() + timedelta(days=tage)).strftime("%Y-%m-%d")
+    # Direkte Google Flights Such-URL
     url = (
         f"https://www.google.com/travel/flights/search"
-        f"?tfs=CBwQAhojEgoyMDI1LTA2LTA1agcIARIDRlJBcgwIAxIIL20vMDVxeHgQAQ"
+        f"?hl=de&curr=EUR"
-    )
+        f"#flt={von}.{nach}.{abflug};c:EUR;e:1;sd:1;t:f"
-    # Direkte URL mit Parametern
-    url = (
-        f"https://www.google.com/travel/flights?"
-        f"q=Flights+from+{von}+to+{nach}+on+{abflug}"
     )
 
-    with SB(uc=True, headless=True, chromium_arg="--no-sandbox") as sb:
+    with SB(uc=True, headless=True, chromium_arg="--no-sandbox --disable-dev-shm-usage") as sb:
         sb.open(url)
+        sb.sleep(6)
+
+        # Cookie-Banner wegklicken
+        for selector in ['button[aria-label*="Alle"]', 'button[aria-label*="Accept"]',
+                         'button[aria-label*="Zustimmen"]', 'button.VfPpkd-LgbsSe']:
+            try:
+                sb.click(selector, timeout=2)
+                sb.sleep(1)
+                break
+            except Exception:
+                continue
+
+        # Noch etwas warten bis Preise laden
         sb.sleep(4)
 
-        # Cookie-Banner wegklicken falls vorhanden
+        results = []
-        try:
-            sb.click('button[aria-label*="Accept"]', timeout=3)
-            sb.sleep(1)
-        except Exception:
-            pass
 
-        # Preise suchen
+        # Versuch 1: Spezifische Preis-Elemente
+        selectors = [
+            'span[data-gs*="price"]',
+            '[data-price]',
+            'div.YMlIz',          # Google Flights Preis-Container (oft genutzt)
+            'div.U3gSDe',
+            'span.nE0Jnd',
+            'div[jsname="MkNb9"] span',
+            'li[data-gs] span',
+        ]
+        for sel in selectors:
             try:
-            preise_elems = sb.find_elements('li[data-price]', timeout=8)
+                elems = sb.find_elements(sel, timeout=2)
-            for elem in preise_elems[:10]:
+                for elem in elems[:15]:
-                preis_str = elem.get_attribute('data-price') or elem.text
+                    preis = _parse_single(elem.text)
-                preis = parse_preis(preis_str)
+                    if preis:
+                        results.append({
+                            "scanner": "google_flights",
+                            "preis": preis,
+                            "waehrung": "EUR",
+                            "airline": "",
+                            "abflug": abflug,
+                            "ankunft": ""
+                        })
+                if results:
+                    break
+            except Exception:
+                continue
+
+        # Versuch 2: Aria-Labels mit Preisangaben
+        if not results:
+            try:
+                elems = sb.find_elements('[aria-label*="€"]', timeout=3)
+                for elem in elems[:20]:
+                    label = elem.get_attribute('aria-label') or elem.text
+                    preis = _parse_single(label)
                     if preis:
                         results.append({
                             "scanner": "google_flights",
@@ -70,42 +125,65 @@ def scrape_google_flights(von, nach, tage=30):
             except Exception:
                 pass
 
-        # Fallback: Alle Preis-Texte auf der Seite
+        # Versuch 3: Regex über ganzen Seitentext
         if not results:
             try:
-                page_text = sb.get_page_source()
+                body = sb.get_text("body", timeout=5)
-                # Suche nach EUR-Preisen im HTML
+                results = parse_preise_aus_text(body, "google_flights", abflug)
-                matches = re.findall(r'(\d{3,4})\s*€', page_text)
-                for m in matches[:5]:
-                    preis = float(m)
-                    if 100 < preis < 5000:
-                        results.append({
-                            "scanner": "google_flights",
-                            "preis": preis,
-                            "waehrung": "EUR",
-                            "airline": "",
-                            "abflug": abflug,
-                            "ankunft": ""
-                        })
             except Exception:
                 pass
 
-    return results
+        # Deduplizieren
+        seen = set()
+        unique = []
+        for r in sorted(results, key=lambda x: x["preis"]):
+            if r["preis"] not in seen:
+                seen.add(r["preis"])
+                unique.append(r)
+
+        return unique[:10]
+
+
+def _parse_single(text):
+    if not text:
+        return None
+    text = text.replace('\xa0', ' ').replace('\u202f', ' ')
+    patterns = [
+        r'(\d{1,2}[.,]\d{3})\s*[€]',
+        r'[€]\s*(\d{3,4})',
+        r'(\d{3,4})\s*[€]',
+        r'EUR\s*(\d{3,4})',
+        r'(\d{3,4})\s*EUR',
+    ]
+    for p in patterns:
+        m = re.search(p, text)
+        if m:
+            clean = m.group(1).replace('.', '').replace(',', '')
+            try:
+                v = float(clean)
+                if 100 < v < 5000:
+                    return round(v, 2)
+            except ValueError:
+                continue
+    return None
 
 
 def scrape_kayak(von, nach, tage=30):
-    results = []
     abflug = (datetime.now() + timedelta(days=tage)).strftime("%Y-%m-%d")
-    url = f"https://www.kayak.com/flights/{von}-{nach}/{abflug}?sort=price_a"
+    url = f"https://www.kayak.de/flights/{von}-{nach}/{abflug}?sort=price_a&currency=EUR"
+    results = []
 
-    with SB(uc=True, headless=True, chromium_arg="--no-sandbox") as sb:
+    with SB(uc=True, headless=True, chromium_arg="--no-sandbox --disable-dev-shm-usage") as sb:
         sb.open(url)
-        sb.sleep(5)
+        sb.sleep(7)
 
+        # Preis-Selektoren Kayak
+        for sel in ['.price-text', '.f8F1-price-text', 'span[class*="price"]',
+                    'div[class*="price"] span', '.Iqt3']:
             try:
-            elems = sb.find_elements('.price-text', timeout=8)
+                elems = sb.find_elements(sel, timeout=3)
                 for elem in elems[:10]:
-                preis = parse_preis(elem.text)
+                    preis = _parse_single(elem.text)
                     if preis:
                         results.append({
                             "scanner": "kayak",
@@ -115,35 +193,57 @@ def scrape_kayak(von, nach, tage=30):
                             "abflug": abflug,
                             "ankunft": ""
                         })
+                if results:
+                    break
+            except Exception:
+                continue
+
+        # Fallback Regex
+        if not results:
+            try:
+                body = sb.get_text("body", timeout=5)
+                results = parse_preise_aus_text(body, "kayak", abflug)
             except Exception:
                 pass
 
-    return results
+    return results[:10]
 
 
 def scrape_skyscanner(von, nach, tage=30):
+    abflug_fmt = (datetime.now() + timedelta(days=tage)).strftime("%y%m%d")
+    abflug_iso = (datetime.now() + timedelta(days=tage)).strftime("%Y-%m-%d")
+    url = f"https://www.skyscanner.de/transport/flights/{von.lower()}/{nach.lower()}/{abflug_fmt}/?currency=EUR"
     results = []
-    abflug = (datetime.now() + timedelta(days=tage)).strftime("%y%m%d")
-    url = f"https://www.skyscanner.de/flights/{von.lower()}/{nach.lower()}/{abflug}/"
 
-    with SB(uc=True, headless=True, chromium_arg="--no-sandbox") as sb:
+    with SB(uc=True, headless=True, chromium_arg="--no-sandbox --disable-dev-shm-usage") as sb:
         sb.open(url)
-        sb.sleep(5)
+        sb.sleep(7)
 
+        for sel in ['[data-testid="price-label"]', 'span[class*="Price"]',
+                    'div[class*="price"]', 'span[class*="price"]']:
             try:
-            elems = sb.find_elements('[data-testid="price-label"]', timeout=8)
+                elems = sb.find_elements(sel, timeout=3)
                 for elem in elems[:10]:
-                preis = parse_preis(elem.text)
+                    preis = _parse_single(elem.text)
                     if preis:
                         results.append({
                             "scanner": "skyscanner",
                             "preis": preis,
                             "waehrung": "EUR",
                             "airline": "",
-                        "abflug": abflug,
+                            "abflug": abflug_iso,
                             "ankunft": ""
                         })
+                if results:
+                    break
+            except Exception:
+                continue
+
+        if not results:
+            try:
+                body = sb.get_text("body", timeout=5)
+                results = parse_preise_aus_text(body, "skyscanner", abflug_iso)
             except Exception:
                 pass
 
-    return results
+    return results[:10]