fix(rag): breite Mehrfachsuche + mehr Treffer fuer Uebersichten
- wide_recall: bis 16 ES-Runden mit Sparten-/Gesellschafts-Queries, Merge nach Dedup-Key, bis 25 distinct Treffer, groessere Snippets. - Normale Suche: top_k bis 15, ES bis 150. - Forciertes RAG: top_k 25, Tool-Payload 32k Zeichen. - Hinweis: 100% Vollstaendigkeit haengt von Index/OCR ab.
This commit is contained in:
Homelab Cursor
parent
a9d1069728
commit
dcf70b087b
2 changed files with 162 additions and 21 deletions
|
|
@ -450,7 +450,7 @@ def ask_with_tools(question: str, tool_handlers: dict, session_id: str = None, d
|
|||
question
|
||||
+ " Versicherung Beitrag Beitragsrechnung Jahresbetrag"
|
||||
)
|
||||
_rag_res = _rag_fn(query=_rag_q, top_k=10)
|
||||
_rag_res = _rag_fn(query=_rag_q, top_k=25)
|
||||
if _rag_res and not _rag_res.startswith("Keine"):
|
||||
log.info("RAG-Pflicht: %d Zeichen — loesche Session-History", len(str(_rag_res)))
|
||||
messages = [
|
||||
|
|
@ -467,9 +467,9 @@ def ask_with_tools(question: str, tool_handlers: dict, session_id: str = None, d
|
|||
{"role": "assistant", "content": None,
|
||||
"tool_calls": [{"id": "forced_rag", "type": "function",
|
||||
"function": {"name": "rag_search",
|
||||
"arguments": json.dumps({"query": _rag_q, "top_k": 10})}}]},
|
||||
"arguments": json.dumps({"query": _rag_q, "top_k": 25})}}]},
|
||||
{"role": "tool", "tool_call_id": "forced_rag",
|
||||
"content": str(_rag_res)[:12000]},
|
||||
"content": str(_rag_res)[:32000]},
|
||||
{"role": "user", "content": question},
|
||||
]
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -21,6 +21,10 @@ OLLAMA_EMBED_URL = "http://100.84.255.83:11434/api/embeddings"
|
|||
EMBED_MODEL = "nomic-embed-text"
|
||||
|
||||
MIN_TOP_K = 5
|
||||
# Breite Übersichten: mehr ES-Runden, mehr distinct Treffer
|
||||
MAX_TOP_K_NORMAL = 15
|
||||
MAX_TOP_K_WIDE = 25
|
||||
ES_SIZE_CAP = 150
|
||||
|
||||
TOOLS = [
|
||||
{
|
||||
|
|
@ -33,8 +37,8 @@ TOOLS = [
|
|||
"Anleitungen, Buecher, persoenliche Unterlagen). "
|
||||
"Nutze dieses Tool wenn der User nach einem bestimmten Dokument, "
|
||||
"Vertrag, Brief oder persoenlicher Information fragt. "
|
||||
"Bei breiten Fragen ('welche Versicherungen', 'alle Vertraege') "
|
||||
"immer top_k=10 verwenden."
|
||||
"Bei breiten Fragen ('welche Versicherungen', Jahreskosten, Listen) "
|
||||
"top_k=15 oder hoeher setzen."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
|
|
@ -48,8 +52,8 @@ TOOLS = [
|
|||
},
|
||||
"top_k": {
|
||||
"type": "integer",
|
||||
"description": "Anzahl Ergebnisse (5-10, Standard 8)",
|
||||
"default": 8,
|
||||
"description": "Anzahl Ergebnisse (5-25, Standard 10)",
|
||||
"default": 10,
|
||||
},
|
||||
},
|
||||
"required": ["query"],
|
||||
|
|
@ -73,7 +77,7 @@ SUCHANFRAGE: Kurze Keywords, KEINE ganzen Saetze. Beispiele:
|
|||
- "Familienbuch" / "Grundsteuer Erklaerung" / "Haftpflicht" / "Kindergeld" / "Mietvertrag" / "Arbeitsvertrag" / "Reisepass"
|
||||
|
||||
ERGEBNISSE AUSWERTEN:
|
||||
- Bei breiten Fragen ("welche Versicherungen", "alle Vertraege"): top_k=10
|
||||
- Bei breiten Fragen ("welche Versicherungen", Jahreskosten, Listen): top_k=15-25, ALLE Treffer aus der Tool-Antwort abarbeiten
|
||||
- Liste die gefundenen Dokumente mit Ordner und kurzem Inhalt auf
|
||||
- ERFINDE KEINE Details die nicht im Ergebnis stehen
|
||||
- Der Ordnerpfad (vor dem Dateinamen, getrennt durch __) zeigt die Kategorie
|
||||
|
|
@ -147,6 +151,7 @@ def _es_hybrid_search(query: str, es_size: int) -> dict:
|
|||
if not qvec:
|
||||
return {"_error": "Embedding fehlgeschlagen (Ollama nicht erreichbar?)."}
|
||||
|
||||
es_size = min(ES_SIZE_CAP, max(es_size, 20))
|
||||
kb_filter = {"term": {"kb_id": KB_ID}}
|
||||
body = {
|
||||
"size": es_size,
|
||||
|
|
@ -154,7 +159,7 @@ def _es_hybrid_search(query: str, es_size: int) -> dict:
|
|||
"field": "q_768_vec",
|
||||
"query_vector": qvec,
|
||||
"k": es_size,
|
||||
"num_candidates": min(500, max(es_size * 5, 120)),
|
||||
"num_candidates": min(500, max(es_size * 5, 150)),
|
||||
"filter": [kb_filter],
|
||||
},
|
||||
"query": {
|
||||
|
|
@ -191,25 +196,152 @@ def _es_hybrid_search(query: str, es_size: int) -> dict:
|
|||
return {"_error": str(e)}
|
||||
|
||||
|
||||
def _is_wide_recall_query(q: str) -> bool:
|
||||
"""Übersichts-/Listen-/Kostenfragen: mehrfach suchen und mergen."""
|
||||
ql = (q or "").lower()
|
||||
if any(x in ql for x in ("welche versicherung", "alle versicherung", "versicherungen habe")):
|
||||
return True
|
||||
if "versicherung" in ql and any(
|
||||
x in ql
|
||||
for x in (
|
||||
"welche",
|
||||
"alle",
|
||||
"liste",
|
||||
"überblick",
|
||||
"ueberblick",
|
||||
"kosten",
|
||||
"beitrag",
|
||||
"jähr",
|
||||
"jaehr",
|
||||
"jahres",
|
||||
"gesamt",
|
||||
"summe",
|
||||
"übersicht",
|
||||
"uebersicht",
|
||||
)
|
||||
):
|
||||
return True
|
||||
costish = any(
|
||||
x in ql
|
||||
for x in (
|
||||
"kosten",
|
||||
"beitrag",
|
||||
"beiträge",
|
||||
"beitraege",
|
||||
"eur",
|
||||
"euro",
|
||||
"jähr",
|
||||
"jaehr",
|
||||
"jahreskosten",
|
||||
"prämie",
|
||||
"praemie",
|
||||
)
|
||||
)
|
||||
broad = any(
|
||||
x in ql
|
||||
for x in (
|
||||
"liste",
|
||||
"übersicht",
|
||||
"uebersicht",
|
||||
"alle",
|
||||
"gesamt",
|
||||
"summe",
|
||||
"jährlich",
|
||||
"jaehrlich",
|
||||
)
|
||||
)
|
||||
return costish and broad
|
||||
|
||||
|
||||
# Zusatzanfragen decken Sparten + Gesellschaften ab (Recall)
|
||||
_WIDE_SUBQUERIES = [
|
||||
"Versicherung Beitragsrechnung Jahresbeitrag",
|
||||
"Wohngebäudeversicherung Gebäude Beitrag",
|
||||
"Hausratversicherung Beitrag Ergo",
|
||||
"Haftpflichtversicherung Beitrag GARANTA",
|
||||
"Kfz Kasko Haftpflicht Beitragsrechnung",
|
||||
"Rechtsschutzversicherung Beitrag",
|
||||
"Lebensversicherung Beitrag",
|
||||
"Krankenversicherung PKV Beitrag",
|
||||
"Sachversicherung LVM Beitrag",
|
||||
"LVM AutoPlus Versicherungsschein",
|
||||
"Allianz Versicherung Police",
|
||||
"Nürnberger Versicherung Beitrag",
|
||||
"Ergo Versicherung Police",
|
||||
"Unfallversicherung Berufsunfähigkeit",
|
||||
"Bausparvertrag Bauspar",
|
||||
]
|
||||
|
||||
|
||||
def _merge_hits_from_queries(queries: list[str], es_size: int, pool_cap: int) -> tuple[list, str | None]:
|
||||
"""Führt mehrere Hybrid-Suchen aus; pro Dokument höchster Score."""
|
||||
best: dict[str, dict] = {}
|
||||
last_err: str | None = None
|
||||
|
||||
def absorb(hits: list) -> None:
|
||||
for h in hits:
|
||||
src = h.get("_source") or {}
|
||||
dn = src.get("docnm_kwd") or "?"
|
||||
dk = _dedup_key(dn)
|
||||
sc = float(h.get("_score") or 0.0)
|
||||
old = best.get(dk)
|
||||
if old is None or sc > float(old.get("_score") or 0.0):
|
||||
best[dk] = h
|
||||
|
||||
for q in queries:
|
||||
q = (q or "").strip()
|
||||
if not q:
|
||||
continue
|
||||
data = _es_hybrid_search(q, es_size)
|
||||
if "_error" in data:
|
||||
last_err = str(data["_error"])
|
||||
log.warning("wide_recall subquery fail %s: %s", q[:40], last_err)
|
||||
continue
|
||||
absorb((data.get("hits") or {}).get("hits") or [])
|
||||
|
||||
merged = sorted(best.values(), key=lambda h: float(h.get("_score") or 0.0), reverse=True)
|
||||
return merged[:pool_cap], last_err
|
||||
|
||||
|
||||
def handle_rag_search(query: str, top_k: int = 8, **kw):
|
||||
if not query or not query.strip():
|
||||
return "rag_search: query fehlt."
|
||||
|
||||
top_k = max(MIN_TOP_K, min(int(top_k or 8), 10))
|
||||
es_size = min(120, max(top_k * 12, 50))
|
||||
qstrip = query.strip()
|
||||
wide = _is_wide_recall_query(qstrip)
|
||||
cap = MAX_TOP_K_WIDE if wide else MAX_TOP_K_NORMAL
|
||||
top_k = max(MIN_TOP_K, min(int(top_k or 10), cap))
|
||||
es_size = min(ES_SIZE_CAP, max(top_k * 10, 70))
|
||||
|
||||
data = _es_hybrid_search(query.strip(), es_size)
|
||||
if wide:
|
||||
subqs = [qstrip]
|
||||
for sq in _WIDE_SUBQUERIES:
|
||||
if sq.lower() not in qstrip.lower():
|
||||
subqs.append(sq)
|
||||
pool_cap = max(top_k * 3, 45)
|
||||
hits, err = _merge_hits_from_queries(subqs[:16], es_size, pool_cap=pool_cap)
|
||||
if err and not hits:
|
||||
return f"Fehler bei der Dokumentensuche: {err}"
|
||||
header = (
|
||||
f"**Breitensuche ({len(subqs[:16])} Anfragen gemerged) fuer '{qstrip}' — "
|
||||
f"{len(hits)} Kandidaten, zeige bis {top_k} distinct:**\n"
|
||||
)
|
||||
snip_len = 750
|
||||
else:
|
||||
data = _es_hybrid_search(qstrip, es_size)
|
||||
if "_error" in data:
|
||||
return f"Fehler bei der Dokumentensuche: {data['_error']}"
|
||||
|
||||
hits = (data.get("hits") or {}).get("hits") or []
|
||||
header = f"**Dokumente fuer '{qstrip}' (bis {top_k}):**\n"
|
||||
snip_len = 650
|
||||
|
||||
if not hits:
|
||||
return f"Keine Ergebnisse fuer '{query}' in der Wissensbasis gefunden."
|
||||
return f"Keine Ergebnisse fuer '{qstrip}' in der Wissensbasis gefunden."
|
||||
|
||||
seen_docs: set[str] = set()
|
||||
lines: list[str] = []
|
||||
|
||||
count = 0
|
||||
|
||||
for h in hits:
|
||||
if count >= top_k:
|
||||
break
|
||||
|
|
@ -222,7 +354,7 @@ def handle_rag_search(query: str, top_k: int = 8, **kw):
|
|||
|
||||
score = h.get("_score") or 0.0
|
||||
raw = src.get("content_with_weight") or src.get("content_de") or ""
|
||||
content = raw[:600].strip()
|
||||
content = raw[:snip_len].strip()
|
||||
ocr = _ocr_note(raw)
|
||||
folder = _folder_from_docname(doc_name)
|
||||
filename = doc_name.rsplit("__", 1)[-1] if "__" in doc_name else doc_name
|
||||
|
|
@ -236,10 +368,19 @@ def handle_rag_search(query: str, top_k: int = 8, **kw):
|
|||
count += 1
|
||||
|
||||
if count == 0:
|
||||
return f"Keine Dokumente fuer '{query}' gefunden."
|
||||
return f"Keine Dokumente fuer '{qstrip}' gefunden."
|
||||
|
||||
lines.insert(0, f"**{count} verschiedene Dokumente fuer '{query}':**\n")
|
||||
lines.append("\n---\n(Ende der Ergebnisse. Nur diese Dokumente wurden gefunden.)")
|
||||
lines.insert(0, header)
|
||||
tail = (
|
||||
"\n---\n(Ende der Ergebnisse. Nur diese Dokumente in dieser Runde. "
|
||||
+ (
|
||||
"Bei Summen/Zahlen: alle Treffer prüfen; OCR kann unvollständig sein."
|
||||
if wide
|
||||
else ""
|
||||
)
|
||||
+ ")"
|
||||
)
|
||||
lines.append(tail)
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue