Skip to content

Commit 58ac75f

Browse files
[ST-1803] apps/summarization: add more robust AI response handling
2 parents e1b0280 + c2bea0c commit 58ac75f

10 files changed

Lines changed: 290 additions & 17 deletions

File tree

CHANGELOG.md

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,9 @@ This project (not yet) adheres to [Semantic Versioning](https://semver.org/spec/
1818

1919
### Changed
2020

21+
- Summarization: resilient LLM JSON parsing with json-repair and Text trimming
2122
- Changed BMBF logo to BMFTR
22-
- Adjusted Newsletter emails.
23+
- Adjusted Newsletter emails
2324
- Installed HTMX rather than using script tag
2425

2526

agents.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
11
## Agent Note
22

33
In this project **always run Python from the virtual environment (`venv`)** (e.g. `venv/bin/python` or `source venv/bin/activate`), not the system Python.
4+
5+
**Code style:** Comments and variable names must always be in English.
6+
7+
**Changelog:** Add entries **only** to the root `CHANGELOG.md`. Keep them **very short**; do not use separate changelog files or long prose.

apps/summarization/llm_json.py

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
"""Extract and parse JSON from LLM text (strip wrappers, repair, validate as Pydantic)."""
2+
3+
from __future__ import annotations
4+
5+
import re
6+
from typing import Any
7+
8+
import json_repair
9+
from pydantic import BaseModel
10+
11+
_FENCED_JSON = re.compile(r"```(?:json)?\s*([\s\S]*?)```", re.IGNORECASE)
12+
13+
14+
def extract_llm_json_payload(text: str) -> str:
15+
"""
16+
Remove surrounding prose and markdown fences so the remainder is JSON-like.
17+
18+
- Strips ```json ... ``` (or ``` ... ```) blocks if present.
19+
- Drops any leading characters before the first ``{`` or ``[``.
20+
"""
21+
s = (text or "").strip()
22+
if not s:
23+
return s
24+
25+
m = _FENCED_JSON.search(s)
26+
if m:
27+
s = m.group(1).strip()
28+
29+
for i, ch in enumerate(s):
30+
if ch in "{[":
31+
return s[i:].strip()
32+
33+
return s
34+
35+
36+
def parse_structured_llm_json(raw_text: str, result_type: type[BaseModel]) -> BaseModel:
37+
"""Strip wrappers, repair with json_repair, validate to ``result_type``."""
38+
payload = extract_llm_json_payload(raw_text)
39+
data: Any = json_repair.loads(payload)
40+
return result_type.model_validate(data)
Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
"""Rich error formatting for AI summarization test playground (staff debugging)."""
2+
3+
from __future__ import annotations
4+
5+
from collections import deque
6+
from typing import Any
7+
8+
9+
def format_playground_exception(exc: BaseException, *, max_related: int = 16) -> str:
10+
"""
11+
Build a multi-line, human-readable error for the summarization test UIs.
12+
13+
Works for any exception type. Adds optional Pydantic validation details when present.
14+
"""
15+
lines: list[str] = []
16+
17+
def append_block(title: str, body: str) -> None:
18+
if lines:
19+
lines.append("")
20+
lines.append(f"=== {title} ===")
21+
lines.append(body.strip() if body else "(keine Meldung)")
22+
23+
append_block("Hauptfehler", f"{type(exc).__name__}: {exc}")
24+
25+
related = _collect_related_exceptions(exc, max_total=max_related)
26+
if related:
27+
parts = []
28+
for i, link in enumerate(related, start=1):
29+
parts.append(f"[{i}] {type(link).__name__}: {link}")
30+
append_block("Verknüpfte Fehler (Ursache / Kontext / Gruppe)", "\n".join(parts))
31+
32+
pydantic_bits = _collect_pydantic_error_details(exc)
33+
if pydantic_bits:
34+
append_block("Validierungsdetails", "\n".join(pydantic_bits))
35+
36+
return "\n".join(lines)
37+
38+
39+
def _collect_related_exceptions(root: BaseException, *, max_total: int) -> list[BaseException]:
40+
"""
41+
All exceptions reachable via __cause__, __context__, and ExceptionGroup.subexceptions.
42+
43+
Excludes `root` (already shown as Hauptfehler). Order: BFS, no duplicates.
44+
"""
45+
out: list[BaseException] = []
46+
seen: set[int] = set()
47+
q: deque[BaseException] = deque()
48+
seen.add(id(root))
49+
50+
def enqueue(e: BaseException | None) -> None:
51+
if e is None or id(e) in seen:
52+
return
53+
seen.add(id(e))
54+
q.append(e)
55+
56+
enqueue(getattr(root, "__cause__", None))
57+
enqueue(getattr(root, "__context__", None))
58+
_enqueue_exception_group_children(root, enqueue)
59+
60+
while q and len(out) < max_total:
61+
cur = q.popleft()
62+
out.append(cur)
63+
enqueue(getattr(cur, "__cause__", None))
64+
enqueue(getattr(cur, "__context__", None))
65+
_enqueue_exception_group_children(cur, enqueue)
66+
67+
return out
68+
69+
70+
def _enqueue_exception_group_children(exc: BaseException, enqueue: Any) -> None:
71+
subs = getattr(exc, "exceptions", None)
72+
if not subs:
73+
return
74+
if type(exc).__name__ not in ("ExceptionGroup", "BaseExceptionGroup"):
75+
return
76+
for sub in subs:
77+
if isinstance(sub, BaseException):
78+
enqueue(sub)
79+
80+
81+
def _collect_pydantic_error_details(exc: BaseException, *, max_errors: int = 12) -> list[str]:
82+
"""Extract pydantic v2 ValidationError.errors() entries from an exception chain."""
83+
out: list[str] = []
84+
seen: set[int] = set()
85+
stack: list[BaseException] = [exc]
86+
87+
while stack:
88+
cur = stack.pop()
89+
cid = id(cur)
90+
if cid in seen:
91+
continue
92+
seen.add(cid)
93+
94+
err_fn = getattr(cur, "errors", None)
95+
if callable(err_fn):
96+
try:
97+
raw = err_fn()
98+
except Exception:
99+
raw = None
100+
if isinstance(raw, list) and raw:
101+
for item in raw[:max_errors]:
102+
out.append(_format_one_pydantic_error(item))
103+
if len(raw) > max_errors:
104+
out.append(f"... und {len(raw) - max_errors} weitere Fehler")
105+
return out
106+
107+
for nxt in (getattr(cur, "__cause__", None), getattr(cur, "__context__", None)):
108+
if isinstance(nxt, BaseException):
109+
stack.append(nxt)
110+
111+
return out
112+
113+
114+
def _format_one_pydantic_error(item: Any) -> str:
115+
if not isinstance(item, dict):
116+
return str(item)
117+
loc = item.get("loc")
118+
loc_s = ".".join(str(x) for x in loc) if isinstance(loc, tuple) else str(loc)
119+
msg = item.get("msg", "")
120+
typ = item.get("type", "")
121+
parts = [f"• {loc_s or '(root)'}: {msg}"]
122+
if typ:
123+
parts.append(f" (Typ: {typ})")
124+
inp = item.get("input")
125+
if inp is not None:
126+
snippet = _shorten_for_display(inp, limit=400)
127+
parts.append(f" Eingabe-Ausschnitt: {snippet!r}")
128+
return "\n".join(parts)
129+
130+
131+
def _shorten_for_display(s: Any, *, limit: int) -> str:
132+
text = s if isinstance(s, str) else repr(s)
133+
text = text.replace("\r\n", "\n").replace("\r", "\n")
134+
if len(text) <= limit:
135+
return text
136+
return text[: limit - 3] + "..."

apps/summarization/providers.py

Lines changed: 36 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2,19 +2,34 @@
22

33
import logging
44
from abc import ABC
5+
from typing import TypeVar
6+
from typing import cast
57

68
from django.conf import settings
79
from pydantic import BaseModel
810
from pydantic_ai import Agent
911
from pydantic_ai import ImageUrl
12+
from pydantic_ai import TextOutput
1013
from pydantic_ai.models.mistral import MistralModel
1114
from pydantic_ai.models.openai import OpenAIChatModel
1215
from pydantic_ai.providers.mistral import MistralProvider
1316
from pydantic_ai.providers.openai import OpenAIProvider
1417
from sentry_sdk import capture_exception
1518

19+
from .llm_json import parse_structured_llm_json
20+
from .pydantic_models import DocumentSummaryResponse
21+
1622
logger = logging.getLogger(__name__)
1723

24+
TModel = TypeVar("TModel", bound=BaseModel)
25+
26+
27+
def _make_json_parse_fn(result_type: type[TModel]):
28+
def parse(text: str) -> TModel:
29+
return parse_structured_llm_json(text, result_type)
30+
31+
return parse
32+
1833

1934
class ProviderConfig:
2035
"""Configuration for an AI provider."""
@@ -176,8 +191,8 @@ def text_request(
176191
agent = Agent(
177192
model=model,
178193
system_prompt=self.system_prompt,
179-
output_type=result_type,
180-
tools=[], # Disable tool_calls to avoid validation errors with non-standard providers
194+
output_type=TextOutput(_make_json_parse_fn(result_type)),
195+
tools=[],
181196
)
182197

183198
try:
@@ -195,6 +210,7 @@ def text_request(
195210
capture_exception(e)
196211
raise
197212

213+
# Deprecate ? And Use text_request or vision_request instead ?
198214
def request(self, request: AIRequest, result_type: type[BaseModel]) -> BaseModel:
199215
"""
200216
Automatically determines if it's a text or multimodal request.
@@ -203,24 +219,33 @@ def request(self, request: AIRequest, result_type: type[BaseModel]) -> BaseModel
203219
# Check if request supports vision (multimodal request)
204220
if getattr(request, "vision_support", False):
205221
image_urls = getattr(request, "image_urls", None) or []
206-
return self.multimodal_request(request, result_type, image_urls)
222+
if not issubclass(result_type, DocumentSummaryResponse):
223+
raise TypeError(
224+
"Vision requests require result_type to be DocumentSummaryResponse or a subclass."
225+
)
226+
return self.multimodal_request(
227+
request, cast(type[DocumentSummaryResponse], result_type), image_urls
228+
)
207229
else:
208230
return self.text_request(request, result_type)
209231

210-
# TODO: Deprectaed ? Use separate Vison Requests instead ?
232+
# Rename to vision_request instead and use only DocumentSummaryResponse for the result type?
211233
def multimodal_request(
212-
self, request: AIRequest, result_type: type[BaseModel], image_urls: list[str]
213-
) -> BaseModel:
234+
self,
235+
request: AIRequest,
236+
result_type: type[DocumentSummaryResponse],
237+
image_urls: list[str],
238+
) -> DocumentSummaryResponse:
214239
"""
215240
Execute a multimodal request with images using vision API.
216241
217242
Args:
218243
request: Pydantic BaseModel with request data
219-
result_type: Pydantic BaseModel class for structured output
244+
result_type: DocumentSummaryResponse (or a subclass at runtime)
220245
image_urls: List of image URLs to include in the request
221246
222247
Returns:
223-
Structured response as BaseModel instance
248+
Structured response instance
224249
"""
225250
# Use MistralModel for Mistral, OpenAIChatModel for others
226251
# Note: Mistral may not support vision/multimodal requests
@@ -241,9 +266,9 @@ def multimodal_request(
241266
agent = Agent(
242267
model=model,
243268
system_prompt=self.system_prompt,
244-
output_type=result_type,
245-
output_retries=3, # Allow more retries for vision output validation
246-
tools=[], # Disable tool_calls to avoid validation errors with non-standard providers
269+
output_type=TextOutput(_make_json_parse_fn(result_type)),
270+
output_retries=3,
271+
tools=[],
247272
)
248273

249274
# Build user content with prompt and image URLs

apps/summarization/templates/summarization/test.html

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,19 @@
8080
border-left-color: #dc3545;
8181
color: #721c24;
8282
}
83+
.error pre.error-detail {
84+
margin: 10px 0 0 0;
85+
padding: 12px;
86+
background: #fff;
87+
border: 1px solid #f5c6cb;
88+
border-radius: 4px;
89+
white-space: pre-wrap;
90+
word-break: break-word;
91+
font-family: ui-monospace, "Courier New", monospace;
92+
font-size: 13px;
93+
line-height: 1.45;
94+
color: #491217;
95+
}
8396
.summary {
8497
margin-top: 15px;
8598
margin-bottom: 20px;
@@ -169,7 +182,8 @@ <h1>AI Summarization Test</h1>
169182

170183
{% if error %}
171184
<div class="result error">
172-
<strong>Error:</strong> {{ error }}
185+
<strong>Fehlerdetails</strong>
186+
<pre class="error-detail">{{ error }}</pre>
173187
</div>
174188
{% endif %}
175189

apps/summarization/templates/summarization/test_documents.html

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,19 @@
7070
border-left-color: #dc3545;
7171
color: #721c24;
7272
}
73+
.error pre.error-detail {
74+
margin: 10px 0 0 0;
75+
padding: 12px;
76+
background: #fff;
77+
border: 1px solid #f5c6cb;
78+
border-radius: 4px;
79+
white-space: pre-wrap;
80+
word-break: break-word;
81+
font-family: ui-monospace, "Courier New", monospace;
82+
font-size: 13px;
83+
line-height: 1.45;
84+
color: #491217;
85+
}
7386
.document-item {
7487
margin-top: 15px;
7588
margin-bottom: 20px;
@@ -152,7 +165,8 @@ <h1>Document Summarization Test</h1>
152165

153166
{% if error %}
154167
<div class="result error">
155-
<strong>Error:</strong> {{ error }}
168+
<strong>Fehlerdetails</strong>
169+
<pre class="error-detail">{{ error }}</pre>
156170
</div>
157171
{% endif %}
158172

apps/summarization/views.py

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
from adhocracy4.projects.models import Project
1212

1313
from .export_utils.core import generate_full_export
14+
from .playground_errors import format_playground_exception
1415
from .pydantic_models import DocumentInputItem
1516
from .pydantic_models import ProjectSummaryResponse
1617
from .services import AIService
@@ -80,7 +81,7 @@ def _handle_text_request(
8081
except BaseException as e:
8182
if isinstance(e, (KeyboardInterrupt, SystemExit, GeneratorExit)):
8283
raise
83-
return None, 0, str(e)
84+
return None, 0, format_playground_exception(e)
8485

8586
def _extract_project_from_json(self, text: str):
8687
"""Extract project information from JSON text if available."""
@@ -236,9 +237,9 @@ def post(self, request):
236237
context["summary_response"] = response
237238

238239
except json.JSONDecodeError as e:
239-
context["error"] = f"Invalid JSON: {str(e)}"
240+
context["error"] = format_playground_exception(e)
240241
except Exception as e:
241-
context["error"] = str(e)
242+
context["error"] = format_playground_exception(e)
242243
else:
243244
context["error"] = "Please provide documents in JSON format"
244245

0 commit comments

Comments
 (0)