fix(gooddata-eval): stop transient gen-ai faults from being scored as content failures

packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py

@@ -32,8 +32,15 @@
 # gen-ai's last event, only if at least one item was already emitted (conversations_controller.py).
 _RESPONSE_ENDED_EVENT = "response_ended"
 
-_RETRYABLE_STATUS_CODES: frozenset[int] = frozenset({429, 502, 503, 504})
+# 500 is here on evidence, not on principle: in one visualization eval batch it hard-failed
+# 10 of 56 runs with zero retry attempts, and every affected question scored normally when the
+# same question/model ran again the next day -- i.e. transient gen-ai faults, not deterministic
+# server bugs. A genuinely deterministic 500 still terminates, just after the bounded backoff.
+_RETRYABLE_STATUS_CODES: frozenset[int] = frozenset({429, 500, 502, 503, 504})
 _METADATA_SYNC_MARKER = "METADATA_SYNC_IN_PROGRESS"
+# Stands in for the `id` a persisted visualization would carry, on the fallback path
+# where the agent's create_adhoc_visualization call failed and only its arguments survive.
+_ADHOC_VIZ_ID = "adhoc-visualization-not-persisted"
 
 _KNOWN_PART_TYPES: frozenset[str] = frozenset(
     {
@@ -247,8 +254,14 @@ def _build_chat_result(acc: _SseAccumulator) -> ChatResult:
         # Fallback: the agent produced a correct visualization definition via
         # create_adhoc_visualization but the call failed (e.g. data source not
         # accessible). The last attempt is the agent's best answer.
+        #
+        # These are raw tool-call arguments, so they carry no `id` -- nothing was
+        # ever persisted. CreatedVisualization requires one, so synthesize a
+        # sentinel rather than letting the whole ChatResult fail to validate:
+        # dropping the turn entirely would score a stalled data source as a
+        # content failure, which is exactly what this fallback exists to prevent.
         payload["createdVisualizations"] = {
-            "objects": [acc.adhoc_viz_args[-1]],
+            "objects": [{"id": _ADHOC_VIZ_ID, **acc.adhoc_viz_args[-1]}],
             "reasoning": "\n".join(acc.viz_reasoning_parts),
         }
     result = ChatResult.model_validate(payload)

packages/gooddata-eval/src/gooddata_eval/core/scoring.py

@@ -1,8 +1,10 @@
 # (C) 2026 GoodData Corporation
 """Visualization scoring — ported from gdc-nas tavern-e2e app/vis_assertions/metrics.py."""
 
+import calendar
 import json
 from dataclasses import dataclass
+from datetime import date, timedelta
 
 from gooddata_eval.core.models import AacBucketRef, AacQueryField, CreatedVisualization
 
@@ -106,13 +108,81 @@ def validate_cross_references(viz: CreatedVisualization) -> tuple[bool, list[str
     return len(errors) == 0, errors
 
 
-def _normalize_date_filter(filter_dict: dict, _fields: dict) -> dict:
+def _shift_month(anchor: date, offset: int) -> tuple[date, date]:
+    """First and last day of the calendar month ``offset`` months from ``anchor``."""
+    total = anchor.year * 12 + (anchor.month - 1) + offset
+    year, month = divmod(total, 12)
+    return date(year, month + 1, 1), date(year, month + 1, calendar.monthrange(year, month + 1)[1])
+
+
+def _absolute_span(granularity: str, start_offset: int, end_offset: int, today: date) -> tuple[date, date] | None:
+    """Resolve a relative date filter to the inclusive absolute span it denotes.
+
+    Returns None for granularities this cannot resolve unambiguously -- notably the
+    WEEK family, whose start-of-week convention varies (WEEK vs WEEK_US vs ...).
+    Guessing there would trade a false negative for a false positive.
+    """
+    gran = granularity.upper()
+    if gran == "DAY":
+        return today + timedelta(days=start_offset), today + timedelta(days=end_offset)
+    if gran == "MONTH":
+        return _shift_month(today, start_offset)[0], _shift_month(today, end_offset)[1]
+    if gran == "QUARTER":
+        q_start_month = (today.month - 1) // 3 * 3 + 1
+        anchor = date(today.year, q_start_month, 1)
+        return _shift_month(anchor, start_offset * 3)[0], _shift_month(anchor, end_offset * 3 + 2)[1]
+    if gran == "YEAR":
+        return date(today.year + start_offset, 1, 1), date(today.year + end_offset, 12, 31)
+    return None
+
+
+def _as_date(value: object) -> date | None:
+    if isinstance(value, str):
+        try:
+            return date.fromisoformat(value[:10])
+        except ValueError:
+            return None
+    return None
+
+
+def _normalize_date_filter(filter_dict: dict, _fields: dict, today: date | None = None) -> dict:
+    """Canonicalize a date filter, resolving relative offsets to an absolute span.
+
+    The agent may answer "last month" either relatively (``granularity: MONTH,
+    from: -1, to: -1``) or absolutely (``from: 2026-08-01, to: 2026-08-31``).
+    Compared literally these never match, so a correct answer in the encoding the
+    fixture did not happen to use was scored as a wrong date period. Both forms
+    collapse to the same absolute span here.
+
+    Resolution is relative to today, which is the same "today" the agent resolved
+    against -- scoring runs in the same process as the turn. Re-scoring an archived
+    result at a later date would therefore drift; nothing currently does that.
+    """
+    raw_from, raw_to = filter_dict.get("from"), filter_dict.get("to")
+    granularity = filter_dict.get("granularity")
+    span: tuple[date, date] | None = None
+
+    if isinstance(raw_from, int) and isinstance(raw_to, int) and isinstance(granularity, str):
+        span = _absolute_span(granularity, raw_from, raw_to, today or date.today())
+    else:
+        start, end = _as_date(raw_from), _as_date(raw_to)
+        if start and end:
+            span = (start, end)
+
+    if span is not None:
+        return {
+            "type": "date_filter",
+            "dataset_uri": filter_dict.get("using", ""),
+            "from": span[0].isoformat(),
+            "to": span[1].isoformat(),
+        }
+    # Unresolvable (e.g. the WEEK family): fall back to literal comparison.
     return {
         "type": "date_filter",
         "dataset_uri": filter_dict.get("using", ""),
-        "from": filter_dict.get("from"),
-        "to": filter_dict.get("to"),
-        "granularity": filter_dict.get("granularity"),
+        "from": raw_from,
+        "to": raw_to,
+        "granularity": granularity,
     }
 
 
@@ -171,7 +241,9 @@ def _normalize_attribute_filter(filter_dict: dict, _fields: dict) -> dict:
     }
 
 
-def _split_and_normalize_filters(viz: CreatedVisualization) -> tuple[set[str], set[str], set[str]]:
+def _split_and_normalize_filters(
+    viz: CreatedVisualization, today: date | None = None
+) -> tuple[set[str], set[str], set[str]]:
     date_set: set[str] = set()
     ranking_set: set[str] = set()
     attr_set: set[str] = set()
@@ -180,15 +252,15 @@ def _split_and_normalize_filters(viz: CreatedVisualization) -> tuple[set[str], s
     for filter_dict in viz.query.filter_by.values():
         ft = filter_dict.get("type")
         if ft == "date_filter":
-            date_set.add(json.dumps(_normalize_date_filter(filter_dict, fields), sort_keys=True))
+            date_set.add(json.dumps(_normalize_date_filter(filter_dict, fields, today), sort_keys=True))
         elif ft == "ranking_filter":
             ranking_set.add(json.dumps(_normalize_ranking_filter(filter_dict, fields, sole_dim_uri), sort_keys=True))
         elif ft == "attribute_filter":
             attr_set.add(json.dumps(_normalize_attribute_filter(filter_dict, fields), sort_keys=True))
     return date_set, ranking_set, attr_set
 
 
-def normalized_filters(viz: CreatedVisualization) -> dict[str, list[str]]:
+def normalized_filters(viz: CreatedVisualization, today: date | None = None) -> dict[str, list[str]]:
     """A visualization's filters exactly as `check_filters` compares them.
 
     Grouped by the three categories it scores separately and sorted for stable output.
@@ -197,13 +269,18 @@ def normalized_filters(viz: CreatedVisualization) -> dict[str, list[str]]:
     a `filter_date_score` of False otherwise gives no clue whether the period differed,
     the granularity did, or the dataset the filter hangs off did.
     """
-    date_set, ranking_set, attr_set = _split_and_normalize_filters(viz)
+    date_set, ranking_set, attr_set = _split_and_normalize_filters(viz, today)
     return {"date": sorted(date_set), "ranking": sorted(ranking_set), "attribute": sorted(attr_set)}
 
 
-def check_filters(expected: CreatedVisualization, actual: CreatedVisualization) -> FilterScores:
-    exp_date, exp_rank, exp_attr = _split_and_normalize_filters(expected)
-    act_date, act_rank, act_attr = _split_and_normalize_filters(actual)
+def check_filters(
+    expected: CreatedVisualization, actual: CreatedVisualization, today: date | None = None
+) -> FilterScores:
+    # One anchor for both sides: resolving each against its own date.today() would
+    # score inconsistently for a run that straddles midnight.
+    today = today or date.today()
+    exp_date, exp_rank, exp_attr = _split_and_normalize_filters(expected, today)
+    act_date, act_rank, act_attr = _split_and_normalize_filters(actual, today)
     return FilterScores(
         date_ok=act_date == exp_date,
         ranking_ok=act_rank == exp_rank,

packages/gooddata-eval/tests/test_scoring.py

@@ -1,4 +1,6 @@
 # (C) 2026 GoodData Corporation
+from datetime import date
+
 from gooddata_eval.core.models import CreatedVisualization
 from gooddata_eval.core.scoring import (
     check_filters,
@@ -205,3 +207,58 @@ def test_normalized_filters_is_empty_per_category_when_unfiltered():
         }
     )
     assert normalized_filters(viz) == {"date": [], "ranking": [], "attribute": []}
+
+
+# --- relative vs absolute date filters denote the same period ---
+#
+# The agent answers "last month" either relatively (granularity MONTH, from -1, to -1)
+# or absolutely (from 2026-08-01, to 2026-08-31). Compared literally these never match,
+# so a correct answer in whichever encoding the fixture did not happen to use was scored
+# as a wrong date period. Both forms now collapse to the same absolute span.
+
+_TODAY = date(2026, 9, 9)
+
+
+def _date_viz(**overrides):
+    f = {"using": "dataset/dt_transactions", "type": "date_filter"}
+    f.update(overrides)
+    return _viz(query={"fields": {}, "filter_by": {"f_d": f}})
+
+
+def test_check_filters_relative_and_absolute_last_month_agree():
+    expected = _date_viz(**{"from": -1, "to": -1, "granularity": "MONTH"})
+    actual = _date_viz(**{"from": "2026-08-01", "to": "2026-08-31", "granularity": None})
+    assert check_filters(expected, actual, _TODAY).date_ok is True
+
+
+def test_check_filters_absolute_spanning_two_months_still_differs_from_one():
+    """Normalization must not flatten a genuinely wrong window into a match."""
+    expected = _date_viz(**{"from": -1, "to": -1, "granularity": "MONTH"})
+    actual = _date_viz(**{"from": "2026-07-01", "to": "2026-08-31", "granularity": None})
+    assert check_filters(expected, actual, _TODAY).date_ok is False
+
+
+def test_check_filters_day_offsets_resolve_and_off_by_one_still_fails():
+    expected = _date_viz(**{"from": -89, "to": 0, "granularity": "DAY"})
+    assert check_filters(expected, _date_viz(**{"from": "2026-06-12", "to": "2026-09-09"}), _TODAY).date_ok is True
+    assert check_filters(expected, _date_viz(**{"from": -90, "to": 0, "granularity": "DAY"}), _TODAY).date_ok is False
+
+
+def test_check_filters_quarter_and_year_offsets_resolve():
+    q = _date_viz(**{"from": -1, "to": -1, "granularity": "QUARTER"})
+    assert check_filters(q, _date_viz(**{"from": "2026-04-01", "to": "2026-06-30"}), _TODAY).date_ok is True
+    y = _date_viz(**{"from": 0, "to": 0, "granularity": "YEAR"})
+    assert check_filters(y, _date_viz(**{"from": "2026-01-01", "to": "2026-12-31"}), _TODAY).date_ok is True
+
+
+def test_check_filters_week_granularity_falls_back_to_literal_comparison():
+    """WEEK start-of-week convention varies, so it is compared literally rather than guessed."""
+    expected = _date_viz(**{"from": -2, "to": -1, "granularity": "WEEK"})
+    assert check_filters(expected, _date_viz(**{"from": -2, "to": -1, "granularity": "WEEK"}), _TODAY).date_ok is True
+    assert check_filters(expected, _date_viz(**{"from": -13, "to": 0, "granularity": "DAY"}), _TODAY).date_ok is False
+
+
+def test_check_filters_date_still_distinguishes_the_dataset_it_hangs_off():
+    expected = _date_viz(**{"from": -1, "to": -1, "granularity": "MONTH"})
+    actual = _date_viz(using="dataset/dt_date", **{"from": -1, "to": -1, "granularity": "MONTH"})
+    assert check_filters(expected, actual, _TODAY).date_ok is False

packages/gooddata-eval/tests/test_sse_client.py

@@ -18,8 +18,9 @@ def test_parse_sse_lines_collects_text_and_visualization(fixtures_dir):
 
 
 def test_parse_sse_lines_raises_on_error_event():
-    lines = ['data: {"statusCode": 500, "detail": "boom"}']
-    with pytest.raises(RuntimeError, match="SSE error 500"):
+    # 400: a code outside _RETRYABLE_STATUS_CODES, so this exercises the terminal path.
+    lines = ['data: {"statusCode": 400, "detail": "boom"}']
+    with pytest.raises(RuntimeError, match="SSE error 400"):
         parse_sse_lines(lines)
 
 
@@ -51,7 +52,7 @@ def test_parse_sse_lines_error_carries_partial_result_with_tool_calls_already_se
             }
         ),
         "",
-        json.dumps({"statusCode": 500, "detail": "boom"}),
+        json.dumps({"statusCode": 400, "detail": "boom"}),
     ]
     lines = [f"data: {line}" if line else line for line in lines]
     with pytest.raises(ChatError) as ei:
@@ -362,6 +363,28 @@ def test_parse_sse_lines_falls_back_to_adhoc_viz_when_multipart_viz_is_null():
     assert result.created_visualizations.objects[0].type == "line_chart"
 
 
+def test_parse_sse_lines_adhoc_fallback_synthesizes_id_when_args_have_none():
+    """A create_adhoc_visualization definition carries no `id` -- nothing was persisted.
+
+    CreatedVisualization requires one, so without a synthesized stand-in the whole
+    ChatResult fails to validate and the turn is lost. Regression test: real agent
+    tool arguments have no `id`, unlike the hand-written fixtures above.
+    """
+    viz_def = {
+        "type": "line_chart",
+        "query": {"fields": {"m": {"using": "metric/total_sales"}}, "filter_by": {}},
+        "metrics": ["m"],
+    }
+    lines = [
+        f'data: {{"item": {{"role": "assistant", "content": {{"type": "toolCall", "callId": "c1", "name": "create_adhoc_visualization", "arguments": {{"visualization": {json.dumps(viz_def)}}}}}}}}}',
+        'data: {"item": {"role": "assistant", "content": {"type": "multipart", "parts": [{"type": "visualization", "visualization": null}]}}}',
+    ]
+    result = parse_sse_lines(lines)
+    assert result.created_visualizations is not None
+    assert result.created_visualizations.objects[0].id == "adhoc-visualization-not-persisted"
+    assert result.created_visualizations.objects[0].type == "line_chart"
+
+
 def test_parse_sse_lines_counts_reasoning_steps():
     lines = [
         'data: {"item": {"role": "assistant", "content": {"type": "reasoning", "summary": "step one"}}}',
@@ -450,7 +473,7 @@ def test_parse_sse_lines_has_no_alert_proposals_by_default():
     assert parse_sse_lines(lines).alert_proposals == []
 
 
-@pytest.mark.parametrize("code", [429, 502, 503, 504])
+@pytest.mark.parametrize("code", [429, 500, 502, 503, 504])
 def test_parse_sse_lines_transient_status_codes(code):
     with pytest.raises(TransientChatError) as ei:
         parse_sse_lines([f'data: {{"statusCode": {code}, "detail": null}}'])

packages/gooddata-eval/tests/test_visualization_evaluator.py

@@ -1,4 +1,6 @@
 # (C) 2026 GoodData Corporation
+import re
+
 from gooddata_eval.core.evaluators import get_evaluator
 from gooddata_eval.core.models import ChatResult, DatasetItem
 
@@ -158,8 +160,11 @@ def test_detail_reports_the_filters_that_were_compared():
 
     assert result.detail["filter_date_score"] is False
     expected, actual = result.detail["expected_filters"], result.detail["actual_filters"]
-    assert '"from": -11' in expected["date"][0]
-    assert '"from": -12' in actual["date"][0]
+    # Relative offsets are reported as the absolute span they resolve to, so the two
+    # periods are legible side by side and visibly different -- which is the point.
+    assert re.search(r'"from": "\d{4}-\d{2}-\d{2}"', expected["date"][0])
+    assert re.search(r'"from": "\d{4}-\d{2}-\d{2}"', actual["date"][0])
+    assert expected["date"] != actual["date"]
     assert expected["ranking"] == actual["ranking"] == []
     assert expected["attribute"] == actual["attribute"] == []