AssertionError: {}
assert 0 == 2
 +  where 0 = len({})
self = <tests.pytest.acceptance.evaluations.test_evaluate_flow.TestEvaluatorOrderingInvariant object at 0x7f2029be20d0>
agenta_init = None
evaluators = [<function output_matches at 0x7f2029c828e0>, <function score_dict at 0x7f2029c82ca0>]

    @pytest.mark.parametrize(
        "evaluators",
        [
            [score_dict, output_matches],  # contaminator first
            [output_matches, score_dict],  # contaminator second
        ],
        ids=["contaminator-first", "contaminator-second"],
    )
    async def test_order_does_not_change_evaluator_inputs(
        self, agenta_init, evaluators
    ):
        rev = await _make_testset()
        result = await aevaluate(
            name="sdk-eval-order",
            testsets={str(rev.id): "custom"},
            applications=[echo_app],
            evaluators=evaluators,
        )
        _assert_eval_result(result, expected_scenarios=2)
    
        means = _evaluator_score_means(result)
        # both evaluators scored.
>       assert len(means) == 2, means
E       AssertionError: {}
E       assert 0 == 2
E        +  where 0 = len({})

oss/tests/pytest/acceptance/evaluations/test_evaluate_flow.py:272: AssertionError

AssertionError: {}
assert 0 == 3
 +  where 0 = len({})
self = <tests.pytest.acceptance.evaluations.test_evaluate_flow.TestEvaluatorOrderingInvariant object at 0x7f2029bbe520>
agenta_init = None

    async def test_sensitive_evaluator_on_both_sides_of_contaminator(self, agenta_init):
        # The strongest shape: a contamination-sensitive evaluator sits BEFORE
        # and AFTER the dict-returning one. Both must score 1.0 — the one after
        # the contaminator is the position that regressed in production.
        rev = await _make_testset()
        result = await aevaluate(
            name="sdk-eval-sandwich",
            testsets={str(rev.id): "custom"},
            applications=[echo_app],
            evaluators=[output_matches, score_dict, output_matches_b],
        )
        _assert_eval_result(result, expected_scenarios=2)
        means = _evaluator_score_means(result)
>       assert len(means) == 3, means
E       AssertionError: {}
E       assert 0 == 3
E        +  where 0 = len({})

oss/tests/pytest/acceptance/evaluations/test_evaluate_flow.py:295: AssertionError

AssertionError: no evaluator metrics in run metrics: []
assert {}
self = <tests.pytest.acceptance.evaluations.test_evaluate_flow.TestEvaluateLocalCallable object at 0x7f5b93e36fd0>
agenta_init = None

    async def test_multiple_evaluators(self, agenta_init):
        rev = await _make_testset()
        result = await aevaluate(
            name="sdk-eval-multi",
            testsets={str(rev.id): "custom"},
            applications=[echo_app],
            evaluators=[pass_evaluator, length_evaluator],
        )
        _assert_eval_result(result, expected_scenarios=2)
>       _assert_evaluator_metrics_present(result)

oss/tests/pytest/acceptance/evaluations/test_evaluate_flow.py:210: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

result = {'run': EvaluationRun(id=UUID('019f0069-bc4b-78b3-aa36-802867341ca2')), 'scenarios': [{'scenario': EvaluationScenario(....}}}, 'metrics': None, 'status': <EvaluationStatus.ERRORS: 'errors'>}], 'metrics': {'global': None, 'variational': []}}

    def _assert_evaluator_metrics_present(result):
        # The evaluator must actually be EXECUTED by the SDK runtime (custom origin),
        # so its outputs land in the run metrics. Before the custom-execution fix the
        # evaluator step was skipped (logged pending, trace_id=None) and produced no
        # metrics — this assertion guards that regression.
        data = _metrics_data(result)
        evaluator_steps = {k: v for k, v in data.items() if k.startswith("evaluator-")}
>       assert evaluator_steps, f"no evaluator metrics in run metrics: {list(data.keys())}"
E       AssertionError: no evaluator metrics in run metrics: []
E       assert {}

oss/tests/pytest/acceptance/evaluations/test_evaluate_flow.py:148: AssertionError

AssertionError: {}
assert 0 == 2
 +  where 0 = len({})
self = <tests.pytest.acceptance.evaluations.test_evaluate_flow.TestEvaluatorOrderingInvariant object at 0x7f5b93e56490>
agenta_init = None
evaluators = [<function score_dict at 0x7f5b93d4aac0>, <function output_matches at 0x7f5b93d4a700>]

    @pytest.mark.parametrize(
        "evaluators",
        [
            [score_dict, output_matches],  # contaminator first
            [output_matches, score_dict],  # contaminator second
        ],
        ids=["contaminator-first", "contaminator-second"],
    )
    async def test_order_does_not_change_evaluator_inputs(
        self, agenta_init, evaluators
    ):
        rev = await _make_testset()
        result = await aevaluate(
            name="sdk-eval-order",
            testsets={str(rev.id): "custom"},
            applications=[echo_app],
            evaluators=evaluators,
        )
        _assert_eval_result(result, expected_scenarios=2)
    
        means = _evaluator_score_means(result)
        # both evaluators scored.
>       assert len(means) == 2, means
E       AssertionError: {}
E       assert 0 == 2
E        +  where 0 = len({})

oss/tests/pytest/acceptance/evaluations/test_evaluate_flow.py:272: AssertionError