fix(agent): Claude alias resolution + skill/error tracing + error sanitize (QA F-031/F-029/F-030) - #4855
38 errors, 58 fail, 8 skipped, 558 pass in 6m 23s
662 tests 558 ✅ 6m 23s ⏱️
1 suites 8 💤
1 files 58 ❌ 38 🔥
Results for commit 3a51244.
Annotations
github-actions / Railway API Test Results
test_restart_finished_batch_run_re_dispatches_and_finalizes_again (oss.tests.pytest.acceptance.evaluations.test_evaluation_flows_modify.TestEvaluationModifyFlows) failed
api/oss/tests/results/junit.xml [took 2m 18s]
Raw output
AssertionError: assert 'running' == 'success'
- success
+ running
self = <evaluations.test_evaluation_flows_modify.TestEvaluationModifyFlows object at 0x7f0a278911d0>
authed_api = <function authed_api.<locals>._request at 0x7f0a277027a0>
def test_restart_finished_batch_run_re_dispatches_and_finalizes_again(
self, authed_api
):
# ARRANGE: run a batch eval to terminal success ------------------------
testset = create_testset(authed_api)
application = create_mock_application(authed_api, key="echo")
evaluator = create_mock_evaluator(authed_api, key="pass")
evaluation = create_simple_evaluation(
authed_api,
name="flow-restart-finished",
data={
"testset_steps": [testset["revision_id"]],
"application_steps": [application["revision_id"]],
"evaluator_steps": {evaluator["revision_id"]: "auto"},
},
)
run_id = evaluation["id"]
first = wait_for_run_terminal(authed_api, run_id)
> assert first.json()["run"]["status"] == "success"
E AssertionError: assert 'running' == 'success'
E
E - success
E + running
oss/tests/pytest/acceptance/evaluations/test_evaluation_flows_modify.py:43: AssertionError
github-actions / Railway API Test Results
test_create_annotations (oss.tests.pytest.acceptance.annotations.test_annotations_basics.TestAnnotationsBasics) failed
api/oss/tests/results/junit.xml [took 1s]
Raw output
assert 500 in (200, 202)
+ where 500 = <Response [500]>.status_code
self = <annotations.test_annotations_basics.TestAnnotationsBasics object at 0x7fb00131a710>
authed_api = <function authed_api.<locals>._request at 0x7fb000bece00>
def test_create_annotations(self, authed_api):
# ACT ------------------------------------------------------------------
evaluator_slug = str(uuid4())
annotation_data_outputs = {
"integer": 42,
"float": 3.14,
"string": "Hello, world!",
"boolean": True,
"array": [1],
"object": {"aloha": "mahalo"},
}
annotation_links = {
"invocation": {
"trace_id": "019688a5e2097b80ad12e2907b83e9fc",
"span_id": "25a8e0b9bbf37d30",
}
}
response = authed_api(
"POST",
"/simple/traces/",
json={
"trace": {
"data": {
"outputs": annotation_data_outputs,
},
"tags": {"tag1": "value1", "tag2": "value2"},
"meta": {"meta1": "value1", "meta2": "value2"},
"references": {"evaluator": {"slug": evaluator_slug}},
"links": annotation_links,
}
},
)
# ----------------------------------------------------------------------
# ASSERT ---------------------------------------------------------------
> assert response.status_code in (200, 202)
E assert 500 in (200, 202)
E + where 500 = <Response [500]>.status_code
oss/tests/pytest/acceptance/annotations/test_annotations_basics.py:45: AssertionError
github-actions / Railway API Test Results
test_create_trace_without_links_does_not_create_evaluator (oss.tests.pytest.acceptance.annotations.test_annotations_basics.TestAnnotationsBasics) failed
api/oss/tests/results/junit.xml [took 1s]
Raw output
assert 500 in (200, 202)
+ where 500 = <Response [500]>.status_code
self = <annotations.test_annotations_basics.TestAnnotationsBasics object at 0x7fb00131a850>
authed_api = <function authed_api.<locals>._request at 0x7fb000bece00>
def test_create_trace_without_links_does_not_create_evaluator(self, authed_api):
evaluator_slug = str(uuid4())
response = authed_api(
"POST",
"/simple/traces/",
json={
"trace": {
"data": {"outputs": {"score": 1}},
"references": {"evaluator": {"slug": evaluator_slug}},
"links": {},
}
},
)
> assert response.status_code in (200, 202)
E assert 500 in (200, 202)
E + where 500 = <Response [500]>.status_code
oss/tests/pytest/acceptance/annotations/test_annotations_basics.py:79: AssertionError
github-actions / Railway API Test Results
test_fetch_annotations (oss.tests.pytest.acceptance.annotations.test_annotations_basics.TestAnnotationsBasics) failed
api/oss/tests/results/junit.xml [took 0s]
Raw output
assert 500 in (200, 202)
+ where 500 = <Response [500]>.status_code
self = <annotations.test_annotations_basics.TestAnnotationsBasics object at 0x7fb00129b360>
authed_api = <function authed_api.<locals>._request at 0x7fb000bece00>
def test_fetch_annotations(self, authed_api):
# ARRANGE --------------------------------------------------------------
evaluator_slug = str(uuid4())
annotation_data_outputs = {
"integer": 42,
"float": 3.14,
"string": "Hello, world!",
"boolean": True,
"array": [1],
"object": {"aloha": "mahalo"},
}
annotation_links = {
"invocation": {
"trace_id": "019688a5e2097b80ad12e2907b83e9fc",
"span_id": "25a8e0b9bbf37d30",
}
}
response = authed_api(
"POST",
"/simple/traces/",
json={
"trace": {
"data": {
"outputs": annotation_data_outputs,
},
"tags": {"tag1": "value1", "tag2": "value2"},
"meta": {"meta1": "value1", "meta2": "value2"},
"references": {"evaluator": {"slug": evaluator_slug}},
"links": annotation_links,
}
},
)
> assert response.status_code in (200, 202)
E assert 500 in (200, 202)
E + where 500 = <Response [500]>.status_code
oss/tests/pytest/acceptance/annotations/test_annotations_basics.py:131: AssertionError
github-actions / Railway API Test Results
test_edit_annotations (oss.tests.pytest.acceptance.annotations.test_annotations_basics.TestAnnotationsBasics) failed
api/oss/tests/results/junit.xml [took 0s]
Raw output
assert 500 in (200, 202)
+ where 500 = <Response [500]>.status_code
self = <annotations.test_annotations_basics.TestAnnotationsBasics object at 0x7fb00129b490>
authed_api = <function authed_api.<locals>._request at 0x7fb000bece00>
def test_edit_annotations(self, authed_api):
# ARRANGE --------------------------------------------------------------
evaluator_slug = str(uuid4())
annotation_data_outputs = {
"integer": 42,
"float": 3.14,
"string": "Hello, world!",
"boolean": True,
"array": [1],
"object": {"aloha": "mahalo"},
}
annotation_links = {
"invocation": {
"trace_id": "019688a5e2097b80ad12e2907b83e9fc",
"span_id": "25a8e0b9bbf37d30",
}
}
response = authed_api(
"POST",
"/simple/traces/",
json={
"trace": {
"data": {
"outputs": annotation_data_outputs,
},
"tags": {"tag1": "value1", "tag2": "value2"},
"meta": {"meta1": "value1", "meta2": "value2"},
"references": {"evaluator": {"slug": evaluator_slug}},
"links": annotation_links,
}
},
)
> assert response.status_code in (200, 202)
E assert 500 in (200, 202)
E + where 500 = <Response [500]>.status_code
oss/tests/pytest/acceptance/annotations/test_annotations_basics.py:191: AssertionError
github-actions / Railway API Test Results
test_delete_annotations (oss.tests.pytest.acceptance.annotations.test_annotations_basics.TestAnnotationsBasics) failed
api/oss/tests/results/junit.xml [took 0s]
Raw output
assert 500 in (200, 202)
+ where 500 = <Response [500]>.status_code
self = <annotations.test_annotations_basics.TestAnnotationsBasics object at 0x7fb0011b2690>
authed_api = <function authed_api.<locals>._request at 0x7fb000bece00>
def test_delete_annotations(self, authed_api):
# ARRANGE --------------------------------------------------------------
evaluator_slug = str(uuid4())
annotation_data_outputs = {
"integer": 42,
"float": 3.14,
"string": "Hello, world!",
"boolean": True,
"array": [1],
"object": {"aloha": "mahalo"},
}
annotation_links = {
"invocation": {
"trace_id": "019688a5e2097b80ad12e2907b83e9fc",
"span_id": "25a8e0b9bbf37d30",
}
}
response = authed_api(
"POST",
"/simple/traces/",
json={
"trace": {
"data": {
"outputs": annotation_data_outputs,
},
"tags": {"tag1": "value1", "tag2": "value2"},
"meta": {"meta1": "value1", "meta2": "value2"},
"references": {"evaluator": {"slug": evaluator_slug}},
"links": annotation_links,
}
},
)
> assert response.status_code in (200, 202)
E assert 500 in (200, 202)
E + where 500 = <Response [500]>.status_code
oss/tests/pytest/acceptance/annotations/test_annotations_basics.py:275: AssertionError
github-actions / Railway API Test Results
test_query_annotations_all (oss.tests.pytest.acceptance.annotations.test_annotations_queries.TestAnnotationsQueries) with error
api/oss/tests/results/junit.xml [took 1s]
Raw output
failed on setup with "assert 500 in (200, 202)
+ where 500 = <Response [500]>.status_code"
authed_api = <function authed_api.<locals>._request at 0x7fb000f287c0>
@pytest.fixture(scope="class")
def mock_data(authed_api):
# ACT ------------------------------------------------------------------
evaluator_slug_1 = str(uuid4())
trace_id_1 = uuid4().hex
span_id_1 = uuid4().hex[16:]
trace_id_2 = uuid4().hex
span_id_2 = uuid4().hex[16:]
response = authed_api(
"POST",
"/simple/traces/",
json={
"trace": {
"origin": "custom",
"kind": "adhoc",
"channel": "api",
"data": {
"outputs": {
"integer": 42,
"float": 3.14,
"string": "Hello, world!",
"boolean": True,
"array": [1],
"object": {"aloha": "mahalo"},
},
},
"tags": {"tag1": "value1", "tag2": "value2"},
"meta": {"meta1": "value1", "meta2": "value2"},
"references": {"evaluator": {"slug": evaluator_slug_1}},
"links": {
"invocation": {
"trace_id": trace_id_1,
"span_id": span_id_1,
}
},
}
},
)
> assert response.status_code in (200, 202)
E assert 500 in (200, 202)
E + where 500 = <Response [500]>.status_code
oss/tests/pytest/acceptance/annotations/test_annotations_queries.py:49: AssertionError
github-actions / Railway API Test Results
test_query_annotations_by_annotation_link (oss.tests.pytest.acceptance.annotations.test_annotations_queries.TestAnnotationsQueries) with error
api/oss/tests/results/junit.xml [took 0s]
Raw output
failed on setup with "assert 500 in (200, 202)
+ where 500 = <Response [500]>.status_code"
authed_api = <function authed_api.<locals>._request at 0x7fb000f287c0>
@pytest.fixture(scope="class")
def mock_data(authed_api):
# ACT ------------------------------------------------------------------
evaluator_slug_1 = str(uuid4())
trace_id_1 = uuid4().hex
span_id_1 = uuid4().hex[16:]
trace_id_2 = uuid4().hex
span_id_2 = uuid4().hex[16:]
response = authed_api(
"POST",
"/simple/traces/",
json={
"trace": {
"origin": "custom",
"kind": "adhoc",
"channel": "api",
"data": {
"outputs": {
"integer": 42,
"float": 3.14,
"string": "Hello, world!",
"boolean": True,
"array": [1],
"object": {"aloha": "mahalo"},
},
},
"tags": {"tag1": "value1", "tag2": "value2"},
"meta": {"meta1": "value1", "meta2": "value2"},
"references": {"evaluator": {"slug": evaluator_slug_1}},
"links": {
"invocation": {
"trace_id": trace_id_1,
"span_id": span_id_1,
}
},
}
},
)
> assert response.status_code in (200, 202)
E assert 500 in (200, 202)
E + where 500 = <Response [500]>.status_code
oss/tests/pytest/acceptance/annotations/test_annotations_queries.py:49: AssertionError
github-actions / Railway API Test Results
test_query_annotations_by_link (oss.tests.pytest.acceptance.annotations.test_annotations_queries.TestAnnotationsQueries) with error
api/oss/tests/results/junit.xml [took 0s]
Raw output
failed on setup with "assert 500 in (200, 202)
+ where 500 = <Response [500]>.status_code"
authed_api = <function authed_api.<locals>._request at 0x7fb000f287c0>
@pytest.fixture(scope="class")
def mock_data(authed_api):
# ACT ------------------------------------------------------------------
evaluator_slug_1 = str(uuid4())
trace_id_1 = uuid4().hex
span_id_1 = uuid4().hex[16:]
trace_id_2 = uuid4().hex
span_id_2 = uuid4().hex[16:]
response = authed_api(
"POST",
"/simple/traces/",
json={
"trace": {
"origin": "custom",
"kind": "adhoc",
"channel": "api",
"data": {
"outputs": {
"integer": 42,
"float": 3.14,
"string": "Hello, world!",
"boolean": True,
"array": [1],
"object": {"aloha": "mahalo"},
},
},
"tags": {"tag1": "value1", "tag2": "value2"},
"meta": {"meta1": "value1", "meta2": "value2"},
"references": {"evaluator": {"slug": evaluator_slug_1}},
"links": {
"invocation": {
"trace_id": trace_id_1,
"span_id": span_id_1,
}
},
}
},
)
> assert response.status_code in (200, 202)
E assert 500 in (200, 202)
E + where 500 = <Response [500]>.status_code
oss/tests/pytest/acceptance/annotations/test_annotations_queries.py:49: AssertionError
github-actions / Railway API Test Results
test_query_annotations_by_reference (oss.tests.pytest.acceptance.annotations.test_annotations_queries.TestAnnotationsQueries) with error
api/oss/tests/results/junit.xml [took 0s]
Raw output
failed on setup with "assert 500 in (200, 202)
+ where 500 = <Response [500]>.status_code"
authed_api = <function authed_api.<locals>._request at 0x7fb000f287c0>
@pytest.fixture(scope="class")
def mock_data(authed_api):
# ACT ------------------------------------------------------------------
evaluator_slug_1 = str(uuid4())
trace_id_1 = uuid4().hex
span_id_1 = uuid4().hex[16:]
trace_id_2 = uuid4().hex
span_id_2 = uuid4().hex[16:]
response = authed_api(
"POST",
"/simple/traces/",
json={
"trace": {
"origin": "custom",
"kind": "adhoc",
"channel": "api",
"data": {
"outputs": {
"integer": 42,
"float": 3.14,
"string": "Hello, world!",
"boolean": True,
"array": [1],
"object": {"aloha": "mahalo"},
},
},
"tags": {"tag1": "value1", "tag2": "value2"},
"meta": {"meta1": "value1", "meta2": "value2"},
"references": {"evaluator": {"slug": evaluator_slug_1}},
"links": {
"invocation": {
"trace_id": trace_id_1,
"span_id": span_id_1,
}
},
}
},
)
> assert response.status_code in (200, 202)
E assert 500 in (200, 202)
E + where 500 = <Response [500]>.status_code
oss/tests/pytest/acceptance/annotations/test_annotations_queries.py:49: AssertionError
github-actions / Railway API Test Results
test_query_annotations_by_tags (oss.tests.pytest.acceptance.annotations.test_annotations_queries.TestAnnotationsQueries) with error
api/oss/tests/results/junit.xml [took 0s]
Raw output
failed on setup with "assert 500 in (200, 202)
+ where 500 = <Response [500]>.status_code"
authed_api = <function authed_api.<locals>._request at 0x7fb000f287c0>
@pytest.fixture(scope="class")
def mock_data(authed_api):
# ACT ------------------------------------------------------------------
evaluator_slug_1 = str(uuid4())
trace_id_1 = uuid4().hex
span_id_1 = uuid4().hex[16:]
trace_id_2 = uuid4().hex
span_id_2 = uuid4().hex[16:]
response = authed_api(
"POST",
"/simple/traces/",
json={
"trace": {
"origin": "custom",
"kind": "adhoc",
"channel": "api",
"data": {
"outputs": {
"integer": 42,
"float": 3.14,
"string": "Hello, world!",
"boolean": True,
"array": [1],
"object": {"aloha": "mahalo"},
},
},
"tags": {"tag1": "value1", "tag2": "value2"},
"meta": {"meta1": "value1", "meta2": "value2"},
"references": {"evaluator": {"slug": evaluator_slug_1}},
"links": {
"invocation": {
"trace_id": trace_id_1,
"span_id": span_id_1,
}
},
}
},
)
> assert response.status_code in (200, 202)
E assert 500 in (200, 202)
E + where 500 = <Response [500]>.status_code
oss/tests/pytest/acceptance/annotations/test_annotations_queries.py:49: AssertionError
github-actions / Railway API Test Results
test_query_annotations_by_meta (oss.tests.pytest.acceptance.annotations.test_annotations_queries.TestAnnotationsQueries) with error
api/oss/tests/results/junit.xml [took 0s]
Raw output
failed on setup with "assert 500 in (200, 202)
+ where 500 = <Response [500]>.status_code"
authed_api = <function authed_api.<locals>._request at 0x7fb000f287c0>
@pytest.fixture(scope="class")
def mock_data(authed_api):
# ACT ------------------------------------------------------------------
evaluator_slug_1 = str(uuid4())
trace_id_1 = uuid4().hex
span_id_1 = uuid4().hex[16:]
trace_id_2 = uuid4().hex
span_id_2 = uuid4().hex[16:]
response = authed_api(
"POST",
"/simple/traces/",
json={
"trace": {
"origin": "custom",
"kind": "adhoc",
"channel": "api",
"data": {
"outputs": {
"integer": 42,
"float": 3.14,
"string": "Hello, world!",
"boolean": True,
"array": [1],
"object": {"aloha": "mahalo"},
},
},
"tags": {"tag1": "value1", "tag2": "value2"},
"meta": {"meta1": "value1", "meta2": "value2"},
"references": {"evaluator": {"slug": evaluator_slug_1}},
"links": {
"invocation": {
"trace_id": trace_id_1,
"span_id": span_id_1,
}
},
}
},
)
> assert response.status_code in (200, 202)
E assert 500 in (200, 202)
E + where 500 = <Response [500]>.status_code
oss/tests/pytest/acceptance/annotations/test_annotations_queries.py:49: AssertionError
github-actions / Railway API Test Results
test_query_annotations_by_origin (oss.tests.pytest.acceptance.annotations.test_annotations_queries.TestAnnotationsQueries) with error
api/oss/tests/results/junit.xml [took 0s]
Raw output
failed on setup with "assert 500 in (200, 202)
+ where 500 = <Response [500]>.status_code"
authed_api = <function authed_api.<locals>._request at 0x7fb000f287c0>
@pytest.fixture(scope="class")
def mock_data(authed_api):
# ACT ------------------------------------------------------------------
evaluator_slug_1 = str(uuid4())
trace_id_1 = uuid4().hex
span_id_1 = uuid4().hex[16:]
trace_id_2 = uuid4().hex
span_id_2 = uuid4().hex[16:]
response = authed_api(
"POST",
"/simple/traces/",
json={
"trace": {
"origin": "custom",
"kind": "adhoc",
"channel": "api",
"data": {
"outputs": {
"integer": 42,
"float": 3.14,
"string": "Hello, world!",
"boolean": True,
"array": [1],
"object": {"aloha": "mahalo"},
},
},
"tags": {"tag1": "value1", "tag2": "value2"},
"meta": {"meta1": "value1", "meta2": "value2"},
"references": {"evaluator": {"slug": evaluator_slug_1}},
"links": {
"invocation": {
"trace_id": trace_id_1,
"span_id": span_id_1,
}
},
}
},
)
> assert response.status_code in (200, 202)
E assert 500 in (200, 202)
E + where 500 = <Response [500]>.status_code
oss/tests/pytest/acceptance/annotations/test_annotations_queries.py:49: AssertionError
github-actions / Railway API Test Results
test_query_annotations_by_kind (oss.tests.pytest.acceptance.annotations.test_annotations_queries.TestAnnotationsQueries) with error
api/oss/tests/results/junit.xml [took 0s]
Raw output
failed on setup with "assert 500 in (200, 202)
+ where 500 = <Response [500]>.status_code"
authed_api = <function authed_api.<locals>._request at 0x7fb000f287c0>
@pytest.fixture(scope="class")
def mock_data(authed_api):
# ACT ------------------------------------------------------------------
evaluator_slug_1 = str(uuid4())
trace_id_1 = uuid4().hex
span_id_1 = uuid4().hex[16:]
trace_id_2 = uuid4().hex
span_id_2 = uuid4().hex[16:]
response = authed_api(
"POST",
"/simple/traces/",
json={
"trace": {
"origin": "custom",
"kind": "adhoc",
"channel": "api",
"data": {
"outputs": {
"integer": 42,
"float": 3.14,
"string": "Hello, world!",
"boolean": True,
"array": [1],
"object": {"aloha": "mahalo"},
},
},
"tags": {"tag1": "value1", "tag2": "value2"},
"meta": {"meta1": "value1", "meta2": "value2"},
"references": {"evaluator": {"slug": evaluator_slug_1}},
"links": {
"invocation": {
"trace_id": trace_id_1,
"span_id": span_id_1,
}
},
}
},
)
> assert response.status_code in (200, 202)
E assert 500 in (200, 202)
E + where 500 = <Response [500]>.status_code
oss/tests/pytest/acceptance/annotations/test_annotations_queries.py:49: AssertionError
github-actions / Railway API Test Results
test_query_annotations_by_channel (oss.tests.pytest.acceptance.annotations.test_annotations_queries.TestAnnotationsQueries) with error
api/oss/tests/results/junit.xml [took 0s]
Raw output
failed on setup with "assert 500 in (200, 202)
+ where 500 = <Response [500]>.status_code"
authed_api = <function authed_api.<locals>._request at 0x7fb000f287c0>
@pytest.fixture(scope="class")
def mock_data(authed_api):
# ACT ------------------------------------------------------------------
evaluator_slug_1 = str(uuid4())
trace_id_1 = uuid4().hex
span_id_1 = uuid4().hex[16:]
trace_id_2 = uuid4().hex
span_id_2 = uuid4().hex[16:]
response = authed_api(
"POST",
"/simple/traces/",
json={
"trace": {
"origin": "custom",
"kind": "adhoc",
"channel": "api",
"data": {
"outputs": {
"integer": 42,
"float": 3.14,
"string": "Hello, world!",
"boolean": True,
"array": [1],
"object": {"aloha": "mahalo"},
},
},
"tags": {"tag1": "value1", "tag2": "value2"},
"meta": {"meta1": "value1", "meta2": "value2"},
"references": {"evaluator": {"slug": evaluator_slug_1}},
"links": {
"invocation": {
"trace_id": trace_id_1,
"span_id": span_id_1,
}
},
}
},
)
> assert response.status_code in (200, 202)
E assert 500 in (200, 202)
E + where 500 = <Response [500]>.status_code
oss/tests/pytest/acceptance/annotations/test_annotations_queries.py:49: AssertionError
github-actions / Railway API Test Results
test_reflect_before_contaminator (oss.tests.pytest.acceptance.evaluations.test_evaluation_flows_evaluator_ordering.TestEvaluatorOrderingFlows) failed
api/oss/tests/results/junit.xml [took 5s]
Raw output
AssertionError: {'count': 0}
self = <evaluations.test_evaluation_flows_evaluator_ordering.TestEvaluatorOrderingFlows object at 0x7fd1b840d310>
authed_api = <function authed_api.<locals>._request at 0x7fd1b3d63e20>
def test_reflect_before_contaminator(self, authed_api):
# reflect runs BEFORE the score evaluator.
reflect = create_mock_evaluator(authed_api, key="reflect")
score = create_mock_evaluator(
authed_api, key="score", kwargs={"score": 0.5, "threshold": 0.5}
)
# dict preserves insertion order -> reflect first.
> run_id = self._run_with_evaluator_order(
authed_api,
name="flow-order-reflect-first",
evaluator_steps={
reflect["revision_id"]: "auto",
score["revision_id"]: "auto",
},
)
oss/tests/pytest/acceptance/evaluations/test_evaluation_flows_evaluator_ordering.py:61:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
oss/tests/pytest/acceptance/evaluations/test_evaluation_flows_evaluator_ordering.py:39: in _run_with_evaluator_order
evaluation = create_simple_evaluation(
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
authed_api = <function authed_api.<locals>._request at 0x7fd1b3d63e20>
name = 'flow-order-reflect-first'
data = {'testset_steps': ['019f0068-9ffb-7dc3-8284-8536e938dee5'], 'application_steps': ['019f0068-a33c-77f3-9308-2a1f698ff2b8'], 'evaluator_steps': {'019f0068-98d4-7030-863d-50bcbb14bb4c': 'auto', '019f0068-9c17-77b3-ba6f-e787e896b95d': 'auto'}}
flags = None
def create_simple_evaluation(authed_api, *, name, data, flags=None) -> dict:
"""Create (and auto-start) a simple evaluation. Returns the evaluation dict."""
payload = {"name": name, "data": data}
if flags is not None:
payload["flags"] = flags
response = authed_api(
"POST",
"/simple/evaluations/",
json={"evaluation": payload},
)
assert response.status_code == 200, response.text
body = response.json()
> assert body["count"] == 1, body
^^^^^^^^^^^^^^^^^^
E AssertionError: {'count': 0}
oss/tests/pytest/acceptance/evaluations/_flow_helpers.py:183: AssertionError
github-actions / Railway API Test Results
test_create_testset_to_evaluator_evaluation_does_not_fail_setup (oss.tests.pytest.acceptance.evaluations.test_simple_evaluations_workflows.TestSimpleEvaluationsWorkflowReferences) failed
api/oss/tests/results/junit.xml [took 3s]
Raw output
assert 0 == 1
self = <evaluations.test_simple_evaluations_workflows.TestSimpleEvaluationsWorkflowReferences object at 0x7fb0010f4830>
authed_api = <function authed_api.<locals>._request at 0x7fb000c544a0>
def test_create_testset_to_evaluator_evaluation_does_not_fail_setup(
self, authed_api
):
testset = _create_simple_testset(authed_api)
evaluator = _create_simple_evaluator(authed_api)
response = authed_api(
"POST",
"/simple/evaluations/",
json={
"evaluation": {
"name": "testset-to-evaluator-setup",
"data": {
"testset_steps": [testset["revision_id"]],
"evaluator_steps": [evaluator["revision_id"]],
},
}
},
)
assert response.status_code == 200
body = response.json()
> assert body["count"] == 1
E assert 0 == 1
oss/tests/pytest/acceptance/evaluations/test_simple_evaluations_workflows.py:271: AssertionError
github-actions / Railway API Test Results
test_reflect_after_contaminator (oss.tests.pytest.acceptance.evaluations.test_evaluation_flows_evaluator_ordering.TestEvaluatorOrderingFlows) failed
api/oss/tests/results/junit.xml [took 4s]
Raw output
AssertionError: {'count': 0}
self = <evaluations.test_evaluation_flows_evaluator_ordering.TestEvaluatorOrderingFlows object at 0x7fd1b840d090>
authed_api = <function authed_api.<locals>._request at 0x7fd1b3d63e20>
def test_reflect_after_contaminator(self, authed_api):
# reflect runs AFTER the score evaluator — the position that regressed in
# production. A leaked score-evaluator output would drop reflect to 0.0.
score = create_mock_evaluator(
authed_api, key="score", kwargs={"score": 0.5, "threshold": 0.5}
)
reflect = create_mock_evaluator(authed_api, key="reflect")
> run_id = self._run_with_evaluator_order(
authed_api,
name="flow-order-reflect-second",
evaluator_steps={
score["revision_id"]: "auto",
reflect["revision_id"]: "auto",
},
)
oss/tests/pytest/acceptance/evaluations/test_evaluation_flows_evaluator_ordering.py:81:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
oss/tests/pytest/acceptance/evaluations/test_evaluation_flows_evaluator_ordering.py:39: in _run_with_evaluator_order
evaluation = create_simple_evaluation(
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
authed_api = <function authed_api.<locals>._request at 0x7fd1b3d63e20>
name = 'flow-order-reflect-second'
data = {'testset_steps': ['019f0068-b1cf-7162-8a0c-92e5cac2d1cc'], 'application_steps': ['019f0068-b533-70e2-b66c-6b65b64f2d60'], 'evaluator_steps': {'019f0068-aae7-7db0-92b4-05985671ef55': 'auto', '019f0068-ad7e-73e3-a9e2-1a131de47718': 'auto'}}
flags = None
def create_simple_evaluation(authed_api, *, name, data, flags=None) -> dict:
"""Create (and auto-start) a simple evaluation. Returns the evaluation dict."""
payload = {"name": name, "data": data}
if flags is not None:
payload["flags"] = flags
response = authed_api(
"POST",
"/simple/evaluations/",
json={"evaluation": payload},
)
assert response.status_code == 200, response.text
body = response.json()
> assert body["count"] == 1, body
^^^^^^^^^^^^^^^^^^
E AssertionError: {'count': 0}
oss/tests/pytest/acceptance/evaluations/_flow_helpers.py:183: AssertionError
github-actions / Railway API Test Results
test_create_simple_queue_from_testsets (oss.tests.pytest.acceptance.evaluations.test_simple_queues_basics.TestSimpleQueuesBasics) failed
api/oss/tests/results/junit.xml [took 2s]
Raw output
assert 500 == 200
+ where 500 = <Response [500]>.status_code
self = <evaluations.test_simple_queues_basics.TestSimpleQueuesBasics object at 0x7fb001121ba0>
authed_api = <function authed_api.<locals>._request at 0x7fb000c54ae0>
def test_create_simple_queue_from_testsets(self, authed_api):
evaluator_revision_id = _create_evaluator(authed_api)
testset_revision_id = _create_testset(authed_api)
response = authed_api(
"POST",
"/simple/queues/",
json={
"queue": {
"name": "testset-backed-queue",
"data": {
"testsets": [testset_revision_id],
"evaluators": [evaluator_revision_id],
},
}
},
)
> assert response.status_code == 200
E assert 500 == 200
E + where 500 = <Response [500]>.status_code
oss/tests/pytest/acceptance/evaluations/test_simple_queues_basics.py:193: AssertionError
github-actions / Railway API Test Results
test_create_source_backed_queue_preserves_repeats_and_assignments (oss.tests.pytest.acceptance.evaluations.test_simple_queues_basics.TestSimpleQueuesBasics) failed
api/oss/tests/results/junit.xml [took 2s]
Raw output
assert 500 == 200
+ where 500 = <Response [500]>.status_code
self = <evaluations.test_simple_queues_basics.TestSimpleQueuesBasics object at 0x7fb0010f5370>
authed_api = <function authed_api.<locals>._request at 0x7fb000c54ae0>
def test_create_source_backed_queue_preserves_repeats_and_assignments(
self, authed_api
):
evaluator_revision_id = _create_evaluator(authed_api)
testset_revision_id = _create_testset(authed_api)
user_id_1 = str(uuid4())
user_id_2 = str(uuid4())
response = authed_api(
"POST",
"/simple/queues/",
json={
"queue": {
"name": "testset-backed-queue-with-repeats",
"data": {
"testsets": [testset_revision_id],
"evaluators": {evaluator_revision_id: "human"},
"repeats": 2,
"assignments": [[user_id_1], [user_id_2]],
},
}
},
)
> assert response.status_code == 200
E assert 500 == 200
E + where 500 = <Response [500]>.status_code
oss/tests/pytest/acceptance/evaluations/test_simple_queues_basics.py:224: AssertionError
github-actions / Railway API Test Results
test_probe_returns_results_envelope_for_empty_slice (oss.tests.pytest.acceptance.evaluations.test_run_slice_endpoints.TestRunSliceEndpoints) failed
api/oss/tests/results/junit.xml [took 4s]
Raw output
KeyError: 'evaluation'
self = <evaluations.test_run_slice_endpoints.TestRunSliceEndpoints object at 0x7f6316313d90>
authed_api = <function authed_api.<locals>._request at 0x7f6315c577e0>
def test_probe_returns_results_envelope_for_empty_slice(self, authed_api):
> evaluation = _create_testset_evaluator_evaluation(authed_api)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
oss/tests/pytest/acceptance/evaluations/test_run_slice_endpoints.py:132:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
authed_api = <function authed_api.<locals>._request at 0x7f6315c577e0>
def _create_testset_evaluator_evaluation(authed_api) -> dict:
"""A dispatchable testset -> application -> evaluator run (mock app, no LLM).
The application step is required: testset -> evaluator (no app) is a
`potential`/undispatchable topology that `start` auto-fails-and-closes, which
would 409 every subsequent slice op. The mock app keeps it deterministic.
"""
testset = _create_simple_testset(authed_api)
application = _create_mock_application(authed_api)
evaluator = _create_simple_evaluator(authed_api)
response = authed_api(
"POST",
"/simple/evaluations/",
json={
"evaluation": {
"name": f"run-slice-{uuid4().hex[:8]}",
"flags": {"is_cached": False, "is_split": False},
"data": {
"testset_steps": {testset["revision_id"]: "custom"},
"application_steps": {application["revision_id"]: "custom"},
"evaluator_steps": {evaluator["revision_id"]: "auto"},
"repeats": 1,
},
}
},
)
assert response.status_code == 200, response.text
> return response.json()["evaluation"]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
E KeyError: 'evaluation'
oss/tests/pytest/acceptance/evaluations/test_run_slice_endpoints.py:112: KeyError
github-actions / Railway API Test Results
test_probe_accepts_full_coordinate_body (oss.tests.pytest.acceptance.evaluations.test_run_slice_endpoints.TestRunSliceEndpoints) failed
api/oss/tests/results/junit.xml [took 5s]
Raw output
KeyError: 'evaluation'
self = <evaluations.test_run_slice_endpoints.TestRunSliceEndpoints object at 0x7f6316313ed0>
authed_api = <function authed_api.<locals>._request at 0x7f6315c577e0>
def test_probe_accepts_full_coordinate_body(self, authed_api):
> evaluation = _create_testset_evaluator_evaluation(authed_api)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
oss/tests/pytest/acceptance/evaluations/test_run_slice_endpoints.py:150:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
authed_api = <function authed_api.<locals>._request at 0x7f6315c577e0>
def _create_testset_evaluator_evaluation(authed_api) -> dict:
"""A dispatchable testset -> application -> evaluator run (mock app, no LLM).
The application step is required: testset -> evaluator (no app) is a
`potential`/undispatchable topology that `start` auto-fails-and-closes, which
would 409 every subsequent slice op. The mock app keeps it deterministic.
"""
testset = _create_simple_testset(authed_api)
application = _create_mock_application(authed_api)
evaluator = _create_simple_evaluator(authed_api)
response = authed_api(
"POST",
"/simple/evaluations/",
json={
"evaluation": {
"name": f"run-slice-{uuid4().hex[:8]}",
"flags": {"is_cached": False, "is_split": False},
"data": {
"testset_steps": {testset["revision_id"]: "custom"},
"application_steps": {application["revision_id"]: "custom"},
"evaluator_steps": {evaluator["revision_id"]: "auto"},
"repeats": 1,
},
}
},
)
assert response.status_code == 200, response.text
> return response.json()["evaluation"]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
E KeyError: 'evaluation'
oss/tests/pytest/acceptance/evaluations/test_run_slice_endpoints.py:112: KeyError
github-actions / Railway API Test Results
test_simple_queue_allows_human_mixed_with_non_human_evaluators (oss.tests.pytest.acceptance.evaluations.test_simple_queues_basics.TestSimpleQueuesBasics) failed
api/oss/tests/results/junit.xml [took 3s]
Raw output
assert 500 == 200
+ where 500 = <Response [500]>.status_code
self = <evaluations.test_simple_queues_basics.TestSimpleQueuesBasics object at 0x7fb0010fe850>
authed_api = <function authed_api.<locals>._request at 0x7fb000c54ae0>
def test_simple_queue_allows_human_mixed_with_non_human_evaluators(
self, authed_api
):
# The "at least one human" rule allows a mix: a human alongside any
# non-human origin (auto or custom) is a valid human queue. The explicit
# non-human origin is honored on the underlying run.
for non_human_origin, expected_flag in (
("auto", "has_auto"),
("custom", "has_custom"),
):
human_evaluator_id = _create_evaluator(authed_api)
other_evaluator_id = _create_evaluator(authed_api)
testset_revision_id = _create_testset(authed_api)
response = authed_api(
"POST",
"/simple/queues/",
json={
"queue": {
"name": f"testset-backed-human-plus-{non_human_origin}",
"data": {
"testsets": [testset_revision_id],
"evaluators": {
human_evaluator_id: "human",
other_evaluator_id: non_human_origin,
},
},
}
},
)
> assert response.status_code == 200
E assert 500 == 200
E + where 500 = <Response [500]>.status_code
oss/tests/pytest/acceptance/evaluations/test_simple_queues_basics.py:340: AssertionError
github-actions / Railway API Test Results
test_probe_accepts_empty_body (oss.tests.pytest.acceptance.evaluations.test_run_slice_endpoints.TestRunSliceEndpoints) failed
api/oss/tests/results/junit.xml [took 3s]
Raw output
KeyError: 'evaluation'
self = <evaluations.test_run_slice_endpoints.TestRunSliceEndpoints object at 0x7f631611d220>
authed_api = <function authed_api.<locals>._request at 0x7f6315c577e0>
def test_probe_accepts_empty_body(self, authed_api):
> evaluation = _create_testset_evaluator_evaluation(authed_api)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
oss/tests/pytest/acceptance/evaluations/test_run_slice_endpoints.py:168:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
authed_api = <function authed_api.<locals>._request at 0x7f6315c577e0>
def _create_testset_evaluator_evaluation(authed_api) -> dict:
"""A dispatchable testset -> application -> evaluator run (mock app, no LLM).
The application step is required: testset -> evaluator (no app) is a
`potential`/undispatchable topology that `start` auto-fails-and-closes, which
would 409 every subsequent slice op. The mock app keeps it deterministic.
"""
testset = _create_simple_testset(authed_api)
application = _create_mock_application(authed_api)
evaluator = _create_simple_evaluator(authed_api)
response = authed_api(
"POST",
"/simple/evaluations/",
json={
"evaluation": {
"name": f"run-slice-{uuid4().hex[:8]}",
"flags": {"is_cached": False, "is_split": False},
"data": {
"testset_steps": {testset["revision_id"]: "custom"},
"application_steps": {application["revision_id"]: "custom"},
"evaluator_steps": {evaluator["revision_id"]: "auto"},
"repeats": 1,
},
}
},
)
assert response.status_code == 200, response.text
> return response.json()["evaluation"]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
E KeyError: 'evaluation'
oss/tests/pytest/acceptance/evaluations/test_run_slice_endpoints.py:112: KeyError
github-actions / Railway API Test Results
test_add_testcases_rejects_testset_backed_source_queue (oss.tests.pytest.acceptance.evaluations.test_simple_queues_basics.TestSimpleQueuesBasics) failed
api/oss/tests/results/junit.xml [took 2s]
Raw output
assert 500 == 200
+ where 500 = <Response [500]>.status_code
self = <evaluations.test_simple_queues_basics.TestSimpleQueuesBasics object at 0x7fb000f91310>
authed_api = <function authed_api.<locals>._request at 0x7fb000c54ae0>
def test_add_testcases_rejects_testset_backed_source_queue(self, authed_api):
evaluator_revision_id = _create_evaluator(authed_api)
testset_revision_id = _create_testset(authed_api)
create_response = authed_api(
"POST",
"/simple/queues/",
json={
"queue": {
"data": {
"testsets": [testset_revision_id],
"evaluators": [evaluator_revision_id],
},
}
},
)
> assert create_response.status_code == 200
E assert 500 == 200
E + where 500 = <Response [500]>.status_code
oss/tests/pytest/acceptance/evaluations/test_simple_queues_basics.py:423: AssertionError