Skip to content

fix(agent): Claude alias resolution + skill/error tracing + error sanitize (QA F-031/F-029/F-030) - #4855

Merged
mmabrouk merged 1 commit into
big-agentsfrom
fix/agent-sdk-tracing-findings
Jun 25, 2026
Merged

fix(agent): Claude alias resolution + skill/error tracing + error sanitize (QA F-031/F-029/F-030)#4855
mmabrouk merged 1 commit into
big-agentsfrom
fix/agent-sdk-tracing-findings

fix(agent): Claude alias resolution + skill/error tracing + error san…

3a51244
Select commit
Loading
Failed to load commit list.
Sign in for the full log view
GitHub Actions / Railway API Test Results failed Jun 25, 2026 in 0s

38 errors, 58 fail, 8 skipped, 558 pass in 6m 23s

662 tests   558 ✅  6m 23s ⏱️
  1 suites    8 💤
  1 files     58 ❌  38 🔥

Results for commit 3a51244.

Annotations

Check warning on line 0 in oss.tests.pytest.acceptance.evaluations.test_evaluation_flows_modify.TestEvaluationModifyFlows

See this annotation in the file changed.

@github-actions github-actions / Railway API Test Results

test_restart_finished_batch_run_re_dispatches_and_finalizes_again (oss.tests.pytest.acceptance.evaluations.test_evaluation_flows_modify.TestEvaluationModifyFlows) failed

api/oss/tests/results/junit.xml [took 2m 18s]
Raw output
AssertionError: assert 'running' == 'success'
  
  - success
  + running
self = <evaluations.test_evaluation_flows_modify.TestEvaluationModifyFlows object at 0x7f0a278911d0>
authed_api = <function authed_api.<locals>._request at 0x7f0a277027a0>

    def test_restart_finished_batch_run_re_dispatches_and_finalizes_again(
        self, authed_api
    ):
        # ARRANGE: run a batch eval to terminal success ------------------------
        testset = create_testset(authed_api)
        application = create_mock_application(authed_api, key="echo")
        evaluator = create_mock_evaluator(authed_api, key="pass")
        evaluation = create_simple_evaluation(
            authed_api,
            name="flow-restart-finished",
            data={
                "testset_steps": [testset["revision_id"]],
                "application_steps": [application["revision_id"]],
                "evaluator_steps": {evaluator["revision_id"]: "auto"},
            },
        )
        run_id = evaluation["id"]
        first = wait_for_run_terminal(authed_api, run_id)
>       assert first.json()["run"]["status"] == "success"
E       AssertionError: assert 'running' == 'success'
E         
E         - success
E         + running

oss/tests/pytest/acceptance/evaluations/test_evaluation_flows_modify.py:43: AssertionError

Check warning on line 0 in oss.tests.pytest.acceptance.annotations.test_annotations_basics.TestAnnotationsBasics

See this annotation in the file changed.

@github-actions github-actions / Railway API Test Results

test_create_annotations (oss.tests.pytest.acceptance.annotations.test_annotations_basics.TestAnnotationsBasics) failed

api/oss/tests/results/junit.xml [took 1s]
Raw output
assert 500 in (200, 202)
 +  where 500 = <Response [500]>.status_code
self = <annotations.test_annotations_basics.TestAnnotationsBasics object at 0x7fb00131a710>
authed_api = <function authed_api.<locals>._request at 0x7fb000bece00>

    def test_create_annotations(self, authed_api):
        # ACT ------------------------------------------------------------------
        evaluator_slug = str(uuid4())
    
        annotation_data_outputs = {
            "integer": 42,
            "float": 3.14,
            "string": "Hello, world!",
            "boolean": True,
            "array": [1],
            "object": {"aloha": "mahalo"},
        }
    
        annotation_links = {
            "invocation": {
                "trace_id": "019688a5e2097b80ad12e2907b83e9fc",
                "span_id": "25a8e0b9bbf37d30",
            }
        }
    
        response = authed_api(
            "POST",
            "/simple/traces/",
            json={
                "trace": {
                    "data": {
                        "outputs": annotation_data_outputs,
                    },
                    "tags": {"tag1": "value1", "tag2": "value2"},
                    "meta": {"meta1": "value1", "meta2": "value2"},
                    "references": {"evaluator": {"slug": evaluator_slug}},
                    "links": annotation_links,
                }
            },
        )
        # ----------------------------------------------------------------------
    
        # ASSERT ---------------------------------------------------------------
>       assert response.status_code in (200, 202)
E       assert 500 in (200, 202)
E        +  where 500 = <Response [500]>.status_code

oss/tests/pytest/acceptance/annotations/test_annotations_basics.py:45: AssertionError

Check warning on line 0 in oss.tests.pytest.acceptance.annotations.test_annotations_basics.TestAnnotationsBasics

See this annotation in the file changed.

@github-actions github-actions / Railway API Test Results

test_create_trace_without_links_does_not_create_evaluator (oss.tests.pytest.acceptance.annotations.test_annotations_basics.TestAnnotationsBasics) failed

api/oss/tests/results/junit.xml [took 1s]
Raw output
assert 500 in (200, 202)
 +  where 500 = <Response [500]>.status_code
self = <annotations.test_annotations_basics.TestAnnotationsBasics object at 0x7fb00131a850>
authed_api = <function authed_api.<locals>._request at 0x7fb000bece00>

    def test_create_trace_without_links_does_not_create_evaluator(self, authed_api):
        evaluator_slug = str(uuid4())
    
        response = authed_api(
            "POST",
            "/simple/traces/",
            json={
                "trace": {
                    "data": {"outputs": {"score": 1}},
                    "references": {"evaluator": {"slug": evaluator_slug}},
                    "links": {},
                }
            },
        )
    
>       assert response.status_code in (200, 202)
E       assert 500 in (200, 202)
E        +  where 500 = <Response [500]>.status_code

oss/tests/pytest/acceptance/annotations/test_annotations_basics.py:79: AssertionError

Check warning on line 0 in oss.tests.pytest.acceptance.annotations.test_annotations_basics.TestAnnotationsBasics

See this annotation in the file changed.

@github-actions github-actions / Railway API Test Results

test_fetch_annotations (oss.tests.pytest.acceptance.annotations.test_annotations_basics.TestAnnotationsBasics) failed

api/oss/tests/results/junit.xml [took 0s]
Raw output
assert 500 in (200, 202)
 +  where 500 = <Response [500]>.status_code
self = <annotations.test_annotations_basics.TestAnnotationsBasics object at 0x7fb00129b360>
authed_api = <function authed_api.<locals>._request at 0x7fb000bece00>

    def test_fetch_annotations(self, authed_api):
        # ARRANGE --------------------------------------------------------------
        evaluator_slug = str(uuid4())
    
        annotation_data_outputs = {
            "integer": 42,
            "float": 3.14,
            "string": "Hello, world!",
            "boolean": True,
            "array": [1],
            "object": {"aloha": "mahalo"},
        }
    
        annotation_links = {
            "invocation": {
                "trace_id": "019688a5e2097b80ad12e2907b83e9fc",
                "span_id": "25a8e0b9bbf37d30",
            }
        }
    
        response = authed_api(
            "POST",
            "/simple/traces/",
            json={
                "trace": {
                    "data": {
                        "outputs": annotation_data_outputs,
                    },
                    "tags": {"tag1": "value1", "tag2": "value2"},
                    "meta": {"meta1": "value1", "meta2": "value2"},
                    "references": {"evaluator": {"slug": evaluator_slug}},
                    "links": annotation_links,
                }
            },
        )
    
>       assert response.status_code in (200, 202)
E       assert 500 in (200, 202)
E        +  where 500 = <Response [500]>.status_code

oss/tests/pytest/acceptance/annotations/test_annotations_basics.py:131: AssertionError

Check warning on line 0 in oss.tests.pytest.acceptance.annotations.test_annotations_basics.TestAnnotationsBasics

See this annotation in the file changed.

@github-actions github-actions / Railway API Test Results

test_edit_annotations (oss.tests.pytest.acceptance.annotations.test_annotations_basics.TestAnnotationsBasics) failed

api/oss/tests/results/junit.xml [took 0s]
Raw output
assert 500 in (200, 202)
 +  where 500 = <Response [500]>.status_code
self = <annotations.test_annotations_basics.TestAnnotationsBasics object at 0x7fb00129b490>
authed_api = <function authed_api.<locals>._request at 0x7fb000bece00>

    def test_edit_annotations(self, authed_api):
        # ARRANGE --------------------------------------------------------------
        evaluator_slug = str(uuid4())
    
        annotation_data_outputs = {
            "integer": 42,
            "float": 3.14,
            "string": "Hello, world!",
            "boolean": True,
            "array": [1],
            "object": {"aloha": "mahalo"},
        }
    
        annotation_links = {
            "invocation": {
                "trace_id": "019688a5e2097b80ad12e2907b83e9fc",
                "span_id": "25a8e0b9bbf37d30",
            }
        }
    
        response = authed_api(
            "POST",
            "/simple/traces/",
            json={
                "trace": {
                    "data": {
                        "outputs": annotation_data_outputs,
                    },
                    "tags": {"tag1": "value1", "tag2": "value2"},
                    "meta": {"meta1": "value1", "meta2": "value2"},
                    "references": {"evaluator": {"slug": evaluator_slug}},
                    "links": annotation_links,
                }
            },
        )
    
>       assert response.status_code in (200, 202)
E       assert 500 in (200, 202)
E        +  where 500 = <Response [500]>.status_code

oss/tests/pytest/acceptance/annotations/test_annotations_basics.py:191: AssertionError

Check warning on line 0 in oss.tests.pytest.acceptance.annotations.test_annotations_basics.TestAnnotationsBasics

See this annotation in the file changed.

@github-actions github-actions / Railway API Test Results

test_delete_annotations (oss.tests.pytest.acceptance.annotations.test_annotations_basics.TestAnnotationsBasics) failed

api/oss/tests/results/junit.xml [took 0s]
Raw output
assert 500 in (200, 202)
 +  where 500 = <Response [500]>.status_code
self = <annotations.test_annotations_basics.TestAnnotationsBasics object at 0x7fb0011b2690>
authed_api = <function authed_api.<locals>._request at 0x7fb000bece00>

    def test_delete_annotations(self, authed_api):
        # ARRANGE --------------------------------------------------------------
        evaluator_slug = str(uuid4())
    
        annotation_data_outputs = {
            "integer": 42,
            "float": 3.14,
            "string": "Hello, world!",
            "boolean": True,
            "array": [1],
            "object": {"aloha": "mahalo"},
        }
    
        annotation_links = {
            "invocation": {
                "trace_id": "019688a5e2097b80ad12e2907b83e9fc",
                "span_id": "25a8e0b9bbf37d30",
            }
        }
    
        response = authed_api(
            "POST",
            "/simple/traces/",
            json={
                "trace": {
                    "data": {
                        "outputs": annotation_data_outputs,
                    },
                    "tags": {"tag1": "value1", "tag2": "value2"},
                    "meta": {"meta1": "value1", "meta2": "value2"},
                    "references": {"evaluator": {"slug": evaluator_slug}},
                    "links": annotation_links,
                }
            },
        )
    
>       assert response.status_code in (200, 202)
E       assert 500 in (200, 202)
E        +  where 500 = <Response [500]>.status_code

oss/tests/pytest/acceptance/annotations/test_annotations_basics.py:275: AssertionError

Check failure on line 0 in oss.tests.pytest.acceptance.annotations.test_annotations_queries.TestAnnotationsQueries

See this annotation in the file changed.

@github-actions github-actions / Railway API Test Results

test_query_annotations_all (oss.tests.pytest.acceptance.annotations.test_annotations_queries.TestAnnotationsQueries) with error

api/oss/tests/results/junit.xml [took 1s]
Raw output
failed on setup with "assert 500 in (200, 202)
 +  where 500 = <Response [500]>.status_code"
authed_api = <function authed_api.<locals>._request at 0x7fb000f287c0>

    @pytest.fixture(scope="class")
    def mock_data(authed_api):
        # ACT ------------------------------------------------------------------
        evaluator_slug_1 = str(uuid4())
    
        trace_id_1 = uuid4().hex
        span_id_1 = uuid4().hex[16:]
        trace_id_2 = uuid4().hex
        span_id_2 = uuid4().hex[16:]
    
        response = authed_api(
            "POST",
            "/simple/traces/",
            json={
                "trace": {
                    "origin": "custom",
                    "kind": "adhoc",
                    "channel": "api",
                    "data": {
                        "outputs": {
                            "integer": 42,
                            "float": 3.14,
                            "string": "Hello, world!",
                            "boolean": True,
                            "array": [1],
                            "object": {"aloha": "mahalo"},
                        },
                    },
                    "tags": {"tag1": "value1", "tag2": "value2"},
                    "meta": {"meta1": "value1", "meta2": "value2"},
                    "references": {"evaluator": {"slug": evaluator_slug_1}},
                    "links": {
                        "invocation": {
                            "trace_id": trace_id_1,
                            "span_id": span_id_1,
                        }
                    },
                }
            },
        )
    
>       assert response.status_code in (200, 202)
E       assert 500 in (200, 202)
E        +  where 500 = <Response [500]>.status_code

oss/tests/pytest/acceptance/annotations/test_annotations_queries.py:49: AssertionError

Check failure on line 0 in oss.tests.pytest.acceptance.annotations.test_annotations_queries.TestAnnotationsQueries

See this annotation in the file changed.

@github-actions github-actions / Railway API Test Results

test_query_annotations_by_annotation_link (oss.tests.pytest.acceptance.annotations.test_annotations_queries.TestAnnotationsQueries) with error

api/oss/tests/results/junit.xml [took 0s]
Raw output
failed on setup with "assert 500 in (200, 202)
 +  where 500 = <Response [500]>.status_code"
authed_api = <function authed_api.<locals>._request at 0x7fb000f287c0>

    @pytest.fixture(scope="class")
    def mock_data(authed_api):
        # ACT ------------------------------------------------------------------
        evaluator_slug_1 = str(uuid4())
    
        trace_id_1 = uuid4().hex
        span_id_1 = uuid4().hex[16:]
        trace_id_2 = uuid4().hex
        span_id_2 = uuid4().hex[16:]
    
        response = authed_api(
            "POST",
            "/simple/traces/",
            json={
                "trace": {
                    "origin": "custom",
                    "kind": "adhoc",
                    "channel": "api",
                    "data": {
                        "outputs": {
                            "integer": 42,
                            "float": 3.14,
                            "string": "Hello, world!",
                            "boolean": True,
                            "array": [1],
                            "object": {"aloha": "mahalo"},
                        },
                    },
                    "tags": {"tag1": "value1", "tag2": "value2"},
                    "meta": {"meta1": "value1", "meta2": "value2"},
                    "references": {"evaluator": {"slug": evaluator_slug_1}},
                    "links": {
                        "invocation": {
                            "trace_id": trace_id_1,
                            "span_id": span_id_1,
                        }
                    },
                }
            },
        )
    
>       assert response.status_code in (200, 202)
E       assert 500 in (200, 202)
E        +  where 500 = <Response [500]>.status_code

oss/tests/pytest/acceptance/annotations/test_annotations_queries.py:49: AssertionError

Check failure on line 0 in oss.tests.pytest.acceptance.annotations.test_annotations_queries.TestAnnotationsQueries

See this annotation in the file changed.

@github-actions github-actions / Railway API Test Results

test_query_annotations_by_link (oss.tests.pytest.acceptance.annotations.test_annotations_queries.TestAnnotationsQueries) with error

api/oss/tests/results/junit.xml [took 0s]
Raw output
failed on setup with "assert 500 in (200, 202)
 +  where 500 = <Response [500]>.status_code"
authed_api = <function authed_api.<locals>._request at 0x7fb000f287c0>

    @pytest.fixture(scope="class")
    def mock_data(authed_api):
        # ACT ------------------------------------------------------------------
        evaluator_slug_1 = str(uuid4())
    
        trace_id_1 = uuid4().hex
        span_id_1 = uuid4().hex[16:]
        trace_id_2 = uuid4().hex
        span_id_2 = uuid4().hex[16:]
    
        response = authed_api(
            "POST",
            "/simple/traces/",
            json={
                "trace": {
                    "origin": "custom",
                    "kind": "adhoc",
                    "channel": "api",
                    "data": {
                        "outputs": {
                            "integer": 42,
                            "float": 3.14,
                            "string": "Hello, world!",
                            "boolean": True,
                            "array": [1],
                            "object": {"aloha": "mahalo"},
                        },
                    },
                    "tags": {"tag1": "value1", "tag2": "value2"},
                    "meta": {"meta1": "value1", "meta2": "value2"},
                    "references": {"evaluator": {"slug": evaluator_slug_1}},
                    "links": {
                        "invocation": {
                            "trace_id": trace_id_1,
                            "span_id": span_id_1,
                        }
                    },
                }
            },
        )
    
>       assert response.status_code in (200, 202)
E       assert 500 in (200, 202)
E        +  where 500 = <Response [500]>.status_code

oss/tests/pytest/acceptance/annotations/test_annotations_queries.py:49: AssertionError

Check failure on line 0 in oss.tests.pytest.acceptance.annotations.test_annotations_queries.TestAnnotationsQueries

See this annotation in the file changed.

@github-actions github-actions / Railway API Test Results

test_query_annotations_by_reference (oss.tests.pytest.acceptance.annotations.test_annotations_queries.TestAnnotationsQueries) with error

api/oss/tests/results/junit.xml [took 0s]
Raw output
failed on setup with "assert 500 in (200, 202)
 +  where 500 = <Response [500]>.status_code"
authed_api = <function authed_api.<locals>._request at 0x7fb000f287c0>

    @pytest.fixture(scope="class")
    def mock_data(authed_api):
        # ACT ------------------------------------------------------------------
        evaluator_slug_1 = str(uuid4())
    
        trace_id_1 = uuid4().hex
        span_id_1 = uuid4().hex[16:]
        trace_id_2 = uuid4().hex
        span_id_2 = uuid4().hex[16:]
    
        response = authed_api(
            "POST",
            "/simple/traces/",
            json={
                "trace": {
                    "origin": "custom",
                    "kind": "adhoc",
                    "channel": "api",
                    "data": {
                        "outputs": {
                            "integer": 42,
                            "float": 3.14,
                            "string": "Hello, world!",
                            "boolean": True,
                            "array": [1],
                            "object": {"aloha": "mahalo"},
                        },
                    },
                    "tags": {"tag1": "value1", "tag2": "value2"},
                    "meta": {"meta1": "value1", "meta2": "value2"},
                    "references": {"evaluator": {"slug": evaluator_slug_1}},
                    "links": {
                        "invocation": {
                            "trace_id": trace_id_1,
                            "span_id": span_id_1,
                        }
                    },
                }
            },
        )
    
>       assert response.status_code in (200, 202)
E       assert 500 in (200, 202)
E        +  where 500 = <Response [500]>.status_code

oss/tests/pytest/acceptance/annotations/test_annotations_queries.py:49: AssertionError

Check failure on line 0 in oss.tests.pytest.acceptance.annotations.test_annotations_queries.TestAnnotationsQueries

See this annotation in the file changed.

@github-actions github-actions / Railway API Test Results

test_query_annotations_by_tags (oss.tests.pytest.acceptance.annotations.test_annotations_queries.TestAnnotationsQueries) with error

api/oss/tests/results/junit.xml [took 0s]
Raw output
failed on setup with "assert 500 in (200, 202)
 +  where 500 = <Response [500]>.status_code"
authed_api = <function authed_api.<locals>._request at 0x7fb000f287c0>

    @pytest.fixture(scope="class")
    def mock_data(authed_api):
        # ACT ------------------------------------------------------------------
        evaluator_slug_1 = str(uuid4())
    
        trace_id_1 = uuid4().hex
        span_id_1 = uuid4().hex[16:]
        trace_id_2 = uuid4().hex
        span_id_2 = uuid4().hex[16:]
    
        response = authed_api(
            "POST",
            "/simple/traces/",
            json={
                "trace": {
                    "origin": "custom",
                    "kind": "adhoc",
                    "channel": "api",
                    "data": {
                        "outputs": {
                            "integer": 42,
                            "float": 3.14,
                            "string": "Hello, world!",
                            "boolean": True,
                            "array": [1],
                            "object": {"aloha": "mahalo"},
                        },
                    },
                    "tags": {"tag1": "value1", "tag2": "value2"},
                    "meta": {"meta1": "value1", "meta2": "value2"},
                    "references": {"evaluator": {"slug": evaluator_slug_1}},
                    "links": {
                        "invocation": {
                            "trace_id": trace_id_1,
                            "span_id": span_id_1,
                        }
                    },
                }
            },
        )
    
>       assert response.status_code in (200, 202)
E       assert 500 in (200, 202)
E        +  where 500 = <Response [500]>.status_code

oss/tests/pytest/acceptance/annotations/test_annotations_queries.py:49: AssertionError

Check failure on line 0 in oss.tests.pytest.acceptance.annotations.test_annotations_queries.TestAnnotationsQueries

See this annotation in the file changed.

@github-actions github-actions / Railway API Test Results

test_query_annotations_by_meta (oss.tests.pytest.acceptance.annotations.test_annotations_queries.TestAnnotationsQueries) with error

api/oss/tests/results/junit.xml [took 0s]
Raw output
failed on setup with "assert 500 in (200, 202)
 +  where 500 = <Response [500]>.status_code"
authed_api = <function authed_api.<locals>._request at 0x7fb000f287c0>

    @pytest.fixture(scope="class")
    def mock_data(authed_api):
        # ACT ------------------------------------------------------------------
        evaluator_slug_1 = str(uuid4())
    
        trace_id_1 = uuid4().hex
        span_id_1 = uuid4().hex[16:]
        trace_id_2 = uuid4().hex
        span_id_2 = uuid4().hex[16:]
    
        response = authed_api(
            "POST",
            "/simple/traces/",
            json={
                "trace": {
                    "origin": "custom",
                    "kind": "adhoc",
                    "channel": "api",
                    "data": {
                        "outputs": {
                            "integer": 42,
                            "float": 3.14,
                            "string": "Hello, world!",
                            "boolean": True,
                            "array": [1],
                            "object": {"aloha": "mahalo"},
                        },
                    },
                    "tags": {"tag1": "value1", "tag2": "value2"},
                    "meta": {"meta1": "value1", "meta2": "value2"},
                    "references": {"evaluator": {"slug": evaluator_slug_1}},
                    "links": {
                        "invocation": {
                            "trace_id": trace_id_1,
                            "span_id": span_id_1,
                        }
                    },
                }
            },
        )
    
>       assert response.status_code in (200, 202)
E       assert 500 in (200, 202)
E        +  where 500 = <Response [500]>.status_code

oss/tests/pytest/acceptance/annotations/test_annotations_queries.py:49: AssertionError

Check failure on line 0 in oss.tests.pytest.acceptance.annotations.test_annotations_queries.TestAnnotationsQueries

See this annotation in the file changed.

@github-actions github-actions / Railway API Test Results

test_query_annotations_by_origin (oss.tests.pytest.acceptance.annotations.test_annotations_queries.TestAnnotationsQueries) with error

api/oss/tests/results/junit.xml [took 0s]
Raw output
failed on setup with "assert 500 in (200, 202)
 +  where 500 = <Response [500]>.status_code"
authed_api = <function authed_api.<locals>._request at 0x7fb000f287c0>

    @pytest.fixture(scope="class")
    def mock_data(authed_api):
        # ACT ------------------------------------------------------------------
        evaluator_slug_1 = str(uuid4())
    
        trace_id_1 = uuid4().hex
        span_id_1 = uuid4().hex[16:]
        trace_id_2 = uuid4().hex
        span_id_2 = uuid4().hex[16:]
    
        response = authed_api(
            "POST",
            "/simple/traces/",
            json={
                "trace": {
                    "origin": "custom",
                    "kind": "adhoc",
                    "channel": "api",
                    "data": {
                        "outputs": {
                            "integer": 42,
                            "float": 3.14,
                            "string": "Hello, world!",
                            "boolean": True,
                            "array": [1],
                            "object": {"aloha": "mahalo"},
                        },
                    },
                    "tags": {"tag1": "value1", "tag2": "value2"},
                    "meta": {"meta1": "value1", "meta2": "value2"},
                    "references": {"evaluator": {"slug": evaluator_slug_1}},
                    "links": {
                        "invocation": {
                            "trace_id": trace_id_1,
                            "span_id": span_id_1,
                        }
                    },
                }
            },
        )
    
>       assert response.status_code in (200, 202)
E       assert 500 in (200, 202)
E        +  where 500 = <Response [500]>.status_code

oss/tests/pytest/acceptance/annotations/test_annotations_queries.py:49: AssertionError

Check failure on line 0 in oss.tests.pytest.acceptance.annotations.test_annotations_queries.TestAnnotationsQueries

See this annotation in the file changed.

@github-actions github-actions / Railway API Test Results

test_query_annotations_by_kind (oss.tests.pytest.acceptance.annotations.test_annotations_queries.TestAnnotationsQueries) with error

api/oss/tests/results/junit.xml [took 0s]
Raw output
failed on setup with "assert 500 in (200, 202)
 +  where 500 = <Response [500]>.status_code"
authed_api = <function authed_api.<locals>._request at 0x7fb000f287c0>

    @pytest.fixture(scope="class")
    def mock_data(authed_api):
        # ACT ------------------------------------------------------------------
        evaluator_slug_1 = str(uuid4())
    
        trace_id_1 = uuid4().hex
        span_id_1 = uuid4().hex[16:]
        trace_id_2 = uuid4().hex
        span_id_2 = uuid4().hex[16:]
    
        response = authed_api(
            "POST",
            "/simple/traces/",
            json={
                "trace": {
                    "origin": "custom",
                    "kind": "adhoc",
                    "channel": "api",
                    "data": {
                        "outputs": {
                            "integer": 42,
                            "float": 3.14,
                            "string": "Hello, world!",
                            "boolean": True,
                            "array": [1],
                            "object": {"aloha": "mahalo"},
                        },
                    },
                    "tags": {"tag1": "value1", "tag2": "value2"},
                    "meta": {"meta1": "value1", "meta2": "value2"},
                    "references": {"evaluator": {"slug": evaluator_slug_1}},
                    "links": {
                        "invocation": {
                            "trace_id": trace_id_1,
                            "span_id": span_id_1,
                        }
                    },
                }
            },
        )
    
>       assert response.status_code in (200, 202)
E       assert 500 in (200, 202)
E        +  where 500 = <Response [500]>.status_code

oss/tests/pytest/acceptance/annotations/test_annotations_queries.py:49: AssertionError

Check failure on line 0 in oss.tests.pytest.acceptance.annotations.test_annotations_queries.TestAnnotationsQueries

See this annotation in the file changed.

@github-actions github-actions / Railway API Test Results

test_query_annotations_by_channel (oss.tests.pytest.acceptance.annotations.test_annotations_queries.TestAnnotationsQueries) with error

api/oss/tests/results/junit.xml [took 0s]
Raw output
failed on setup with "assert 500 in (200, 202)
 +  where 500 = <Response [500]>.status_code"
authed_api = <function authed_api.<locals>._request at 0x7fb000f287c0>

    @pytest.fixture(scope="class")
    def mock_data(authed_api):
        # ACT ------------------------------------------------------------------
        evaluator_slug_1 = str(uuid4())
    
        trace_id_1 = uuid4().hex
        span_id_1 = uuid4().hex[16:]
        trace_id_2 = uuid4().hex
        span_id_2 = uuid4().hex[16:]
    
        response = authed_api(
            "POST",
            "/simple/traces/",
            json={
                "trace": {
                    "origin": "custom",
                    "kind": "adhoc",
                    "channel": "api",
                    "data": {
                        "outputs": {
                            "integer": 42,
                            "float": 3.14,
                            "string": "Hello, world!",
                            "boolean": True,
                            "array": [1],
                            "object": {"aloha": "mahalo"},
                        },
                    },
                    "tags": {"tag1": "value1", "tag2": "value2"},
                    "meta": {"meta1": "value1", "meta2": "value2"},
                    "references": {"evaluator": {"slug": evaluator_slug_1}},
                    "links": {
                        "invocation": {
                            "trace_id": trace_id_1,
                            "span_id": span_id_1,
                        }
                    },
                }
            },
        )
    
>       assert response.status_code in (200, 202)
E       assert 500 in (200, 202)
E        +  where 500 = <Response [500]>.status_code

oss/tests/pytest/acceptance/annotations/test_annotations_queries.py:49: AssertionError

Check warning on line 0 in oss.tests.pytest.acceptance.evaluations.test_evaluation_flows_evaluator_ordering.TestEvaluatorOrderingFlows

See this annotation in the file changed.

@github-actions github-actions / Railway API Test Results

test_reflect_before_contaminator (oss.tests.pytest.acceptance.evaluations.test_evaluation_flows_evaluator_ordering.TestEvaluatorOrderingFlows) failed

api/oss/tests/results/junit.xml [took 5s]
Raw output
AssertionError: {'count': 0}
self = <evaluations.test_evaluation_flows_evaluator_ordering.TestEvaluatorOrderingFlows object at 0x7fd1b840d310>
authed_api = <function authed_api.<locals>._request at 0x7fd1b3d63e20>

    def test_reflect_before_contaminator(self, authed_api):
        # reflect runs BEFORE the score evaluator.
        reflect = create_mock_evaluator(authed_api, key="reflect")
        score = create_mock_evaluator(
            authed_api, key="score", kwargs={"score": 0.5, "threshold": 0.5}
        )
        # dict preserves insertion order -> reflect first.
>       run_id = self._run_with_evaluator_order(
            authed_api,
            name="flow-order-reflect-first",
            evaluator_steps={
                reflect["revision_id"]: "auto",
                score["revision_id"]: "auto",
            },
        )

oss/tests/pytest/acceptance/evaluations/test_evaluation_flows_evaluator_ordering.py:61: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
oss/tests/pytest/acceptance/evaluations/test_evaluation_flows_evaluator_ordering.py:39: in _run_with_evaluator_order
    evaluation = create_simple_evaluation(
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

authed_api = <function authed_api.<locals>._request at 0x7fd1b3d63e20>
name = 'flow-order-reflect-first'
data = {'testset_steps': ['019f0068-9ffb-7dc3-8284-8536e938dee5'], 'application_steps': ['019f0068-a33c-77f3-9308-2a1f698ff2b8'], 'evaluator_steps': {'019f0068-98d4-7030-863d-50bcbb14bb4c': 'auto', '019f0068-9c17-77b3-ba6f-e787e896b95d': 'auto'}}
flags = None

    def create_simple_evaluation(authed_api, *, name, data, flags=None) -> dict:
        """Create (and auto-start) a simple evaluation. Returns the evaluation dict."""
        payload = {"name": name, "data": data}
        if flags is not None:
            payload["flags"] = flags
        response = authed_api(
            "POST",
            "/simple/evaluations/",
            json={"evaluation": payload},
        )
        assert response.status_code == 200, response.text
        body = response.json()
>       assert body["count"] == 1, body
               ^^^^^^^^^^^^^^^^^^
E       AssertionError: {'count': 0}

oss/tests/pytest/acceptance/evaluations/_flow_helpers.py:183: AssertionError

Check warning on line 0 in oss.tests.pytest.acceptance.evaluations.test_simple_evaluations_workflows.TestSimpleEvaluationsWorkflowReferences

See this annotation in the file changed.

@github-actions github-actions / Railway API Test Results

test_create_testset_to_evaluator_evaluation_does_not_fail_setup (oss.tests.pytest.acceptance.evaluations.test_simple_evaluations_workflows.TestSimpleEvaluationsWorkflowReferences) failed

api/oss/tests/results/junit.xml [took 3s]
Raw output
assert 0 == 1
self = <evaluations.test_simple_evaluations_workflows.TestSimpleEvaluationsWorkflowReferences object at 0x7fb0010f4830>
authed_api = <function authed_api.<locals>._request at 0x7fb000c544a0>

    def test_create_testset_to_evaluator_evaluation_does_not_fail_setup(
        self, authed_api
    ):
        testset = _create_simple_testset(authed_api)
        evaluator = _create_simple_evaluator(authed_api)
    
        response = authed_api(
            "POST",
            "/simple/evaluations/",
            json={
                "evaluation": {
                    "name": "testset-to-evaluator-setup",
                    "data": {
                        "testset_steps": [testset["revision_id"]],
                        "evaluator_steps": [evaluator["revision_id"]],
                    },
                }
            },
        )
    
        assert response.status_code == 200
        body = response.json()
>       assert body["count"] == 1
E       assert 0 == 1

oss/tests/pytest/acceptance/evaluations/test_simple_evaluations_workflows.py:271: AssertionError

Check warning on line 0 in oss.tests.pytest.acceptance.evaluations.test_evaluation_flows_evaluator_ordering.TestEvaluatorOrderingFlows

See this annotation in the file changed.

@github-actions github-actions / Railway API Test Results

test_reflect_after_contaminator (oss.tests.pytest.acceptance.evaluations.test_evaluation_flows_evaluator_ordering.TestEvaluatorOrderingFlows) failed

api/oss/tests/results/junit.xml [took 4s]
Raw output
AssertionError: {'count': 0}
self = <evaluations.test_evaluation_flows_evaluator_ordering.TestEvaluatorOrderingFlows object at 0x7fd1b840d090>
authed_api = <function authed_api.<locals>._request at 0x7fd1b3d63e20>

    def test_reflect_after_contaminator(self, authed_api):
        # reflect runs AFTER the score evaluator — the position that regressed in
        # production. A leaked score-evaluator output would drop reflect to 0.0.
        score = create_mock_evaluator(
            authed_api, key="score", kwargs={"score": 0.5, "threshold": 0.5}
        )
        reflect = create_mock_evaluator(authed_api, key="reflect")
>       run_id = self._run_with_evaluator_order(
            authed_api,
            name="flow-order-reflect-second",
            evaluator_steps={
                score["revision_id"]: "auto",
                reflect["revision_id"]: "auto",
            },
        )

oss/tests/pytest/acceptance/evaluations/test_evaluation_flows_evaluator_ordering.py:81: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
oss/tests/pytest/acceptance/evaluations/test_evaluation_flows_evaluator_ordering.py:39: in _run_with_evaluator_order
    evaluation = create_simple_evaluation(
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

authed_api = <function authed_api.<locals>._request at 0x7fd1b3d63e20>
name = 'flow-order-reflect-second'
data = {'testset_steps': ['019f0068-b1cf-7162-8a0c-92e5cac2d1cc'], 'application_steps': ['019f0068-b533-70e2-b66c-6b65b64f2d60'], 'evaluator_steps': {'019f0068-aae7-7db0-92b4-05985671ef55': 'auto', '019f0068-ad7e-73e3-a9e2-1a131de47718': 'auto'}}
flags = None

    def create_simple_evaluation(authed_api, *, name, data, flags=None) -> dict:
        """Create (and auto-start) a simple evaluation. Returns the evaluation dict."""
        payload = {"name": name, "data": data}
        if flags is not None:
            payload["flags"] = flags
        response = authed_api(
            "POST",
            "/simple/evaluations/",
            json={"evaluation": payload},
        )
        assert response.status_code == 200, response.text
        body = response.json()
>       assert body["count"] == 1, body
               ^^^^^^^^^^^^^^^^^^
E       AssertionError: {'count': 0}

oss/tests/pytest/acceptance/evaluations/_flow_helpers.py:183: AssertionError

Check warning on line 0 in oss.tests.pytest.acceptance.evaluations.test_simple_queues_basics.TestSimpleQueuesBasics

See this annotation in the file changed.

@github-actions github-actions / Railway API Test Results

test_create_simple_queue_from_testsets (oss.tests.pytest.acceptance.evaluations.test_simple_queues_basics.TestSimpleQueuesBasics) failed

api/oss/tests/results/junit.xml [took 2s]
Raw output
assert 500 == 200
 +  where 500 = <Response [500]>.status_code
self = <evaluations.test_simple_queues_basics.TestSimpleQueuesBasics object at 0x7fb001121ba0>
authed_api = <function authed_api.<locals>._request at 0x7fb000c54ae0>

    def test_create_simple_queue_from_testsets(self, authed_api):
        evaluator_revision_id = _create_evaluator(authed_api)
        testset_revision_id = _create_testset(authed_api)
    
        response = authed_api(
            "POST",
            "/simple/queues/",
            json={
                "queue": {
                    "name": "testset-backed-queue",
                    "data": {
                        "testsets": [testset_revision_id],
                        "evaluators": [evaluator_revision_id],
                    },
                }
            },
        )
    
>       assert response.status_code == 200
E       assert 500 == 200
E        +  where 500 = <Response [500]>.status_code

oss/tests/pytest/acceptance/evaluations/test_simple_queues_basics.py:193: AssertionError

Check warning on line 0 in oss.tests.pytest.acceptance.evaluations.test_simple_queues_basics.TestSimpleQueuesBasics

See this annotation in the file changed.

@github-actions github-actions / Railway API Test Results

test_create_source_backed_queue_preserves_repeats_and_assignments (oss.tests.pytest.acceptance.evaluations.test_simple_queues_basics.TestSimpleQueuesBasics) failed

api/oss/tests/results/junit.xml [took 2s]
Raw output
assert 500 == 200
 +  where 500 = <Response [500]>.status_code
self = <evaluations.test_simple_queues_basics.TestSimpleQueuesBasics object at 0x7fb0010f5370>
authed_api = <function authed_api.<locals>._request at 0x7fb000c54ae0>

    def test_create_source_backed_queue_preserves_repeats_and_assignments(
        self, authed_api
    ):
        evaluator_revision_id = _create_evaluator(authed_api)
        testset_revision_id = _create_testset(authed_api)
        user_id_1 = str(uuid4())
        user_id_2 = str(uuid4())
    
        response = authed_api(
            "POST",
            "/simple/queues/",
            json={
                "queue": {
                    "name": "testset-backed-queue-with-repeats",
                    "data": {
                        "testsets": [testset_revision_id],
                        "evaluators": {evaluator_revision_id: "human"},
                        "repeats": 2,
                        "assignments": [[user_id_1], [user_id_2]],
                    },
                }
            },
        )
    
>       assert response.status_code == 200
E       assert 500 == 200
E        +  where 500 = <Response [500]>.status_code

oss/tests/pytest/acceptance/evaluations/test_simple_queues_basics.py:224: AssertionError

Check warning on line 0 in oss.tests.pytest.acceptance.evaluations.test_run_slice_endpoints.TestRunSliceEndpoints

See this annotation in the file changed.

@github-actions github-actions / Railway API Test Results

test_probe_returns_results_envelope_for_empty_slice (oss.tests.pytest.acceptance.evaluations.test_run_slice_endpoints.TestRunSliceEndpoints) failed

api/oss/tests/results/junit.xml [took 4s]
Raw output
KeyError: 'evaluation'
self = <evaluations.test_run_slice_endpoints.TestRunSliceEndpoints object at 0x7f6316313d90>
authed_api = <function authed_api.<locals>._request at 0x7f6315c577e0>

    def test_probe_returns_results_envelope_for_empty_slice(self, authed_api):
>       evaluation = _create_testset_evaluator_evaluation(authed_api)
                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

oss/tests/pytest/acceptance/evaluations/test_run_slice_endpoints.py:132: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

authed_api = <function authed_api.<locals>._request at 0x7f6315c577e0>

    def _create_testset_evaluator_evaluation(authed_api) -> dict:
        """A dispatchable testset -> application -> evaluator run (mock app, no LLM).
    
        The application step is required: testset -> evaluator (no app) is a
        `potential`/undispatchable topology that `start` auto-fails-and-closes, which
        would 409 every subsequent slice op. The mock app keeps it deterministic.
        """
        testset = _create_simple_testset(authed_api)
        application = _create_mock_application(authed_api)
        evaluator = _create_simple_evaluator(authed_api)
        response = authed_api(
            "POST",
            "/simple/evaluations/",
            json={
                "evaluation": {
                    "name": f"run-slice-{uuid4().hex[:8]}",
                    "flags": {"is_cached": False, "is_split": False},
                    "data": {
                        "testset_steps": {testset["revision_id"]: "custom"},
                        "application_steps": {application["revision_id"]: "custom"},
                        "evaluator_steps": {evaluator["revision_id"]: "auto"},
                        "repeats": 1,
                    },
                }
            },
        )
        assert response.status_code == 200, response.text
>       return response.json()["evaluation"]
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
E       KeyError: 'evaluation'

oss/tests/pytest/acceptance/evaluations/test_run_slice_endpoints.py:112: KeyError

Check warning on line 0 in oss.tests.pytest.acceptance.evaluations.test_run_slice_endpoints.TestRunSliceEndpoints

See this annotation in the file changed.

@github-actions github-actions / Railway API Test Results

test_probe_accepts_full_coordinate_body (oss.tests.pytest.acceptance.evaluations.test_run_slice_endpoints.TestRunSliceEndpoints) failed

api/oss/tests/results/junit.xml [took 5s]
Raw output
KeyError: 'evaluation'
self = <evaluations.test_run_slice_endpoints.TestRunSliceEndpoints object at 0x7f6316313ed0>
authed_api = <function authed_api.<locals>._request at 0x7f6315c577e0>

    def test_probe_accepts_full_coordinate_body(self, authed_api):
>       evaluation = _create_testset_evaluator_evaluation(authed_api)
                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

oss/tests/pytest/acceptance/evaluations/test_run_slice_endpoints.py:150: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

authed_api = <function authed_api.<locals>._request at 0x7f6315c577e0>

    def _create_testset_evaluator_evaluation(authed_api) -> dict:
        """A dispatchable testset -> application -> evaluator run (mock app, no LLM).
    
        The application step is required: testset -> evaluator (no app) is a
        `potential`/undispatchable topology that `start` auto-fails-and-closes, which
        would 409 every subsequent slice op. The mock app keeps it deterministic.
        """
        testset = _create_simple_testset(authed_api)
        application = _create_mock_application(authed_api)
        evaluator = _create_simple_evaluator(authed_api)
        response = authed_api(
            "POST",
            "/simple/evaluations/",
            json={
                "evaluation": {
                    "name": f"run-slice-{uuid4().hex[:8]}",
                    "flags": {"is_cached": False, "is_split": False},
                    "data": {
                        "testset_steps": {testset["revision_id"]: "custom"},
                        "application_steps": {application["revision_id"]: "custom"},
                        "evaluator_steps": {evaluator["revision_id"]: "auto"},
                        "repeats": 1,
                    },
                }
            },
        )
        assert response.status_code == 200, response.text
>       return response.json()["evaluation"]
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
E       KeyError: 'evaluation'

oss/tests/pytest/acceptance/evaluations/test_run_slice_endpoints.py:112: KeyError

Check warning on line 0 in oss.tests.pytest.acceptance.evaluations.test_simple_queues_basics.TestSimpleQueuesBasics

See this annotation in the file changed.

@github-actions github-actions / Railway API Test Results

test_simple_queue_allows_human_mixed_with_non_human_evaluators (oss.tests.pytest.acceptance.evaluations.test_simple_queues_basics.TestSimpleQueuesBasics) failed

api/oss/tests/results/junit.xml [took 3s]
Raw output
assert 500 == 200
 +  where 500 = <Response [500]>.status_code
self = <evaluations.test_simple_queues_basics.TestSimpleQueuesBasics object at 0x7fb0010fe850>
authed_api = <function authed_api.<locals>._request at 0x7fb000c54ae0>

    def test_simple_queue_allows_human_mixed_with_non_human_evaluators(
        self, authed_api
    ):
        # The "at least one human" rule allows a mix: a human alongside any
        # non-human origin (auto or custom) is a valid human queue. The explicit
        # non-human origin is honored on the underlying run.
        for non_human_origin, expected_flag in (
            ("auto", "has_auto"),
            ("custom", "has_custom"),
        ):
            human_evaluator_id = _create_evaluator(authed_api)
            other_evaluator_id = _create_evaluator(authed_api)
            testset_revision_id = _create_testset(authed_api)
    
            response = authed_api(
                "POST",
                "/simple/queues/",
                json={
                    "queue": {
                        "name": f"testset-backed-human-plus-{non_human_origin}",
                        "data": {
                            "testsets": [testset_revision_id],
                            "evaluators": {
                                human_evaluator_id: "human",
                                other_evaluator_id: non_human_origin,
                            },
                        },
                    }
                },
            )
    
>           assert response.status_code == 200
E           assert 500 == 200
E            +  where 500 = <Response [500]>.status_code

oss/tests/pytest/acceptance/evaluations/test_simple_queues_basics.py:340: AssertionError

Check warning on line 0 in oss.tests.pytest.acceptance.evaluations.test_run_slice_endpoints.TestRunSliceEndpoints

See this annotation in the file changed.

@github-actions github-actions / Railway API Test Results

test_probe_accepts_empty_body (oss.tests.pytest.acceptance.evaluations.test_run_slice_endpoints.TestRunSliceEndpoints) failed

api/oss/tests/results/junit.xml [took 3s]
Raw output
KeyError: 'evaluation'
self = <evaluations.test_run_slice_endpoints.TestRunSliceEndpoints object at 0x7f631611d220>
authed_api = <function authed_api.<locals>._request at 0x7f6315c577e0>

    def test_probe_accepts_empty_body(self, authed_api):
>       evaluation = _create_testset_evaluator_evaluation(authed_api)
                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

oss/tests/pytest/acceptance/evaluations/test_run_slice_endpoints.py:168: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

authed_api = <function authed_api.<locals>._request at 0x7f6315c577e0>

    def _create_testset_evaluator_evaluation(authed_api) -> dict:
        """A dispatchable testset -> application -> evaluator run (mock app, no LLM).
    
        The application step is required: testset -> evaluator (no app) is a
        `potential`/undispatchable topology that `start` auto-fails-and-closes, which
        would 409 every subsequent slice op. The mock app keeps it deterministic.
        """
        testset = _create_simple_testset(authed_api)
        application = _create_mock_application(authed_api)
        evaluator = _create_simple_evaluator(authed_api)
        response = authed_api(
            "POST",
            "/simple/evaluations/",
            json={
                "evaluation": {
                    "name": f"run-slice-{uuid4().hex[:8]}",
                    "flags": {"is_cached": False, "is_split": False},
                    "data": {
                        "testset_steps": {testset["revision_id"]: "custom"},
                        "application_steps": {application["revision_id"]: "custom"},
                        "evaluator_steps": {evaluator["revision_id"]: "auto"},
                        "repeats": 1,
                    },
                }
            },
        )
        assert response.status_code == 200, response.text
>       return response.json()["evaluation"]
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
E       KeyError: 'evaluation'

oss/tests/pytest/acceptance/evaluations/test_run_slice_endpoints.py:112: KeyError

Check warning on line 0 in oss.tests.pytest.acceptance.evaluations.test_simple_queues_basics.TestSimpleQueuesBasics

See this annotation in the file changed.

@github-actions github-actions / Railway API Test Results

test_add_testcases_rejects_testset_backed_source_queue (oss.tests.pytest.acceptance.evaluations.test_simple_queues_basics.TestSimpleQueuesBasics) failed

api/oss/tests/results/junit.xml [took 2s]
Raw output
assert 500 == 200
 +  where 500 = <Response [500]>.status_code
self = <evaluations.test_simple_queues_basics.TestSimpleQueuesBasics object at 0x7fb000f91310>
authed_api = <function authed_api.<locals>._request at 0x7fb000c54ae0>

    def test_add_testcases_rejects_testset_backed_source_queue(self, authed_api):
        evaluator_revision_id = _create_evaluator(authed_api)
        testset_revision_id = _create_testset(authed_api)
        create_response = authed_api(
            "POST",
            "/simple/queues/",
            json={
                "queue": {
                    "data": {
                        "testsets": [testset_revision_id],
                        "evaluators": [evaluator_revision_id],
                    },
                }
            },
        )
>       assert create_response.status_code == 200
E       assert 500 == 200
E        +  where 500 = <Response [500]>.status_code

oss/tests/pytest/acceptance/evaluations/test_simple_queues_basics.py:423: AssertionError