Add agent ref to agentic_ol system config

docs/agentic_lightspeed_evaluation.md

@@ -28,6 +28,7 @@ agents:
   openshift_agentic_lightspeed:
     type: openshift_agentic_run
     namespace: openshift-lightspeed
+    agent_ref: default
     auto_approve: true
     cleanup_openshift_agentic_runs: true
     timeout: 900
@@ -39,6 +40,7 @@ agents:
 | Field | Type | Default | Description |
 |-------|------|---------|-------------|
 | `namespace` | string | *(required)* | Kubernetes namespace containing AgenticRun resources |
+| `agent_ref` | string | `null` | Name of the Agent CR on the cluster — injected into all stages defined in eval data |
 | `auto_approve` | bool | `true` | Automatically approve AgenticRuns when phase is Proposed |
 | `cleanup_openshift_agentic_runs` | bool | `true` | Delete eval AgenticRun CRs after status is captured |
 | `timeout` | int | `900` | Total timeout in seconds for the AgenticRun lifecycle |
@@ -47,6 +49,35 @@ agents:
 | `cache_dir` | string | `null` | Location of cached queries |
 | `cache_enabled` | bool | `true` | Enable caching |
 
+#### `agent_ref` and NxM evaluation
+
+For HTTP API agents, the model/provider is a config-level field — different agent configs naturally produce different runs. For agentic evaluation, the model is determined by the Agent CR on the cluster, and the Agent CR name lives in the eval data spec (`analysis.agent`, `execution.agent`, etc.).
+
+`agent_ref` bridges this gap: when set, it overrides the agent name in all stages defined in the eval data. This enables NxM behavioral evaluation with different cluster-side Agent CRs:
+
+```yaml
+agents:
+  default:
+    agent: [eval_fast, eval_smart]
+    repeat: 3
+  eval_fast:
+    type: openshift_agentic_run
+    namespace: openshift-lightspeed
+    agent_ref: fast-agent
+  eval_smart:
+    type: openshift_agentic_run
+    namespace: openshift-lightspeed
+    agent_ref: smart-agent
+```
+
+**Override rules:**
+
+- `agent_ref` set, stage defined in eval data — config overrides eval data's agent
+- `agent_ref` not set, stage defined in eval data — eval data's agent is used as-is
+- Stage not defined in eval data — `agent_ref` does not inject the stage
+
+Note: `agent_ref` overriding eval data is an inversion of the normal pattern (where eval data overrides config). This is intentional — eval data agent names are placeholders for stage selection, while `agent_ref` represents the actual agent choice. The CRD spec couples stage selection with agent selection in the same field, so the config override is the pragmatic way to separate them for NxM.
+
 ### Turn Data Structure
 
 For agentic workflows, each turn uses `openshift_agentic_run_spec` to define the AgenticRun and `expected_openshift_agentic_run_status` to define success criteria.

src/lightspeed_evaluation/core/models/agents.py

@@ -180,6 +180,12 @@ class OpenshiftAgenticRunAgentConfig(BaseModel):
         pattern=r"\S+",
         description="Kubernetes namespace containing AgenticRun resources",
     )
+    agent_ref: Optional[str] = Field(
+        default=None,
+        min_length=1,
+        pattern=r"^\S+$",
+        description="Name of the Agent CR on the cluster to use for all stages",
+    )
     auto_approve: bool = True
     cleanup_openshift_agentic_runs: bool = True
     timeout: int = Field(default=900, gt=0)

src/lightspeed_evaluation/pipeline/evaluation/driver.py

@@ -173,6 +173,7 @@ def execute_turn(
             else ""
         )
         cr_name = f"eval-{safe_id}-{suffix}" if safe_id else f"eval-{suffix}"
+        self._apply_config_overrides(turn_data)
         openshift_agentic_run_spec = turn_data.openshift_agentic_run_spec or {}
         manifest = self._build_agentic_run_cr(turn_data, cr_name)
         deadline = time.monotonic() + self._config.timeout
@@ -218,6 +219,18 @@ def execute_turn(
         logger.info("AgenticRun '%s' reached terminal state: %s", cr_name, outcome)
         return (None, None)
 
+    def _apply_config_overrides(self, turn_data: TurnData) -> None:
+        """Enrich turn_data spec with agent config overrides."""
+        if not self._config.agent_ref:
+            return
+        spec = turn_data.openshift_agentic_run_spec
+        if spec is None:
+            spec = {}
+            turn_data.openshift_agentic_run_spec = spec
+        for stage in ("analysis", "execution", "verification"):
+            if stage in spec and isinstance(spec[stage], dict):
+                spec[stage]["agent"] = self._config.agent_ref
+
     def _amend_turn_data(
         self, turn_data: TurnData, status_dict: dict[str, Any]
     ) -> None:

tests/unit/pipeline/evaluation/test_openshift_agentic_run_driver.py

@@ -126,6 +126,19 @@ def test_invalid_poll_interval_zero(self) -> None:
                 {**VALID_CONFIG, "poll_interval": 0}
             )
 
+    def test_agent_ref_default_none(self) -> None:
+        """Test agent_ref defaults to None."""
+        config = OpenshiftAgenticRunAgentConfig.model_validate(VALID_CONFIG)
+        assert config.agent_ref is None
+
+    @pytest.mark.parametrize("value", ["", " ", "  "])
+    def test_agent_ref_rejects_empty_and_whitespace(self, value: str) -> None:
+        """Test agent_ref rejects empty and whitespace-only strings."""
+        with pytest.raises(ValidationError):
+            OpenshiftAgenticRunAgentConfig.model_validate(
+                {**VALID_CONFIG, "agent_ref": value}
+            )
+
 
 # ── Condition helpers ────────────────────────────────────────────────
 
@@ -351,6 +364,73 @@ def test_approval_cr_with_agent_refs(self, mocker: MockerFixture) -> None:
         assert cr["spec"]["stages"][2]["verification"] == {"agent": "eval-default"}
 
 
+# ── Config overrides ────────────────────────────────────────────────
+
+
+class TestApplyConfigOverrides:
+    """Unit tests for OpenshiftAgenticRunDriver._apply_config_overrides."""
+
+    def _make_driver(
+        self, mocker: MockerFixture, agent_ref: str | None = None
+    ) -> OpenshiftAgenticRunDriver:
+        """Create a driver with optional agent_ref."""
+        mocker.patch(f"{MODULE}.shutil").which.return_value = "/usr/bin/oc"
+        config = {**VALID_CONFIG}
+        if agent_ref:
+            config["agent_ref"] = agent_ref
+        return OpenshiftAgenticRunDriver(config)
+
+    def test_agent_ref_injected_into_stages(self, mocker: MockerFixture) -> None:
+        """Test agent_ref is set as default agent in each stage."""
+        driver = self._make_driver(mocker, agent_ref="fast-agent")
+        turn = TurnData(
+            turn_id="t1",
+            query="Q",
+            openshift_agentic_run_spec={
+                "analysis": {},
+                "execution": {},
+                "verification": {},
+            },
+        )
+        driver._apply_config_overrides(turn)
+
+        spec: dict[str, Any] = turn.openshift_agentic_run_spec or {}
+        assert spec["analysis"]["agent"] == "fast-agent"
+        assert spec["execution"]["agent"] == "fast-agent"
+        assert spec["verification"]["agent"] == "fast-agent"
+
+    def test_agent_ref_overrides_eval_data_agent(self, mocker: MockerFixture) -> None:
+        """Test agent_ref from config overrides eval data agent."""
+        driver = self._make_driver(mocker, agent_ref="fast-agent")
+        turn = TurnData(
+            turn_id="t1",
+            query="Q",
+            openshift_agentic_run_spec={
+                "analysis": {"agent": "custom-agent"},
+                "execution": {},
+            },
+        )
+        driver._apply_config_overrides(turn)
+
+        spec: dict[str, Any] = turn.openshift_agentic_run_spec or {}
+        assert spec["analysis"]["agent"] == "fast-agent"
+        assert spec["execution"]["agent"] == "fast-agent"
+
+    def test_no_agent_ref_no_change(self, mocker: MockerFixture) -> None:
+        """Test no modification when agent_ref is None."""
+        driver = self._make_driver(mocker)
+        turn = TurnData(
+            turn_id="t1",
+            query="Q",
+            openshift_agentic_run_spec={"analysis": {}, "execution": {}},
+        )
+        driver._apply_config_overrides(turn)
+
+        spec: dict[str, Any] = turn.openshift_agentic_run_spec or {}
+        assert "agent" not in spec["analysis"]
+        assert "agent" not in spec["execution"]
+
+
 # ── Extract summary ─────────────────────────────────────────────────