feat(evals): ground-truth store, predicate library, xarm7 tabletop suite

dimos/core/global_config.py

@@ -99,6 +99,9 @@ class GlobalConfig(BaseSettings):
     mujoco_global_map_from_pointcloud: str | None = None
     mujoco_start_pos: str = "-1.0, 1.0"
     mujoco_steps_per_frame: int = 7
+    # Global on-switch for MujocoSimModule's GT object pose stream — eval runs
+    # need it on blueprints whose module config doesn't set publish_ground_truth.
+    mujoco_publish_ground_truth: bool = False
     scene_package: str | None = None
     robot_model: str | None = None
     robot_id: str | None = None

dimos/evals/gt_recorder.py

@@ -0,0 +1,41 @@
+# Copyright 2026 Dimensional Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""GT recorder: the simulator's privileged object poses, recorded for scoring.
+
+Subscribes ``/gt_object_poses`` (MujocoSimModule ``publish_ground_truth``)
+over LCM and records it to its own db, separate from the agent-visible
+recording — ground truth must never leak into the agent's memory. The eval
+runner deploys this per case that declares ``ground_truth=True``; scorers
+read it back through ``EvalRunner.gt_store()`` and
+:mod:`dimos.evals.predicates`.
+"""
+
+from __future__ import annotations
+
+from dimos.core.stream import In
+from dimos.memory.module import Recorder
+from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped
+
+
+class GTRecorder(Recorder):
+    """Records the GT object pose stream to ``db_path``.
+
+    GT poses are already world-frame (frame_id = body name, no tf anchor),
+    so deploy with ``poseless_streams=[predicates.GT_STREAM]`` and
+    ``record_tf=False``. The stream keeps the port name — the wiring layer
+    names topics after it.
+    """
+
+    gt_object_poses: In[PoseStamped]

dimos/evals/predicates.py

@@ -0,0 +1,130 @@
+# Copyright 2026 Dimensional Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Ground-truth predicates: physical outcomes from simulator truth, not agent memory.
+
+Interactive cases that declare ``ground_truth=True`` get a second store in
+their score callable — the GT recorder's db of world poses the sim publishes
+for every free-joint scene body (MujocoSimModule ``publish_ground_truth``).
+Predicates here turn a role's pose history into a 0.0/1.0 outcome that
+composes with the usual scorers and aggregates::
+
+    ROLES = {"cup": "cup", "apple": "apple"}
+
+    def score(store: Store, gt: Store) -> float:
+        picked = lifted(gt, ROLES, "cup", min_delta=0.05)
+        collateral = displaced(gt, ROLES, "apple", threshold=0.10)
+        return picked * (1.0 - collateral)
+
+    InteractiveEval(..., score=score, ground_truth=True, aggregate=final)
+
+``roles`` maps the case's semantic names to GT body names (the pose's
+``frame_id``). All predicates raise LookupError while the GT stream has no
+data for the role — the runner's sampler treats LookupError as "not yet,
+keep waiting".
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import TYPE_CHECKING
+
+from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped
+
+if TYPE_CHECKING:
+    from dimos.memory.store.base import Store
+
+GT_STREAM = "gt_object_poses"
+"""Stream (and module port) the simulator's ground-truth poses flow on."""
+
+# Body up-axis alignment below which a pose counts as toppled (cos 60°).
+_UP_DOT_MIN = 0.5
+
+
+def gt_poses(
+    gt: Store, roles: Mapping[str, str], role: str, *, stream: str = GT_STREAM
+) -> list[PoseStamped]:
+    """Time-ordered world poses of the body *role* maps to.
+
+    The GT stream multiplexes every scene body on one stream; ``frame_id``
+    carries the body name.
+    """
+    body = roles[role]
+    poses = [obs.data for obs in gt.streams[stream] if obs.data.frame_id == body]
+    if not poses:
+        raise LookupError(f"no GT poses for role {role!r} (body {body!r}) on stream {stream!r}")
+    return poses
+
+
+def lifted(gt: Store, roles: Mapping[str, str], role: str, *, min_delta: float) -> float:
+    """1.0 once the body's z rises ``min_delta`` above its episode-start z."""
+    poses = gt_poses(gt, roles, role)
+    gain = max(p.position.z for p in poses) - poses[0].position.z
+    return float(gain > min_delta)
+
+
+def displaced(gt: Store, roles: Mapping[str, str], role: str, *, threshold: float) -> float:
+    """1.0 once the body's xy distance from its episode-start xy exceeds
+    ``threshold`` — the knock-down/knock-aside detector."""
+    poses = gt_poses(gt, roles, role)
+    x0, y0 = poses[0].position.x, poses[0].position.y
+    farthest = max(((p.position.x - x0) ** 2 + (p.position.y - y0) ** 2) ** 0.5 for p in poses)
+    return float(farthest > threshold)
+
+
+def knocked_over(gt: Store, roles: Mapping[str, str], role: str) -> float:
+    """1.0 once the body's up axis tips more than 60° off world-up."""
+    poses = gt_poses(gt, roles, role)
+    for p in poses:
+        up_z = p.orientation.to_rotation_matrix()[2, 2]
+        if up_z < _UP_DOT_MIN:
+            return 1.0
+    return 0.0
+
+
+def near(gt: Store, roles: Mapping[str, str], role_a: str, role_b: str, *, dist: float) -> float:
+    """1.0 when the two bodies' latest positions are within ``dist`` (3D)."""
+    a, b = gt_poses(gt, roles, role_a)[-1], gt_poses(gt, roles, role_b)[-1]
+    return float((a.position - b.position).length() <= dist)
+
+
+def contained(
+    gt: Store,
+    roles: Mapping[str, str],
+    role_a: str,
+    role_b: str,
+    *,
+    xy_tol: float = 0.1,
+) -> float:
+    """1.0 when a sits inside b: within ``xy_tol`` of b's xy and above b's z.
+
+    Pose-derived approximation — GT carries body poses, not shape extents,
+    so b's "footprint" is the tolerance disc around its center and its "top"
+    is its origin height. Size ``xy_tol`` to the container, not the content.
+    """
+    a, b = gt_poses(gt, roles, role_a)[-1], gt_poses(gt, roles, role_b)[-1]
+    dx, dy = a.position.x - b.position.x, a.position.y - b.position.y
+    return float((dx**2 + dy**2) ** 0.5 <= xy_tol and a.position.z > b.position.z)
+
+
+def grasped(gt: Store, roles: Mapping[str, str], role: str) -> float:
+    """Whether the gripper holds the object — placeholder.
+
+    Not decidable from poses alone: a pose-only heuristic (object tracks the
+    end effector) also fires on pushes and drags. Blocked on the contact /
+    gripper-force channel from issue #3594 phase 2.
+    """
+    raise NotImplementedError(
+        "grasped() needs the contact channel (issue #3594 phase 2); poses alone can't tell a grasp from a push"
+    )

dimos/evals/runner.py

@@ -22,23 +22,33 @@
 
 from __future__ import annotations
 
-from collections.abc import Callable, Mapping, Sequence
+from collections.abc import Mapping, Sequence
 from dataclasses import asdict, dataclass, replace
 import json
 from pathlib import Path
 import subprocess
 import time
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, cast
 
 from dimos.constants import STATE_DIR
 from dimos.core.resource import CompositeResource
-from dimos.evals.types import EvalCase, EvalResult, InteractiveEval, ResponseT, Suite
+from dimos.evals.types import (
+    EvalCase,
+    EvalResult,
+    GTScore,
+    InteractiveEval,
+    ResponseT,
+    Score,
+    Suite,
+    uses_gt_store,
+)
 from dimos.protocol.service.spec import BaseConfig, Configurable
 from dimos.utils.logging_config import setup_logger
 
 if TYPE_CHECKING:
     from langchain_core.language_models.chat_models import BaseChatModel
 
+    from dimos.core.coordination.module_coordinator import ModuleCoordinator
     from dimos.e2e_tests.dim_sim_client import DimSimClient
     from dimos.e2e_tests.dimos_cli_call import DimosCliCall
     from dimos.memory.store.base import Store
@@ -104,6 +114,8 @@ def __init__(self, **kwargs: Any) -> None:
         self._proc: DimosCliCall | None = None
         self._sim: DimSimClient | None = None
         self._run_dir: Path | None = None
+        self._gt_db: Path | None = None  # current case's GT db, when ground_truth
+        self._gt_coordinator: ModuleCoordinator | None = None
 
     # -- run lifecycle -----------------------------------------------------------
 
@@ -206,6 +218,14 @@ def live_store(self) -> Store:
 
         return SqliteStore(path=self.config.live_db, must_exist=True)
 
+    def gt_store(self) -> Store:
+        """Store over the per-case GT db the GT recorder writes (ground_truth cases)."""
+        from dimos.memory.store.sqlite import SqliteStore
+
+        if self._gt_db is None:
+            raise RuntimeError("gt_store() needs a case with ground_truth=True under run()")
+        return SqliteStore(path=str(self._gt_db), must_exist=True)
+
     def encode(self, stream: Stream[Any, Any]) -> list[dict[str, Any]]:
         """mem2 Stream -> model-legible content blocks (the surface under test).
 
@@ -345,10 +365,17 @@ def setup_env(self, case: InteractiveEval) -> None:
 
             proc = DimosCliCall()
             proc.simulator = case.simulator
-            proc.global_args = ["--dimsim-scene", case.scene]
+            # --dimsim-scene only exists for the dimsim launcher; mujoco sims
+            # name their scene in the blueprint itself.
+            if case.simulator == "dimsim":
+                proc.global_args = ["--dimsim-scene", case.scene]
+            if case.ground_truth:
+                proc.global_args.append("--mujoco-publish-ground-truth")
             proc.demo_args = ["run", *case.blueprint.split()]
             proc.start()
             self._proc = proc
+        if case.ground_truth:
+            self._start_gt_recorder(case)
         if not self._wait_mcp(self.config.launch_timeout_s):
             raise RuntimeError(f"MCP at {self.mcp_url} not ready — is dimos up?")
         if case.setup is not _no_setup:
@@ -359,6 +386,29 @@ def setup_env(self, case: InteractiveEval) -> None:
             self._sim = sim
             case.setup(sim)
 
+    def _start_gt_recorder(self, case: InteractiveEval) -> None:
+        """Run a GT recorder blueprint in-process: subscribe the sim's GT
+        stream over LCM, record it to <run_dir>/<case_id>.gt.db.
+
+        The blueprint has no local Out for the recorder's In port — the
+        coordinator gives unmatched ports their ``/<name>`` LCM topic, which
+        is exactly what the sim publishes on.
+        """
+        from dimos.core.coordination.blueprints import autoconnect
+        from dimos.core.coordination.module_coordinator import ModuleCoordinator
+        from dimos.evals.gt_recorder import GTRecorder
+        from dimos.evals.predicates import GT_STREAM
+
+        self._gt_db = self.run_dir / f"{case.id}.gt.db"
+        blueprint = autoconnect(
+            GTRecorder.blueprint(
+                db_path=self._gt_db,
+                record_tf=False,
+                poseless_streams=[GT_STREAM],
+            )
+        )
+        self._gt_coordinator = ModuleCoordinator.build(blueprint)
+
     def teardown_env(self) -> None:
         """Per-case cleanup — the runner owns env lifecycle, cases just declare it."""
         if self._sim is not None:
@@ -367,6 +417,10 @@ def teardown_env(self) -> None:
         if self._proc is not None:
             self._proc.stop()
             self._proc = None
+        if self._gt_coordinator is not None:
+            self._gt_coordinator.stop()
+            self._gt_coordinator = None
+        self._gt_db = None
 
     def check_env(self, case: InteractiveEval) -> None:
         if self.config.attach or not case.simulator:
@@ -396,18 +450,28 @@ def instruct(self, text: str) -> None:
             transport.stop()
 
     def sample(
-        self, score: Callable[[Store], float], interval_s: float, timeout_s: float
+        self, score: Score | GTScore, interval_s: float, timeout_s: float
     ) -> list[tuple[float, float]]:
         """Score the live Recorder store on an interval — the mem2 analogue of
-        lcm_spy.wait_until_odom_position, but it returns a graded series."""
+        lcm_spy.wait_until_odom_position, but it returns a graded series.
+
+        Two-arg scores additionally receive the GT store (the case declared
+        ``ground_truth=True``, so ``_start_gt_recorder`` ran in setup_env).
+        """
         deadline = time.monotonic() + timeout_s
         t0 = time.monotonic()
         series: list[tuple[float, float]] = []
         store = self._wait_live_store(deadline)
+        gt: Store | None = None
         try:
+            if uses_gt_store(score):
+                gt = self._wait_gt_store(deadline)
             while time.monotonic() < deadline:
                 try:
-                    value = score(store)
+                    if gt is not None:
+                        value = cast("GTScore", score)(store, gt)
+                    else:
+                        value = cast("Score", score)(store)
                 except LookupError:
                     value = None  # stream not written yet — keep waiting
                 if value is not None:
@@ -417,6 +481,8 @@ def sample(
                 time.sleep(interval_s)
         finally:
             store.stop()
+            if gt is not None:
+                gt.stop()
         return series
 
     def _wait_live_store(self, deadline: float) -> Store:
@@ -425,6 +491,13 @@ def _wait_live_store(self, deadline: float) -> Store:
             time.sleep(1.0)
         return self.live_store()
 
+    def _wait_gt_store(self, deadline: float) -> Store:
+        """GT db appears once the GT recorder module starts — poll like the live db."""
+        assert self._gt_db is not None, "two-arg score but no GT recorder — ground_truth=True?"
+        while not self._gt_db.exists() and time.monotonic() < deadline:
+            time.sleep(1.0)
+        return self.gt_store()
+
 
 def _git_sha() -> str:
     try: