feat(microduck): simulation cockpit and hosted-world extension points

MUJOCO_LOG.TXT

@@ -0,0 +1,14 @@
+Fri Sep  4 00:23:55 2026
+WARNING: ARB_clip_control unavailable while mjDEPTH_ZEROFAR requested, depth accuracy will be limited
+
+Fri Sep  4 09:09:16 2026
+WARNING: ARB_clip_control unavailable while mjDEPTH_ZEROFAR requested, depth accuracy will be limited
+
+Fri Sep  4 11:09:46 2026
+WARNING: ARB_clip_control unavailable while mjDEPTH_ZEROFAR requested, depth accuracy will be limited
+
+Fri Sep  4 13:23:42 2026
+WARNING: ARB_clip_control unavailable while mjDEPTH_ZEROFAR requested, depth accuracy will be limited
+
+Fri Sep  4 13:23:58 2026
+WARNING: ARB_clip_control unavailable while mjDEPTH_ZEROFAR requested, depth accuracy will be limited

dimos/agents/mcp/mcp_client.py

@@ -55,6 +55,10 @@
 _RESPONSES_REASONING_MODEL_PREFIXES = ("gpt-5", "o1", "o3", "o4")
 
 
+def _uses_responses_api(model_name: str) -> bool:
+    return ":" not in model_name and model_name.startswith(_RESPONSES_REASONING_MODEL_PREFIXES)
+
+
 def _init_model(model_name: str, trace_dir: Path | None = None) -> Any:
     """Initialize a model while preserving LangChain provider resolution.
 
@@ -63,7 +67,7 @@ def _init_model(model_name: str, trace_dir: Path | None = None) -> Any:
     ``http_client``; other providers keep working, untraced at the wire.
     """
     client = None if trace_dir is None else tracing_http_client(trace_dir)
-    if ":" in model_name or not model_name.startswith(_RESPONSES_REASONING_MODEL_PREFIXES):
+    if not _uses_responses_api(model_name):
         model = init_chat_model(model=model_name)
         if client is not None and isinstance(model, ChatOpenAI):
             return init_chat_model(model=model_name, http_client=client)
@@ -205,15 +209,20 @@ def _mcp_tool_to_langchain(self, mcp_tool: dict[str, Any]) -> StructuredTool:
         description = mcp_tool.get("description", "")
         input_schema = mcp_tool.get("inputSchema", {"type": "object", "properties": {}})
 
-        def call_tool(**kwargs: Any) -> str:
+        def call_tool(**kwargs: Any) -> str | list[dict[str, Any]]:
             result = self._mcp_tool_call(name, kwargs)
-            content = result.get("content", [])
+            content: list[dict[str, Any]] = result.get("content", [])
+            # Responses accepts image/file blocks in function outputs. Keep them
+            # in this tool turn so the model can observe before answering.
+            if _uses_responses_api(self.config.model) and any(
+                item.get("type") != "text" for item in content
+            ):
+                return content
             parts = [c.get("text", "") for c in content if c.get("type") == "text"]
             text = "\n".join(parts)
 
-            # Images need to be added to the history separately because they
-            # cannot be included in the tool response for OpenAI models and
-            # probably others.
+            # Preserve the separate-message fallback for model adapters that
+            # do not support multimodal tool outputs.
             for item in content:
                 if item.get("type") != "text":
                     uuid_ = str(uuid.uuid4())

dimos/agents/mcp/mcp_server.py

@@ -381,15 +381,16 @@ def stop(self) -> None:
     def on_system_modules(self, modules: list[RPCClient]) -> None:
         # TODO: this is a bit hacky, also not thread-safe
         assert self.rpc is not None
-        app.state.skills = [
-            skill_info for module in modules for skill_info in (module.get_skills() or [])
+        bound_skills = [
+            (module.remote_name, skill_info)
+            for module in modules
+            for skill_info in (module.get_skills() or [])
         ]
+        app.state.skills = [skill_info for _, skill_info in bound_skills]
         app.state.skills_by_name = {s.func_name: s for s in app.state.skills}
         app.state.rpc_calls = {
-            skill_info.func_name: RpcCall(
-                None, self.rpc, skill_info.func_name, skill_info.class_name, []
-            )
-            for skill_info in app.state.skills
+            skill_info.func_name: RpcCall(None, self.rpc, skill_info.func_name, instance_name, [])
+            for instance_name, skill_info in bound_skills
         }
 
     @skill

dimos/agents/mcp/test_mcp_client_unit.py

@@ -18,7 +18,7 @@
 from threading import RLock
 from unittest.mock import MagicMock, create_autospec, patch
 
-from langchain_core.messages import HumanMessage
+from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
 from langchain_core.messages.base import BaseMessage
 from langchain_openai import ChatOpenAI
 import pytest
@@ -265,3 +265,53 @@ def test_on_system_modules_resolves_non_reasoning_models(
         configured_mcp_client.on_system_modules([])
 
     init.assert_called_once_with(model=model_name)
+
+
+@pytest.mark.parametrize("include_text", [False, True])
+def test_observation_is_in_tool_response_before_next_model_turn(
+    mcp_client: McpClient, monkeypatch: pytest.MonkeyPatch, include_text: bool
+) -> None:
+    """The Responses adapter receives the camera in the tool output, not a later user turn."""
+    image = {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,/9j/2Q=="}}
+    content = ([{"type": "text", "text": "Current camera"}] if include_text else []) + [image]
+    monkeypatch.setattr(mcp_client, "_mcp_tool_call", lambda name, args: {"content": content})
+    tool = mcp_client._mcp_tool_to_langchain(
+        {"name": "observe", "description": "Get the current camera."}
+    )
+    call = {"name": "observe", "args": {}, "id": "camera-call", "type": "tool_call"}
+
+    result = tool.invoke(call)
+
+    assert isinstance(result, ToolMessage)
+    assert result.content == content
+    with pytest.raises(Empty):
+        mcp_client._message_queue.get_nowait()
+    model = ChatOpenAI(model="gpt-5.6-luna", use_responses_api=True, api_key="test-key")
+    payload = model._get_request_payload(
+        [HumanMessage("Observe."), AIMessage(content="", tool_calls=[call]), result]
+    )
+    expected = ([{"type": "input_text", "text": "Current camera"}] if include_text else []) + [
+        {"type": "input_image", "image_url": "data:image/jpeg;base64,/9j/2Q=="}
+    ]
+    assert payload["input"][-1] == {
+        "type": "function_call_output",
+        "call_id": "camera-call",
+        "output": expected,
+    }
+
+
+def test_legacy_model_keeps_separate_image_fallback(
+    mcp_client: McpClient, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    monkeypatch.setattr(mcp_client.config, "model", "gpt-4.1")
+    image = {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,/9j/2Q=="}}
+    monkeypatch.setattr(mcp_client, "_mcp_tool_call", lambda name, args: {"content": [image]})
+    tool = mcp_client._mcp_tool_to_langchain(
+        {"name": "observe", "description": "Get the current camera."}
+    )
+
+    result = tool.invoke({})
+
+    assert "Tool call started with UUID:" in result
+    message = mcp_client._message_queue.get_nowait()
+    assert message.content[1] == image

dimos/agents/mcp/tool_stream.py

@@ -39,6 +39,7 @@
 import uuid
 
 from dimos.agents.annotation import current_skill_context
+from dimos.core.global_config import global_config
 from dimos.core.transport import PubSubTransport
 from dimos.core.transport_factory import make_transport
 from dimos.utils.logging_config import setup_logger
@@ -97,7 +98,7 @@ def make_stopped_notification(tool_name: str, token: str | None = None) -> dict[
 
 def subscribe(callback: ToolStreamCallback) -> Callable[[], None]:
     """Subscribe to the tool-stream topic and return a cleanup callable."""
-    transport: PubSubTransport[dict[str, Any]] = make_transport(TOOL_STREAM_TOPIC)
+    transport: PubSubTransport[dict[str, Any]] = make_transport(global_config.tool_stream_topic)
     transport.start()
     unsubscribe = transport.subscribe(callback)
 
@@ -138,6 +139,7 @@ class ToolStream:
 
     def __init__(self, tool_name: str) -> None:
         self.tool_name: str = tool_name
+        self._topic = global_config.tool_stream_topic
         self.id: str = str(uuid.uuid4())
         self._closed: threading.Event = threading.Event()
         self._lock = threading.Lock()
@@ -179,7 +181,7 @@ def send(self, message: str) -> None:
                 logger.warning("send on closed ToolStream", stream_id=self.id)
                 return
             if self._transport is None:
-                self._transport = make_transport(TOOL_STREAM_TOPIC)
+                self._transport = make_transport(self._topic)
                 self._transport.start()
             self._progress += 1
             progress = self._progress
@@ -205,7 +207,7 @@ def stop(self) -> None:
         # If no `send()` ever happened we spin up a transport here so the
         # lifecycle signal isn't lost.
         if transport is None:
-            transport = make_transport(TOOL_STREAM_TOPIC)
+            transport = make_transport(self._topic)
             transport.start()
         try:
             transport.publish(make_stopped_notification(self.tool_name, self._acquire_token))

dimos/core/global_config.py

@@ -106,6 +106,8 @@ class GlobalConfig(BaseSettings):
     robot_rotation_diameter: float = 0.6
     nerf_speed: float = 1.0
     mcp_port: int = 9990
+    # Scope background skill updates when several robot runtimes share a transport.
+    tool_stream_topic: str = "/tool_streams"
     # Seconds an MCP client waits for a tool to answer. A skill that thinks
     # for longer than this is cut off at the client, not the server, so the
     # caller owns the number.

dimos/navigation/replanning_a_star/global_planner.py

@@ -74,7 +74,12 @@ class GlobalPlanner(Resource):
     _max_path_deviation: float = 0.9
     _replanning_enabled: bool = True
 
-    def __init__(self, global_config: GlobalConfig) -> None:
+    def __init__(
+        self,
+        global_config: GlobalConfig,
+        stuck_time_window: float | None = None,
+        stuck_threshold: float | None = None,
+    ) -> None:
         self.path = Subject()
         self.goal_reached = Subject()
 
@@ -85,9 +90,17 @@ def __init__(self, global_config: GlobalConfig) -> None:
             self._global_config, self._navigation_map, self._goal_tolerance
         )
 
-        stuck_threshold = self._stuck_threshold
-        if global_config.simulation:
-            stuck_threshold = 1.0
+        # Stuck detection: the robot is stuck when every position over the
+        # last `stuck_time_window` seconds stays within `stuck_threshold` of
+        # their centroid. The defaults suit a ~0.5 m/s robot; slow platforms
+        # pass their own values so normal progress doesn't read as stuck.
+        if stuck_time_window is not None:
+            self._stuck_time_window = stuck_time_window
+        if stuck_threshold is None:
+            stuck_threshold = self._stuck_threshold
+            if global_config.simulation:
+                stuck_threshold = 1.0
+        self._stuck_threshold = stuck_threshold
 
         self._position_tracker = PositionTracker(self._stuck_time_window, stuck_threshold)
         self._replan_limiter = ReplanLimiter()

dimos/navigation/replanning_a_star/module.py

@@ -37,6 +37,10 @@
 class ReplanningAStarPlannerConfig(ModuleConfig):
     robot_width: float | None = None
     robot_rotation_diameter: float | None = None
+    # Stuck detector overrides (seconds, metres); None keeps the planner's
+    # defaults, which assume a robot that covers well over 0.4 m in 8 s.
+    stuck_time_window: float | None = None
+    stuck_threshold: float | None = None
 
 
 class ReplanningAStarPlanner(Module, NavigationInterface):
@@ -71,7 +75,11 @@ def __init__(self, **kwargs: Any) -> None:
         effective_global_config = (
             self.config.g.model_copy(update=overrides) if overrides else self.config.g
         )
-        self._planner = GlobalPlanner(effective_global_config)
+        self._planner = GlobalPlanner(
+            effective_global_config,
+            stuck_time_window=self.config.stuck_time_window,
+            stuck_threshold=self.config.stuck_threshold,
+        )
 
     @rpc
     def start(self) -> None:

dimos/navigation/replanning_a_star/test_global_planner.py

@@ -44,3 +44,47 @@ def test_find_wide_path_with_start_inside_inflation() -> None:
 
     assert path is not None
     assert len(path.poses) > 0
+
+
+def test_stuck_detector_defaults_and_simulation_override() -> None:
+    planner = GlobalPlanner(GlobalConfig())
+    assert planner._position_tracker._time_window == 8.0
+    assert planner._position_tracker._threshold == 0.4
+
+    sim = GlobalPlanner(GlobalConfig(simulation="mujoco"))
+    assert sim._position_tracker._threshold == 1.0
+
+
+def test_stuck_detector_accepts_explicit_window_and_threshold() -> None:
+    """Slow robots (the microduck walks ~0.06 m/s) pass their own values so
+    normal progress does not trip the detector; explicit values also beat
+    the simulation override."""
+    planner = GlobalPlanner(
+        GlobalConfig(simulation="mujoco"), stuck_time_window=10.0, stuck_threshold=0.15
+    )
+    assert planner._stuck_time_window == 10.0
+    assert planner._position_tracker._time_window == 10.0
+    assert planner._position_tracker._threshold == 0.15
+
+    # Either value may be given on its own.
+    window_only = GlobalPlanner(GlobalConfig(), stuck_time_window=12.0)
+    assert window_only._position_tracker._time_window == 12.0
+    assert window_only._position_tracker._threshold == 0.4
+
+
+def test_module_config_threads_stuck_detector_into_global_planner() -> None:
+    from dimos.navigation.replanning_a_star.module import ReplanningAStarPlanner
+
+    module = ReplanningAStarPlanner(stuck_time_window=10.0, stuck_threshold=0.15)
+    try:
+        assert module._planner._position_tracker._time_window == 10.0
+        assert module._planner._position_tracker._threshold == 0.15
+    finally:
+        module._close_module()
+
+    default = ReplanningAStarPlanner()
+    try:
+        assert default._planner._position_tracker._time_window == 8.0
+        assert default._planner._position_tracker._threshold == 0.4
+    finally:
+        default._close_module()

dimos/robot/all_blueprints.py

@@ -81,6 +81,11 @@
     "keyboard-teleop-xarm7": "dimos.robot.manipulators.xarm.blueprints.teleop:keyboard_teleop_xarm7",
     "learning-collect-quest-piper": "dimos.imitation.collection.blueprint:learning_collect_quest_piper",
     "learning-collect-quest-xarm7": "dimos.imitation.collection.blueprint:learning_collect_quest_xarm7",
+    "microduck-agentic-sim": "dimos.robot.pollen.microduck.blueprints.microduck_agentic_sim:microduck_agentic_sim",
+    "microduck-agentic-sim-ollama": "dimos.robot.pollen.microduck.blueprints.microduck_agentic_sim:microduck_agentic_sim_ollama",
+    "microduck-cockpit-sim": "dimos.robot.pollen.microduck.blueprints.microduck_cockpit_sim:microduck_cockpit_sim",
+    "microduck-cockpit-sim-ollama": "dimos.robot.pollen.microduck.blueprints.microduck_cockpit_sim:microduck_cockpit_sim_ollama",
+    "microduck-sim": "dimos.robot.pollen.microduck.blueprints.microduck_sim:microduck_sim",
     "mid360": "dimos.hardware.sensors.lidar.livox.livox_blueprints:mid360",
     "mid360-fastlio": "dimos.hardware.sensors.lidar.fastlio2.fastlio_blueprints:mid360_fastlio",
     "mid360-fastlio-ray-trace": "dimos.hardware.sensors.lidar.fastlio2.fastlio_blueprints:mid360_fastlio_ray_trace",
@@ -210,6 +215,7 @@
     "drone-connection-module": "dimos.robot.drone.connection_module.DroneConnectionModule",
     "drone-tracking-module": "dimos.robot.drone.drone_tracking_module.DroneTrackingModule",
     "dual-open-yam-coordinator": "dimos.robot.manipulators.dual_openyam.blueprints.basic.DualOpenYamCoordinator",
+    "duck-control-module": "dimos.robot.pollen.microduck.control_module.DuckControlModule",
     "emitter-module": "dimos.utils.demo_image_encoding.EmitterModule",
     "episode-monitor-module": "dimos.imitation.collection.episode_monitor.EpisodeMonitorModule",
     "eval-module": "dimos.evals.module.EvalModule",
@@ -261,6 +267,8 @@
     "mcp-client": "dimos.agents.mcp.mcp_client.McpClient",
     "mcp-server": "dimos.agents.mcp.mcp_server.McpServer",
     "memory-module": "dimos.memory.module.MemoryModule",
+    "microduck-sim-module": "dimos.robot.pollen.microduck.sim_module.MicroduckSimModule",
+    "microduck-skill-container": "dimos.robot.pollen.microduck.skills.MicroduckSkillContainer",
     "mid360-pcap-recorder": "dimos.hardware.sensors.lidar.virtual_mid360.recorder.Mid360PcapRecorder",
     "mid360-realsense-recorder": "dimos.robot.assembly.mid360_realsense_30.Mid360RealsenseRecorder",
     "mid360-realsense-static-tf": "dimos.robot.assembly.mid360_realsense_30.Mid360RealsenseStaticTf",