diff --git a/MUJOCO_LOG.TXT b/MUJOCO_LOG.TXT new file mode 100644 index 0000000000..b8e04a2a2e --- /dev/null +++ b/MUJOCO_LOG.TXT @@ -0,0 +1,14 @@ +Fri Sep 4 00:23:55 2026 +WARNING: ARB_clip_control unavailable while mjDEPTH_ZEROFAR requested, depth accuracy will be limited + +Fri Sep 4 09:09:16 2026 +WARNING: ARB_clip_control unavailable while mjDEPTH_ZEROFAR requested, depth accuracy will be limited + +Fri Sep 4 11:09:46 2026 +WARNING: ARB_clip_control unavailable while mjDEPTH_ZEROFAR requested, depth accuracy will be limited + +Fri Sep 4 13:23:42 2026 +WARNING: ARB_clip_control unavailable while mjDEPTH_ZEROFAR requested, depth accuracy will be limited + +Fri Sep 4 13:23:58 2026 +WARNING: ARB_clip_control unavailable while mjDEPTH_ZEROFAR requested, depth accuracy will be limited diff --git a/dimos/agents/mcp/mcp_client.py b/dimos/agents/mcp/mcp_client.py index 0a7d3ebb2b..cbfe4579fb 100644 --- a/dimos/agents/mcp/mcp_client.py +++ b/dimos/agents/mcp/mcp_client.py @@ -55,6 +55,10 @@ _RESPONSES_REASONING_MODEL_PREFIXES = ("gpt-5", "o1", "o3", "o4") +def _uses_responses_api(model_name: str) -> bool: + return ":" not in model_name and model_name.startswith(_RESPONSES_REASONING_MODEL_PREFIXES) + + def _init_model(model_name: str, trace_dir: Path | None = None) -> Any: """Initialize a model while preserving LangChain provider resolution. @@ -63,7 +67,7 @@ def _init_model(model_name: str, trace_dir: Path | None = None) -> Any: ``http_client``; other providers keep working, untraced at the wire. """ client = None if trace_dir is None else tracing_http_client(trace_dir) - if ":" in model_name or not model_name.startswith(_RESPONSES_REASONING_MODEL_PREFIXES): + if not _uses_responses_api(model_name): model = init_chat_model(model=model_name) if client is not None and isinstance(model, ChatOpenAI): return init_chat_model(model=model_name, http_client=client) @@ -205,15 +209,20 @@ def _mcp_tool_to_langchain(self, mcp_tool: dict[str, Any]) -> StructuredTool: description = mcp_tool.get("description", "") input_schema = mcp_tool.get("inputSchema", {"type": "object", "properties": {}}) - def call_tool(**kwargs: Any) -> str: + def call_tool(**kwargs: Any) -> str | list[dict[str, Any]]: result = self._mcp_tool_call(name, kwargs) - content = result.get("content", []) + content: list[dict[str, Any]] = result.get("content", []) + # Responses accepts image/file blocks in function outputs. Keep them + # in this tool turn so the model can observe before answering. + if _uses_responses_api(self.config.model) and any( + item.get("type") != "text" for item in content + ): + return content parts = [c.get("text", "") for c in content if c.get("type") == "text"] text = "\n".join(parts) - # Images need to be added to the history separately because they - # cannot be included in the tool response for OpenAI models and - # probably others. + # Preserve the separate-message fallback for model adapters that + # do not support multimodal tool outputs. for item in content: if item.get("type") != "text": uuid_ = str(uuid.uuid4()) diff --git a/dimos/agents/mcp/mcp_server.py b/dimos/agents/mcp/mcp_server.py index 61d7572af8..4bbbd5c138 100644 --- a/dimos/agents/mcp/mcp_server.py +++ b/dimos/agents/mcp/mcp_server.py @@ -381,15 +381,16 @@ def stop(self) -> None: def on_system_modules(self, modules: list[RPCClient]) -> None: # TODO: this is a bit hacky, also not thread-safe assert self.rpc is not None - app.state.skills = [ - skill_info for module in modules for skill_info in (module.get_skills() or []) + bound_skills = [ + (module.remote_name, skill_info) + for module in modules + for skill_info in (module.get_skills() or []) ] + app.state.skills = [skill_info for _, skill_info in bound_skills] app.state.skills_by_name = {s.func_name: s for s in app.state.skills} app.state.rpc_calls = { - skill_info.func_name: RpcCall( - None, self.rpc, skill_info.func_name, skill_info.class_name, [] - ) - for skill_info in app.state.skills + skill_info.func_name: RpcCall(None, self.rpc, skill_info.func_name, instance_name, []) + for instance_name, skill_info in bound_skills } @skill diff --git a/dimos/agents/mcp/test_mcp_client_unit.py b/dimos/agents/mcp/test_mcp_client_unit.py index a49df130ff..857aded192 100644 --- a/dimos/agents/mcp/test_mcp_client_unit.py +++ b/dimos/agents/mcp/test_mcp_client_unit.py @@ -18,7 +18,7 @@ from threading import RLock from unittest.mock import MagicMock, create_autospec, patch -from langchain_core.messages import HumanMessage +from langchain_core.messages import AIMessage, HumanMessage, ToolMessage from langchain_core.messages.base import BaseMessage from langchain_openai import ChatOpenAI import pytest @@ -265,3 +265,53 @@ def test_on_system_modules_resolves_non_reasoning_models( configured_mcp_client.on_system_modules([]) init.assert_called_once_with(model=model_name) + + +@pytest.mark.parametrize("include_text", [False, True]) +def test_observation_is_in_tool_response_before_next_model_turn( + mcp_client: McpClient, monkeypatch: pytest.MonkeyPatch, include_text: bool +) -> None: + """The Responses adapter receives the camera in the tool output, not a later user turn.""" + image = {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,/9j/2Q=="}} + content = ([{"type": "text", "text": "Current camera"}] if include_text else []) + [image] + monkeypatch.setattr(mcp_client, "_mcp_tool_call", lambda name, args: {"content": content}) + tool = mcp_client._mcp_tool_to_langchain( + {"name": "observe", "description": "Get the current camera."} + ) + call = {"name": "observe", "args": {}, "id": "camera-call", "type": "tool_call"} + + result = tool.invoke(call) + + assert isinstance(result, ToolMessage) + assert result.content == content + with pytest.raises(Empty): + mcp_client._message_queue.get_nowait() + model = ChatOpenAI(model="gpt-5.6-luna", use_responses_api=True, api_key="test-key") + payload = model._get_request_payload( + [HumanMessage("Observe."), AIMessage(content="", tool_calls=[call]), result] + ) + expected = ([{"type": "input_text", "text": "Current camera"}] if include_text else []) + [ + {"type": "input_image", "image_url": "data:image/jpeg;base64,/9j/2Q=="} + ] + assert payload["input"][-1] == { + "type": "function_call_output", + "call_id": "camera-call", + "output": expected, + } + + +def test_legacy_model_keeps_separate_image_fallback( + mcp_client: McpClient, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(mcp_client.config, "model", "gpt-4.1") + image = {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,/9j/2Q=="}} + monkeypatch.setattr(mcp_client, "_mcp_tool_call", lambda name, args: {"content": [image]}) + tool = mcp_client._mcp_tool_to_langchain( + {"name": "observe", "description": "Get the current camera."} + ) + + result = tool.invoke({}) + + assert "Tool call started with UUID:" in result + message = mcp_client._message_queue.get_nowait() + assert message.content[1] == image diff --git a/dimos/agents/mcp/tool_stream.py b/dimos/agents/mcp/tool_stream.py index 7a8d0882ed..a2344c762a 100644 --- a/dimos/agents/mcp/tool_stream.py +++ b/dimos/agents/mcp/tool_stream.py @@ -39,6 +39,7 @@ import uuid from dimos.agents.annotation import current_skill_context +from dimos.core.global_config import global_config from dimos.core.transport import PubSubTransport from dimos.core.transport_factory import make_transport from dimos.utils.logging_config import setup_logger @@ -97,7 +98,7 @@ def make_stopped_notification(tool_name: str, token: str | None = None) -> dict[ def subscribe(callback: ToolStreamCallback) -> Callable[[], None]: """Subscribe to the tool-stream topic and return a cleanup callable.""" - transport: PubSubTransport[dict[str, Any]] = make_transport(TOOL_STREAM_TOPIC) + transport: PubSubTransport[dict[str, Any]] = make_transport(global_config.tool_stream_topic) transport.start() unsubscribe = transport.subscribe(callback) @@ -138,6 +139,7 @@ class ToolStream: def __init__(self, tool_name: str) -> None: self.tool_name: str = tool_name + self._topic = global_config.tool_stream_topic self.id: str = str(uuid.uuid4()) self._closed: threading.Event = threading.Event() self._lock = threading.Lock() @@ -179,7 +181,7 @@ def send(self, message: str) -> None: logger.warning("send on closed ToolStream", stream_id=self.id) return if self._transport is None: - self._transport = make_transport(TOOL_STREAM_TOPIC) + self._transport = make_transport(self._topic) self._transport.start() self._progress += 1 progress = self._progress @@ -205,7 +207,7 @@ def stop(self) -> None: # If no `send()` ever happened we spin up a transport here so the # lifecycle signal isn't lost. if transport is None: - transport = make_transport(TOOL_STREAM_TOPIC) + transport = make_transport(self._topic) transport.start() try: transport.publish(make_stopped_notification(self.tool_name, self._acquire_token)) diff --git a/dimos/core/global_config.py b/dimos/core/global_config.py index ff7b39edf2..fd8351a699 100644 --- a/dimos/core/global_config.py +++ b/dimos/core/global_config.py @@ -106,6 +106,8 @@ class GlobalConfig(BaseSettings): robot_rotation_diameter: float = 0.6 nerf_speed: float = 1.0 mcp_port: int = 9990 + # Scope background skill updates when several robot runtimes share a transport. + tool_stream_topic: str = "/tool_streams" # Seconds an MCP client waits for a tool to answer. A skill that thinks # for longer than this is cut off at the client, not the server, so the # caller owns the number. diff --git a/dimos/navigation/replanning_a_star/global_planner.py b/dimos/navigation/replanning_a_star/global_planner.py index 7658589e30..e85c39617d 100644 --- a/dimos/navigation/replanning_a_star/global_planner.py +++ b/dimos/navigation/replanning_a_star/global_planner.py @@ -74,7 +74,12 @@ class GlobalPlanner(Resource): _max_path_deviation: float = 0.9 _replanning_enabled: bool = True - def __init__(self, global_config: GlobalConfig) -> None: + def __init__( + self, + global_config: GlobalConfig, + stuck_time_window: float | None = None, + stuck_threshold: float | None = None, + ) -> None: self.path = Subject() self.goal_reached = Subject() @@ -85,9 +90,17 @@ def __init__(self, global_config: GlobalConfig) -> None: self._global_config, self._navigation_map, self._goal_tolerance ) - stuck_threshold = self._stuck_threshold - if global_config.simulation: - stuck_threshold = 1.0 + # Stuck detection: the robot is stuck when every position over the + # last `stuck_time_window` seconds stays within `stuck_threshold` of + # their centroid. The defaults suit a ~0.5 m/s robot; slow platforms + # pass their own values so normal progress doesn't read as stuck. + if stuck_time_window is not None: + self._stuck_time_window = stuck_time_window + if stuck_threshold is None: + stuck_threshold = self._stuck_threshold + if global_config.simulation: + stuck_threshold = 1.0 + self._stuck_threshold = stuck_threshold self._position_tracker = PositionTracker(self._stuck_time_window, stuck_threshold) self._replan_limiter = ReplanLimiter() diff --git a/dimos/navigation/replanning_a_star/module.py b/dimos/navigation/replanning_a_star/module.py index 9ea8607bb2..3796a45c93 100644 --- a/dimos/navigation/replanning_a_star/module.py +++ b/dimos/navigation/replanning_a_star/module.py @@ -37,6 +37,10 @@ class ReplanningAStarPlannerConfig(ModuleConfig): robot_width: float | None = None robot_rotation_diameter: float | None = None + # Stuck detector overrides (seconds, metres); None keeps the planner's + # defaults, which assume a robot that covers well over 0.4 m in 8 s. + stuck_time_window: float | None = None + stuck_threshold: float | None = None class ReplanningAStarPlanner(Module, NavigationInterface): @@ -71,7 +75,11 @@ def __init__(self, **kwargs: Any) -> None: effective_global_config = ( self.config.g.model_copy(update=overrides) if overrides else self.config.g ) - self._planner = GlobalPlanner(effective_global_config) + self._planner = GlobalPlanner( + effective_global_config, + stuck_time_window=self.config.stuck_time_window, + stuck_threshold=self.config.stuck_threshold, + ) @rpc def start(self) -> None: diff --git a/dimos/navigation/replanning_a_star/test_global_planner.py b/dimos/navigation/replanning_a_star/test_global_planner.py index 9916874a7b..2b0fd79eb5 100644 --- a/dimos/navigation/replanning_a_star/test_global_planner.py +++ b/dimos/navigation/replanning_a_star/test_global_planner.py @@ -44,3 +44,47 @@ def test_find_wide_path_with_start_inside_inflation() -> None: assert path is not None assert len(path.poses) > 0 + + +def test_stuck_detector_defaults_and_simulation_override() -> None: + planner = GlobalPlanner(GlobalConfig()) + assert planner._position_tracker._time_window == 8.0 + assert planner._position_tracker._threshold == 0.4 + + sim = GlobalPlanner(GlobalConfig(simulation="mujoco")) + assert sim._position_tracker._threshold == 1.0 + + +def test_stuck_detector_accepts_explicit_window_and_threshold() -> None: + """Slow robots (the microduck walks ~0.06 m/s) pass their own values so + normal progress does not trip the detector; explicit values also beat + the simulation override.""" + planner = GlobalPlanner( + GlobalConfig(simulation="mujoco"), stuck_time_window=10.0, stuck_threshold=0.15 + ) + assert planner._stuck_time_window == 10.0 + assert planner._position_tracker._time_window == 10.0 + assert planner._position_tracker._threshold == 0.15 + + # Either value may be given on its own. + window_only = GlobalPlanner(GlobalConfig(), stuck_time_window=12.0) + assert window_only._position_tracker._time_window == 12.0 + assert window_only._position_tracker._threshold == 0.4 + + +def test_module_config_threads_stuck_detector_into_global_planner() -> None: + from dimos.navigation.replanning_a_star.module import ReplanningAStarPlanner + + module = ReplanningAStarPlanner(stuck_time_window=10.0, stuck_threshold=0.15) + try: + assert module._planner._position_tracker._time_window == 10.0 + assert module._planner._position_tracker._threshold == 0.15 + finally: + module._close_module() + + default = ReplanningAStarPlanner() + try: + assert default._planner._position_tracker._time_window == 8.0 + assert default._planner._position_tracker._threshold == 0.4 + finally: + default._close_module() diff --git a/dimos/robot/all_blueprints.py b/dimos/robot/all_blueprints.py index c92c90c349..37e5b66e30 100644 --- a/dimos/robot/all_blueprints.py +++ b/dimos/robot/all_blueprints.py @@ -81,6 +81,11 @@ "keyboard-teleop-xarm7": "dimos.robot.manipulators.xarm.blueprints.teleop:keyboard_teleop_xarm7", "learning-collect-quest-piper": "dimos.imitation.collection.blueprint:learning_collect_quest_piper", "learning-collect-quest-xarm7": "dimos.imitation.collection.blueprint:learning_collect_quest_xarm7", + "microduck-agentic-sim": "dimos.robot.pollen.microduck.blueprints.microduck_agentic_sim:microduck_agentic_sim", + "microduck-agentic-sim-ollama": "dimos.robot.pollen.microduck.blueprints.microduck_agentic_sim:microduck_agentic_sim_ollama", + "microduck-cockpit-sim": "dimos.robot.pollen.microduck.blueprints.microduck_cockpit_sim:microduck_cockpit_sim", + "microduck-cockpit-sim-ollama": "dimos.robot.pollen.microduck.blueprints.microduck_cockpit_sim:microduck_cockpit_sim_ollama", + "microduck-sim": "dimos.robot.pollen.microduck.blueprints.microduck_sim:microduck_sim", "mid360": "dimos.hardware.sensors.lidar.livox.livox_blueprints:mid360", "mid360-fastlio": "dimos.hardware.sensors.lidar.fastlio2.fastlio_blueprints:mid360_fastlio", "mid360-fastlio-ray-trace": "dimos.hardware.sensors.lidar.fastlio2.fastlio_blueprints:mid360_fastlio_ray_trace", @@ -210,6 +215,7 @@ "drone-connection-module": "dimos.robot.drone.connection_module.DroneConnectionModule", "drone-tracking-module": "dimos.robot.drone.drone_tracking_module.DroneTrackingModule", "dual-open-yam-coordinator": "dimos.robot.manipulators.dual_openyam.blueprints.basic.DualOpenYamCoordinator", + "duck-control-module": "dimos.robot.pollen.microduck.control_module.DuckControlModule", "emitter-module": "dimos.utils.demo_image_encoding.EmitterModule", "episode-monitor-module": "dimos.imitation.collection.episode_monitor.EpisodeMonitorModule", "eval-module": "dimos.evals.module.EvalModule", @@ -261,6 +267,8 @@ "mcp-client": "dimos.agents.mcp.mcp_client.McpClient", "mcp-server": "dimos.agents.mcp.mcp_server.McpServer", "memory-module": "dimos.memory.module.MemoryModule", + "microduck-sim-module": "dimos.robot.pollen.microduck.sim_module.MicroduckSimModule", + "microduck-skill-container": "dimos.robot.pollen.microduck.skills.MicroduckSkillContainer", "mid360-pcap-recorder": "dimos.hardware.sensors.lidar.virtual_mid360.recorder.Mid360PcapRecorder", "mid360-realsense-recorder": "dimos.robot.assembly.mid360_realsense_30.Mid360RealsenseRecorder", "mid360-realsense-static-tf": "dimos.robot.assembly.mid360_realsense_30.Mid360RealsenseStaticTf", diff --git a/dimos/robot/pollen/microduck/README.md b/dimos/robot/pollen/microduck/README.md new file mode 100644 index 0000000000..8426046ce1 --- /dev/null +++ b/dimos/robot/pollen/microduck/README.md @@ -0,0 +1,136 @@ +# Microduck simulation + +[Microduck](https://github.com/pollen-robotics/microduck) — pollen-robotics's +~25 cm, ~800 g open-source biped — walking in a small MuJoCo room with the +standard dimOS navigation stack and agent on top of its pretrained RL gait. + +``` +humancli -> /human_input -> McpClient (LLM) -> skills + begin_exploration / go_to_object / move_to / ... + -> WavefrontFrontierExplorer / ReplanningAStarPlanner + -> MovementManager -> cmd_vel + -> alpha_walking.onnx (50 Hz) -> MuJoCo (200 Hz) +``` + +## Quick start + +```bash +# Simulation + nav only (drive it from `dimos shell` / planner RPCs): +dimos --viewer none run microduck-sim + +# With the agent, using a local LLM via ollama (pulls qwen3:8b on first use): +dimos --viewer none run microduck-agentic-sim-ollama +# ...or with OPENAI_API_KEY set: +dimos --viewer none run microduck-agentic-sim + +# In a second terminal: +humancli +> explore the room for 30 seconds, then walk to the red ball +``` + +On first start the robot model/meshes and walking policy (~26 MB) are +downloaded from the public pollen-robotics GitHub repos into +`~/.cache/dimos/microduck` (see `assets_fetch.py`; `DIMOS_MICRODUCK_ASSETS` +overrides the location, and the fetch is pinned to upstream commits). + +If module startup hangs with `RPC call ... timed out` on macOS with +Tailscale (or other VPNs that own the multicast route), zenoh's +loopback-only peer discovery is broken on your machine; run everything with +`--zenoh-scouting` (and `ZENOH_SCOUTING=true humancli`), or put +`zenoh_scouting=true` in your `.env`. + +## Web cockpit (`microduck-cockpit-sim`) + +A four-room flat (kitchen = "space A", living = "space B", bedroom = +"space C", office = "space D", around an open hub) driven from the browser: +WASDQE teleop, one button per RL policy, a teleop/agent switch, the agent's +humancli transcript, and click-to-goal on the costmap through the standard +`ReplanningAStarPlanner`. + +```bash +# OPENAI_API_KEY must be in the environment (the ollama variant needs no key): +dimos --viewer none run microduck-cockpit-sim --local-relay +dimos --viewer none run microduck-cockpit-sim-ollama --local-relay +# then open http://127.0.0.1:7780 (localhost only; the URL is also logged) + +MICRODUCK_VARIANT=rollers dimos --viewer none run microduck-cockpit-sim --local-relay +``` + +Stop any other microduck blueprint first - they share topic names on the +zenoh router. On macOS with Tailscale, add `--zenoh-scouting` as above. + +Panels: + +- **Control strip** - `Teleop` / `Agent` mode, one button per policy + (`walk`, `stand`, `roller*`, `sitstand`, `kick_left/right`, `roulade`, + `ground_pick`; greyed with a reason when the variant or the asset lacks + it), the nav state chip and its cancel (✕). +- **Nav map** - costmap, rooms, landmark objects, remembered places, the + planner's path and goal. Click anywhere to send a goal, click a room label + to go to that room's target, `Esc` (map focused) or ✕ to cancel. +- **Teleop (WASDQE)** - W/S forward-back, A/D strafe, Q/E yaw; only acts in + teleop mode, and only while the tab is in the foreground (browsers + throttle background tabs; the deadman then zeroes the twist). +- **Chat** - the agent transcript (tool calls and results included) with an + input box, live in agent mode. Try `go to space A`, `go to the kitchen`, + `walk to the blue box`, `remember this place as fridge`, `what do you + see?`, `do a roulade`. +- **Chase cam / Head cam** - decoded only while the tab is visible. + +In teleop mode the planner's `nav_cmd_vel` is muted (a click still plans +and draws the path); switching to agent mode hands `cmd_vel` to the planner +and enables the chat. Navigation is locked (and any active goal cancelled) +while a one-shot policy runs, while the duck is seated, fallen or standing +back up; the chip shows the reason. + +## What's in the box + +- `sim_module.py` — `MicroduckSimModule(MujocoSimModule)`: composes the room + scene + robot MJCF (adding three trunk-mounted raycast-lidar cameras), + runs the ONNX walking policy in the engine's step hook at 50 Hz, and maps + `cmd_vel` twists into the policy's command space. No ControlCoordinator: + the whole robot is one module. Odom/tf/IMU/pointcloud publishing is + inherited from `MujocoSimModule`. +- `gait.py` — the 61-dim observation contract of the alpha policies + (documented in microduck's `duck-control/src/obs.rs`), with joint order, + home pose and action scale read from the ONNX metadata. +- `skills.py` — `MicroduckSkillContainer`: `go_to_object` / `list_objects` / + `move_to` / `where_am_i` / `stop_moving` / `wait`. Object positions are the + scene's ground truth (configured in the blueprint), not perception — this + is the deliberately-basic demo. +- `assets/room_scene.xml` — a 4 x 3 m walled room with four colored objects. + World geometry is geom group 0; the lidar only raycasts group 0, so the + robot (groups 2/3) never sees itself. +- `web_codecs.py` — the cockpit's own `@web_encoder`s (the transcript, the + planner path, the JSON-string state streams). The blueprint declares the + matching streams as `Channel(...)` and `cockpit()` generates a relay-bridge + subclass with a typed port for each, so none of this robot's vocabulary - + or its langchain dependency - lands in `dimos.web`. +- `blueprints/` — `microduck-sim` (sim + nav + explorer), + `microduck-agentic-sim[-ollama]` (adds McpServer/McpClient + skills) and + `microduck-cockpit-sim[-ollama]` (the web cockpit above). + +## Quirks worth knowing + +- **Command shaping** (`sim_module.py`): the policy tracks its velocity + command with a ~2.5x undershoot, so requested twists are multiplied up + (`cmd_gain_linear/angular`), and it has a yaw deadband — pure-turn + commands below the top of its range barely rotate it (~3-9 deg/s at + 1.0 rad/s vs 25-31 deg/s at 1.5) — so turn commands are bumped to + `min_effective_wz` (1.5, the range maximum). +- **Falls**: only the plain walking policy is published (no fall recovery), + and the walk-optimized model has no trunk collision geoms. When the trunk + stays tilted > ~55 degrees for 2 s the module stands the duck back up, + nudged toward the room origin so it doesn't re-spawn wedged inside + whatever it tripped over. +- **Rendered cameras are Linux-only**: `mujoco.Renderer` needs a GL context + that macOS only allows on the main thread; creating one from the engine's + sim thread deadlocks the worker. The raycast lidar is pure `mj_ray` and + works everywhere; the blueprint enables `color_image` only off-macOS. +- The planner floors commands at 0.2 m/s; the duck actually walks ~0.1 m/s, + so room crossings take a minute or two. That's the robot, not a bug. + +## Licensing + +Code in the upstream microduck repos is Apache-2.0; the 3D model files are +CC BY-NC-SA — they are downloaded to a local cache, not redistributed here. diff --git a/dimos/robot/pollen/microduck/assets/four_room_scene.xml b/dimos/robot/pollen/microduck/assets/four_room_scene.xml new file mode 100644 index 0000000000..584edec166 --- /dev/null +++ b/dimos/robot/pollen/microduck/assets/four_room_scene.xml @@ -0,0 +1,97 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dimos/robot/pollen/microduck/assets/room_scene.xml b/dimos/robot/pollen/microduck/assets/room_scene.xml new file mode 100644 index 0000000000..cdd5461c3d --- /dev/null +++ b/dimos/robot/pollen/microduck/assets/room_scene.xml @@ -0,0 +1,42 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dimos/robot/pollen/microduck/assets_fetch.py b/dimos/robot/pollen/microduck/assets_fetch.py new file mode 100644 index 0000000000..336360bfdf --- /dev/null +++ b/dimos/robot/pollen/microduck/assets_fetch.py @@ -0,0 +1,292 @@ +# Copyright 2025-2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Download-on-first-use assets for the Microduck simulation. + +The Microduck MJCF models and meshes (~25 MB of STLs) live in +`pollen-robotics/microduck_rl`, and the pretrained policies ship in +`pollen-robotics/microduck`. Neither belongs in this git repo, so this module +pulls both from GitHub at pinned commits into a local cache the first time a +Microduck simulation starts. + +Cache layout (``$DIMOS_MICRODUCK_ASSETS`` overrides the default root +``~/.cache/dimos/microduck``):: + + /robot/robot_walk.xml # MJCF + joints_properties.xml + assets/*.stl + /robot/robot_allcollisions.xml # "default" variant (ROBOT_MJCF_BY_VARIANT) + /robot/robot_allcollisions_rollers.xml # "rollers" variant + /policies/.onnx # one per policy (POLICY_FILES) + /.complete- # informational marker + +Fetching is per file and additive: whatever is already cached is neither +re-downloaded nor deleted, only missing files are fetched, and each policy +download is preceded by a HEAD request confirming the pinned source serves +that file. A policy that cannot be obtained is reported in +``MicroduckAssets.missing`` (the cockpit greys it out) instead of failing the +start; only the walking policy and the robot model are mandatory. Callers +that only run the walking policy can pass ``ensure_assets(policies=("walk",))`` +to skip the probes for the other eight. +""" + +from __future__ import annotations + +from collections.abc import Iterable +from dataclasses import dataclass +import io +import os +from pathlib import Path +import shutil +import tarfile +import tempfile +import urllib.error +import urllib.request + +from dimos.robot.pollen.microduck.policies import DEFAULT_VARIANT, POLICY_SPECS, PolicyName +from dimos.utils.logging_config import setup_logger + +logger = setup_logger() + +# Pinned upstream commits (Apache-2.0 code, CC BY-NC-SA meshes). +_MICRODUCK_RL_COMMIT = "d424a0c899f6b33cbd3daeb279913134349c0b63" +_MICRODUCK_COMMIT = "590b986bd8c0d50ae02cb3ea2f59c463b6828168" + +_RL_TARBALL = ( + f"https://codeload.github.com/pollen-robotics/microduck_rl/tar.gz/{_MICRODUCK_RL_COMMIT}" +) +_RL_ROBOT_SUBDIR = "src/mjlab_microduck/robot/microduck" +_POLICY_BASE_URL = ( + f"https://raw.githubusercontent.com/pollen-robotics/microduck/{_MICRODUCK_COMMIT}/policies" +) + +_CACHE_VERSION = "1" +_MIN_POLICY_BYTES = 100_000 +# The HEAD probe is the only network call a start with a complete-but-for-one +# cache makes; keep it short so an offline machine is not held up for long. +_HEAD_TIMEOUT_S = 5 +_GET_TIMEOUT_S = 120 + +ROBOT_MJCF_NAME = "robot_walk.xml" +WALKING_POLICY_NAME = "alpha_walking.onnx" + +# Robot MJCF per variant: the wheeled model is needed for the roller policies, +# the plain all-collisions model for everything else (see policies.py). +ROBOT_MJCF_BY_VARIANT: dict[str, str] = { + "default": "robot_allcollisions.xml", + "rollers": "robot_allcollisions_rollers.xml", +} + +# Policy -> ONNX file name under /policies/. +POLICY_FILES: dict[PolicyName, str] = {name: spec.onnx for name, spec in POLICY_SPECS.items()} + + +@dataclass(frozen=True) +class MicroduckAssets: + """What ``ensure_assets`` found or fetched.""" + + robot_dir: Path + policy_dir: Path + missing: tuple[PolicyName, ...] # policies whose ONNX could not be obtained + + def robot_mjcf(self, variant: str = DEFAULT_VARIANT) -> Path: + return self.robot_dir / ROBOT_MJCF_BY_VARIANT[variant] + + def policy_path(self, name: PolicyName | str) -> Path: + return self.policy_dir / POLICY_FILES[PolicyName(name)] + + +def assets_root() -> Path: + override = os.environ.get("DIMOS_MICRODUCK_ASSETS") + if override: + return Path(override).expanduser() + return Path.home() / ".cache" / "dimos" / "microduck" + + +def robot_dir() -> Path: + return assets_root() / "robot" + + +def policy_dir() -> Path: + return assets_root() / "policies" + + +def robot_mjcf_path() -> Path: + """The walk-only model the plain ``microduck-sim`` blueprint uses.""" + return robot_dir() / ROBOT_MJCF_NAME + + +def variant_mjcf_path(variant: str = DEFAULT_VARIANT) -> Path: + if variant not in ROBOT_MJCF_BY_VARIANT: + raise ValueError( + f"unknown Microduck variant {variant!r}; expected one of {tuple(ROBOT_MJCF_BY_VARIANT)}" + ) + return robot_dir() / ROBOT_MJCF_BY_VARIANT[variant] + + +def walking_policy_path() -> Path: + return policy_dir() / WALKING_POLICY_NAME + + +def policy_path(name: PolicyName | str) -> Path: + return policy_dir() / POLICY_FILES[PolicyName(name)] + + +def _marker() -> Path: + return assets_root() / f".complete-{_CACHE_VERSION}" + + +def ensure_assets( + variant: str = DEFAULT_VARIANT, + *, + policies: Iterable[PolicyName | str] | None = None, +) -> MicroduckAssets: + """Fetch whatever Microduck files are missing for ``variant``; never deletes. + + Mandatory: the robot models (one tarball holds every variant) and the + walking policy - failing to get them raises with a hint about the manual + fallback (cloning the repos and pointing ``DIMOS_MICRODUCK_ASSETS`` at a + prepared directory). Every other policy is best effort and ends up in + ``MicroduckAssets.missing`` when it cannot be downloaded (offline, or the + pinned source does not serve it). + + ``policies`` restricts which policies may be downloaded (default: all + nine); a walk-only caller passes ``policies=("walk",)`` to avoid any + network probe for the others. Policies not in the cache are always + reported in ``missing``, whether or not they were requested. + """ + variant_mjcf = variant_mjcf_path(variant) # validates the variant + root = assets_root() + robots = robot_dir() + policy_root = policy_dir() + wanted = {PolicyName(p) for p in policies} if policies is not None else set(POLICY_FILES) + wanted.add(PolicyName.WALK) + + required_mjcf = {ROBOT_MJCF_NAME, variant_mjcf.name} + if not all((robots / name).exists() for name in required_mjcf): + logger.info("Fetching Microduck robot models", root=str(root)) + root.mkdir(parents=True, exist_ok=True) + try: + _fetch_robot_dir(robots, required_mjcf) + except Exception as exc: + raise RuntimeError(_MANUAL_HINT.format(what="robot models")) from exc + + missing: list[PolicyName] = [] + unreachable = False + for name, filename in POLICY_FILES.items(): + dest = policy_root / filename + if dest.exists(): + continue + if unreachable or name not in wanted: + missing.append(name) + continue + try: + _fetch_policy(dest) + except _SourceUnreachableError as exc: + logger.warning( + "Microduck policy source unreachable; skipping remaining downloads", + error=str(exc), + ) + unreachable = True + missing.append(name) + except Exception as exc: + logger.warning("Microduck policy unavailable", policy=str(name), error=str(exc)) + missing.append(name) + + if PolicyName.WALK in missing: + raise RuntimeError(_MANUAL_HINT.format(what="walking policy")) + + if not _marker().exists(): + root.mkdir(parents=True, exist_ok=True) + _marker().touch() + missing_wanted = [str(m) for m in missing if m in wanted] + if missing_wanted: + logger.warning( + "Microduck assets ready with missing policies", root=str(root), missing=missing_wanted + ) + else: + logger.info("Microduck assets ready", root=str(root)) + return MicroduckAssets(robot_dir=robots, policy_dir=policy_root, missing=tuple(missing)) + + +_MANUAL_HINT = ( + "Failed to download the Microduck {what} from GitHub. To prepare the assets " + "manually, copy microduck_rl's src/mjlab_microduck/robot/microduck/ to " + "/robot/ and microduck's policies/*.onnx to /policies/, then set " + "DIMOS_MICRODUCK_ASSETS=." +) + + +class _SourceUnreachableError(RuntimeError): + """The download host could not be reached at all (offline / DNS / timeout).""" + + +def _fetch_robot_dir(dest: Path, required: Iterable[str] = (ROBOT_MJCF_NAME,)) -> None: + """Download the microduck_rl robot directory into ``dest`` (additive copy).""" + logger.info("Downloading microduck_rl robot models", url=_RL_TARBALL) + with urllib.request.urlopen(_RL_TARBALL, timeout=_GET_TIMEOUT_S) as resp: + payload = resp.read() + + prefix = None + with tempfile.TemporaryDirectory() as tmp: + with tarfile.open(fileobj=io.BytesIO(payload), mode="r:gz") as tar: + members = [] + for member in tar.getmembers(): + if prefix is None: + prefix = member.name.split("/", 1)[0] + rel = member.name.split("/", 1)[-1] + if rel.startswith(_RL_ROBOT_SUBDIR) and member.isfile(): + members.append(member) + tar.extractall(tmp, members=members, filter="data") + src = Path(tmp) / (prefix or "") / _RL_ROBOT_SUBDIR + for name in required: + if not (src / name).exists(): + raise RuntimeError(f"{name} not found in tarball dir {src}") + # Never wipe an existing cache: overlay the tarball on top of it. + shutil.copytree(src, dest, dirs_exist_ok=True) + + +def _source_serves(url: str) -> bool: + """HEAD ``url``: True if served, False if the source lacks it; raises when offline.""" + request = urllib.request.Request(url, method="HEAD") + try: + with urllib.request.urlopen(request, timeout=_HEAD_TIMEOUT_S) as resp: + return 200 <= int(resp.status) < 300 + except urllib.error.HTTPError as exc: + if exc.code == 404 or exc.code == 410: + return False + raise + except (urllib.error.URLError, TimeoutError, OSError) as exc: + raise _SourceUnreachableError(f"{url}: {exc}") from exc + + +def _fetch_policy(dest: Path) -> None: + """Download one policy file, first confirming the pinned source serves it.""" + if dest.exists(): + return + url = f"{_POLICY_BASE_URL}/{dest.name}" + if not _source_serves(url): + raise RuntimeError(f"pinned source does not serve {dest.name}: {url}") + logger.info("Downloading Microduck policy", url=url) + dest.parent.mkdir(parents=True, exist_ok=True) + try: + with urllib.request.urlopen(url, timeout=_GET_TIMEOUT_S) as resp: + payload = resp.read() + except urllib.error.HTTPError: + raise + except (urllib.error.URLError, TimeoutError, OSError) as exc: + raise _SourceUnreachableError(f"{url}: {exc}") from exc + if len(payload) < _MIN_POLICY_BYTES: + raise RuntimeError(f"policy download looks truncated ({len(payload)} bytes)") + partial = dest.with_name(dest.name + ".part") + partial.write_bytes(payload) + partial.replace(dest) diff --git a/dimos/robot/pollen/microduck/blueprints/microduck_agentic_sim.py b/dimos/robot/pollen/microduck/blueprints/microduck_agentic_sim.py new file mode 100644 index 0000000000..aa1d195508 --- /dev/null +++ b/dimos/robot/pollen/microduck/blueprints/microduck_agentic_sim.py @@ -0,0 +1,79 @@ +# Copyright 2025-2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Agentic Microduck room simulation - drive the duck from humancli. + +Adds the MCP agent stack on top of ``microduck-sim`` so natural-language +commands ("explore the room", "walk to the red ball") become skill calls: + + humancli -> /human_input -> McpClient (LLM) -> McpServer tools + -> begin_exploration / go_to_object / move_to / ... + +Usage: + dimos run microduck-agentic-sim # needs OPENAI_API_KEY + dimos run microduck-agentic-sim-ollama # local LLM via ollama + # in a second terminal: + humancli +""" + +from __future__ import annotations + +from dimos.agents.mcp.mcp_client import McpClient +from dimos.agents.mcp.mcp_server import McpServer +from dimos.agents.ollama_agent import ollama_installed +from dimos.core.coordination.blueprints import autoconnect +from dimos.robot.pollen.microduck.blueprints.microduck_sim import ( + MICRODUCK_ROOM_OBJECTS, + microduck_sim, +) +from dimos.robot.pollen.microduck.skills import MicroduckSkillContainer + +MICRODUCK_SYSTEM_PROMPT = """\ +You are the brain of Microduck, a tiny (25 cm tall) two-legged duck robot +walking around a small room in simulation. You walk slowly (about 0.1 m/s), +so trips across the room take a minute or two - that is normal. + +You can: +- list_objects / where_am_i to orient yourself +- begin_exploration / end_exploration to roam and map the room +- go_to_object(name) to walk right up to a known object +- move_to(x, y) for raw coordinates, stop_moving to halt, wait(seconds) + +IMPORTANT: call movement tools strictly ONE AT A TIME and wait for each +result before calling the next - never combine begin_exploration, +go_to_object, or move_to in the same step. To explore "for a while", call +begin_exploration, then wait(seconds), then end_exploration, each as its +own step. + +Keep answers short and playful - you are a duck. When asked to find or go +to something, prefer go_to_object. Report what you did once actions finish. +""" + +_skills = MicroduckSkillContainer.blueprint(objects=MICRODUCK_ROOM_OBJECTS) + +microduck_agentic_sim = autoconnect( + microduck_sim, + McpServer.blueprint(), + McpClient.blueprint(system_prompt=MICRODUCK_SYSTEM_PROMPT), + _skills, +) + +microduck_agentic_sim_ollama = autoconnect( + microduck_sim, + McpServer.blueprint(), + McpClient.blueprint(system_prompt=MICRODUCK_SYSTEM_PROMPT, model="ollama:qwen3:8b"), + _skills, +).requirements( + ollama_installed, +) diff --git a/dimos/robot/pollen/microduck/blueprints/microduck_cockpit_sim.py b/dimos/robot/pollen/microduck/blueprints/microduck_cockpit_sim.py new file mode 100644 index 0000000000..ca3641ed9f --- /dev/null +++ b/dimos/robot/pollen/microduck/blueprints/microduck_cockpit_sim.py @@ -0,0 +1,374 @@ +# Copyright 2025-2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Microduck cockpit: a four-room MuJoCo apartment driven from the browser. + +The duck walks a compact 4 x 4 m flat with four rooms (kitchen = "space A", +living = "space B", bedroom = "space C", office = "space D") around a +central hub. The web cockpit shows the chase and head cameras, the costmap +with named places, WASDQE teleop, one button per RL policy (walk, kicks, +roulade, sit/stand, ...) and - in agent mode - the chat with the LLM that +drives the duck through the places memory and the nav stack: + + browser --tele_cmd_vel/goal_request/ui_command/human_input--> relay bridge + -> DuckControlModule (teleop | agent arbitration, nav <-> policies) + -> MicroduckSimModule (policies in the loop) / ReplanningAStarPlanner + -> McpClient (LLM) -> McpServer -> MicroduckSkillContainer skills + +Usage: + dimos run microduck-cockpit-sim --local-relay # needs OPENAI_API_KEY + dimos run microduck-cockpit-sim-ollama --local-relay + # then open the cockpit URL the relay prints (http://127.0.0.1:7780) + +``MICRODUCK_VARIANT=rollers`` selects the wheeled-feet robot and its policies. +""" + +from __future__ import annotations + +import os + +from langchain_core.messages.base import BaseMessage + +from dimos.agents.mcp.mcp_client import McpClient +from dimos.agents.mcp.mcp_server import McpServer +from dimos.agents.ollama_agent import ollama_installed +from dimos.agents.skills.observe_skill import ObserveSkill +from dimos.core.coordination.blueprints import autoconnect +from dimos.mapping.costmapper import CostMapper +from dimos.mapping.pointclouds.occupancy import HeightCostConfig +from dimos.mapping.voxels.module import VoxelGridMapper +from dimos.msgs.nav_msgs.Path import Path as NavPath +from dimos.msgs.sensor_msgs.Image import Image +from dimos.navigation.replanning_a_star.module import ReplanningAStarPlanner + +# Imported for the registration side effect: the cockpit blueprint resolves +# these encoders by id when it compiles MICRODUCK_COCKPIT_CHANNELS. +from dimos.robot.pollen.microduck import web_codecs # noqa: F401 +from dimos.robot.pollen.microduck.config import MICRODUCK +from dimos.robot.pollen.microduck.control_module import DuckControlModule +from dimos.robot.pollen.microduck.places import ( + FOUR_ROOM_XML, + MICRODUCK_OBJECTS, + MICRODUCK_ROOMS, +) +from dimos.robot.pollen.microduck.sim_module import ( + LIDAR_CAMERA_SPECS, + POV_CAMERA_NAME, + MicroduckSimModule, +) +from dimos.robot.pollen.microduck.skills import MicroduckSkillContainer +from dimos.web.cockpit import ( + Channel, + Chat, + Col, + Control, + NavMap, + Row, + Teleop, + Video, + cockpit, +) + +# Robot model + policy set; see policies.py for the two variants. +_VARIANT = os.environ.get("MICRODUCK_VARIANT", "default") + +# The gait covers ~0.06 m/s, so the planner's default stuck detector (< 0.4 m +# in 8 s) fires on every leg and the replan limiter gives up mid-room. Over +# 10 s a walking duck moves ~0.6 m (>= 0.3 m from the window's centroid); +# a real stall stays under 0.15 m. +_DUCK_STUCK_TIME_WINDOW = 10.0 +_DUCK_STUCK_THRESHOLD = 0.15 + +# The walking policy is trained for |vx| <= 0.3 m/s (after the sim module's +# command gain); faster teleop requests only saturate. +_TELEOP_MAX_LINEAR = 0.15 +_TELEOP_MAX_ANGULAR = 0.6 + +# Camera rates: what the sim renders at, and the cockpit's cap on top. +# +# The cap must sit ABOVE the render rate, never on it. The bridge's rate gate +# drops any frame arriving less than 1/max_hz after the last, so a cap equal +# to the source aliases against publisher jitter and silently loses ~30% of +# frames (measured: 11.5 Hz in, 8.4 fps out under a 12 Hz cap). +# +# Two budgets bound these numbers, and they respond to different knobs. +# +# RENDER (sim thread). Cameras render inline in the sim loop, so every +# millisecond spent rendering is one physics does not advance. Cost here is +# GEOMETRY, not pixels: the duck's own 215k-vertex body dominates, so a +# 64x48 render measures ~27 ms against ~29 ms for 640x360. Only the frame +# RATE moves this. At these rates the real-time factor stays 1.00 and the +# gait, policies and nav all pass end to end. +# +# DELIVERY (relay -> browser). This is the one that produces "stale", and it +# is BYTES, not stream count. Measured against the live cockpit while the +# duck walked, watching the relay's own counters: the sim never hitches (bus +# gaps p50 56 ms, worst 79 ms) and the robot->relay leg never hitches, but +# relay->browser freezes for seconds at a time, ALL CHANNELS AT ONCE - the +# signature of connection-level flow control rather than a per-stream +# problem. Halving JPEG quality, which leaves the stream count untouched, +# cut the freezes from 2.1-4.1 s to 0.8-1.3 s; shrinking the inset (~7% of +# the bytes) barely moved them. So quality and rate on the CHASE camera are +# the levers that matter - it is ~90% of the byte budget - and 40 is the +# highest quality measured to keep freezes under the 2 s that trips the +# panel badge. +# +# This is a workaround, not a fix. The freeze is the relay blocking in +# createUnidirectionalStream (web/relay/session.ts) with waitUntilAvailable +# and no timeout, which stops every channel rather than dropping a frame on +# a latest-wins video feed. Fixing that upstream would let all of these +# numbers go back up. +_CHASE_CAM_SIZE = (640, 360) +_CHASE_CAM_FPS = 30.0 +_CHASE_CAM_MAX_HZ = 40.0 +_CHASE_CAM_QUALITY = 40 +# The head camera only fills a ~260 px inset, so it ships at a quarter of the +# pixels. That costs nothing in render time (geometry-bound, see above) but a +# lot in BYTES, which is the constraint that actually bites - see the +# flow-control note below. It stays the frame `observe` reads; 320x180 is +# ample for describing a room. +_HEAD_CAM_SIZE = (320, 180) +_HEAD_CAM_FPS = 6 +_HEAD_CAM_MAX_HZ = 12.0 +_HEAD_CAM_QUALITY = 40 + +MICRODUCK_COCKPIT_SYSTEM_PROMPT = """\ +You are the brain of Microduck, a tiny (25 cm tall) two-legged duck robot +living in a small simulated flat. You walk slowly (about 0.1 m/s), so +crossing the flat takes a minute or two - that is normal, and navigation +tools block until the duck arrives or gives up. + +The flat has four rooms around a central hub; each room also goes by a +"space" letter: +- kitchen = space A (red_box, orange_crate) +- living = space B (blue_box) +- bedroom = space C (green_cylinder) +- office = space D (yellow_pillar) + +Tools: +- list_places / where_am_i to orient yourself (rooms, objects, remembered spots) +- go_to_room(name) for a room or space letter, go_to_object(name) for a landmark, + go_to_place(name) for anything by name (rooms, objects, remembered spots) +- move_to(x, y) for raw coordinates, stop_moving to halt, wait(seconds) +- remember_place(name) to save where you stand for later +- list_policies / perform(name) for tricks (kicks, roulade, ...), sit / stand_up +- observe to look through the head camera + +Call movement tools strictly ONE AT A TIME and wait for each result before +the next; never combine go_to_*, move_to or perform in the same step. When +the human names a room, space letter or object, prefer go_to_room / +go_to_object over raw coordinates. Sit only when asked; the duck cannot walk +while seated, so stand_up first. + +Keep answers short and playful - you are a duck. Report what you did once +actions finish. +""" + +# The control strip sizes to its content; the row below takes the rest. +MICRODUCK_COCKPIT_LAYOUT = Col( + Control(), + Row( + # The chase camera is the main view - you need to see the duck to + # drive it - with the duck's own view inset in the corner, so both + # are visible at once instead of trading places. Rate/quality are + # pinned here and mirrored by the chase_image Channel below (a panel + # and a channel for one stream must agree on everything but max_hz); + # color_image is a built-in bridge port, so it needs no Channel. + Video( + "chase_image", + title="Chase cam (duck view inset)", + max_hz=_CHASE_CAM_MAX_HZ, + quality=_CHASE_CAM_QUALITY, + inset="color_image", + inset_max_hz=_HEAD_CAM_MAX_HZ, + inset_quality=_HEAD_CAM_QUALITY, + ), + Col( + NavMap(), + Teleop( + max_linear=_TELEOP_MAX_LINEAR, + max_angular=_TELEOP_MAX_ANGULAR, + # DuckControl drops teleop twists in agent mode; naming + # the stream lets the pad say so instead of arming. + mode="mode", + title="Teleop (WASDQE)", + ), + shares=[3, 2], + ), + Chat(), + # The transcript wraps tool calls and their results, so the agent + # column earns more width than the map/teleop stack beside it. + shares=[5, 3, 4], + ), +) + +# The streams the panels above need that no robot bridge has a built-in port +# for. Declaring them here (rather than in the bridge) is what keeps duck +# vocabulary out of dimos.web: cockpit() generates a typed port per entry and +# resolves the encoder from web_codecs by id. +# +# All of them resend on subscribe - a cockpit opened mid-run must show the +# duck's current mode, places and nav state instead of waiting for the next +# publish - and the event-shaped ones skip the rate gate, because a missed +# sample there is lost data, not a dropped frame. +MICRODUCK_COCKPIT_CHANNELS = ( + Channel( + "chase_image", + Image, + encoding="jpeg.v1", + delivery="latest", + max_hz=_CHASE_CAM_MAX_HZ, + params={"quality": _CHASE_CAM_QUALITY}, + resend_on_subscribe=True, + ), + Channel( + "agent", + BaseMessage, + encoding="chat.json.v1", + max_hz=30.0, + resend_on_subscribe=True, + rate_gate=False, + # A reload must not lose the conversation so far. + replay_depth=200, + ), + Channel( + "agent_idle", + bool, + encoding="flag.json.v1", + max_hz=10.0, + resend_on_subscribe=True, + rate_gate=False, + ), + Channel( + "path", + NavPath, + encoding="path.json.v1", + delivery="latest", + max_hz=5.0, + resend_on_subscribe=True, + ), + Channel( + "nav_state", + str, + encoding="navstate.json.v1", + max_hz=10.0, + resend_on_subscribe=True, + rate_gate=False, + ), + Channel( + "mode", + str, + encoding="mode.json.v1", + max_hz=10.0, + resend_on_subscribe=True, + rate_gate=False, + ), + Channel( + "places", + str, + encoding="places.json.v1", + max_hz=2.0, + resend_on_subscribe=True, + rate_gate=False, + ), + Channel( + "policy_state", + str, + encoding="policy.json.v1", + max_hz=10.0, + resend_on_subscribe=True, + rate_gate=False, + ), +) + + +def _stack(mcp_client_kwargs: dict[str, object]): # type: ignore[no-untyped-def] + return autoconnect( + MicroduckSimModule.blueprint( + scene_xml=FOUR_ROOM_XML, + headless=True, + spawn_xy=(0.0, 0.0), + variant=_VARIANT, + # First-person camera: the cockpit's primary view and the frame + # the observe skill reads. NOT the MJCF's stock `head_camera`, + # which is mounted backwards and renders the duck's own jaw (see + # POV_CAMERA_NAME in sim_module.py). + camera_name=POV_CAMERA_NAME, + width=_HEAD_CAM_SIZE[0], + height=_HEAD_CAM_SIZE[1], + fps=_HEAD_CAM_FPS, + enable_color=True, + enable_depth=False, + # Raycast lidar -> world-frame pointcloud for mapping. World + # geometry is group 0; the robot and the ball are not. + enable_pointcloud=True, + pointcloud_fps=3.0, + enable_mujoco_lidar=True, + mujoco_lidar_camera_names=[name for name, _ in LIDAR_CAMERA_SPECS], + mujoco_lidar_geom_groups=[0], + mujoco_lidar_raycast_width=96, + mujoco_lidar_raycast_height=48, + mujoco_lidar_min_range=0.05, + mujoco_lidar_max_range=6.0, + mujoco_lidar_max_height=0.6, + mujoco_lidar_voxel_size=0.03, + mujoco_lidar_robot_exclusion_radius=0.2, + chase_cam=True, + chase_cam_fps=_CHASE_CAM_FPS, + chase_cam_size=_CHASE_CAM_SIZE, + ), + # CPU voxel grid: the flat is tiny, and this also runs on macOS. + VoxelGridMapper.blueprint(emit_every=1, voxel_size=0.03, device="CPU:0"), + CostMapper.blueprint( + config=HeightCostConfig( + resolution=0.03, + can_pass_under=MICRODUCK.height_clearance + 0.05, + can_climb=0.03, + ), + initial_safe_radius_meters=MICRODUCK.width_clearance + 0.15, + ), + ReplanningAStarPlanner.blueprint( + robot_width=MICRODUCK.width_clearance, + robot_rotation_diameter=MICRODUCK.rotation_diameter, + stuck_time_window=_DUCK_STUCK_TIME_WINDOW, + stuck_threshold=_DUCK_STUCK_THRESHOLD, + ), + # Teleop/agent arbitration; replaces MovementManager (nav_cmd_vel + # only reaches cmd_vel in agent mode). + DuckControlModule.blueprint(), + MicroduckSkillContainer.blueprint(rooms=MICRODUCK_ROOMS, objects=MICRODUCK_OBJECTS), + ObserveSkill.blueprint(), + McpServer.blueprint(), + McpClient.blueprint(system_prompt=MICRODUCK_COCKPIT_SYSTEM_PROMPT, **mcp_client_kwargs), + cockpit(layout=MICRODUCK_COCKPIT_LAYOUT, channels=MICRODUCK_COCKPIT_CHANNELS), + ).remappings([(VoxelGridMapper, "lidar", "pointcloud")]) + + +# The last call in each assignment below has to be a builder method: the +# blueprint registry is generated by a static AST scan +# (test_all_blueprints_generation) that only recognises `autoconnect(...)` by +# name or an expression ending in one of these builders. A bare `_stack({})` +# is invisible to it, and `dimos run microduck-cockpit-sim` then fails with +# "Unknown blueprint" while the ollama variant - which ends in +# .requirements() - resolves fine. +# +# nerf_speed halves the planner's 0.55 m/s default; the sim module's command +# gain maps the rest into the gait's trained velocity range. +microduck_cockpit_sim = _stack({}).global_config(robot_model="microduck", nerf_speed=0.5) + +microduck_cockpit_sim_ollama = ( + _stack({"model": "ollama:qwen3:8b"}) + .global_config(robot_model="microduck", nerf_speed=0.5) + .requirements(ollama_installed) +) diff --git a/dimos/robot/pollen/microduck/blueprints/microduck_sim.py b/dimos/robot/pollen/microduck/blueprints/microduck_sim.py new file mode 100644 index 0000000000..f408ca9b67 --- /dev/null +++ b/dimos/robot/pollen/microduck/blueprints/microduck_sim.py @@ -0,0 +1,116 @@ +# Copyright 2025-2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Microduck room-simulation blueprint. + +A pretrained Microduck (pollen-robotics's ~25 cm biped) walking in a small +MuJoCo room, with the standard dimOS navigation stack on top of its RL gait: + + raycast lidar -> VoxelGridMapper -> CostMapper -> ReplanningAStarPlanner + -> MovementManager -> cmd_vel -> walking policy -> MuJoCo + +Usage: + dimos run microduck-sim + dimos shell # then e.g. app.WavefrontFrontierExplorer.explore() + +Model assets and the walking policy are downloaded from the public +pollen-robotics repos into ~/.cache/dimos/microduck on first start. +""" + +from __future__ import annotations + +from pathlib import Path +import platform + +from dimos.core.coordination.blueprints import autoconnect +from dimos.mapping.costmapper import CostMapper +from dimos.mapping.pointclouds.occupancy import HeightCostConfig +from dimos.mapping.voxels.module import VoxelGridMapper +from dimos.navigation.frontier_exploration.wavefront_frontier_goal_selector import ( + WavefrontFrontierExplorer, +) +from dimos.navigation.movement_manager.movement_manager import MovementManager +from dimos.navigation.replanning_a_star.module import ReplanningAStarPlanner +from dimos.robot.pollen.microduck.config import MICRODUCK +from dimos.robot.pollen.microduck.sim_module import LIDAR_CAMERA_SPECS, MicroduckSimModule + +_SCENE_XML = Path(__file__).resolve().parents[1] / "assets" / "room_scene.xml" + +# Ground-truth object positions; must match assets/room_scene.xml. +MICRODUCK_ROOM_OBJECTS: dict[str, tuple[float, float]] = { + "red_box": (1.5, 0.8), + "blue_box": (-1.5, -0.8), +} + +microduck_sim = ( + autoconnect( + MicroduckSimModule.blueprint( + scene_xml=_SCENE_XML, + headless=True, + spawn_xy=(0.0, 0.0), + # Head camera stream (for viewers / future perception). Rendered + # cameras need mujoco.Renderer, whose GL context cannot be + # created off the main thread on macOS - the sim thread would + # deadlock the worker - so the stream is Linux-only. + width=320, + height=240, + fps=5, + enable_color=platform.system() != "Darwin", + enable_depth=False, + # Raycast lidar -> world-frame pointcloud for mapping. World + # geometry is group 0; the robot (groups 2/3) is invisible to it. + enable_pointcloud=True, + pointcloud_fps=3.0, + enable_mujoco_lidar=True, + mujoco_lidar_camera_names=[name for name, _ in LIDAR_CAMERA_SPECS], + mujoco_lidar_geom_groups=[0], + # Dense enough that the room's small (6-14 cm) objects reliably + # land in the voxel map, not just the walls. + mujoco_lidar_raycast_width=96, + mujoco_lidar_raycast_height=48, + mujoco_lidar_min_range=0.05, + mujoco_lidar_max_range=6.0, + mujoco_lidar_max_height=0.6, + mujoco_lidar_voxel_size=0.03, + mujoco_lidar_robot_exclusion_radius=0.2, + ), + # CPU voxel grid: the room is tiny, and this also runs on machines + # without CUDA (macOS). + VoxelGridMapper.blueprint(emit_every=1, voxel_size=0.03, device="CPU:0"), + CostMapper.blueprint( + config=HeightCostConfig( + resolution=0.03, + can_pass_under=MICRODUCK.height_clearance + 0.05, + can_climb=0.03, + ), + initial_safe_radius_meters=MICRODUCK.width_clearance + 0.15, + ), + ReplanningAStarPlanner.blueprint( + robot_width=MICRODUCK.width_clearance, + robot_rotation_diameter=MICRODUCK.rotation_diameter, + ), + MovementManager.blueprint(), + WavefrontFrontierExplorer.blueprint( + min_frontier_perimeter=0.15, + safe_distance=0.6, + lookahead_distance=2.0, + max_explored_distance=6.0, + goal_timeout=30.0, + ), + ) + .remappings([(VoxelGridMapper, "lidar", "pointcloud")]) + # nerf_speed halves the planner's 0.55 m/s default; the sim module's + # command gain maps the rest into the gait's trained velocity range. + .global_config(robot_model="microduck", nerf_speed=0.5) +) diff --git a/dimos/robot/pollen/microduck/config.py b/dimos/robot/pollen/microduck/config.py new file mode 100644 index 0000000000..42faa56650 --- /dev/null +++ b/dimos/robot/pollen/microduck/config.py @@ -0,0 +1,58 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Microduck physical description, as the navigation stack needs it. + +Mirrors dimos/robot/unitree/g1/config.py: one frozen descriptor the blueprints +read, rather than loose constants copied into each one. + +These are CLEARANCES, not measurements - each carries margin over the duck's +real size, which is why they are named `*_clearance`. Measured from the +composed MJCF (the AABB of the robot root's body subtree, rest pose): + + width 14 cm footprint circle 20 cm + height 26 cm + +They cannot be derived from that model here: a blueprint is a static +declaration evaluated long before the MuJoCo model is loaded, and neither +CostMapper nor ReplanningAStarPlanner can take a footprint at start-up. So +they are constants - but constants in one place, with the margin visible. +""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class MicroduckConfig: + """Physical metadata used by Microduck navigation blueprints.""" + + name: str + #: Planner footprint width. 14 cm of duck plus ~6 cm of margin; the + #: costmap adds its own safe radius on top of this. + width_clearance: float + #: Ceiling the duck fits under. 26 cm of duck plus ~2 cm. + height_clearance: float + #: Diameter the duck sweeps turning in place. Its footprint circle is + #: ~20 cm; the extra 10 cm keeps a turn off the walls. + rotation_diameter: float + + +MICRODUCK = MicroduckConfig( + name="microduck", + width_clearance=0.2, + height_clearance=0.28, + rotation_diameter=0.3, +) diff --git a/dimos/robot/pollen/microduck/control_module.py b/dimos/robot/pollen/microduck/control_module.py new file mode 100644 index 0000000000..b25f6de022 --- /dev/null +++ b/dimos/robot/pollen/microduck/control_module.py @@ -0,0 +1,477 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""DuckControlModule: teleop/nav velocity mux + cockpit command router for +the Microduck. + +Replaces MovementManager in the cockpit blueprint. Three things happen here: + +* ``tele_cmd_vel`` (keyboard pad) and ``nav_cmd_vel`` (planner) are merged + into one ``cmd_vel`` with MovementManager semantics (teleop wins, nav + resumes after a cooldown) plus a mode switch: in ``agent`` mode the pad is + ignored so the LLM owns the body. A locked policy (oneshot running, + seated, fallen...) zeroes everything. +* ``ui_command`` JSON from the cockpit is routed: ``set_mode`` flips the + mode, ``policy`` is re-published as ``policy_request`` for the sim, + ``cancel_nav`` cancels the planner goal over RPC. +* A small state thread publishes ``nav_state`` / ``mode`` JSON at + ``state_hz`` so the cockpit (and late viewers) always know what the duck + is doing. + +Navigation is cancelled through the planner's ``cancel_goal()`` RPC via the +``_navigation`` module ref; nobody publishes ``stop_movement`` here. + +Wire formats (all JSON ``str`` streams carry ``"t": time.time()``): + +* ``mode``: ``{"mode": "teleop"|"agent", "t": t}`` +* ``nav_state``: ``{"state": , "goal": {"x","y","yaw"}|null, "since": t, "t": t}`` + (``goal`` is non-null exactly while a goal is in progress) +* ``policy_request``: ``{"policy": "kick_left", "action": "start"|"stop"|"toggle", "t": t}`` + (``policy`` omitted for a bare ``{"action": "stop"}``) +* ``ui_command`` (in): ``{"name": "set_mode", "args": {"mode": ...}}`` | + ``{"name": "policy", "args": {"policy": ..., "action": ...}}`` | ``{"name": "cancel_nav"}`` +""" + +from __future__ import annotations + +from collections.abc import Callable +import json +import threading +import time +from typing import Any, Literal + +from reactivex.disposable import Disposable + +from dimos.constants import DEFAULT_THREAD_JOIN_TIMEOUT +from dimos.core.core import rpc +from dimos.core.module import Module, ModuleConfig +from dimos.core.stream import In, Out +from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped +from dimos.msgs.geometry_msgs.Twist import Twist +from dimos.msgs.geometry_msgs.Vector3 import Vector3 +from dimos.navigation.base import NavigationState +from dimos.navigation.navigation_spec import NavigationInterfaceSpec +from dimos.utils.logging_config import setup_logger + +logger = setup_logger() + +Mode = Literal["teleop", "agent"] +MODES: tuple[str, ...] = ("teleop", "agent") +UI_COMMANDS: tuple[str, ...] = ("set_mode", "policy", "cancel_nav") +POLICY_ACTIONS: tuple[str, ...] = ("start", "stop", "toggle") + +# nav_state "state" values. The first three mirror NavigationState; the +# rest are terminal outcomes DuckControl derives itself. +NAV_STATES: tuple[str, ...] = ( + "idle", + "following_path", + "recovery", + "reached", + "cancelled", + "no_path", +) + +# A goal whose planner state never leaves IDLE for this long is reported as +# no_path (the planner logs "no path" but exposes nothing over RPC). +NO_PATH_TIMEOUT_SEC = 15.0 +# The planner drops out of FOLLOWING_PATH briefly on every replan; only an +# idle spell longer than this (after following started) counts as the goal +# having been stopped from elsewhere. +STOP_SETTLE_SEC = 2.0 +_LOG_THROTTLE_SEC = 1.0 + + +def _dumps(obj: dict[str, Any]) -> str: + return json.dumps({**obj, "t": time.time()}, separators=(",", ":")) + + +def _is_zero(msg: Twist) -> bool: + return ( + msg.linear.x == 0 + and msg.linear.y == 0 + and msg.linear.z == 0 + and msg.angular.x == 0 + and msg.angular.y == 0 + and msg.angular.z == 0 + ) + + +class DuckControlConfig(ModuleConfig): + tele_cooldown_sec: float = 1.0 + state_hz: float = 5.0 + default_mode: Mode = "teleop" + # (linear.x, linear.y, angular.z) multipliers applied to teleop twists. + tele_cmd_vel_scaling: tuple[float, float, float] = (1.0, 1.0, 1.0) + + +class DuckControlModule(Module): + """Mux tele_cmd_vel/nav_cmd_vel into cmd_vel, route cockpit ui_command, + publish mode + nav_state JSON.""" + + config: DuckControlConfig + + _navigation: NavigationInterfaceSpec + + nav_cmd_vel: In[Twist] + tele_cmd_vel: In[Twist] + ui_command: In[str] + policy_state: In[str] + goal_request: In[PoseStamped] + + cmd_vel: Out[Twist] + mode: Out[str] + nav_state: Out[str] + policy_request: Out[str] + + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + self._lock = threading.Lock() + # Monotonic clock for durations; tests swap it for a fake. + self._clock: Callable[[], float] = time.monotonic + self._mode: str = self.config.default_mode + # A teleop episode is in progress: a nonzero pad twist arrived and no + # zero (release / e-stop) has ended it yet. + self._teleop_active = False + # _clock() of the last pad message (nonzero or e-stop zero); nav + # twists are dropped within tele_cooldown_sec of it. + self._last_teleop_time = -float("inf") + self._locked = False + self._zero_sent_while_locked = False + # Navigation bookkeeping (guarded by _lock). + self._goal: dict[str, float] | None = None + self._goal_set_at = 0.0 # _clock() when the current goal arrived + self._nav_active = False # a goal is in progress (not yet terminal) + self._nav_started = False # planner seen non-idle for this goal + # is_goal_reached() may still be latched from a previous goal when a + # new one arrives; only a True seen after a False (or after + # following) counts. _last_reached is the previous poll's answer. + self._nav_armed = False + self._last_reached = False + self._idle_since: float | None = None # _clock() when idle-after-following began + self._state = "idle" + self._since = time.time() + self._stop_event = threading.Event() + self._state_thread: threading.Thread | None = None + self._last_log: dict[str, float] = {} + + # ------------------------------------------------------------------ lifecycle + + @rpc + def start(self) -> None: + super().start() + self._subscribe(self.nav_cmd_vel, self._on_nav) + self._subscribe(self.tele_cmd_vel, self._on_teleop) + self._subscribe(self.ui_command, self._on_ui_command) + self._subscribe(self.policy_state, self._on_policy_state) + self._subscribe(self.goal_request, self._on_goal_request) + self._publish_mode() + self._stop_event.clear() + self._state_thread = threading.Thread( + target=self._state_loop, name="DuckControl-state", daemon=True + ) + self._state_thread.start() + + @rpc + def stop(self) -> None: + self._stop_event.set() + thread = self._state_thread + if thread is not None and thread is not threading.current_thread(): + thread.join(DEFAULT_THREAD_JOIN_TIMEOUT) + if thread.is_alive(): + logger.error("DuckControl state thread did not stop in time.") + self._state_thread = None + with self._lock: + self._teleop_active = False + super().stop() + + def _subscribe(self, stream: In[Any], callback: Callable[[Any], None]) -> None: + if stream.transport is None: + logger.warning( + "DuckControl input has no transport; not subscribing", stream=stream.name + ) + return + self.register_disposable(Disposable(stream.subscribe(callback))) + + # ------------------------------------------------------------------ helpers + + def _log_throttled(self, key: str, message: str, **kwargs: Any) -> None: + now = self._clock() + if now - self._last_log.get(key, -float("inf")) < _LOG_THROTTLE_SEC: + return + self._last_log[key] = now + logger.warning(message, **kwargs) + + def _nav_rpc(self, name: str) -> Any: + """Call a NavigationInterfaceSpec method; None when the planner is + unavailable (not wired yet, RPC timeout...). Never raises.""" + navigation = getattr(self, "_navigation", None) + if navigation is None: + self._log_throttled("nav_missing", "DuckControl has no _navigation ref", call=name) + return None + try: + return getattr(navigation, name)() + except Exception as e: + self._log_throttled(f"nav_{name}", "navigation RPC failed", call=name, error=str(e)) + return None + + def _publish_zero(self) -> None: + self.cmd_vel.publish(Twist()) + + def _publish_mode(self) -> None: + self.mode.publish(_dumps({"mode": self._mode})) + + def _publish_nav_state(self, snapshot: dict[str, Any]) -> None: + self.nav_state.publish(_dumps(snapshot)) + + def _snapshot_locked(self) -> dict[str, Any]: + return { + "state": self._state, + "goal": dict(self._goal) if self._nav_active and self._goal is not None else None, + "since": self._since, + } + + def _set_state_locked(self, state: str) -> None: + if state != self._state: + self._state = state + self._since = time.time() + + def _set_terminal_locked(self, state: str) -> None: + self._set_state_locked(state) + self._nav_active = False + self._nav_started = False + self._nav_armed = False + self._idle_since = None + + def _cancel_nav(self) -> None: + """Cancel the planner goal over RPC, zero cmd_vel and report + `cancelled` (when a goal was being tracked).""" + self._nav_rpc("cancel_goal") + self._publish_zero() + with self._lock: + if not self._nav_active: + return + self._set_terminal_locked("cancelled") + snapshot = self._snapshot_locked() + self._publish_nav_state(snapshot) + + # ------------------------------------------------------------------ inputs + + def _on_ui_command(self, raw: str) -> None: + try: + command = json.loads(raw) + except (TypeError, ValueError): + self._log_throttled("ui_bad_json", "ui_command is not JSON", raw=str(raw)[:120]) + return + if not isinstance(command, dict) or command.get("name") not in UI_COMMANDS: + self._log_throttled("ui_unknown", "unknown ui_command", raw=str(raw)[:120]) + return + name = command["name"] + args = command.get("args") or {} + if not isinstance(args, dict): + self._log_throttled("ui_bad_args", "ui_command args must be an object", name=name) + return + if name == "set_mode": + self._set_mode(args.get("mode")) + elif name == "policy": + self._forward_policy(args.get("policy"), args.get("action")) + else: + self._cancel_nav() + + def _set_mode(self, mode: Any) -> None: + if mode not in MODES: + self._log_throttled("ui_bad_mode", "set_mode with unknown mode", mode=str(mode)) + return + with self._lock: + changed = mode != self._mode + self._mode = mode + zero = changed and self._teleop_active + if changed: + self._teleop_active = False + if changed: + logger.info("DuckControl mode", mode=mode) + if zero: + # The pad's release zero will be ignored in agent mode; do not + # leave the last held twist running in the sim. + self._publish_zero() + self._publish_mode() + + def _forward_policy(self, policy: Any, action: Any) -> None: + if action not in POLICY_ACTIONS: + self._log_throttled( + "ui_bad_action", "policy command with bad action", action=str(action) + ) + return + if policy is not None and (not isinstance(policy, str) or not policy): + self._log_throttled( + "ui_bad_policy", "policy command with bad policy", policy=str(policy) + ) + return + if policy is None and action != "stop": + self._log_throttled("ui_no_policy", "policy command without a policy", action=action) + return + request: dict[str, Any] = {"action": action} + if policy is not None: + request["policy"] = policy + self.policy_request.publish(_dumps(request)) + + def _on_teleop(self, msg: Twist) -> None: + with self._lock: + locked = self._locked + send_zero = locked and not self._zero_sent_while_locked + if locked: + self._zero_sent_while_locked = True + mode = self._mode + if locked: + if send_zero: + self._publish_zero() + return + if mode == "agent": + self._log_throttled("tele_agent", "tele_cmd_vel ignored in agent mode") + return + + zero = _is_zero(msg) + with self._lock: + goal_active = self._nav_active + if zero: + # e-stop / key release: meaningful only if something was moving. + stop = self._teleop_active or goal_active + self._teleop_active = False + else: + stop = False + self._teleop_active = True + if stop or not zero: + self._last_teleop_time = self._clock() + if zero: + if stop: + self._cancel_nav() # publishes the zero + return + if goal_active: + self._cancel_nav() + sx, sy, sw = self.config.tele_cmd_vel_scaling + self.cmd_vel.publish( + Twist( + linear=Vector3(msg.linear.x * sx, msg.linear.y * sy, msg.linear.z), + angular=Vector3(msg.angular.x, msg.angular.y, msg.angular.z * sw), + ) + ) + + def _on_nav(self, msg: Twist) -> None: + with self._lock: + if self._locked: + return + if self._clock() - self._last_teleop_time < self.config.tele_cooldown_sec: + return + self._teleop_active = False + self.cmd_vel.publish(msg) + + def _on_policy_state(self, raw: str) -> None: + try: + state = json.loads(raw) + except (TypeError, ValueError): + self._log_throttled("policy_bad_json", "policy_state is not JSON") + return + if not isinstance(state, dict): + return + locked = bool(state.get("locked", False)) + with self._lock: + rising = locked and not self._locked + self._locked = locked + if rising: + self._zero_sent_while_locked = True # _cancel_nav below sends it + self._teleop_active = False + elif not locked: + self._zero_sent_while_locked = False + if rising: + logger.info("policy locked; cancelling navigation") + self._cancel_nav() + + def _on_goal_request(self, goal: PoseStamped) -> None: + with self._lock: + self._goal = { + "x": float(goal.position.x), + "y": float(goal.position.y), + "yaw": float(goal.yaw), + } + self._goal_set_at = self._clock() + self._nav_active = True + self._nav_started = False + self._nav_armed = not self._last_reached + self._idle_since = None + self._set_state_locked("idle") + snapshot = self._snapshot_locked() + self._publish_nav_state(snapshot) + + # ------------------------------------------------------------------ state thread + + def _state_loop(self) -> None: + period = 1.0 / max(self.config.state_hz, 0.1) + while not self._stop_event.is_set(): + try: + self._tick() + except Exception: + logger.exception("DuckControl state tick failed") + self._stop_event.wait(period) + + def _tick(self) -> None: + """One state-thread iteration: poll the planner, publish nav_state + mode.""" + planner_state = self._nav_rpc("get_state") + reached = self._nav_rpc("is_goal_reached") + snapshot = self._update_nav_state(planner_state, reached) + self._publish_nav_state(snapshot) + self._publish_mode() + + def _update_nav_state(self, planner_state: Any, reached: Any) -> dict[str, Any]: + """Fold one planner poll into the nav bookkeeping; returns the + nav_state snapshot (without "t"). A poll that failed (None) leaves + the bookkeeping untouched so a flaky RPC cannot fake an outcome.""" + now = self._clock() + with self._lock: + if not isinstance(planner_state, NavigationState): + return self._snapshot_locked() + if isinstance(reached, bool): + self._last_reached = reached + if not reached: + self._nav_armed = True + + if planner_state in (NavigationState.FOLLOWING_PATH, NavigationState.RECOVERY): + # A goal set behind our back (planner RPC set_goal) still + # shows up as navigation in progress, goal unknown. + if not self._nav_active: + self._goal = None + self._goal_set_at = now + self._nav_active = True + self._nav_started = True + self._nav_armed = True + self._idle_since = None + self._set_state_locked(planner_state.value) + return self._snapshot_locked() + + if not self._nav_active: + return self._snapshot_locked() # idle, or a sticky terminal state + + # Goal tracked, planner idle: waiting for a plan, done, or stopped. + if reached is True and self._nav_armed: + self._set_terminal_locked("reached") + elif self._nav_started: + if self._idle_since is None: + self._idle_since = now + if now - self._idle_since >= STOP_SETTLE_SEC: + self._set_terminal_locked("cancelled") + else: + self._set_state_locked(planner_state.value) + elif now - self._goal_set_at >= NO_PATH_TIMEOUT_SEC: + self._set_terminal_locked("no_path") + else: + self._set_state_locked(planner_state.value) + return self._snapshot_locked() diff --git a/dimos/robot/pollen/microduck/gait.py b/dimos/robot/pollen/microduck/gait.py new file mode 100644 index 0000000000..9bbb4b307f --- /dev/null +++ b/dimos/robot/pollen/microduck/gait.py @@ -0,0 +1,263 @@ +# Copyright 2025-2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Microduck ONNX gait policy: observation building and inference. + +The alpha policies from pollen-robotics/microduck share one observation +contract (61 floats), documented in that repo's duck-control/src/obs.rs:: + + 0..3 gyro, trunk frame, rad/s + 3..6 projected gravity, trunk frame, unit vector + 6..20 joint position minus home pose (14, policy order) + 20..34 joint velocity (14) + 34..48 previous raw policy action (14) + 48..61 command: [vx, vy, vyaw, head(4), body_x, body_y, body_z, + body_roll, body_pitch, body_yaw] + +Actions are position offsets from the home pose, applied at 50 Hz. Joint +order, home pose and action scale are read from the ONNX metadata rather +than hardcoded, so a retrained/re-exported policy keeps working as long as +it declares them. + +Every published Microduck policy (walk, stand, sitstand, kicks, roulade, +ground pick, roller, roller crouch) declares the same joint order and home +pose, so the model-bound half of the contract (``MicroduckObserver``) and the +``last_action`` slot can be shared by several loaded policies; only the +13-float command differs per policy (see ``policies.py``). +``MicroduckGaitPolicy`` is the single-policy (walking) wrapper the plain +``microduck-sim`` blueprint runs. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import numpy as np +from numpy.typing import NDArray + +if TYPE_CHECKING: + import mujoco + +OBS_LEN = 61 +COMMAND_LEN = 13 +CONTROL_DT = 0.02 # 50 Hz, the rate every alpha policy was trained at + +# Command clipping, matching the velocity ranges the walking policy was +# trained on (microduck_rl velocity task). +VX_RANGE = (-0.25, 0.3) +VY_RANGE = (-0.2, 0.2) +WZ_RANGE = (-1.5, 1.5) + +_TRUNK_FREEJOINT = "trunk_base_freejoint" +_GYRO_SENSOR = "imu_ang_vel" + + +@dataclass(frozen=True) +class PolicySession: + """One loaded ONNX policy plus the metadata the runner needs from it.""" + + path: Path + session: Any # onnxruntime.InferenceSession (kept untyped: optional dep) + input_name: str + output_name: str + joint_names: tuple[str, ...] + default_pose: NDArray[np.float32] + action_scale: float + + def run(self, obs: NDArray[np.float32]) -> NDArray[np.float32]: + """Raw action (position offsets, policy joint order) for one observation.""" + action = self.session.run([self.output_name], {self.input_name: obs.reshape(1, -1)})[0] + return np.asarray(action, dtype=np.float32).reshape(-1) + + +def load_policy_session(onnx_path: str | Path) -> PolicySession: + """Load an alpha-family ONNX policy, validating its metadata and obs width.""" + import onnxruntime as ort + + path = Path(onnx_path) + session = ort.InferenceSession(str(path)) + meta = session.get_modelmeta().custom_metadata_map + try: + joint_names = tuple(meta["joint_names"].split(",")) + default_pose = np.array( + [float(v) for v in meta["default_joint_pos"].split(",")], dtype=np.float32 + ) + action_scale = float(meta.get("action_scale", "1.0")) + except KeyError as exc: + raise RuntimeError( + f"Microduck policy {path} is missing ONNX metadata {exc}; " + "use a policy exported by microduck_rl's scripts/export.py" + ) from exc + + obs_dim = session.get_inputs()[0].shape[-1] + if obs_dim != OBS_LEN: + raise RuntimeError( + f"Microduck policy {path} expects obs dim {obs_dim}, this runner builds {OBS_LEN}; " + "only unified-61D alpha policies are supported" + ) + if len(default_pose) != len(joint_names): + raise RuntimeError( + f"Microduck policy {path} declares {len(joint_names)} joints but a " + f"{len(default_pose)}-long default pose" + ) + return PolicySession( + path=path, + session=session, + input_name=session.get_inputs()[0].name, + output_name=session.get_outputs()[0].name, + joint_names=joint_names, + default_pose=default_pose, + action_scale=action_scale, + ) + + +class MicroduckObserver: + """Model-bound half of the observation contract. + + Resolves the policy's joints and sensors by name in a composed MuJoCo + model (the robot may be embedded in an arbitrary scene) and builds the + 61-float observation from ``MjData``. Holds no per-policy state, so one + observer serves every loaded policy. + """ + + def __init__( + self, + model: mujoco.MjModel, + joint_names: Sequence[str], + default_pose: NDArray[np.float32], + *, + prefix: str = "", + ) -> None: + import mujoco + + self.joint_names: list[str] = list(joint_names) + self.default_pose = np.asarray(default_pose, dtype=np.float32) + n = len(self.joint_names) + if len(self.default_pose) != n: + raise ValueError(f"default_pose has {len(self.default_pose)} entries for {n} joints") + + # Resolve joint addresses in policy order. + self._qpos_adr = np.empty(n, dtype=np.int64) + self._qvel_adr = np.empty(n, dtype=np.int64) + for i, name in enumerate(self.joint_names): + jid = mujoco.mj_name2id(model, mujoco.mjtObj.mjOBJ_JOINT, prefix + name) + if jid < 0: + raise RuntimeError(f"Microduck joint '{name}' not found in composed model") + self._qpos_adr[i] = model.jnt_qposadr[jid] + self._qvel_adr[i] = model.jnt_dofadr[jid] + + gyro_sid = mujoco.mj_name2id(model, mujoco.mjtObj.mjOBJ_SENSOR, prefix + _GYRO_SENSOR) + if gyro_sid < 0: + raise RuntimeError(f"Microduck model has no '{_GYRO_SENSOR}' sensor") + adr = int(model.sensor_adr[gyro_sid]) + self._gyro_slice = slice(adr, adr + 3) + + trunk_jid = mujoco.mj_name2id(model, mujoco.mjtObj.mjOBJ_JOINT, prefix + _TRUNK_FREEJOINT) + if trunk_jid < 0: + raise RuntimeError(f"Microduck model has no '{_TRUNK_FREEJOINT}'") + root_adr = int(model.jnt_qposadr[trunk_jid]) + self.root_qpos_adr = root_adr + self.root_qvel_adr = int(model.jnt_dofadr[trunk_jid]) + self._root_quat_slice = slice(root_adr + 3, root_adr + 7) + + @property + def num_joints(self) -> int: + return len(self.joint_names) + + def initial_qpos(self, data: mujoco.MjData) -> None: + """Pose the robot at the standing home pose, keeping its base x/y.""" + adr = self.root_qpos_adr + data.qpos[adr + 2] = 0.125 + data.qpos[adr + 3 : adr + 7] = (1.0, 0.0, 0.0, 0.0) + data.qpos[self._qpos_adr] = self.default_pose + data.qvel[self._qvel_adr] = 0.0 + data.qvel[self.root_qvel_adr : self.root_qvel_adr + 6] = 0.0 + + def root_yaw(self, data: mujoco.MjData) -> float: + """Trunk yaw (rad) in the world frame.""" + w, x, y, z = data.qpos[self._root_quat_slice] + return float(np.arctan2(2.0 * (w * z + x * y), 1.0 - 2.0 * (y * y + z * z))) + + def projected_gravity(self, data: mujoco.MjData) -> NDArray[np.float32]: + w, x, y, z = data.qpos[self._root_quat_slice] + quat = np.array([w, x, y, z], dtype=np.float32) + down = np.array([0.0, 0.0, -1.0], dtype=np.float32) + # v rotated by quat^-1 + xyz = quat[1:4] + t = np.cross(xyz, down) * 2.0 + return np.asarray(down - quat[0] * t + np.cross(xyz, t), dtype=np.float32) + + def build( + self, + data: mujoco.MjData, + last_action: NDArray[np.float32], + command: NDArray[np.float32], + ) -> NDArray[np.float32]: + gyro = data.sensordata[self._gyro_slice].astype(np.float32) + gravity = self.projected_gravity(data) + joint_pos = data.qpos[self._qpos_adr].astype(np.float32) - self.default_pose + joint_vel = data.qvel[self._qvel_adr].astype(np.float32) + return np.concatenate([gyro, gravity, joint_pos, joint_vel, last_action, command]) + + +class MicroduckGaitPolicy: + """Runs an alpha-family ONNX policy against a composed MuJoCo model. + + The model may embed the robot in an arbitrary scene; joints and sensors + are resolved by name (policy order from ONNX metadata). + """ + + def __init__(self, onnx_path: str | Path, model: mujoco.MjModel) -> None: + self._policy = load_policy_session(onnx_path) + self.joint_names: list[str] = list(self._policy.joint_names) + self.default_pose = self._policy.default_pose + self.action_scale = self._policy.action_scale + self._observer = MicroduckObserver(model, self.joint_names, self.default_pose) + self.root_qpos_adr = self._observer.root_qpos_adr + + n = len(self.joint_names) + self.last_action = np.zeros(n, dtype=np.float32) + self._command = np.zeros(COMMAND_LEN, dtype=np.float32) + + @property + def num_joints(self) -> int: + return len(self.joint_names) + + def set_twist(self, vx: float, vy: float, wz: float) -> None: + self._command[0] = float(np.clip(vx, *VX_RANGE)) + self._command[1] = float(np.clip(vy, *VY_RANGE)) + self._command[2] = float(np.clip(wz, *WZ_RANGE)) + + def reset(self) -> None: + self.last_action[:] = 0.0 + self._command[:] = 0.0 + + def initial_qpos(self, data: mujoco.MjData) -> None: + """Pose the robot at the standing home pose, keeping its base x/y.""" + self._observer.initial_qpos(data) + + def projected_gravity(self, data: mujoco.MjData) -> NDArray[np.float32]: + return self._observer.projected_gravity(data) + + def build_observation(self, data: mujoco.MjData) -> NDArray[np.float32]: + return self._observer.build(data, self.last_action, self._command) + + def step(self, data: mujoco.MjData) -> NDArray[np.float32]: + """One 50 Hz control step: returns position targets in policy order.""" + action = self._policy.run(self.build_observation(data)) + self.last_action = action.copy() + return self.default_pose + action * self.action_scale diff --git a/dimos/robot/pollen/microduck/places.py b/dimos/robot/pollen/microduck/places.py new file mode 100644 index 0000000000..0e6977f549 --- /dev/null +++ b/dimos/robot/pollen/microduck/places.py @@ -0,0 +1,561 @@ +# Copyright 2025-2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Places the Microduck knows about: rooms, landmark objects, tagged spots. + +The four-room arena (``assets/four_room_scene.xml``) is described here as +plain data (:data:`MICRODUCK_ROOMS`, :data:`MICRODUCK_OBJECTS`) and +persisted through :class:`PlacesMemory`, a thin wrapper over a dimos +``SqliteStore`` stream of :class:`RobotLocation` records. The memory is what +the skills query ("go to space A", "where am I?") and what the cockpit's +``places`` channel is rendered from. A unit test cross-checks the constants +against the scene XML; keep the two in sync. + +Several blueprints share one database file (``~/.cache/dimos/microduck/ +places.db``) but describe different worlds, so every record is tagged with a +*scene id* and a :class:`PlacesMemory` only ever sees its own scene's rooms, +objects and remembered spots. The id is normally derived from the room / +object tables (:func:`scene_id`), so two blueprints with different tables +never leak places into each other. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +import hashlib +import json +import math +from pathlib import Path +import re +import threading +import time +from typing import Any + +from dimos.memory.store.sqlite import SqliteStore +from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped +from dimos.msgs.geometry_msgs.Quaternion import Quaternion +from dimos.msgs.geometry_msgs.Vector3 import Vector3 +from dimos.types.robot_location import RobotLocation + +# Scene shipped with the package; the cockpit blueprint spawns the duck here. +FOUR_ROOM_XML = Path(__file__).resolve().parent / "assets" / "four_room_scene.xml" + +# Frame every place is expressed in (world == MuJoCo == odom for the sim). +PLACES_FRAME = "world" + +# The open central hub is not part of any room. +HUB_HALF_EXTENT = 0.8 + +# Inner faces of the four-room arena walls. +ARENA_HALF_EXTENT = 2.0 + +# Scene id used when a PlacesMemory is opened without one (direct use). +DEFAULT_SCENE = "default" + +# The kickable ball. It is deliberately NOT declared in the scene XML: MuJoCo +# numbers joints in body order and the simulation engine takes joint 0 as the +# robot's root free joint (odom, spawn pose, IMU fallback, lidar exclusion), +# so a jointed body declared before the robot is attached would be mistaken +# for it. The sim module adds the ball with :func:`add_ball_body` *after* +# ``MjSpec.attach`` and later teleports it in front of a foot for kicks. +BALL_BODY = "ball" +BALL_FREEJOINT = "ball_freejoint" +BALL_GEOM = "ball_geom" +BALL_START: tuple[float, float, float] = (1.2, -0.6, 0.035) +BALL_RADIUS = 0.035 +BALL_MASS = 0.05 +BALL_GEOM_GROUP = 1 # invisible to the raycast lidar (group 0), so never in the costmap +_BALL_RGBA = (0.95, 0.95, 0.95, 1.0) +_BALL_FRICTION = (0.5, 0.005, 0.0001) + + +def add_ball_body( + spec: Any, + *, + name: str = BALL_BODY, + pos: tuple[float, float, float] = BALL_START, +) -> Any: + """Add the kickable ball to a ``mujoco.MjSpec`` scene and return its body. + + Call this **after** the robot has been attached to ``spec`` so the ball's + free joint is numbered after the robot's root joint (see :data:`BALL_BODY`). + The joint is named ``f"{name}_freejoint"`` and the geom ``f"{name}_geom"``; + with the defaults that is :data:`BALL_FREEJOINT` / :data:`BALL_GEOM`. + ``mujoco`` is imported lazily so this module stays import-light for the + agent process. + """ + import mujoco + + body = spec.worldbody.add_body(name=name, pos=[float(v) for v in pos]) + body.add_freejoint(name=f"{name}_freejoint") + body.add_geom( + name=f"{name}_geom", + type=mujoco.mjtGeom.mjGEOM_SPHERE, + size=[BALL_RADIUS, 0.0, 0.0], + mass=BALL_MASS, + group=BALL_GEOM_GROUP, + rgba=list(_BALL_RGBA), + friction=list(_BALL_FRICTION), + ) + return body + + +@dataclass(frozen=True) +class RoomSpec: + """A rectangular room of the arena. + + Attributes: + name: Canonical room name (e.g. ``"kitchen"``). + aliases: Other names people use for it (e.g. ``"space A"``). + bounds: ``(xmin, xmax, ymin, ymax)`` in metres, world frame. + target: ``(x, y, yaw)`` the robot navigates to for "go to ": + a point inside the room just past the hub opening, with the yaw + looking into the room (away from the central hub) so the head + camera sees the room's landmark after arriving. + """ + + name: str + aliases: tuple[str, ...] + bounds: tuple[float, float, float, float] + target: tuple[float, float, float] + + def contains(self, x: float, y: float) -> bool: + xmin, xmax, ymin, ymax = self.bounds + return xmin <= x <= xmax and ymin <= y <= ymax + + @property + def center(self) -> tuple[float, float]: + xmin, xmax, ymin, ymax = self.bounds + return ((xmin + xmax) / 2.0, (ymin + ymax) / 2.0) + + +@dataclass(frozen=True) +class PlaceRecord: + """One entry of the places memory, whatever its kind. + + ``x``/``y``/``yaw`` are the navigation target of the place: the object's + own position for objects and tagged spots, the room's ``target`` for rooms. + ``bounds`` is only set for rooms. + """ + + kind: str + name: str + aliases: tuple[str, ...] + x: float + y: float + yaw: float + bounds: tuple[float, float, float, float] | None = None + + @property + def is_room(self) -> bool: + return self.kind == "room" + + def distance_to(self, x: float, y: float) -> float: + return math.hypot(self.x - x, self.y - y) + + +MICRODUCK_ROOMS: dict[str, RoomSpec] = { + "kitchen": RoomSpec( + name="kitchen", + aliases=("space A",), + bounds=(0.0, 2.0, 0.0, 2.0), + target=(1.2, 1.0, 0.0), + ), + "living": RoomSpec( + name="living", + aliases=("space B", "living room", "lounge"), + bounds=(-2.0, 0.0, 0.0, 2.0), + target=(-1.2, 1.0, 3.14159), + ), + "bedroom": RoomSpec( + name="bedroom", + aliases=("space C",), + bounds=(-2.0, 0.0, -2.0, 0.0), + target=(-1.2, -1.0, 3.14159), + ), + "office": RoomSpec( + name="office", + aliases=("space D", "study"), + bounds=(0.0, 2.0, -2.0, 0.0), + target=(1.2, -1.0, 0.0), + ), +} + +# name -> (x, y) of the static landmark objects in four_room_scene.xml. +MICRODUCK_OBJECTS: dict[str, tuple[float, float]] = { + "red_box": (1.5, 1.5), + "blue_box": (-1.5, 1.5), + "green_cylinder": (-1.5, -1.5), + "yellow_pillar": (1.5, -1.5), + "orange_crate": (0.6, 1.7), +} + + +def scene_id( + rooms: Mapping[str, RoomSpec] | None = None, + objects: Mapping[str, tuple[float, float]] | None = None, +) -> str: + """Stable identifier of a world described by its room / object tables. + + Two blueprints that seed different tables get different ids and therefore + never see each other's records in a shared database; the same tables + always map to the same id, so remembered places survive restarts. + """ + canonical = { + "rooms": { + name: { + "aliases": list(room.aliases), + "bounds": [float(b) for b in room.bounds], + "target": [float(t) for t in room.target], + } + for name, room in sorted((rooms or {}).items()) + }, + "objects": {name: [float(x), float(y)] for name, (x, y) in sorted((objects or {}).items())}, + } + digest = hashlib.sha1(json.dumps(canonical, sort_keys=True).encode()).hexdigest() + return f"scene-{digest[:12]}" + + +# Words that carry no meaning when someone names a place: "the kitchen", +# "space A please", "go to room B", "kitchen area". +_FILLER_WORDS = frozenset( + {"please", "go", "goto", "to", "the", "room", "space", "area", "zone", "spot"} +) +_NON_ALNUM = re.compile(r"[^a-z0-9]+") + + +def normalize_place_query(query: str) -> tuple[str, ...]: + """Return candidate normalised forms of ``query``, most specific first. + + Lower-cases, replaces every run of non-alphanumerics (spaces, underscores, + punctuation) by a single space, then also yields the form with leading and + trailing filler words ("the", "space", "room", "please", "go to", ...) + stripped. ``"Space a"`` -> ``("space a", "a")`` and ``"the kitchen + please"`` -> ``("the kitchen please", "kitchen")``. Empty forms are + dropped. + """ + base = _NON_ALNUM.sub(" ", query.lower()).strip() + forms: list[str] = [] + if base: + forms.append(base) + words = base.split() + while words and words[0] in _FILLER_WORDS: + words.pop(0) + while words and words[-1] in _FILLER_WORDS: + words.pop() + stripped = " ".join(words) + if stripped and stripped not in forms: + forms.append(stripped) + return tuple(forms) + + +def place_key(name: str) -> str: + """The identity of a place name: its normalised form (case, punctuation + and spacing folded), so ``"Charger"``, ``"charger"`` and ``"charger!"`` + are one place and re-tagging overwrites instead of duplicating. + """ + forms = normalize_place_query(name) + return forms[0] if forms else name.strip().lower() + + +def _contains_words(haystack: str, needle: str) -> bool: + """Whole-word containment on already normalised strings.""" + return f" {needle} " in f" {haystack} " + + +def _yaw_quaternion(yaw: float) -> Quaternion: + return Quaternion(0.0, 0.0, math.sin(yaw / 2.0), math.cos(yaw / 2.0)) + + +class PlacesMemory: + """dimos memory-backed place store: SqliteStore stream ``places`` of RobotLocation. + + Every place is one :class:`RobotLocation` observation tagged with + ``{"kind": kind, "name": name, "scene": scene}`` and carrying its pose so + the store's spatial index (``near``) works. Appending is the only write + the store offers, so re-adding a name appends a newer record and readers + keep the latest per ``(kind, place_key(name))`` - the newest record also + decides the display name, so re-tagging "charger" as "Charger" renames + rather than duplicates; :meth:`seed` skips records that already exist + unchanged, which makes it idempotent across restarts. + + All reads are restricted to ``scene`` (see :func:`scene_id`), so a + database shared by several blueprints never shows one world's rooms, + objects or remembered spots to another. + """ + + STREAM = "places" + + def __init__(self, db_path: str | Path, scene: str = DEFAULT_SCENE) -> None: + self.db_path = Path(db_path).expanduser() + self.db_path.parent.mkdir(parents=True, exist_ok=True) + self.scene = str(scene) or DEFAULT_SCENE + self._lock = threading.RLock() + self._store = SqliteStore(path=str(self.db_path)) + self._stream = self._store.stream(self.STREAM, RobotLocation) + + def _view(self) -> Any: + """The stream restricted to this memory's scene (tag filter, pushed to SQL).""" + return self._stream.tags(scene=self.scene) + + # -- lifecycle ----------------------------------------------------------- + + def close(self) -> None: + with self._lock: + self._store.stop() + + def __enter__(self) -> PlacesMemory: + return self + + def __exit__(self, *exc: object) -> None: + self.close() + + # -- writes -------------------------------------------------------------- + + def seed( + self, + rooms: Mapping[str, RoomSpec] | None = None, + objects: Mapping[str, tuple[float, float]] | None = None, + ) -> int: + """Insert the static rooms and objects that are not already stored. + + Idempotent by ``(kind, name)`` within this scene: a record is only + appended when no record with that key exists or the stored one + differs (moved object, changed alias list). Returns the number of + records written. + """ + written = 0 + with self._lock: + existing = {(r.kind, place_key(r.name)): r for r in self.all()} + for name, room in (rooms or {}).items(): + x, y, yaw = room.target + record = PlaceRecord( + "room", name, tuple(room.aliases), float(x), float(y), float(yaw), room.bounds + ) + if existing.get(("room", place_key(name))) != record: + self.add( + name, + x, + y, + yaw, + kind="room", + metadata={ + "aliases": list(room.aliases), + "bounds": list(room.bounds), + "target": list(room.target), + }, + ) + written += 1 + for name, (x, y) in (objects or {}).items(): + record = PlaceRecord("object", name, (), float(x), float(y), 0.0, None) + if existing.get(("object", place_key(name))) != record: + self.add(name, x, y, 0.0, kind="object") + written += 1 + return written + + def add( + self, + name: str, + x: float, + y: float, + yaw: float = 0.0, + *, + kind: str = "tagged", + metadata: dict[str, Any] | None = None, + ) -> RobotLocation: + """Persist a place in this scene and return the stored :class:`RobotLocation`.""" + x, y, yaw = float(x), float(y), float(yaw) + meta: dict[str, Any] = { + "kind": kind, + "scene": self.scene, + "aliases": [], + "bounds": None, + "target": None, + } + meta.update(metadata or {}) + location = RobotLocation( + name=name, + position=(x, y, 0.0), + rotation=(0.0, 0.0, yaw), + frame_id=PLACES_FRAME, + metadata=meta, + ) + pose = PoseStamped( + ts=location.timestamp, + frame_id=PLACES_FRAME, + position=Vector3(x, y, 0.0), + orientation=_yaw_quaternion(yaw), + ) + with self._lock: + self._stream.append( + location, + ts=location.timestamp, + pose=pose, + tags={"kind": kind, "name": name, "scene": self.scene}, + ) + return location + + # -- reads --------------------------------------------------------------- + + @staticmethod + def _to_record(location: RobotLocation) -> PlaceRecord: + meta = location.metadata or {} + kind = str(meta.get("kind", "tagged")) + bounds = meta.get("bounds") + return PlaceRecord( + kind=kind, + name=location.name, + aliases=tuple(str(a) for a in (meta.get("aliases") or ())), + x=float(location.position[0]), + y=float(location.position[1]), + yaw=float(location.rotation[2]), + bounds=tuple(float(b) for b in bounds) if bounds else None, # type: ignore[arg-type] + ) + + @staticmethod + def _latest_per_key(observations: list[Any]) -> list[PlaceRecord]: + """Collapse duplicates to the newest record per ``(kind, place_key(name))``. + + Keeps first-insertion order so seeded rooms list in display order; + the newest record supplies the display name. + """ + by_key: dict[tuple[str, str], tuple[float, int, PlaceRecord]] = {} + for obs in sorted(observations, key=lambda o: (o.ts, o.id)): + record = PlacesMemory._to_record(obs.data) + key = (record.kind, place_key(record.name)) + if key in by_key: + by_key[key] = (by_key[key][0], by_key[key][1], record) + else: + by_key[key] = (obs.ts, obs.id, record) + return [rec for _ts, _id, rec in sorted(by_key.values(), key=lambda v: (v[0], v[1]))] + + def all(self) -> list[PlaceRecord]: + """Every place of this scene: rooms first (insertion order), then objects, then tagged.""" + with self._lock: + records = self._latest_per_key(self._view().to_list()) + order = {"room": 0, "object": 1} + return sorted(records, key=lambda r: order.get(r.kind, 2)) + + def rooms(self) -> list[PlaceRecord]: + return [r for r in self.all() if r.kind == "room"] + + def matches(self, query: str, kind: str | None = None) -> list[PlaceRecord]: + """Every record a free-form place name could mean. + + An exact match on a canonical name or an alias (after + :func:`normalize_place_query`) yields exactly that record. Otherwise + whole-word containment either way is tried and *all* distinct hits + are returned in table order - so ``"box"`` yields ``red_box`` and + ``blue_box`` and the caller can ask which one was meant. A query made + of filler words only (``"room"``, ``"space"``) matches nothing by + containment. + """ + forms = normalize_place_query(query) + if not forms: + return [] + records = [r for r in self.all() if kind is None or r.kind == kind] + # (normalised key, record, key-is-canonical-name). Aliases contribute + # their filler-stripped form too, so "room A" / "A" hit "space A". + keyed: list[tuple[str, PlaceRecord, bool]] = [] + for record in records: + for key in normalize_place_query(record.name): + keyed.append((key, record, True)) + for alias in record.aliases: + for key in normalize_place_query(alias): + keyed.append((key, record, False)) + + for form in forms: + for key, record, is_name in keyed: + if is_name and key == form: + return [record] + for key, record, is_name in keyed: + if not is_name and key == form: + return [record] + + if all(word in _FILLER_WORDS for word in forms[0].split()): + return [] # "room" / "space" alone would hit every alias + + for form in forms: + hits: dict[tuple[str, str], PlaceRecord] = {} + for key, record, _ in keyed: + if len(key) < 3: + continue # "a" from "space A" would match everything + if _contains_words(form, key) or _contains_words(key, form): + hits.setdefault((record.kind, record.name), record) + if hits: + return list(hits.values()) + return [] + + def find(self, query: str, kind: str | None = None) -> PlaceRecord | None: + """Resolve a free-form place name to one record, or ``None``. + + Matching order: exact canonical name, exact alias, then whole-word + containment either way. ``"space A"``, ``"Space a"``, ``"the + kitchen"`` and ``"kitchen please"`` all resolve to the kitchen; + ``"red box"`` resolves to ``red_box``. A query that only matches by + containment and fits several places (``"box"``) is ambiguous and + yields ``None``; use :meth:`matches` to list the candidates. + """ + candidates = self.matches(query, kind) + return candidates[0] if len(candidates) == 1 else None + + def room_at(self, x: float, y: float) -> PlaceRecord | None: + """The room containing ``(x, y)``; ``None`` in the open hub or outside.""" + if abs(x) < HUB_HALF_EXTENT and abs(y) < HUB_HALF_EXTENT: + return None + for record in self.rooms(): + if record.bounds is None: + continue + xmin, xmax, ymin, ymax = record.bounds + if xmin <= x <= xmax and ymin <= y <= ymax: + return record + return None + + def near(self, x: float, y: float, radius: float) -> list[PlaceRecord]: + """Places of this scene whose pose lies within ``radius`` metres, nearest first.""" + with self._lock: + observations = self._view().near((float(x), float(y), 0.0), float(radius)).to_list() + records = self._latest_per_key(observations) + return sorted(records, key=lambda r: r.distance_to(x, y)) + + def to_json(self, t: float | None = None) -> str: + """The ``places`` channel payload (compact JSON, world frame).""" + rooms: list[dict[str, Any]] = [] + objects: list[dict[str, Any]] = [] + tagged: list[dict[str, Any]] = [] + for record in self.all(): + if record.kind == "room": + rooms.append( + { + "name": record.name, + "aliases": list(record.aliases), + "bounds": list(record.bounds or ()), + "target": [record.x, record.y, record.yaw], + } + ) + elif record.kind == "object": + objects.append({"name": record.name, "x": record.x, "y": record.y}) + else: + tagged.append( + {"name": record.name, "x": record.x, "y": record.y, "yaw": record.yaw} + ) + payload = { + "frame": PLACES_FRAME, + "rooms": rooms, + "objects": objects, + "tagged": tagged, + "t": time.time() if t is None else t, + } + return json.dumps(payload, separators=(",", ":")) diff --git a/dimos/robot/pollen/microduck/policies.py b/dimos/robot/pollen/microduck/policies.py new file mode 100644 index 0000000000..0f3c08ca92 --- /dev/null +++ b/dimos/robot/pollen/microduck/policies.py @@ -0,0 +1,909 @@ +# Copyright 2025-2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Microduck policy catalogue, bank and scheduler. + +Three layers, each usable without the one above it: + +* ``PolicyName`` / ``POLICY_SPECS`` - the nine published Microduck ONNX + policies: kind (base / posture / oneshot), file name, the robot variant(s) + they physically work on, and their timing constants. +* ``PolicyBank`` - one ONNX session per loadable policy, all sharing a single + ``MicroduckObserver`` and ``last_action`` slot (valid across switches: every + published net declares the same joint order and home pose), stepped at the + policies' 50 Hz. +* ``PolicyScheduler`` - the pure-python state machine that decides which + policy runs with which 13-float command ``[vx, vy, wz, head(4), body(6)]``: + base selection (with roller braking), sit/stand posture, timed one-shots + (kicks, roulade, ground pick), fall handling and the ``policy_state`` JSON + snapshot for the cockpit. Requests arrive from any thread, are resolved + against the state they were made in and parked in a single pending slot; + ``tick()`` runs on the sim thread only. + +Command layouts, windows and the variant matrix follow what was measured in +MuJoCo with the published policies (kicks: zero command for 0.5 s with the +ball at a yaw-frame offset; roulade: 2.2 s window with the fall detector +suspended; ground pick: phase-encoded command until phase 0.7; roller: zero +throttle for 2 s before handing over to a legged policy) and upstream's +``scripts/infer_policy.py``. +""" + +from __future__ import annotations + +from collections.abc import Callable, Iterable, Mapping +from copy import copy +from dataclasses import dataclass +import math +from pathlib import Path +import threading +import time +from typing import TYPE_CHECKING, Any + +import numpy as np +from numpy.typing import NDArray + +from dimos.robot.pollen.microduck.gait import ( + COMMAND_LEN, + VX_RANGE, + VY_RANGE, + WZ_RANGE, + MicroduckObserver, + PolicySession, + load_policy_session, +) +from dimos.utils.logging_config import setup_logger + +logger = setup_logger() + +try: + from enum import StrEnum +except ImportError: # pragma: no cover - Python 3.10 floor + from enum import Enum + + class StrEnum(str, Enum): # type: ignore[no-redef] + """Minimal ``enum.StrEnum`` stand-in for Python 3.10.""" + + def __str__(self) -> str: + return str(self.value) + + +if TYPE_CHECKING: + import mujoco + + +class PolicyName(StrEnum): + """The published Microduck policies, in cockpit display order.""" + + WALK = "walk" + STAND = "stand" + ROLLER = "roller" + ROLLER_CROUCH = "roller_crouch" + SITSTAND = "sitstand" + KICK_LEFT = "kick_left" + KICK_RIGHT = "kick_right" + ROULADE = "roulade" + GROUND_PICK = "ground_pick" + + +class PolicyKind(StrEnum): + BASE = "base" # runs until another base is selected; walk/roller follow the twist + POSTURE = "posture" # sitstand: seated until asked to stand back up + ONESHOT = "oneshot" # timed trick, then back to the base + + +# Robot MJCF variants. Roller policies need the wheeled model; sitstand falls +# over on it (the duck rolls away while seated) and so does roulade (the roll +# completes but the duck topples within a second of the walk hand-over, at +# any window length from 2.2 to 4 s - measured in the headless matrix). +DEFAULT_VARIANT = "default" +ROLLERS_VARIANT = "rollers" +VARIANTS: tuple[str, ...] = (DEFAULT_VARIANT, ROLLERS_VARIANT) + +_ALL_VARIANTS = frozenset(VARIANTS) +_ROLLERS_ONLY = frozenset({ROLLERS_VARIANT}) +_DEFAULT_ONLY = frozenset({DEFAULT_VARIANT}) + +# Timing (sim seconds; advanced by tick(dt), never by wall clock). +KICK_DURATION_S = 0.5 +ROULADE_DURATION_S = 2.2 # 2.0-2.5 s valid; shorter leaves the duck on its back +ROULADE_GRACE_S = 1.0 # fall detector stays suspended this long after the roll +GROUND_PICK_PERIOD_S = 4.0 # phase += dt / period, command = [cos 2pi*phi, sin 2pi*phi] +GROUND_PICK_END_PHASE = 0.7 # upstream runtime cut-off (~2.8 s) +ROLLER_CROUCH_PERIOD_S = 3.0 # same phase encoding, looping while the base is selected +STAND_UP_DURATION_S = 2.0 # sitstand with cmd[0]=0 before handing back to the base +# Leaving the wheels for a legged base: keep the roller-family base that is +# being left for this long first - ``roller`` at ROLLER_BRAKE_THROTTLE, +# ``roller_crouch`` on its phase loop - so the duck is not walking at speed. +# This is open-loop and the legged policies cannot hold free wheels, so a +# residual glide survives the hand-over. Measured headless on the rollers +# model (brake window + the 3 s after walk takes over): from a 0.8 m/s push, +# zero throttle for 2 s hands over at <=0.3 m/s and the duck still travels +# 0.7-1.5 m forward; a roller that merely crept from rest (0.2 m/s) travels +# ~0.65 m; leaving roller_crouch ~0.8 m. It never reverses and never falls. +# The design minimum of 1 s hands over at up to 0.57 m/s and lets 1.1-2.1 m +# through. A negative throttle (-0.3) sheds speed faster but is reverse +# thrust when there is no speed to shed: 0.7-0.95 m *backwards* from rest, +# which is why it is not used. +BRAKE_DURATION_S = 2.0 +ROLLER_BRAKE_THROTTLE = 0.0 + +# Roller command ranges (microduck_rl velocity_rollers task): throttle +# 0 = coast, >0 = push, <0 = brake; slot 2 is a heading error in rad. +ROLLER_THROTTLE_RANGE = (-0.5, 0.6) +ROLLER_HEADING_RANGE = (-1.0, 1.0) + +# Ball spawn offsets in the trunk yaw frame (x forward, y left), matching the +# kick tasks' reset_ball_in_front_of_foot. +KICK_BALL_OFFSETS: dict[str, tuple[float, float]] = { + str(PolicyName.KICK_LEFT): (0.09, 0.042), + str(PolicyName.KICK_RIGHT): (0.09, -0.042), +} + +# Pseudo-states reported in ``active`` while the scheduler is between policies. +ACTIVE_BRAKING = "braking" +ACTIVE_STANDING_UP = "standing_up" + +# Fall detector threshold on the trunk's projected gravity z (-1 = upright, +# +1 = upside down): past this the duck is lying down. The sim module owns +# the detector (with its own debounce); the scheduler only consumes its +# verdict through ``notify_fall``. +FALL_GRAVITY_Z = -0.55 + +# A rejected request stays visible in snapshot()["last_error"] this long. +LAST_ERROR_TTL_S = 5.0 + +ACTIONS: tuple[str, ...] = ("start", "stop", "toggle") + + +@dataclass(frozen=True) +class PolicySpec: + name: PolicyName + kind: PolicyKind + onnx: str + variants: frozenset[str] + duration: float | None = None # oneshots: sim seconds before handing back + + +POLICY_SPECS: dict[PolicyName, PolicySpec] = { + spec.name: spec + for spec in ( + PolicySpec(PolicyName.WALK, PolicyKind.BASE, "alpha_walking.onnx", _ALL_VARIANTS), + PolicySpec(PolicyName.STAND, PolicyKind.BASE, "alpha_stand.onnx", _ALL_VARIANTS), + PolicySpec(PolicyName.ROLLER, PolicyKind.BASE, "roller.onnx", _ROLLERS_ONLY), + PolicySpec(PolicyName.ROLLER_CROUCH, PolicyKind.BASE, "roller_crouch.onnx", _ROLLERS_ONLY), + PolicySpec(PolicyName.SITSTAND, PolicyKind.POSTURE, "alpha_sitstand.onnx", _DEFAULT_ONLY), + PolicySpec( + PolicyName.KICK_LEFT, + PolicyKind.ONESHOT, + "ball_kick_left.onnx", + _ALL_VARIANTS, + KICK_DURATION_S, + ), + PolicySpec( + PolicyName.KICK_RIGHT, + PolicyKind.ONESHOT, + "ball_kick_right.onnx", + _ALL_VARIANTS, + KICK_DURATION_S, + ), + PolicySpec( + PolicyName.ROULADE, + PolicyKind.ONESHOT, + "roulade.onnx", + _DEFAULT_ONLY, + ROULADE_DURATION_S, + ), + PolicySpec( + PolicyName.GROUND_PICK, + PolicyKind.ONESHOT, + "alpha_ground_pick.onnx", + _ALL_VARIANTS, + GROUND_PICK_PERIOD_S * GROUND_PICK_END_PHASE, + ), + ) +} + +POLICY_NAMES: tuple[str, ...] = tuple(str(name) for name in POLICY_SPECS) +# Plain-str spellings for the names the scheduler stores and emits (keeps +# snapshot()/tick() free of enum members). +_WALK = str(PolicyName.WALK) +_ROLLER = str(PolicyName.ROLLER) +_ROLLER_CROUCH = str(PolicyName.ROLLER_CROUCH) +_SITSTAND = str(PolicyName.SITSTAND) +_ROULADE = str(PolicyName.ROULADE) +_GROUND_PICK = str(PolicyName.GROUND_PICK) +BASE_POLICIES: frozenset[str] = frozenset( + str(n) for n, s in POLICY_SPECS.items() if s.kind is PolicyKind.BASE +) +ONESHOT_POLICIES: frozenset[str] = frozenset( + str(n) for n, s in POLICY_SPECS.items() if s.kind is PolicyKind.ONESHOT +) +_ROLLER_FAMILY: frozenset[str] = frozenset({_ROLLER, _ROLLER_CROUCH}) + +ASSET_MISSING_REASON = "asset missing" + +# What a request resolves to (the pending slot holds one of these, never the +# raw (policy, action) pair - a "toggle" that meant "abort" when it was made +# must not turn into "start" because the oneshot finished before the tick). +_OP_SELECT_BASE = "select_base" # target: base name +_OP_SIT = "sit" +_OP_STAND_UP = "stand_up" +_OP_START_ONESHOT = "start_oneshot" # target: oneshot name +_OP_ABORT_ONESHOT = "abort_oneshot" # target: oneshot name +_OP_ABORT_ANY = "abort_any" # policy-less stop + + +@dataclass(frozen=True) +class _Op: + kind: str + target: str = "" # policy name for select_base / start_oneshot / abort_oneshot + + +# Resolution result meaning "empty the pending slot": the request undoes the +# operation still parked there (a stop right behind an unconsumed start, a +# start right behind an unconsumed abort). Never stored in the slot itself. +_CANCEL = _Op("cancel") + + +def policy_availability(variant: str, missing: Iterable[str] = ()) -> dict[str, str | None]: + """name -> ``None`` (runnable) or the reason it is not, for every policy. + + ``missing`` lists policies whose ONNX file could not be obtained; they are + reported as ``"asset missing"``. + """ + if variant not in _ALL_VARIANTS: + raise ValueError(f"unknown Microduck variant {variant!r}; expected one of {VARIANTS}") + missing_names = {str(m) for m in missing} + out: dict[str, str | None] = {} + for name, spec in POLICY_SPECS.items(): + key = str(name) + if key in missing_names: + out[key] = ASSET_MISSING_REASON + elif variant not in spec.variants: + if spec.variants == _ROLLERS_ONLY: + out[key] = "requires the rollers variant (start with MICRODUCK_VARIANT=rollers)" + else: + out[key] = f"not supported on the {variant} variant" + else: + out[key] = None + return out + + +class PolicyBank: + """Every runnable Microduck policy for one robot variant, sharing one observation. + + Loads the ONNX session of each policy that ``policy_availability`` allows + on ``variant`` and whose file exists in ``policy_dir`` (eager; ~800 KB + each). ``step()`` is the 50 Hz control step: the caller decides which + policy runs (``PolicyScheduler.tick``), the bank builds the shared + observation, runs that net and updates the shared ``last_action``. + + Positional order is (policy_dir, model), like ``MicroduckGaitPolicy``:: + + assets = ensure_assets(variant) + bank = PolicyBank(assets.policy_dir, model, variant=variant, missing=assets.missing) + sched = PolicyScheduler(bank.availability, variant, spawn_ball=...) + # sim thread, 50 Hz: + name, command = sched.tick(dt) + targets = bank.step(name, command, data) + """ + + def __init__( + self, + policy_dir: str | Path, + model: mujoco.MjModel, + *, + variant: str = DEFAULT_VARIANT, + missing: Iterable[str] = (), + ) -> None: + self.variant = variant + self.policy_dir = Path(policy_dir) + wanted = policy_availability(variant, missing) + + self._sessions: dict[str, PolicySession] = {} + not_found: list[str] = [] + for name, reason in wanted.items(): + if reason is not None: + continue + path = self.policy_dir / POLICY_SPECS[PolicyName(name)].onnx + if not path.exists(): + not_found.append(name) + continue + self._sessions[name] = load_policy_session(path) + if not self._sessions: + raise RuntimeError(f"no Microduck policy ONNX files found in {self.policy_dir}") + + reference = self._sessions.get(_WALK) or next(iter(self._sessions.values())) + for name, session in self._sessions.items(): + if session.joint_names != reference.joint_names or not np.allclose( + session.default_pose, reference.default_pose, atol=1e-6 + ): + raise RuntimeError( + f"Microduck policy {name} declares a different joint order or home pose " + f"than {reference.path.name}; the policies cannot share one observation" + ) + + self._observer = MicroduckObserver(model, reference.joint_names, reference.default_pose) + self.joint_names: list[str] = list(reference.joint_names) + self.default_pose: NDArray[np.float32] = reference.default_pose + self.root_qpos_adr = self._observer.root_qpos_adr + self.last_action = np.zeros(len(self.joint_names), dtype=np.float32) + + seen = {str(m) for m in missing} | set(not_found) + self.missing: tuple[PolicyName, ...] = tuple( + PolicyName(n) for n in POLICY_NAMES if n in seen + ) + self._availability = policy_availability(variant, self.missing) + + def for_robot(self, model: mujoco.MjModel, prefix: str) -> PolicyBank: + """Bind shared inference sessions to another robot with independent action history.""" + bank = copy(self) + bank._observer = MicroduckObserver( + model, self.joint_names, self.default_pose, prefix=prefix + ) + bank.root_qpos_adr = bank._observer.root_qpos_adr + bank.last_action = np.zeros(len(self.joint_names), dtype=np.float32) + return bank + + @property + def names(self) -> tuple[str, ...]: + """Loaded policies, display order.""" + return tuple(n for n in POLICY_NAMES if n in self._sessions) + + @property + def availability(self) -> dict[str, str | None]: + """What ``PolicyScheduler`` needs: name -> None | reason (incl. missing assets).""" + return dict(self._availability) + + @property + def num_joints(self) -> int: + return len(self.joint_names) + + def has(self, name: str) -> bool: + return str(name) in self._sessions + + def reset(self) -> None: + """Forget the previous action (after a teleport / fall recovery).""" + self.last_action[:] = 0.0 + + def initial_qpos(self, data: mujoco.MjData) -> None: + self._observer.initial_qpos(data) + + def projected_gravity(self, data: mujoco.MjData) -> NDArray[np.float32]: + return self._observer.projected_gravity(data) + + def root_yaw(self, data: mujoco.MjData) -> float: + return self._observer.root_yaw(data) + + def step(self, name: str, command: Any, data: mujoco.MjData) -> NDArray[np.float32]: + """One 50 Hz step of policy ``name``: position targets in policy joint order.""" + session = self._sessions.get(str(name)) + if session is None: + raise KeyError(f"Microduck policy {name!r} is not loaded (loaded: {self.names})") + cmd = np.asarray(command, dtype=np.float32).reshape(-1) + if cmd.shape[0] != COMMAND_LEN: + raise ValueError(f"command must have {COMMAND_LEN} entries, got {cmd.shape[0]}") + obs = self._observer.build(data, self.last_action, cmd) + action = session.run(obs) + self.last_action = action.copy() + return self.default_pose + action * session.action_scale + + +class PolicyScheduler: + """Decides which Microduck policy runs, with which command, and reports it. + + Thread model: ``set_twist`` / ``request`` may be called from any thread + (a request is resolved against the current state into one concrete + operation that fills the single pending slot, the latest wins); ``tick`` + and ``notify_fall`` run on the sim thread and are the only methods that + change the running policy; ``snapshot`` and the properties are safe + anywhere. Time is sim time fed through ``tick(dt)``; the injected + ``clock`` only ages ``last_error``. + + States (``snapshot()["active"]``): a base name (``walk``, ``stand``, + ``roller``, ``roller_crouch``), ``sitstand`` while seated, + ``standing_up`` (sitstand with the posture flag cleared for 2 s), + ``braking`` (the roller-family base being left keeps running for + ``BRAKE_DURATION_S`` - ``roller`` at ``ROLLER_BRAKE_THROTTLE``, + ``roller_crouch`` on its phase loop - before a legged base takes over) + or a running oneshot's name. ``locked`` covers everything but an idle, + upright base. + """ + + def __init__( + self, + available: Mapping[str, str | None], + variant: str, + *, + spawn_ball: Callable[[float, float], None] | None = None, + clock: Callable[[], float] = time.monotonic, + ) -> None: + if variant not in _ALL_VARIANTS: + raise ValueError(f"unknown Microduck variant {variant!r}; expected one of {VARIANTS}") + self._variant = variant + self._available: dict[str, str | None] = { + name: available.get(name, "not loaded") for name in POLICY_NAMES + } + self._spawn_ball = spawn_ball + self._clock = clock + # RLock: spawn_ball runs inside tick() and may read the scheduler back. + self._lock = threading.RLock() + + bases = [n for n in POLICY_NAMES if n in BASE_POLICIES and self._available[n] is None] + if not bases: + raise ValueError("PolicyScheduler needs at least one available base policy") + self._base: str = _WALK if self._available[_WALK] is None else bases[0] + self._active: str = self._base + self._seated = False + self._fallen = False + self._twist: tuple[float, float, float] = (0.0, 0.0, 0.0) + self._pending: _Op | None = None + self._elapsed = 0.0 # sim seconds in the current timed state + self._phase = 0.0 # ground_pick / roller_crouch phase in [0, 1) + self._brake_policy: str | None = None # roller-family base kept while braking + self._brake_target: str | None = None # legged base to switch to afterwards + self._roulade_grace_left = 0.0 + self._last_error: str | None = None + self._last_error_at = 0.0 + + # ------------------------------------------------------------------ inputs + + def set_twist(self, vx: float, vy: float, wz: float) -> None: + """Latest velocity request (already gain-shaped by the caller); any thread.""" + with self._lock: + self._twist = (float(vx), float(vy), float(wz)) + + def request(self, policy: str | None, action: str) -> tuple[bool, str]: + """Queue ``action`` (start | stop | toggle) on ``policy``; any thread. + + ``policy=None`` with ``stop`` aborts whatever is running. The request + is resolved right here against the current state (so a ``toggle`` + means abort or start depending on what is running *now*) and the + resulting operation waits for the next ``tick``. Returns + ``(accepted, reason)``; a rejection also lands in + ``snapshot()["last_error"]``. Only one operation is pending at a + time and the newest intent wins: a newer operation replaces an older + unconsumed one, and a request that reverses the parked operation (a + stop behind a start, a start behind a stop) empties the slot instead. + Accepted no-ops (``"already seated"`` etc.) leave the slot alone. + """ + name = None if policy is None else str(policy) + with self._lock: + ok, reason, op = self._resolve(name, action) + if not ok: + self._set_error(reason) + return ok, reason + self._last_error = None + if op is _CANCEL: + self._pending = None + elif op is not None: + self._pending = op + return ok, reason + + def notify_fall(self, fallen: bool) -> None: + """Sim thread: the fall detector's verdict; ignored while suspended. + + Edge-triggered: the first ``True`` after an upright period aborts a + running oneshot / seat / stand-up / brake and locks the scheduler + (base policy with a zero command) until ``False``; repeats are + no-ops. Pass a debounced verdict (the sim module's tilt timer), not + the raw per-tick tilt, or a one-tick spike aborts tricks and + cancels navigation through ``locked``. + """ + with self._lock: + if not fallen: + self._fallen = False + return + if self._fallen or self._suspend_fall_detector(): + return + self._fallen = True + active = self._active + if active in ONESHOT_POLICIES: + self._end_oneshot(active) + elif self._seated or active == ACTIVE_STANDING_UP: + self._seated = False + self._active = self._base + elif active == ACTIVE_BRAKING: + self._finish_braking() + self._elapsed = 0.0 + self._phase = 0.0 + + # -------------------------------------------------------------- sim thread + + def tick(self, dt: float) -> tuple[str, NDArray[np.float32]]: + """Sim thread only: consume the pending request, return (policy, command13). + + The returned policy is always a real, available policy name (during + ``braking`` the roller-family base being left, during ``standing_up`` + ``sitstand``), so ``PolicyBank.step`` can always run it. + """ + with self._lock: + self._expire() + op, self._pending = self._pending, None + if op is not None: + self._apply(op) + name, command = self._command() + self._accumulate(dt) + return name, command + + # -------------------------------------------------------------- read side + + @property + def variant(self) -> str: + return self._variant + + @property + def active(self) -> str: + with self._lock: + return self._active + + @property + def base(self) -> str: + with self._lock: + return self._base + + @property + def seated(self) -> bool: + with self._lock: + return self._seated + + @property + def fallen(self) -> bool: + with self._lock: + return self._fallen + + @property + def availability(self) -> dict[str, str | None]: + return dict(self._available) + + @property + def suspend_fall_detector(self) -> bool: + """True while the roulade runs and for ``ROULADE_GRACE_S`` after it.""" + with self._lock: + return self._suspend_fall_detector() + + @property + def locked(self) -> bool: + """Oneshot running, seated / standing up, braking, or fallen.""" + with self._lock: + return self._lock_reason() is not None + + def snapshot(self) -> dict[str, Any]: + """The ``policy_state`` JSON payload (see the module docstring).""" + with self._lock: + active = self._active + oneshot = None + if active in ONESHOT_POLICIES: + oneshot = {"name": active, "progress": round(self._progress(active), 2)} + return { + "variant": self._variant, + "active": active, + "base": self._base, + "seated": self._seated, + "fallen": self._fallen, + "locked": self._lock_reason() is not None, + "oneshot": oneshot, + "policies": [ + { + "name": name, + "kind": str(POLICY_SPECS[PolicyName(name)].kind), + "available": self._available[name] is None, + "reason": self._available[name], + } + for name in POLICY_NAMES + ], + "last_error": self._current_error(), + "t": time.time(), + } + + # --------------------------------------------------------------- internals + + def _suspend_fall_detector(self) -> bool: + return self._active == _ROULADE or self._roulade_grace_left > 0.0 + + def _lock_reason(self) -> str | None: + if self._fallen: + return "fallen" + if self._active in ONESHOT_POLICIES: + return f"{self._active} running" + if self._seated: + return "seated" + if self._active == ACTIVE_STANDING_UP: + return "standing up" + if self._active == ACTIVE_BRAKING: + return "braking" + return None + + def _set_error(self, reason: str) -> None: + self._last_error = reason + self._last_error_at = self._clock() + + def _current_error(self) -> str | None: + if self._last_error is not None and self._clock() - self._last_error_at > LAST_ERROR_TTL_S: + self._last_error = None + return self._last_error + + def _progress(self, name: str) -> float: + if name == _GROUND_PICK: + value = self._phase / GROUND_PICK_END_PHASE + else: + duration = POLICY_SPECS[PolicyName(name)].duration or 1.0 + value = self._elapsed / duration + return float(min(max(value, 0.0), 1.0)) + + def _resolve(self, policy: str | None, action: str) -> tuple[bool, str, _Op | None]: + """Turn a request into (accepted, reason, operation) against the current state. + + Pure apart from reading the state. ``operation`` is ``None`` for an + accepted no-op and ``_CANCEL`` when the request reverses what is + still parked in the pending slot - symmetric in both directions: a + stop / toggle behind an unconsumed start (or sit) cancels it, and a + start / toggle behind an unconsumed abort (or stand-up) of the same + thing cancels that abort, so the running oneshot / seat is kept. + """ + if action not in ACTIONS: + return False, f"unknown action {action!r}", None + pending = self._pending + if policy is None: + if action != "stop": + return False, f"{action} needs a policy name", None + if pending is not None and pending.kind in (_OP_START_ONESHOT, _OP_SIT): + return True, "", _CANCEL + if self._active in ONESHOT_POLICIES or self._seated: + return True, "", _Op(_OP_ABORT_ANY) + return True, "", None # nothing to abort; a parked base select survives + if policy not in self._available: + return False, f"unknown policy {policy!r}", None + reason = self._available[policy] + if reason is not None: + return False, f"{policy} unavailable: {reason}", None + + kind = POLICY_SPECS[PolicyName(policy)].kind + lock_reason = self._lock_reason() + + if kind is PolicyKind.BASE: + target = _WALK if action == "stop" else policy + if self._available[target] is not None: + return False, f"{target} unavailable: {self._available[target]}", None + if lock_reason is not None: + return False, f"locked: {lock_reason}", None + return True, "", _Op(_OP_SELECT_BASE, target) + + if kind is PolicyKind.POSTURE: + seated = self._seated + pending_sit = pending is not None and pending.kind == _OP_SIT + pending_stand_up = ( + seated and pending is not None and pending.kind in (_OP_STAND_UP, _OP_ABORT_ANY) + ) + if action == "stop": + if pending_sit: + return True, "", _CANCEL + if seated: + return True, "", _Op(_OP_STAND_UP) + if self._active == ACTIVE_STANDING_UP: + return True, "already standing up", None + return False, "not seated", None + if action == "toggle": + if pending_sit or pending_stand_up: + return True, "", _CANCEL + if seated: + return True, "", _Op(_OP_STAND_UP) + else: # start + if pending_stand_up: + return True, "already seated", _CANCEL + if seated or pending_sit: + return True, "already seated", None + if lock_reason is not None: + return False, f"locked: {lock_reason}", None + return True, "", _Op(_OP_SIT) + + # oneshot + running = self._active == policy + pending_start = pending == _Op(_OP_START_ONESHOT, policy) + pending_abort = ( + running + and pending is not None + and (pending == _Op(_OP_ABORT_ONESHOT, policy) or pending.kind == _OP_ABORT_ANY) + ) + if action == "stop": + if pending_start: + return True, "", _CANCEL + if running: + return True, "", _Op(_OP_ABORT_ONESHOT, policy) + return False, f"{policy} is not running", None + if action == "toggle": + if pending_start or pending_abort: + return True, "", _CANCEL + if running: + return True, "", _Op(_OP_ABORT_ONESHOT, policy) + else: # start + if pending_abort: + return True, "already running", _CANCEL + if running or pending_start: + return True, "already running", None + if lock_reason is not None: + return False, f"locked: {lock_reason}", None + return True, "", _Op(_OP_START_ONESHOT, policy) + + def _apply(self, op: _Op) -> None: + """Run a resolved operation (sim thread, start of a tick). + + Stops whose subject already ended on its own (the oneshot expired, + the duck stood up) are silently dropped. Starts are re-checked + against the lock because a fall may have beaten them to the tick; + that late rejection is reported through ``last_error``. + """ + if op.kind == _OP_ABORT_ANY: + if self._active in ONESHOT_POLICIES: + self._end_oneshot(self._active) + elif self._seated: + self._begin_standing_up() + return + if op.kind == _OP_ABORT_ONESHOT: + if self._active == op.target: + self._end_oneshot(op.target) + return + if op.kind == _OP_STAND_UP: + if self._seated: + self._begin_standing_up() + return + + if op.kind == _OP_SELECT_BASE and op.target == self._base and self._active == self._base: + return + if op.kind == _OP_SIT and self._seated: + return + if op.kind == _OP_START_ONESHOT and self._active == op.target: + return + lock_reason = self._lock_reason() + if lock_reason is not None: + self._set_error(f"locked: {lock_reason}") + return + if op.kind == _OP_SELECT_BASE: + self._select_base(op.target) + elif op.kind == _OP_SIT: + self._sit() + elif op.kind == _OP_START_ONESHOT: + self._start_oneshot(op.target) + + def _select_base(self, target: str) -> None: + if target == self._base: + return + if self._base in _ROLLER_FAMILY and target not in _ROLLER_FAMILY: + # Leaving the wheels at speed drops the duck: keep the base being + # left (always a loaded policy) for BRAKE_DURATION_S first. + self._active = ACTIVE_BRAKING + self._brake_policy = self._base + self._brake_target = target + self._elapsed = 0.0 + return + self._base = target + self._active = target + self._phase = 0.0 + + def _finish_braking(self) -> None: + self._base = self._brake_target or _WALK + self._brake_policy = None + self._brake_target = None + self._active = self._base + self._elapsed = 0.0 + self._phase = 0.0 + + def _sit(self) -> None: + self._seated = True + self._active = _SITSTAND + self._elapsed = 0.0 + + def _begin_standing_up(self) -> None: + self._seated = False + self._active = ACTIVE_STANDING_UP + self._elapsed = 0.0 + + def _start_oneshot(self, name: str) -> None: + self._active = name + self._elapsed = 0.0 + self._phase = 0.0 + offset = KICK_BALL_OFFSETS.get(name) + if offset is not None and self._spawn_ball is not None: + # The callback pokes MuJoCo from inside the engine's step hook; a + # bug there must not take the sim loop down - the kick still runs. + try: + self._spawn_ball(*offset) + except Exception: + logger.exception("Microduck ball spawn failed; kicking without a ball", policy=name) + + def _end_oneshot(self, name: str) -> None: + self._active = self._base + self._elapsed = 0.0 + self._phase = 0.0 + if name == _ROULADE: + self._roulade_grace_left = ROULADE_GRACE_S + + def _phase_command(self, cmd: NDArray[np.float32]) -> None: + """ground_pick / roller_crouch encode their progress as (cos, sin) of the phase.""" + cmd[0] = math.cos(2.0 * math.pi * self._phase) + cmd[1] = math.sin(2.0 * math.pi * self._phase) + + def _brake_with(self) -> str: + """The loaded roller-family policy kept running while ``braking``.""" + return self._brake_policy or self._base + + def _command(self) -> tuple[str, NDArray[np.float32]]: + """(policy to run, command13) for the current state.""" + cmd = np.zeros(COMMAND_LEN, dtype=np.float32) + active = self._active + if active == ACTIVE_BRAKING: + policy = self._brake_with() + if policy == _ROLLER_CROUCH: + self._phase_command(cmd) + else: + cmd[0] = ROLLER_BRAKE_THROTTLE + return policy, cmd + if active == ACTIVE_STANDING_UP: + return _SITSTAND, cmd # posture flag 0 = stand + if active == _SITSTAND: + cmd[0] = 1.0 # posture flag 1 = sit + return active, cmd + if active in ONESHOT_POLICIES: + if active == _GROUND_PICK: + self._phase_command(cmd) + return active, cmd + # A base: twist-driven unless locked (fallen). + if active == _ROLLER_CROUCH: + self._phase_command(cmd) + return active, cmd + if self._fallen: + return active, cmd + vx, vy, wz = self._twist + if active == _WALK: + cmd[0] = float(np.clip(vx, *VX_RANGE)) + cmd[1] = float(np.clip(vy, *VY_RANGE)) + cmd[2] = float(np.clip(wz, *WZ_RANGE)) + elif active == _ROLLER: + cmd[0] = float(np.clip(vx, *ROLLER_THROTTLE_RANGE)) + cmd[2] = float(np.clip(wz, *ROLLER_HEADING_RANGE)) + # stand ignores the twist + return active, cmd + + def _oneshot_done(self, name: str) -> bool: + if name == _GROUND_PICK: + return self._phase >= GROUND_PICK_END_PHASE - 1e-9 + duration = POLICY_SPECS[PolicyName(name)].duration or 0.0 + return self._elapsed >= duration - 1e-9 + + def _expire(self) -> None: + """Leave timed states whose window elapsed (start of a tick).""" + active = self._active + if active == ACTIVE_BRAKING: + if self._elapsed >= BRAKE_DURATION_S - 1e-9: + self._finish_braking() + elif active == ACTIVE_STANDING_UP: + if self._elapsed >= STAND_UP_DURATION_S - 1e-9: + self._active = self._base + self._elapsed = 0.0 + elif active in ONESHOT_POLICIES and self._oneshot_done(active): + self._end_oneshot(active) + + def _accumulate(self, dt: float) -> None: + """Age the timed states by ``dt`` sim seconds (after the command was built).""" + if self._roulade_grace_left > 0.0: + self._roulade_grace_left = max(0.0, self._roulade_grace_left - dt) + active = self._active + if active in (ACTIVE_BRAKING, ACTIVE_STANDING_UP) or active in ONESHOT_POLICIES: + self._elapsed += dt + if active == _GROUND_PICK: + self._phase += dt / GROUND_PICK_PERIOD_S + elif active == _ROLLER_CROUCH or ( + active == ACTIVE_BRAKING and self._brake_with() == _ROLLER_CROUCH + ): + self._phase = (self._phase + dt / ROLLER_CROUCH_PERIOD_S) % 1.0 diff --git a/dimos/robot/pollen/microduck/sim_module.py b/dimos/robot/pollen/microduck/sim_module.py new file mode 100644 index 0000000000..f1098a887c --- /dev/null +++ b/dimos/robot/pollen/microduck/sim_module.py @@ -0,0 +1,607 @@ +# Copyright 2025-2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Microduck MuJoCo simulation module. + +``MicroduckSimModule`` extends ``MujocoSimModule`` with an in-process RL +locomotion layer: every pretrained Microduck policy (ONNX, 50 Hz) is loaded +into a ``PolicyBank`` and a ``PolicyScheduler`` decides which one runs and +with which command. The position targets drive the MJCF's servo actuators +through the engine's normal command path. No ControlCoordinator or SHM +adapter is involved - the whole robot fits in one module: + + cmd_vel (Twist) ------------> PolicyScheduler --> PolicyBank --> targets --> MuJoCo + policy_request (JSON str) ----^ | + v + policy_state (JSON str) + +Everything else (odom, tf, IMU, head camera, raycast lidar pointcloud) is +inherited from ``MujocoSimModule``. On top of that this module renders a +third-person chase camera (``chase_image``) and drops the kickable ball the +kick policies aim at into the scene. + +JSON contracts (the state's fields are documented in ``policies.py``):: + + policy_request {"action": "start" | "stop" | "toggle", "policy": "", "t": } + ("policy" may be absent for a bare "stop": abort whatever runs) + policy_state PolicyScheduler.snapshot(), published whenever it changes + and at least every 1 / state_hz seconds +""" + +from __future__ import annotations + +import json +import math +from pathlib import Path +import time +from typing import Any + +import mujoco +import numpy as np +from reactivex.disposable import Disposable + +from dimos.core.core import rpc +from dimos.core.stream import In, Out +from dimos.msgs.geometry_msgs.Twist import Twist +from dimos.msgs.sensor_msgs.Image import Image, ImageFormat +from dimos.msgs.sensor_msgs.JointState import JointState +from dimos.robot.pollen.microduck import assets_fetch +from dimos.robot.pollen.microduck.gait import CONTROL_DT +from dimos.robot.pollen.microduck.places import BALL_BODY, BALL_RADIUS, add_ball_body +from dimos.robot.pollen.microduck.policies import ( + DEFAULT_VARIANT, + FALL_GRAVITY_Z, + PolicyBank, + PolicyScheduler, +) +from dimos.simulation.engines.mujoco_engine import MujocoEngine +from dimos.simulation.engines.mujoco_sim_module import ( + MujocoSimModule, + MujocoSimModuleConfig, +) +from dimos.utils.logging_config import setup_logger + +logger = setup_logger() + +# One physics step is 5 ms (forced below); the policy runs every 4th step. +PHYSICS_TIMESTEP = 0.005 +POLICY_DECIMATION = 4 + +# Trunk-mounted raycast-lidar cameras, mirroring the G1 sim convention +# (front/left/right, wide fovy). Positions are in the trunk_base frame of a +# ~25 cm tall robot whose trunk origin sits ~12 cm above ground. +LIDAR_CAMERA_SPECS: tuple[tuple[str, float], ...] = ( + ("lidar_front_camera", 0.0), + ("lidar_left_camera", math.radians(120.0)), + ("lidar_right_camera", math.radians(-120.0)), +) +_LIDAR_CAM_POS = (0.0, 0.0, 0.08) +_LIDAR_CAM_FOVY = 140.0 + +# Forward-facing first-person camera: what the duck sees. +# +# The robot MJCF ships a `head_camera` on the jaw, but it is rotated +# `quat="0 0 -1 0"` and so looks along body -x - backwards, straight into the +# duck's own jaw (measured: 16 of its own geoms in front of the lens, the +# nearest at 0.000 m). Anything rendering it gets a close-up of the inside of +# a beak, including the agent's `observe` skill. This camera sits at the same +# place, on the trunk (steadier than the jaw, which the head policies move) +# and pointing where the duck walks. +POV_CAMERA_NAME = "pov_camera" +# The stock head_camera's position expressed in the trunk frame. +_POV_CAM_POS = (0.081, 0.0, 0.131) +_POV_CAM_FOVY = 70.0 + +# Third-person camera following the trunk; published on ``chase_image``. +CHASE_CAMERA_NAME = "chase_camera" +_CHASE_OPTICAL_FRAME = f"{CHASE_CAMERA_NAME}_optical_frame" + + +def _camera_quat_wxyz(yaw: float, pitch: float = 0.0) -> tuple[float, float, float, float]: + """MuJoCo camera quat looking along `yaw` (0 = body +x), tilted by `pitch`. + + ``pitch`` is in radians, negative = looking down; ``|pitch|`` must stay + below 90 degrees. A MuJoCo camera looks along its -z axis with +y up. + Columns of the rotation are the camera axes in the parent frame. + """ + cp = math.cos(pitch) + fwd = np.array([cp * math.cos(yaw), cp * math.sin(yaw), math.sin(pitch)]) + right = np.cross(fwd, (0.0, 0.0, 1.0)) # right = forward x up ... kept right-handed + right /= np.linalg.norm(right) + up = np.cross(right, fwd) + rot = np.stack([right, up, -fwd], axis=1) # columns: camera x, y, z (looks along -z) + from scipy.spatial.transform import Rotation as R + + x, y, z, w = R.from_matrix(rot).as_quat() + return (float(w), float(x), float(y), float(z)) + + +def _ball_spawn_xy( + trunk_x: float, trunk_y: float, trunk_yaw: float, dx: float, dy: float +) -> tuple[float, float]: + """World x/y of a point ``(dx, dy)`` in the trunk's yaw frame (x forward, y left).""" + cy, sy = math.cos(trunk_yaw), math.sin(trunk_yaw) + return (trunk_x + cy * dx - sy * dy, trunk_y + sy * dx + cy * dy) + + +def _state_key(snapshot: dict[str, Any]) -> dict[str, Any]: + """A ``policy_state`` snapshot without its timestamp, for change detection.""" + return {key: value for key, value in snapshot.items() if key != "t"} + + +def _shape_twist( + config: MicroduckSimModuleConfig, vx: float, vy: float, wz: float +) -> tuple[float, float, float]: + """Map a requested twist onto the walking policy's command. + + Gains compensate the policy's velocity-tracking undershoot. The policy + also has a yaw deadband: pure-turn commands below the top of its range + barely rotate the robot (measured in sim: ~3-9 deg/s at 1.0 rad/s vs + 25-31 deg/s at 1.5; while stepping forward it mostly vanishes, 23-27 + deg/s at 1.0 and 31-43 deg/s at 1.5, no falls). The planner's + rotate-in-place twists (0.2-0.3 rad/s) land around 0.7 after the gain, + so any real turn request is bumped to ``min_effective_wz``. + """ + vx *= config.cmd_gain_linear + vy *= config.cmd_gain_linear + wz *= config.cmd_gain_angular + if 0.05 < abs(wz) < config.min_effective_wz: + wz = math.copysign(config.min_effective_wz, wz) + return vx, vy, wz + + +shape_twist = _shape_twist + + +class MicroduckSimModuleConfig(MujocoSimModuleConfig): + # 14 servo joints and no gripper; a smaller dof would make the parent + # misread joint 14 onward as a gripper. + dof: int = 14 + camera_name: str = "head_camera" + base_frame_id: str = "trunk_base" + imu_gyro_sensor_names: list[str] = ["imu_ang_vel"] + imu_accel_sensor_names: list[str] = ["imu_accel"] + # Robot model and policy set: "default" (legged) or "rollers" (wheeled + # feet); see policies.py. Selects the MJCF unless robot_mjcf is given. + variant: str = DEFAULT_VARIANT + # Directory holding the policy ONNX files; None = the asset cache. + policy_dir: str | Path | None = None + # Gains mapping the requested twist to the policy's command, compensating + # the policy's velocity-tracking undershoot (measured ~2.5x in sim). + cmd_gain_linear: float = 2.4 + cmd_gain_angular: float = 2.6 + # Minimum |yaw-rate| command actually sent to the policy when turning + # (the top of the walk policy's WZ_RANGE; see the deadband note below). + min_effective_wz: float = 1.5 + # Zero the command when nothing published cmd_vel for this long. + cmd_timeout: float = 1.0 + # Stand the robot back up in place when it has been on the ground this + # long (no fall-recovery policy ships with the public Microduck repos). + auto_stand: bool = True + auto_stand_after: float = 2.0 + # Third-person chase camera: a TRACK-mode camera on trunk_base, offset + # (x, y, z) m from it in the world frame, looking along +x tilted by + # pitch. Rendered in the sim thread at chase_cam_fps. + chase_cam: bool = True + chase_cam_size: tuple[int, int] = (640, 360) + chase_cam_fps: float = 12.0 + chase_cam_offset: tuple[float, float, float] = (-0.8, 0.0, 0.45) + chase_cam_pitch_deg: float = -20.0 + chase_cam_fovy: float = 60.0 + # Shadow-casting lights re-draw the robot's 215k-vertex meshes once per + # light per render; off, a 640x360 chase frame costs ~11 ms instead of + # ~22 ms and the chase camera no longer halves the sim rate. + cast_shadows: bool = False + # Body name of the kickable ball added to the scene; "" leaves it out + # (the kick policies then run without a ball). + ball_body: str = BALL_BODY + # policy_state is published on every change and at least this often. + state_hz: float = 5.0 + + +class MicroduckSimModule(MujocoSimModule): + """MuJoCo sim of the Microduck biped with its RL policies in the loop.""" + + config: MicroduckSimModuleConfig + cmd_vel: In[Twist] + policy_request: In[str] + policy_state: Out[str] + chase_image: Out[Image] + + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + self._bank: PolicyBank | None = None + self._scheduler: PolicyScheduler | None = None + self._engine_target_perm: np.ndarray | None = None + self._phys_step = 0 + self._pose_initialized = False + self._latest_twist: tuple[float, float, float] = (0.0, 0.0, 0.0) + self._latest_twist_ts = 0.0 + self._fallen_since: float | None = None + self._ball_qpos_adr: int | None = None + self._ball_qvel_adr: int | None = None + self._last_state_key: dict[str, Any] | None = None + self._last_state_ts = 0.0 + self._last_chase_ts = 0.0 + + @rpc + def start(self) -> None: + variant = self.config.variant + assets = assets_fetch.ensure_assets(variant) + if not self.config.robot_mjcf: + self.config.robot_mjcf = assets.robot_mjcf(variant) + policy_dir = self.config.policy_dir or assets.policy_dir + if self.config.chase_cam: + width, height = self.config.chase_cam_size + self.config.extra_cameras = { + **self.config.extra_cameras, + CHASE_CAMERA_NAME: (int(width), int(height), float(self.config.chase_cam_fps)), + } + + super().start() + + engine = self._engine + assert engine is not None + model = engine.model + bank = PolicyBank(policy_dir, model, variant=variant, missing=assets.missing) + scheduler = PolicyScheduler(bank.availability, variant, spawn_ball=self._spawn_ball) + + # The parent takes joint 0 as the robot root; pin odom/IMU to the + # trunk's free joint by name instead (the ball has a free joint too). + self._root_base_qpos_adr = bank.root_qpos_adr + self._imu_base_qpos_slice = slice(bank.root_qpos_adr + 3, bank.root_qpos_adr + 7) + + # write_joint_command() consumes targets in engine joint-mapping order + # (actuator order for composed models); the policies emit policy order. + engine_names = engine.joint_names + policy_index = {name: i for i, name in enumerate(bank.joint_names)} + try: + self._engine_target_perm = np.array( + [policy_index[name] for name in engine_names], dtype=np.int64 + ) + except KeyError as exc: + raise RuntimeError( + f"engine joint {exc} not in Microduck policy joints {bank.joint_names}" + ) from exc + + self._ball_qpos_adr = self._ball_qvel_adr = None + if self.config.ball_body: + ball_joint = f"{self.config.ball_body}_freejoint" + jid = mujoco.mj_name2id(model, mujoco.mjtObj.mjOBJ_JOINT, ball_joint) + if jid >= 0 and int(model.jnt_type[jid]) == int(mujoco.mjtJoint.mjJNT_FREE): + self._ball_qpos_adr = int(model.jnt_qposadr[jid]) + self._ball_qvel_adr = int(model.jnt_dofadr[jid]) + else: + logger.warning( + "Microduck ball free joint not in the model; kicks run without a ball", + joint=ball_joint, + ) + + self._phys_step = 0 + self._pose_initialized = False + self._fallen_since = None + self._last_state_key = None + self._last_state_ts = 0.0 + self._last_chase_ts = 0.0 + self._bank = bank + self._scheduler = scheduler + + # Splice the policies into the engine's step hooks, keeping the + # parent's post-step publishing (odom/imu/SHM state). + engine.set_step_hooks(before=self._gait_pre_step, after=self._after_step) + + self._subscribe(self.cmd_vel, self._on_cmd_vel) + self._subscribe(self.policy_request, self._on_policy_request) + logger.info( + "MicroduckSimModule started", + variant=variant, + robot_mjcf=str(self.config.robot_mjcf), + policies=list(bank.names), + missing=[str(name) for name in bank.missing], + joints=len(engine_names), + chase_cam=self.config.chase_cam, + ball=self.config.ball_body or None, + ) + + def _subscribe(self, stream: In[Any], callback: Any) -> None: + if stream.transport is None: + logger.warning( + "MicroduckSimModule input has no transport; not subscribing", stream=stream.name + ) + return + self.register_disposable(Disposable(stream.subscribe(callback))) + + # ------------------------------------------------------------------ inputs + + def _on_cmd_vel(self, twist: Twist) -> None: + self._latest_twist = ( + float(twist.linear.x), + float(twist.linear.y), + float(twist.angular.z), + ) + self._latest_twist_ts = time.time() + + def _on_policy_request(self, raw: str) -> None: + """``policy_request`` subscriber (transport thread): parse and queue only. + + Never touches MuJoCo - the scheduler applies the request on the sim + thread at its next tick; a rejection is logged and lands in + ``policy_state.last_error``. + """ + scheduler = self._scheduler + if scheduler is None: + return + try: + request = json.loads(raw) + except (TypeError, ValueError) as exc: + logger.warning("Ignoring malformed policy_request", error=str(exc)) + return + if not isinstance(request, dict): + logger.warning("Ignoring policy_request that is not a JSON object", request=raw) + return + action = request.get("action") + policy = request.get("policy") + if not isinstance(action, str) or not (policy is None or isinstance(policy, str)): + logger.warning("Ignoring policy_request with bad fields", request=request) + return + accepted, reason = scheduler.request(policy, action) + if accepted: + logger.info("Microduck policy request", policy=policy, action=action) + else: + logger.warning( + "Microduck policy request rejected", policy=policy, action=action, reason=reason + ) + + # -------------------------------------------------------------- sim thread + + def _gait_pre_step(self, engine: MujocoEngine) -> None: + """Engine sim-thread hook, before each physics step.""" + bank = self._bank + scheduler = self._scheduler + if bank is None or scheduler is None: + return + + if not self._pose_initialized: + self._stand_in_place(engine) + self._pose_initialized = True + + if self._phys_step % POLICY_DECIMATION == 0: + now = time.time() + if self._check_fall(engine, now): + # Just teleported upright: let the physics settle one tick. + self._phys_step += 1 + if self._sim_hooks is not None: + self._sim_hooks.pre_step(engine) + return + + vx, vy, wz = self._latest_twist + if now - self._latest_twist_ts > self.config.cmd_timeout: + vx, vy, wz = 0.0, 0.0, 0.0 + scheduler.set_twist(*_shape_twist(self.config, vx, vy, wz)) + + name, command = scheduler.tick(CONTROL_DT) + targets = bank.step(name, command, engine.data) + engine.write_joint_command( + JointState(position=targets[self._engine_target_perm].tolist()) + ) + self._publish_policy_state(scheduler.snapshot(), now) + self._phys_step += 1 + + # Keep the SHM bridge fed (harmless no-op without a coordinator). + if self._sim_hooks is not None: + self._sim_hooks.pre_step(engine) + + def _check_fall(self, engine: MujocoEngine, now: float) -> bool: + """Debounced fall detector; True when the duck was just stood back up. + + A trunk tilted past ~55 degrees for ``auto_stand_after`` seconds is + a fall: the scheduler is told (it aborts tricks and locks) and, with + ``auto_stand``, the duck is teleported upright - the sim equivalent + of a human picking it up. Skipped while the scheduler runs the + roulade, which is upside down on purpose. + """ + bank = self._bank + scheduler = self._scheduler + assert bank is not None and scheduler is not None + if scheduler.suspend_fall_detector: + self._fallen_since = None + return False + + gravity_z = float(bank.projected_gravity(engine.data)[2]) + if gravity_z <= FALL_GRAVITY_Z: + self._fallen_since = None + scheduler.notify_fall(False) + return False + if self._fallen_since is None: + self._fallen_since = now + if now - self._fallen_since <= self.config.auto_stand_after: + return False + + scheduler.notify_fall(True) + if not self.config.auto_stand: + return False + logger.warning("Microduck fell over; standing it back up") + # Falls usually happen tripping over an obstacle; standing back up + # exactly in place can wedge the robot inside it. Nudge toward the + # configured spawn point, which the scene author chose as clear floor. + data = engine.data + adr = bank.root_qpos_adr + x = float(data.qpos[adr]) + y = float(data.qpos[adr + 1]) + spawn_x, spawn_y = self.config.spawn_xy or (0.0, 0.0) + dx, dy = x - spawn_x, y - spawn_y + dist = math.hypot(dx, dy) + if dist > 1e-3: + shift = min(0.25, dist) + data.qpos[adr] = x - shift * dx / dist + data.qpos[adr + 1] = y - shift * dy / dist + self._stand_in_place(engine) + self._fallen_since = None + return True + + def _stand_in_place(self, engine: MujocoEngine) -> None: + """Sim thread: pose the duck at its standing home pose where it is.""" + bank = self._bank + assert bank is not None + bank.initial_qpos(engine.data) + bank.reset() + mujoco.mj_forward(engine.model, engine.data) + engine.write_joint_command( + JointState(position=bank.default_pose[self._engine_target_perm].tolist()) + ) + + def _publish_policy_state(self, snapshot: dict[str, Any], now: float) -> bool: + """Publish ``snapshot`` if it changed (ignoring ``t``) or the periodic slot is due.""" + key = _state_key(snapshot) + due = now - self._last_state_ts >= 1.0 / self.config.state_hz + if key == self._last_state_key and not due: + return False + self._last_state_key = key + self._last_state_ts = now + self.policy_state.publish(json.dumps(snapshot, separators=(",", ":"))) + return True + + def _spawn_ball(self, dx: float, dy: float) -> None: + """Drop the ball at rest ``(dx, dy)`` m from the trunk, in its yaw frame. + + Called by the scheduler from inside ``tick`` when a kick starts, so + it runs on the sim thread (the only place MjData may be written). + """ + engine = self._engine + bank = self._bank + if engine is None or bank is None: + return + if self._ball_qpos_adr is None or self._ball_qvel_adr is None: + logger.warning("Microduck ball not in the scene; kicking without a ball") + return + data = engine.data + adr = bank.root_qpos_adr + x, y = _ball_spawn_xy( + float(data.qpos[adr]), float(data.qpos[adr + 1]), bank.root_yaw(data), dx, dy + ) + data.qpos[self._ball_qpos_adr : self._ball_qpos_adr + 7] = ( + x, + y, + BALL_RADIUS, + 1.0, + 0.0, + 0.0, + 0.0, + ) + data.qvel[self._ball_qvel_adr : self._ball_qvel_adr + 6] = 0.0 + mujoco.mj_forward(engine.model, data) + + def _after_step(self, engine: MujocoEngine) -> None: + """Engine sim-thread hook after each physics step.""" + self._publish_shm_and_lcm(engine) + self._publish_chase(engine) + + def _publish_chase(self, engine: MujocoEngine) -> None: + """Publish the chase camera's latest frame once, when a new one was rendered.""" + if not self.config.chase_cam: + return + frame = engine.read_camera(CHASE_CAMERA_NAME) + if frame is None or frame.timestamp <= self._last_chase_ts: + return + self._last_chase_ts = frame.timestamp + self.chase_image.publish( + Image( + data=frame.rgb, + format=ImageFormat.RGB, + frame_id=_CHASE_OPTICAL_FRAME, + ts=frame.timestamp, + ) + ) + + # ------------------------------------------------------------------- model + + def _compose_model(self) -> mujoco.MjModel: + return self._compose_spec().compile() + + def _compose_spec(self) -> mujoco.MjSpec: + """Compose scene + robot, adding the lidar and chase cameras and the ball. + + Simplified from the parent: no scene-package entities, and the + physics timestep is pinned to the 5 ms the policies expect. The ball + is added after the robot so the trunk's free joint stays joint 0 + (the engine's notion of the robot root). + """ + if self.config.robot_mjcf is None: + raise RuntimeError("MicroduckSimModule: robot_mjcf is required") + + if self.config.scene_xml is not None: + spec_scene = mujoco.MjSpec.from_file(str(self.config.scene_xml)) + else: + spec_scene = mujoco.MjSpec() + + spec_robot = mujoco.MjSpec.from_file(str(self.config.robot_mjcf)) + if self.config.robot_meshdir is not None: + spec_robot.meshdir = str(self.config.robot_meshdir) + + trunk = spec_robot.body("trunk_base") + if trunk is None: + raise RuntimeError("Microduck robot MJCF has no 'trunk_base' body") + for name, yaw in LIDAR_CAMERA_SPECS: + trunk.add_camera( + name=name, + pos=list(_LIDAR_CAM_POS), + quat=list(_camera_quat_wxyz(yaw)), + fovy=_LIDAR_CAM_FOVY, + ) + trunk.add_camera( + name=POV_CAMERA_NAME, + pos=list(_POV_CAM_POS), + quat=list(_camera_quat_wxyz(0.0)), + fovy=_POV_CAM_FOVY, + ) + if self.config.chase_cam: + # TRACK: the offset and the orientation stay fixed in the world + # frame while the camera follows the trunk's position. + trunk.add_camera( + name=CHASE_CAMERA_NAME, + mode=mujoco.mjtCamLight.mjCAMLIGHT_TRACK, + pos=[float(v) for v in self.config.chase_cam_offset], + quat=list(_camera_quat_wxyz(0.0, math.radians(self.config.chase_cam_pitch_deg))), + fovy=float(self.config.chase_cam_fovy), + ) + # mujoco.Renderer needs the offscreen buffer at least as large as + # the biggest camera it renders. + width, height = self.config.chase_cam_size + visual = spec_scene.visual.global_ + visual.offwidth = max(int(visual.offwidth), int(width)) + visual.offheight = max(int(visual.offheight), int(height)) + + spec_scene.option.timestep = PHYSICS_TIMESTEP + spec_robot.option.timestep = PHYSICS_TIMESTEP + + spawn_xy = self.config.spawn_xy or (0.0, 0.0) + spawn_z = self.config.spawn_z if self.config.spawn_z is not None else 0.0 + frame_kwargs: dict[str, Any] = { + "pos": [float(spawn_xy[0]), float(spawn_xy[1]), float(spawn_z)], + } + if self.config.spawn_yaw is not None: + yaw = float(self.config.spawn_yaw) + frame_kwargs["quat"] = [math.cos(yaw * 0.5), 0.0, 0.0, math.sin(yaw * 0.5)] + frame = spec_scene.worldbody.add_frame(**frame_kwargs) + spec_scene.attach(spec_robot, prefix="", frame=frame) + if self.config.ball_body: + add_ball_body(spec_scene, name=self.config.ball_body) + if not self.config.cast_shadows: + for light in spec_scene.lights: + light.castshadow = False + return spec_scene + + +# The base policies hold a stand on a zero command, so the module needs no +# explicit idle handling; CONTROL_DT is re-exported for tests. diff --git a/dimos/robot/pollen/microduck/skills.py b/dimos/robot/pollen/microduck/skills.py new file mode 100644 index 0000000000..35d8f27849 --- /dev/null +++ b/dimos/robot/pollen/microduck/skills.py @@ -0,0 +1,1049 @@ +# Copyright 2025-2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Agent skills for the simulated Microduck. + +Two families of skills live here: + +* **Navigation** over the abstract ``NavigationInterfaceSpec``: rooms, + landmark objects and remembered spots come from :class:`PlacesMemory` + (seeded from the blueprint's ground-truth tables of the simulation scene, + not from a perception pipeline - this is honest about being in sim). Goals + are published on ``goal_request`` so the planner, the control module and + the cockpit map all see them, and arrival is awaited through the planner's + RPCs. +* **Policies** ("tricks"): ``perform`` publishes a ``policy_request`` for the + simulator's policy scheduler and watches ``policy_state`` until the + requested one-shot finished, the posture toggled or the base policy + switched, then reports the outcome (or the scheduler's ``last_error``). + +Every skill returns a short human-readable string; the docstrings are what +the LLM sees when it decides which skill to call. +""" + +from __future__ import annotations + +from collections import deque +from collections.abc import Callable +import json +import math +import threading +import time +from typing import Any + +import numpy as np +from reactivex.disposable import Disposable + +from dimos.agents.annotation import skill +from dimos.core.core import rpc +from dimos.core.module import Module, ModuleConfig +from dimos.core.stream import In, Out +from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped +from dimos.msgs.geometry_msgs.Quaternion import Quaternion +from dimos.msgs.geometry_msgs.Vector3 import Vector3 +from dimos.navigation.base import NavigationState +from dimos.navigation.navigation_spec import NavigationInterfaceSpec +from dimos.robot.pollen.microduck.places import ( + ARENA_HALF_EXTENT, + PlaceRecord, + PlacesMemory, + RoomSpec, + normalize_place_query, + scene_id, +) +from dimos.utils.logging_config import setup_logger + +logger = setup_logger() + +# Stop this far from an object's center so the duck ends up next to it +# instead of trying to stand inside it. +_APPROACH_DISTANCE = 0.35 + +# Smallest arena move_to accepts goals in; a scene whose rooms / objects +# reach further widens it (see ``_arena_half_extent``). +_ARENA_HALF_EXTENT = ARENA_HALF_EXTENT + +# Navigation polling (module-level so tests can shrink them). +_NAV_POLL_S = 0.2 +_NAV_START_TIMEOUT_S = 15.0 +_NAV_SETTLE_S = 3.0 + +# Policy polling. The simulator publishes policy_state at 5 Hz and on every +# change, and the scheduler applies an accepted request on its next tick, so +# a request whose transition has not begun within the grace window was +# dropped or rejected. +_POLICY_POLL_S = 0.05 +_POLICY_START_GRACE_S = 2.5 +_POLICY_FIRST_STATE_WAIT_S = 1.0 + +# Recent policy_state messages kept so a short trick whose start and end +# arrive back-to-back between two polls is still seen to have run. +_POLICY_HISTORY = 256 + +_POLICY_ACTIONS = ("start", "stop", "toggle") + +# Static fallback (display order) used until the simulator has reported its +# own policy table; kinds match dimos.robot.pollen.microduck.policies. +_POLICY_KINDS: dict[str, str] = { + "walk": "base", + "stand": "base", + "roller": "base", + "roller_crouch": "base", + "sitstand": "posture", + "kick_left": "oneshot", + "kick_right": "oneshot", + "roulade": "oneshot", + "ground_pick": "oneshot", +} + +_POLICY_DESCRIPTIONS: dict[str, str] = { + "walk": "default walking gait, follows velocity commands", + "stand": "stand still and balance (ignores velocity commands)", + "roller": "drive on the roller wheels (rollers variant only)", + "roller_crouch": "crouched roller driving (rollers variant only)", + "sitstand": "sit down / stand back up (posture toggle)", + "kick_left": "kick the ball with the left foot", + "kick_right": "kick the ball with the right foot", + "roulade": "forward roll (somersault) and get back up", + "ground_pick": "bend down and pick at the ground", +} + +# Free-form names people use -> (policy, forced action or None). +_POLICY_ALIASES: dict[str, tuple[str, str | None]] = { + "walking": ("walk", None), + "standing": ("stand", None), + "stand still": ("stand", None), + "balance": ("stand", None), + "kick": ("kick_left", None), + "kick the ball": ("kick_left", None), + "left kick": ("kick_left", None), + "kick left": ("kick_left", None), + "right kick": ("kick_right", None), + "kick right": ("kick_right", None), + "roll": ("roulade", None), + "forward roll": ("roulade", None), + "somersault": ("roulade", None), + "flip": ("roulade", None), + "tumble": ("roulade", None), + "pick": ("ground_pick", None), + "pick up": ("ground_pick", None), + "ground pick": ("ground_pick", None), + "peck": ("ground_pick", None), + "crouch": ("roller_crouch", None), + "roller crouch": ("roller_crouch", None), + "skate": ("roller", None), + "wheels": ("roller", None), + "rollers": ("roller", None), + "sit": ("sitstand", "start"), + "sit down": ("sitstand", "start"), + "sitting": ("sitstand", "start"), + "stand up": ("sitstand", "stop"), + "standing up": ("sitstand", "stop"), + "get up": ("sitstand", "stop"), + "getting up": ("sitstand", "stop"), + "rise": ("sitstand", "stop"), + "sit stand": ("sitstand", None), +} + + +def _yaw_quaternion(yaw: float) -> Quaternion: + return Quaternion(0.0, 0.0, math.sin(yaw / 2.0), math.cos(yaw / 2.0)) + + +def _normalize_policy_name(name: str) -> str: + return " ".join(name.strip().lower().replace("_", " ").replace("-", " ").split()) + + +def _now_json(payload: dict[str, Any]) -> str: + payload.setdefault("t", time.time()) + return json.dumps(payload, separators=(",", ":")) + + +class MicroduckSkillContainerConfig(ModuleConfig): + # name -> (x, y) world coordinates of landmark objects in the scene. + objects: dict[str, tuple[float, float]] = {} + # name -> RoomSpec of the rooms the scene is divided into. + rooms: dict[str, RoomSpec] = {} + # Sqlite file the places memory persists to (``~`` is expanded). Several + # blueprints share it; records are scoped by ``scene``. + places_db: str = "~/.cache/dimos/microduck/places.db" + # Scene id the memory is scoped to. Empty = derive it from the ``rooms`` + # and ``objects`` tables (``places.scene_id``), so blueprints describing + # different worlds never see each other's rooms, objects or tagged spots. + scene: str = "" + # How long a single navigation skill may block waiting for arrival. + nav_timeout_s: float = 100.0 + # How long ``perform`` waits for a policy to finish / a posture to toggle. + policy_timeout_s: float = 15.0 + # Re-publish the ``places`` snapshot this often so a consumer that + # subscribed after start still receives it (0 disables the heartbeat). + places_republish_s: float = 5.0 + + +class MicroduckSkillContainer(Module): + """Skills the agent can call to move the Microduck around its rooms and + make it perform its policies.""" + + config: MicroduckSkillContainerConfig + + _navigation: NavigationInterfaceSpec + + odom: In[PoseStamped] + policy_state: In[str] + + goal_request: Out[PoseStamped] + policy_request: Out[str] + places: Out[str] + + def __init__(self, **kwargs: object) -> None: + super().__init__(**kwargs) + self._latest_odom: PoseStamped | None = None + self._policy_state: dict[str, Any] | None = None + # Every policy_state gets a sequence number; a skill remembers the + # number current when it sent its request and only trusts newer ones. + self._policy_state_seq: int = 0 + self._policy_history: deque[tuple[int, dict[str, Any]]] = deque(maxlen=_POLICY_HISTORY) + self._state_cond = threading.Condition() + self._memory: PlacesMemory | None = None + self._memory_lock = threading.Lock() + self._stop_event = threading.Event() + self._republish_thread: threading.Thread | None = None + + # -- lifecycle ----------------------------------------------------------- + + @rpc + def start(self) -> None: + super().start() + self._stop_event.clear() + self._places_memory() # open + seed + if self.odom.transport is not None: + self.register_disposable(Disposable(self.odom.subscribe(self._on_odom))) + else: + logger.warning( + "MicroduckSkillContainer: odom is not connected; navigation skills need it" + ) + if self.policy_state.transport is not None: + self.register_disposable(Disposable(self.policy_state.subscribe(self._on_policy_state))) + self._publish_places() + if self.config.places_republish_s > 0: + self._republish_thread = threading.Thread( + target=self._republish_loop, name="microduck-places", daemon=True + ) + self._republish_thread.start() + + @rpc + def stop(self) -> None: + self._stop_event.set() + with self._state_cond: + self._state_cond.notify_all() + thread = self._republish_thread + if thread is not None: + thread.join(timeout=2.0) + self._republish_thread = None + super().stop() + with self._memory_lock: + if self._memory is not None: + self._memory.close() + self._memory = None + + def _scene(self) -> str: + return self.config.scene or scene_id(self.config.rooms, self.config.objects) + + def _places_memory(self) -> PlacesMemory: + """The (lazily opened, seeded) places memory. + + Raises ``RuntimeError`` once :meth:`stop` has run, so a skill thread + still unwinding after shutdown cannot reopen the database that + ``stop`` just closed; :meth:`start` re-arms it. + """ + with self._memory_lock: + if self._stop_event.is_set(): + raise RuntimeError("MicroduckSkillContainer is stopped; places memory is closed") + if self._memory is None: + memory = PlacesMemory(self.config.places_db, scene=self._scene()) + memory.seed(self.config.rooms, self.config.objects) + self._memory = memory + return self._memory + + def _publish_places(self) -> None: + if self._stop_event.is_set(): + return + try: + self.places.publish(self._places_memory().to_json()) + except RuntimeError: + pass # stopped between the check and the publish + except Exception: + logger.exception("Failed to publish places") + + def _republish_loop(self) -> None: + while not self._stop_event.wait(self.config.places_republish_s): + self._publish_places() + + # -- stream callbacks ---------------------------------------------------- + + def _on_odom(self, pose: PoseStamped) -> None: + self._latest_odom = pose + + def _on_policy_state(self, msg: str) -> None: + try: + state = json.loads(msg) + except (TypeError, ValueError): + logger.warning("Ignoring malformed policy_state", msg=str(msg)[:120]) + return + if not isinstance(state, dict): + return + with self._state_cond: + self._policy_state_seq += 1 + self._policy_state = state + self._policy_history.append((self._policy_state_seq, state)) + self._state_cond.notify_all() + + # -- robot pose helpers -------------------------------------------------- + + def _robot_xy(self) -> tuple[float, float] | None: + odom = self._latest_odom + if odom is None: + return None + return (float(odom.position.x), float(odom.position.y)) + + def _robot_yaw(self) -> float | None: + odom = self._latest_odom + if odom is None: + return None + try: + return float(odom.yaw) + except Exception: + return None + + def _where_summary(self) -> str: + robot = self._robot_xy() + if robot is None: + return "Robot position unknown." + try: + memory = self._places_memory() + except RuntimeError: + return f"Robot was at ({robot[0]:.2f}, {robot[1]:.2f}) (module stopping)." + where = self._room_phrase(memory, *robot) + return f"Robot is now at ({robot[0]:.2f}, {robot[1]:.2f}){where}." + + @staticmethod + def _room_phrase(memory: PlacesMemory, x: float, y: float) -> str: + """`` in the kitchen`` / `` in the central hub``; empty when the scene has no rooms.""" + if not memory.rooms(): + return "" + room = memory.room_at(x, y) + return f" in the {room.name}" if room is not None else " in the central hub" + + def _arena_half_extent(self) -> float: + """Goals beyond this (per axis) are refused before reaching the planner. + + The four-room arena is +/-2 m; a scene whose rooms or objects reach + further widens the limit so move_to stays usable there. + """ + reach = [_ARENA_HALF_EXTENT] + reach.extend(abs(b) for room in self.config.rooms.values() for b in room.bounds) + reach.extend(abs(c) for xy in self.config.objects.values() for c in xy) + return max(reach) + + def _distance_suffix(self, record: PlaceRecord) -> str: + robot = self._robot_xy() + if robot is None: + return "" + return f", {record.distance_to(*robot):.2f} m away" + + def _object_lines(self, memory: PlacesMemory, objects: list[PlaceRecord]) -> list[str]: + lines: list[str] = [] + for r in objects: + room = memory.room_at(r.x, r.y) + where = f" in the {room.name}" if room is not None else "" + lines.append(f"- {r.name} at ({r.x:.2f}, {r.y:.2f}){where}{self._distance_suffix(r)}") + return lines + + @staticmethod + def _ambiguous(query: str, candidates: list[PlaceRecord], what: str) -> str: + names = ", ".join(r.name for r in candidates) + return f"Ambiguous {what} '{query}': did you mean {names}? Use the exact name." + + # -- places skills ------------------------------------------------------- + + @skill + def list_places(self) -> str: + """List every place the duck knows: the rooms (with their aliases such + as "space A"), the landmark objects with their coordinates, and any + spots remembered with remember_place. Includes the distance from the + robot when its position is known. Call this when unsure what a room + or object is called before using go_to_room / go_to_object. + """ + memory = self._places_memory() + records = memory.all() + if not records: + return "No places are registered in this scene." + + rooms = [r for r in records if r.kind == "room"] + objects = [r for r in records if r.kind == "object"] + tagged = [r for r in records if r.kind not in ("room", "object")] + lines: list[str] = [] + if rooms: + lines.append("Rooms:") + for r in rooms: + aka = f" (aka {', '.join(r.aliases)})" if r.aliases else "" + xmin, xmax, ymin, ymax = r.bounds or (0.0, 0.0, 0.0, 0.0) + lines.append( + f"- {r.name}{aka}: x {xmin:g}..{xmax:g}, y {ymin:g}..{ymax:g}, " + f"entry point ({r.x:.2f}, {r.y:.2f}){self._distance_suffix(r)}" + ) + if objects: + lines.append("Objects:") + lines.extend(self._object_lines(memory, objects)) + lines.append("Remembered places:") + if tagged: + for r in tagged: + lines.append(f"- {r.name} at ({r.x:.2f}, {r.y:.2f}){self._distance_suffix(r)}") + else: + lines.append("- (none yet; use remember_place to tag the current spot)") + lines.append(self._where_summary()) + return "\n".join(lines) + + @skill + def list_objects(self) -> str: + """List the known landmark objects with their positions in meters (and + the distance from the duck when known). list_places additionally + shows the rooms and remembered spots.""" + memory = self._places_memory() + objects = [r for r in memory.all() if r.kind == "object"] + if not objects: + return "No objects are registered in this scene." + return "Objects in the room:\n" + "\n".join(self._object_lines(memory, objects)) + + @skill + def go_to_room(self, name: str) -> str: + """Walk into a room and stop at its entry point, facing into the room. + Accepts the room's name or any of its aliases as listed by + list_places (e.g. "kitchen" or "space A"). Blocks until the duck + arrives (or navigation gives up) and reports where the duck ended up. + + Args: + name: The room's name or alias, e.g. "space A" or "kitchen". + """ + memory = self._places_memory() + candidates = memory.matches(name, kind="room") + if len(candidates) > 1: + return self._ambiguous(name, candidates, "room") + if not candidates: + known = ", ".join( + f"{r.name} ({', '.join(r.aliases)})" if r.aliases else r.name + for r in memory.rooms() + ) + return f"Unknown room '{name}'. Known rooms: {known or '(none)'}" + record = candidates[0] + robot = self._robot_xy() + if robot is not None and record.distance_to(*robot) <= 0.25: + return f"Already at the {record.name} entry point. {self._where_summary()}" + outcome = self._navigate_to(record.x, record.y, record.yaw) + return f"{outcome} while walking to the {record.name}. {self._where_summary()}" + + @skill + def go_to_object(self, name: str) -> str: + """Walk to a named landmark object and stop right next to it (about + 0.35 m away), facing it. Use list_places to see what exists. Blocks + until arrival and reports where the duck ended up. + + Args: + name: The object's exact name, e.g. "red_box" or "red box" (a bare + "box" is ambiguous when several boxes exist). + """ + memory = self._places_memory() + candidates = memory.matches(name, kind="object") + if len(candidates) > 1: + return self._ambiguous(name, candidates, "object") + if not candidates: + known = ", ".join(r.name for r in memory.all() if r.kind == "object") or "(none)" + return f"Unknown object '{name}'. Known objects: {known}" + return self._approach(candidates[0]) + + @skill + def go_to_place(self, name: str) -> str: + """Walk to any known place by name: a room (or alias like "space B"), + a landmark object, or a spot saved with remember_place. Rooms are + entered at their entry point, objects are approached to 0.35 m, and + remembered spots are reached exactly with the saved heading. Blocks + until arrival and reports where the duck ended up. + + Args: + name: The place's name or alias (exact enough to be unambiguous). + """ + memory = self._places_memory() + candidates = memory.matches(name) + if len(candidates) > 1: + return self._ambiguous(name, candidates, "place") + if not candidates: + return f"Unknown place '{name}'. Use list_places to see what the duck knows." + record = candidates[0] + if record.kind == "room": + return self.go_to_room(record.name) + if record.kind == "object": + return self._approach(record) + outcome = self._navigate_to(record.x, record.y, record.yaw) + return f"{outcome} while walking to {record.name}. {self._where_summary()}" + + @skill + def move_to(self, x: float, y: float) -> str: + """Walk to world coordinates (x, y) in meters and wait for arrival. + The world frame is the map shown in the cockpit; list_places gives + the coordinates of every room, object and remembered spot, which + bound the reachable area. Prefer go_to_room / go_to_object / + go_to_place when the user names a place. + + Args: + x: Target x in meters (world frame). + y: Target y in meters (world frame). + """ + try: + x, y = float(x), float(y) + except (TypeError, ValueError): + return f"Invalid coordinates ({x!r}, {y!r}); give numbers in meters." + if not (math.isfinite(x) and math.isfinite(y)): + return "Invalid coordinates; give finite numbers in meters." + half_extent = self._arena_half_extent() + if abs(x) > half_extent or abs(y) > half_extent: + return ( + f"({x:.2f}, {y:.2f}) is outside the arena walls " + f"(x and y must be within +/-{half_extent:g} m)." + ) + robot = self._robot_xy() + heading = None + if robot is not None: + heading = math.atan2(y - robot[1], x - robot[0]) + outcome = self._navigate_to(x, y, heading) + return f"{outcome}. {self._where_summary()}" + + @skill + def stop_moving(self) -> str: + """Cancel the current navigation goal; the duck stops and stands in + place. Does not stop a trick in progress (use perform with action + "stop" for that).""" + cancelled = self._navigation.cancel_goal() + return "Stopped." if cancelled else "There was no active navigation goal." + + @skill + def where_am_i(self) -> str: + """Report the duck's position, heading, which room it is in (when the + scene has rooms) and the known places within 1 m.""" + robot = self._robot_xy() + if robot is None: + return "Robot position unknown (no odometry yet)." + memory = self._places_memory() + yaw = self._robot_yaw() + heading = "" if yaw is None else f", heading {math.degrees(yaw):.0f} deg" + where = "" + if memory.rooms(): + room = memory.room_at(*robot) + where = ( + f", in the {room.name}" if room is not None else ", in the central hub (no room)" + ) + if room is not None and room.aliases: + where += f" (aka {room.aliases[0]})" + nearby = [r for r in memory.near(robot[0], robot[1], 1.0) if r.kind != "room"] + near_txt = "" + if nearby: + near_txt = ( + " Nearby: " + + ", ".join(f"{r.name} ({r.distance_to(*robot):.2f} m)" for r in nearby[:4]) + + "." + ) + return f"Robot is at ({robot[0]:.2f}, {robot[1]:.2f}){heading}{where}.{near_txt}" + + @skill + def remember_place(self, name: str) -> str: + """Save the duck's current position and heading under a name so it + can be revisited later with go_to_place. The place is persisted and + appears in list_places. + + Args: + name: A short label for this spot, e.g. "charger" or "front door". + """ + label = " ".join(str(name).split()) + if not label: + return "Give the place a name, e.g. remember_place('charger')." + robot = self._robot_xy() + if robot is None: + return "Robot position unknown (no odometry yet); cannot remember this spot." + memory = self._places_memory() + clash = self._reserved_name_clash(memory, label) + if clash is not None: + return f"'{label}' is already the name of a {clash.kind}; pick another name." + yaw = self._robot_yaw() or 0.0 + memory.add(label, robot[0], robot[1], yaw) + self._publish_places() + where = self._room_phrase(memory, *robot) + return f"Remembered '{label}' at ({robot[0]:.2f}, {robot[1]:.2f}){where}." + + @skill + def wait(self, seconds: float) -> str: + """Wait in place for the given number of seconds (0 to 30), e.g. to + let a trick finish or to give the user time to look. + + Args: + seconds: How long to wait, in seconds. + """ + try: + seconds = float(np.clip(float(seconds), 0.0, 30.0)) + except (TypeError, ValueError): + return f"Invalid duration {seconds!r}; give a number of seconds." + deadline = time.monotonic() + seconds + while not self._stop_event.is_set(): + remaining = deadline - time.monotonic() + if remaining <= 0: + break + time.sleep(min(remaining, 0.25)) + return f"Waited {seconds:.0f} s." + + # -- navigation internals ------------------------------------------------ + + @staticmethod + def _reserved_name_clash(memory: PlacesMemory, label: str) -> PlaceRecord | None: + """The room / object whose name or alias ``label`` would shadow, if any. + + Compared on the normalised forms ``find`` matches on, so a tagged spot + called "red box" cannot hide the object ``red_box`` (or vice versa). + """ + forms = set(normalize_place_query(label)) + if not forms: + return None + for record in memory.all(): + if record.kind not in ("room", "object"): + continue + keys = set(normalize_place_query(record.name)) + for alias in record.aliases: + keys.update(normalize_place_query(alias)) + if forms & keys: + return record + return None + + def _approach(self, record: PlaceRecord) -> str: + robot = self._robot_xy() + if robot is None: + return "Robot position unknown (no odometry yet); try again shortly." + dx, dy = record.x - robot[0], record.y - robot[1] + distance = math.hypot(dx, dy) + if distance <= _APPROACH_DISTANCE + 0.05: + return f"Already next to {record.name} ({distance:.2f} m away). {self._where_summary()}" + heading = math.atan2(dy, dx) + gx = record.x - math.cos(heading) * _APPROACH_DISTANCE + gy = record.y - math.sin(heading) * _APPROACH_DISTANCE + outcome = self._navigate_to(gx, gy, heading) + return f"{outcome} while walking to {record.name}. {self._where_summary()}" + + def _navigate_to(self, x: float, y: float, heading: float | None) -> str: + goal = PoseStamped( + ts=time.time(), + frame_id="world", + position=Vector3(float(x), float(y), 0.0), + orientation=_yaw_quaternion(heading) if heading is not None else Quaternion(), + ) + # Publish so the planner, the control module and the cockpit map all + # see the goal. Without a wired transport (direct use, unit tests) + # hand it to the planner RPC instead so the goal is not lost. + self.goal_request.publish(goal) + if self.goal_request.transport is None and not self._navigation.set_goal(goal): + return "Navigation rejected the goal (is the map ready?)" + return self._wait_for_goal(timeout=self.config.nav_timeout_s) + + def _wait_for_goal(self, timeout: float | None = None, settle: float | None = None) -> str: + """Block until arrival, cancellation, or timeout. + + Arrival (`is_goal_reached`) only counts after the planner has been + seen FOLLOWING_PATH for *this* goal - the flag may still be latched + from a previous goal (e.g. one an exploration cycle just finished). + The planner also leaves FOLLOWING_PATH briefly on every replan, so a + pause only counts as the end after `settle` seconds. + """ + timeout = self.config.nav_timeout_s if timeout is None else timeout + settle = _NAV_SETTLE_S if settle is None else settle + deadline = time.monotonic() + timeout + started = False + start_deadline = time.monotonic() + min(_NAV_START_TIMEOUT_S, timeout) + while not started and time.monotonic() < start_deadline: + if self._stop_event.is_set(): + return "Navigation aborted (module stopping)" + if self._navigation.get_state() == NavigationState.FOLLOWING_PATH: + started = True + else: + time.sleep(_NAV_POLL_S) + if not started: + return "Navigation never started following a path (no route found?)" + + idle_since: float | None = None + while time.monotonic() < deadline: + if self._stop_event.is_set(): + return "Navigation aborted (module stopping)" + if self._navigation.is_goal_reached(): + return "Arrived" + if self._navigation.get_state() == NavigationState.FOLLOWING_PATH: + idle_since = None + elif idle_since is None: + idle_since = time.monotonic() + elif time.monotonic() - idle_since > settle: + return "Navigation stopped early (cancelled or no path)" + time.sleep(_NAV_POLL_S) + return "Navigation timed out" + + # -- policy skills ------------------------------------------------------- + + @skill + def list_policies(self) -> str: + """List the duck's policies (tricks and gaits) with what each does and + whether it is available on this robot variant, plus what the duck is + doing right now (active policy, sitting, fallen). Use the listed names + with perform.""" + state = self._current_policy_state() + entries: list[dict[str, Any]] = [] + if state is not None and isinstance(state.get("policies"), list): + entries = [e for e in state["policies"] if isinstance(e, dict) and "name" in e] + if not entries: + entries = [ + {"name": n, "kind": k, "available": True, "reason": None} + for n, k in _POLICY_KINDS.items() + ] + lines = ["Policies:"] + for entry in entries: + name = str(entry["name"]) + kind = str(entry.get("kind") or _POLICY_KINDS.get(name, "?")) + desc = _POLICY_DESCRIPTIONS.get(name, "") + line = f"- {name} [{kind}]" + if desc: + line += f": {desc}" + if not entry.get("available", True): + line += f" (NOT available: {entry.get('reason') or 'unsupported on this variant'})" + lines.append(line) + lines.append( + "Kinds: oneshot = a trick that runs once and ends by itself; posture = " + "sitstand toggles sitting; base = the gait used when idle." + ) + if state is None: + lines.append("The simulator has not reported its policy state yet.") + else: + lines.append(self._policy_summary(state)) + return "\n".join(lines) + + @skill + def perform(self, policy: str, action: str = "start") -> str: + """Make the duck perform a policy and wait for the outcome. Tricks + (kick_left, kick_right, roulade, ground_pick) run once and this + returns when they finish; "sitstand" sits the duck down (or use sit / + stand_up); "walk", "stand", "roller", "roller_crouch" switch the base + gait. Use list_policies for the exact names and availability. The + duck must be standing still and idle; a running trick can be aborted + with action "stop". If the request is refused the reason is returned + (e.g. locked while seated or fallen) - fix that first instead of + retrying blindly. + + Args: + policy: Policy name, e.g. "kick_left", "roulade", "sitstand". + action: "start" (default), "stop" (abort a running trick / stand + up / back to walking) or "toggle" (start, or abort if that + trick is already running; sit or stand for sitstand). + """ + return self._perform(policy, action) + + @skill + def sit(self) -> str: + """Make the duck sit down (it stays seated and cannot walk until + stand_up is called). Returns once it is seated.""" + return self._perform("sitstand", "start") + + @skill + def stand_up(self) -> str: + """Make a seated duck stand back up. Returns once it is standing and + ready to walk again.""" + return self._perform("sitstand", "stop") + + # -- policy internals ---------------------------------------------------- + + def _current_policy_state(self) -> dict[str, Any] | None: + with self._state_cond: + return self._policy_state + + def _known_policies(self, state: dict[str, Any] | None) -> dict[str, dict[str, Any]]: + """name -> entry, from the simulator's table when it has one.""" + table: dict[str, dict[str, Any]] = { + n: {"name": n, "kind": k, "available": True, "reason": None} + for n, k in _POLICY_KINDS.items() + } + if state is not None and isinstance(state.get("policies"), list): + for entry in state["policies"]: + if isinstance(entry, dict) and "name" in entry: + merged = dict(table.get(str(entry["name"]), {})) + merged.update(entry) + table[str(entry["name"])] = merged + return table + + def _resolve_policy( + self, policy: str, known: dict[str, dict[str, Any]] + ) -> tuple[str | None, str | None]: + """Map a free-form name to ``(policy_name, forced_action)``.""" + key = _normalize_policy_name(str(policy)) + if not key: + return None, None + by_norm = {_normalize_policy_name(n): n for n in known} + if key in by_norm: + return by_norm[key], None + if key in _POLICY_ALIASES: + name, forced = _POLICY_ALIASES[key] + return name, forced + for alias, (name, forced) in sorted(_POLICY_ALIASES.items(), key=lambda kv: -len(kv[0])): + if len(alias) >= 4 and (f" {alias} " in f" {key} "): + return name, forced + return None, None + + def _policy_summary(self, state: dict[str, Any]) -> str: + active = state.get("active") + parts = [f"Now: active policy {active!r} (base {state.get('base')!r})"] + if state.get("seated"): + parts.append("seated") + if state.get("fallen"): + parts.append("FALLEN (auto-recovering)") + oneshot = state.get("oneshot") + if isinstance(oneshot, dict): + progress = oneshot.get("progress") + pct = f" {float(progress) * 100:.0f}%" if isinstance(progress, (int, float)) else "" + parts.append(f"running {oneshot.get('name')}{pct}") + elif state.get("locked"): + parts.append("busy") + if state.get("last_error"): + parts.append(f"last error: {state['last_error']}") + return ", ".join(parts) + "." + + def _await_state( + self, predicate: Callable[[dict[str, Any]], bool], timeout: float + ) -> dict[str, Any] | None: + """Block until a policy_state satisfying ``predicate`` arrives.""" + deadline = time.monotonic() + timeout + with self._state_cond: + while True: + state = self._policy_state + if state is not None and predicate(state): + return state + remaining = deadline - time.monotonic() + if remaining <= 0 or self._stop_event.is_set(): + return None + self._state_cond.wait(min(remaining, _POLICY_POLL_S)) + + def _perform(self, policy: str, action: str = "start") -> str: + action_key = str(action).strip().lower() + if action_key not in _POLICY_ACTIONS: + return f"Unknown action '{action}'. Use one of: {', '.join(_POLICY_ACTIONS)}." + state = self._current_policy_state() + known = self._known_policies(state) + name, forced_action = self._resolve_policy(policy, known) + if name is None: + return f"Unknown policy '{policy}'. Known policies: {', '.join(known)}." + if forced_action is not None and action_key == "start": + action_key = forced_action + entry = known.get(name, {}) + kind = str(entry.get("kind") or _POLICY_KINDS.get(name, "oneshot")) + if not entry.get("available", True): + reason = entry.get("reason") or "unsupported on this robot variant" + return f"'{name}' is not available on this robot ({reason})." + + def running(s: dict[str, Any]) -> bool: + one = s.get("oneshot") + return isinstance(one, dict) and one.get("name") == name + + # Fast paths / pre-checks from the latest state. + if state is not None: + seated = bool(state.get("seated")) + oneshot = state.get("oneshot") if isinstance(state.get("oneshot"), dict) else None + if kind == "posture": + if action_key == "start" and seated: + return f"Already sitting. {self._policy_summary(state)}" + if action_key == "stop" and not seated and state.get("active") != "standing_up": + return f"Already standing. {self._policy_summary(state)}" + elif kind == "base" and action_key in ("start", "toggle"): + if state.get("base") == name and state.get("active") == name: + return f"Already using the {name} policy. {self._policy_summary(state)}" + elif kind == "oneshot": + if action_key == "toggle": + # A toggle means "abort" while this trick runs and "start" + # otherwise; decide now so the outcome can be awaited. + action_key = "stop" if running(state) else "start" + if action_key == "stop" and not running(state): + return f"{name} is not running. {self._policy_summary(state)}" + if action_key == "start" and oneshot is not None: + return ( + f"The duck is busy with {oneshot.get('name')}; wait for it to finish " + f"or perform('{oneshot.get('name')}', 'stop') first." + ) + if action_key == "start" and seated: + return "The duck is sitting; call stand_up first." + if state.get("fallen") and action_key != "stop": + return "The duck has fallen and is getting back up; try again in a moment." + + prev_error = state.get("last_error") if state is not None else None + prev_seated = bool(state.get("seated")) if state is not None else False + with self._state_cond: + prev_seq = self._policy_state_seq + self.policy_request.publish(_now_json({"policy": name, "action": action_key})) + logger.info("Policy request", policy=name, action=action_key) + + if self.policy_state.transport is None and state is None: + return f"Sent '{name}' {action_key} request (no policy feedback available)." + if state is None: + state = self._await_state(lambda _s: True, _POLICY_FIRST_STATE_WAIT_S) + if state is None: + return ( + f"Sent '{name}' {action_key} request but the simulator has not reported " + "any policy state; cannot confirm the outcome." + ) + + timeout = float(self.config.policy_timeout_s) + + if kind == "oneshot": + if action_key == "stop": + done = self._await_state(lambda s: not running(s), timeout) + if done is None: + return f"Timed out waiting for {name} to stop." + return f"Stopped {name}. {self._policy_summary(done)}" + + # Only the trick actually running counts as acknowledged; the + # history scan catches one that already finished before we looked. + ack, err = self._await_ack(running, prev_seq, prev_error) + if ack is None: + latest = self._current_policy_state() or state + return f"Could not start {name}: {err}. {self._policy_summary(latest)}" + finished = self._await_state(lambda s: not running(s), timeout) + if finished is None: + return f"Timed out waiting for {name} to finish." + outcome = f"Finished {name}." + if finished.get("fallen"): + outcome += " The duck fell over and is getting back up." + return f"{outcome} {self._policy_summary(finished)}" + + if kind == "posture": + if action_key == "toggle": + target_seated = not prev_seated + else: + target_seated = action_key == "start" + what = "sit down" if target_seated else "stand up" + + def settled(s: dict[str, Any]) -> bool: + return bool(s.get("seated")) == target_seated and s.get("active") != "standing_up" + + def begun(s: dict[str, Any]) -> bool: + if settled(s): + return True + if target_seated: + return bool(s.get("seated")) or s.get("active") == "sitstand" + return s.get("active") == "standing_up" + + final, err = self._await_transition(begun, settled, prev_seq, prev_error, timeout) + if err is not None: + return f"Could not {what}: {err}." + if final is None: + return f"Timed out waiting for the duck to {what}." + return ( + f"The duck is now {'sitting' if target_seated else 'standing'}. " + f"{self._policy_summary(final)}" + ) + + # base policy switch + target = "walk" if action_key == "stop" else name + + def switched(s: dict[str, Any]) -> bool: + return s.get("base") == target and s.get("active") == target + + def base_begun(s: dict[str, Any]) -> bool: + return switched(s) or s.get("base") == target or s.get("active") == "braking" + + final, err = self._await_transition(base_begun, switched, prev_seq, prev_error, timeout) + if err is not None: + return f"Could not switch to {target}: {err}." + if final is None: + return f"Timed out waiting for the base policy to switch to {target}." + return f"Base policy is now {target}. {self._policy_summary(final)}" + + def _await_ack( + self, + begun: Callable[[dict[str, Any]], bool], + prev_seq: int, + prev_error: str | None, + ) -> tuple[dict[str, Any] | None, str | None]: + """Wait up to ``_POLICY_START_GRACE_S`` for the scheduler to take a request. + + Every policy_state received after the request (sequence number above + ``prev_seq``) is examined, not just the latest one, so a transition + that begins and ends between two polls - a short trick, a stand-up + that settles at once - is still seen. Returns ``(state, None)`` for + the first such state in which ``begun`` holds, ``(None, reason)`` as + soon as one carries a different ``last_error`` than before the + request, and otherwise, once the grace period has passed, ``(None, + reason)`` with the latest ``last_error`` - even when that is the very + same string as before (the scheduler re-issues identical rejections + without changing anything but the timestamp) - or "no + acknowledgement". The scheduler applies an accepted request on its + next tick, so a request it never saw fails after the grace period + instead of the full ``policy_timeout_s``. + """ + deadline = time.monotonic() + _POLICY_START_GRACE_S + scanned = prev_seq + with self._state_cond: + while True: + for seq, s in self._policy_history: + if seq <= scanned: + continue + scanned = seq + if begun(s): + return s, None + err = s.get("last_error") + if err and err != prev_error: + return None, str(err) + remaining = deadline - time.monotonic() + if remaining <= 0 or self._stop_event.is_set(): + break + self._state_cond.wait(min(remaining, _POLICY_POLL_S)) + latest = self._policy_state + err = (latest.get("last_error") if latest else None) or "no acknowledgement" + return None, str(err) + + def _await_transition( + self, + begun: Callable[[dict[str, Any]], bool], + settled: Callable[[dict[str, Any]], bool], + prev_seq: int, + prev_error: str | None, + timeout: float, + ) -> tuple[dict[str, Any] | None, str | None]: + """Acknowledge a request (see ``_await_ack``) then wait for ``settled``. + + Returns ``(final_state, None)`` on success, ``(None, reason)`` when the + scheduler rejected the request or reported an error after taking it, + and ``(None, None)`` on timeout. + """ + ack, err = self._await_ack(begun, prev_seq, prev_error) + if ack is None: + return None, err + if settled(ack): + return ack, None + ack_error = ack.get("last_error") + + def later_error(s: dict[str, Any]) -> str | None: + e = s.get("last_error") + return str(e) if e and e != ack_error else None + + final = self._await_state(lambda s: settled(s) or later_error(s) is not None, timeout) + if final is None: + return None, None + if settled(final): + return final, None + return None, later_error(final) diff --git a/dimos/robot/pollen/microduck/test_config.py b/dimos/robot/pollen/microduck/test_config.py new file mode 100644 index 0000000000..c8ba718746 --- /dev/null +++ b/dimos/robot/pollen/microduck/test_config.py @@ -0,0 +1,124 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The nav clearances have to keep covering the robot they describe.""" + +from __future__ import annotations + +import numpy as np +import pytest + +from dimos.robot.pollen.microduck import assets_fetch +from dimos.robot.pollen.microduck.config import MICRODUCK +from dimos.robot.pollen.microduck.places import FOUR_ROOM_XML +from dimos.robot.pollen.microduck.sim_module import MicroduckSimModule + + +def _duck_extent() -> tuple[float, float, float]: + """(width, height, footprint circle) of the robot, in metres. + + AABB over the body subtree hanging off the robot's free joint, in the rest + pose. Deliberately not `geom_rbound`: that is a bounding-SPHERE radius, so + for this mesh-heavy model it reads ~1.3 m for a 25 cm duck. + """ + import mujoco + + robot_xml = assets_fetch.variant_mjcf_path("default") + if not robot_xml.exists(): + pytest.skip(f"Microduck asset cache not present ({assets_fetch.assets_root()})") + module = MicroduckSimModule( + scene_xml=FOUR_ROOM_XML, + robot_mjcf=str(robot_xml), + headless=True, + spawn_xy=(0.0, 0.0), + ) + try: + model = module._compose_model() + finally: + module.stop() # the module spins up threads even for a bare compose + data = mujoco.MjData(model) + mujoco.mj_forward(model, data) + + root = int(model.jnt_bodyid[0]) # joint 0 is the trunk free joint + subtree = {root} + for b in range(model.nbody): + p = b + while p != 0: + if p == root: + subtree.add(b) + break + p = int(model.body_parentid[p]) + + lo = np.full(3, np.inf) + hi = np.full(3, -np.inf) + for gid in range(model.ngeom): + if int(model.geom_bodyid[gid]) not in subtree: + continue + pos = data.geom_xpos[gid] + mat = data.geom_xmat[gid].reshape(3, 3) + if int(model.geom_type[gid]) == int(mujoco.mjtGeom.mjGEOM_MESH): + mid = int(model.geom_dataid[gid]) + start = int(model.mesh_vertadr[mid]) + verts = model.mesh_vert[start : start + int(model.mesh_vertnum[mid])].reshape(-1, 3) + world = verts @ mat.T + pos + lo = np.minimum(lo, world.min(axis=0)) + hi = np.maximum(hi, world.max(axis=0)) + else: + half = np.abs(mat) @ np.asarray(model.geom_size[gid], dtype=float) + lo = np.minimum(lo, pos - half) + hi = np.maximum(hi, pos + half) + size = hi - lo + return float(max(size[0], size[1])), float(size[2]), float(np.hypot(size[0], size[1])) + + +@pytest.mark.mujoco +def test_clearances_cover_the_actual_robot() -> None: + """Every clearance must exceed what it clears. + + These were loose constants copied into two blueprints, with a comment + claiming they were the duck's size - they are not, they carry margin. The + margins are the point, so assert both directions: big enough to be safe, + small enough that nobody has quietly turned a 25 cm duck into a metre-wide + one the planner refuses to route through a doorway. + """ + pytest.importorskip("mujoco") + width, height, circle = _duck_extent() + + assert MICRODUCK.width_clearance > width, ( + f"width_clearance {MICRODUCK.width_clearance} does not cover the duck's {width:.3f} m" + ) + assert MICRODUCK.height_clearance > height, ( + f"height_clearance {MICRODUCK.height_clearance} does not cover the duck's {height:.3f} m" + ) + assert MICRODUCK.rotation_diameter > circle, ( + f"rotation_diameter {MICRODUCK.rotation_diameter} does not cover the duck's " + f"{circle:.3f} m turning circle" + ) + # Margin, not a different robot. The four-room doorways are ~0.7 m. + assert MICRODUCK.width_clearance < width + 0.15 + assert MICRODUCK.rotation_diameter < circle + 0.25 + + +def test_blueprints_take_their_clearances_from_the_descriptor() -> None: + """The duplication this file exists to prevent: the same three numbers + were spelled out in both blueprints, so tuning one silently desynced the + other.""" + from dimos.robot.pollen.microduck.blueprints import microduck_cockpit_sim, microduck_sim + + for module in (microduck_cockpit_sim, microduck_sim): + source = module.__file__ + assert source is not None + text = open(source).read() + assert "MICRODUCK.width_clearance" in text, f"{source} does not use the descriptor" + assert "_DUCK_WIDTH" not in text, f"{source} still carries a local copy" diff --git a/dimos/robot/pollen/microduck/test_control_module.py b/dimos/robot/pollen/microduck/test_control_module.py new file mode 100644 index 0000000000..941c85886d --- /dev/null +++ b/dimos/robot/pollen/microduck/test_control_module.py @@ -0,0 +1,684 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""DuckControlModule: mux/mode/lock semantics and nav_state derivation, +driven through the handlers with a fake planner and a fake clock.""" + +from __future__ import annotations + +from collections.abc import Callable, Generator +from dataclasses import dataclass, field +import json +import math +import time +from typing import Any + +from pydantic import ValidationError +import pytest + +from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped +from dimos.msgs.geometry_msgs.Quaternion import Quaternion +from dimos.msgs.geometry_msgs.Twist import Twist +from dimos.msgs.geometry_msgs.Vector3 import Vector3 +from dimos.navigation.base import NavigationState +from dimos.robot.pollen.microduck.control_module import ( + NAV_STATES, + NO_PATH_TIMEOUT_SEC, + STOP_SETTLE_SEC, + DuckControlModule, +) + + +class FakeClock: + def __init__(self) -> None: + self.now = 100.0 + + def __call__(self) -> float: + return self.now + + def advance(self, seconds: float) -> None: + self.now += seconds + + +class FakeNavigation: + """NavigationInterfaceSpec stand-in: scripted state, recorded calls.""" + + def __init__(self) -> None: + self.state = NavigationState.IDLE + self.reached = False + self.calls: list[str] = [] + self.raise_on: set[str] = set() + + def _call(self, name: str) -> None: + self.calls.append(name) + if name in self.raise_on: + raise TimeoutError(f"{name} timed out") + + def set_goal(self, goal: PoseStamped) -> bool: + self._call("set_goal") + return True + + def get_state(self) -> NavigationState: + self._call("get_state") + return self.state + + def is_goal_reached(self) -> bool: + self._call("is_goal_reached") + return self.reached + + def cancel_goal(self) -> bool: + self._call("cancel_goal") + self.state = NavigationState.IDLE + return True + + @property + def cancels(self) -> int: + return self.calls.count("cancel_goal") + + +class FakeTransport: + """In-stream transport stub for start()/stop(): publishes synchronously.""" + + def __init__(self) -> None: + self.subscribers: list[Callable[[Any], Any]] = [] + + def subscribe(self, cb: Callable[[Any], Any], stream: Any = None) -> Callable[[], None]: + self.subscribers.append(cb) + + def unsubscribe() -> None: + self.subscribers.remove(cb) + + return unsubscribe + + def publish(self, msg: Any) -> None: + for cb in list(self.subscribers): + cb(msg) + + def stop(self) -> None: + self.subscribers.clear() + + +@dataclass +class Captured: + cmd_vel: list[Twist] = field(default_factory=list) + mode: list[dict[str, Any]] = field(default_factory=list) + nav_state: list[dict[str, Any]] = field(default_factory=list) + policy_request: list[dict[str, Any]] = field(default_factory=list) + + def clear(self) -> None: + self.cmd_vel.clear() + self.mode.clear() + self.nav_state.clear() + self.policy_request.clear() + + +def _attach(module: DuckControlModule) -> tuple[Captured, list[Callable[[], None]]]: + captured = Captured() + unsubs = [ + module.cmd_vel.subscribe(captured.cmd_vel.append), + module.mode.subscribe(lambda raw: captured.mode.append(json.loads(raw))), + module.nav_state.subscribe(lambda raw: captured.nav_state.append(json.loads(raw))), + module.policy_request.subscribe( + lambda raw: captured.policy_request.append(json.loads(raw)) + ), + ] + return captured, unsubs + + +@dataclass +class Rig: + module: DuckControlModule + captured: Captured + nav: FakeNavigation + clock: FakeClock + + +@pytest.fixture() +def rig() -> Generator[Rig, None, None]: + module = DuckControlModule(tele_cooldown_sec=1.0) + clock = FakeClock() + nav = FakeNavigation() + module._clock = clock + module._navigation = nav # what _connect_module_refs does at deploy time + captured, unsubs = _attach(module) + try: + yield Rig(module, captured, nav, clock) + finally: + for unsub in unsubs: + unsub() + module._close_module() + + +def _twist(lx: float = 0.0, ly: float = 0.0, wz: float = 0.0) -> Twist: + return Twist(linear=Vector3(lx, ly, 0.0), angular=Vector3(0.0, 0.0, wz)) + + +def _goal(x: float = 1.0, y: float = 2.0, yaw: float = 0.5) -> PoseStamped: + return PoseStamped( + ts=time.time(), + frame_id="world", + position=Vector3(x, y, 0.0), + orientation=Quaternion(0.0, 0.0, math.sin(yaw / 2.0), math.cos(yaw / 2.0)), + ) + + +def _command(name: str, **args: Any) -> str: + payload: dict[str, Any] = {"name": name} + if args: + payload["args"] = args + return json.dumps(payload) + + +def _policy_state(locked: bool) -> str: + return json.dumps({"variant": "default", "active": "walk", "locked": locked, "t": 0.0}) + + +def _is_zero(msg: Twist) -> bool: + return ( + msg.linear.x, + msg.linear.y, + msg.linear.z, + msg.angular.x, + msg.angular.y, + msg.angular.z, + ) == ( + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + ) + + +# ---------------------------------------------------------------- ui_command + + +def test_set_mode_publishes_mode_json(rig: Rig) -> None: + rig.module._on_ui_command(_command("set_mode", mode="agent")) + (msg,) = rig.captured.mode + assert set(msg) == {"mode", "t"} + assert msg["mode"] == "agent" + assert isinstance(msg["t"], float) + + rig.module._on_ui_command(_command("set_mode", mode="teleop")) + assert rig.captured.mode[-1]["mode"] == "teleop" + + +def test_set_mode_rejects_unknown_mode(rig: Rig) -> None: + rig.module._on_ui_command(_command("set_mode", mode="autopilot")) + rig.module._on_ui_command(_command("set_mode")) + assert rig.captured.mode == [] + assert rig.module._mode == "teleop" + + +def test_switch_to_agent_mid_teleop_zeroes_cmd_vel(rig: Rig) -> None: + rig.module._on_teleop(_twist(lx=0.3)) + rig.captured.clear() + rig.module._on_ui_command(_command("set_mode", mode="agent")) + (zero,) = rig.captured.cmd_vel + assert _is_zero(zero) + + +def test_policy_command_is_republished_as_policy_request(rig: Rig) -> None: + rig.module._on_ui_command(_command("policy", policy="kick_left", action="start")) + rig.module._on_ui_command(_command("policy", action="stop")) + rig.module._on_ui_command(_command("policy", policy="sitstand", action="toggle")) + first, bare_stop, toggle = rig.captured.policy_request + assert {k: v for k, v in first.items() if k != "t"} == { + "policy": "kick_left", + "action": "start", + } + assert {k: v for k, v in bare_stop.items() if k != "t"} == {"action": "stop"} + assert toggle["policy"] == "sitstand" and toggle["action"] == "toggle" + assert all(isinstance(m["t"], float) for m in rig.captured.policy_request) + + +@pytest.mark.parametrize( + "args", + [ + {"policy": "kick_left", "action": "launch"}, + {"policy": "kick_left"}, + {"action": "start"}, # start needs a policy + {"policy": "", "action": "start"}, + {"policy": 7, "action": "start"}, + ], + ids=["bad_action", "no_action", "start_without_policy", "empty_policy", "non_str_policy"], +) +def test_invalid_policy_commands_are_dropped(rig: Rig, args: dict[str, Any]) -> None: + rig.module._on_ui_command(_command("policy", **args)) + assert rig.captured.policy_request == [] + + +def test_cancel_nav_command_cancels_goal_and_zeroes(rig: Rig) -> None: + rig.module._on_goal_request(_goal()) + rig.captured.clear() + + rig.module._on_ui_command(_command("cancel_nav")) + + assert rig.nav.cancels == 1 + (zero,) = rig.captured.cmd_vel + assert _is_zero(zero) + (state,) = rig.captured.nav_state + assert state["state"] == "cancelled" + assert state["goal"] is None + + +def test_cancel_nav_without_goal_still_stops_but_keeps_state(rig: Rig) -> None: + rig.module._on_ui_command(_command("cancel_nav")) + assert rig.nav.cancels == 1 + assert len(rig.captured.cmd_vel) == 1 and _is_zero(rig.captured.cmd_vel[0]) + assert rig.captured.nav_state == [] # nothing was in progress: no 'cancelled' flash + + +@pytest.mark.parametrize( + "raw", + [ + "not json", + "[1, 2]", + json.dumps({"name": "reboot"}), + json.dumps({"name": "set_mode", "args": ["agent"]}), + json.dumps({"args": {"mode": "agent"}}), + ], + ids=["not_json", "list", "unknown_name", "args_not_object", "no_name"], +) +def test_invalid_ui_commands_are_ignored(rig: Rig, raw: str) -> None: + rig.module._on_ui_command(raw) + assert rig.captured.cmd_vel == [] + assert rig.captured.mode == [] + assert rig.captured.policy_request == [] + assert rig.nav.cancels == 0 + + +# ---------------------------------------------------------------- teleop mux + + +def test_teleop_nonzero_cancels_active_goal_and_forwards(rig: Rig) -> None: + rig.module._on_goal_request(_goal()) + rig.captured.clear() + + rig.module._on_teleop(_twist(lx=0.3, wz=0.2)) + + assert rig.nav.cancels == 1 + zero, forwarded = rig.captured.cmd_vel + assert _is_zero(zero) + assert forwarded.linear.x == pytest.approx(0.3) + assert forwarded.angular.z == pytest.approx(0.2) + assert rig.captured.nav_state[-1]["state"] == "cancelled" + + # No goal any more: the next twist just forwards, no second cancel. + rig.captured.clear() + rig.module._on_teleop(_twist(lx=0.4)) + assert rig.nav.cancels == 1 + (forwarded,) = rig.captured.cmd_vel + assert forwarded.linear.x == pytest.approx(0.4) + + +def test_teleop_scaling_applies_to_vx_vy_wz(rig: Rig) -> None: + rig.module.config.tele_cmd_vel_scaling = (0.5, 2.0, 0.25) + rig.module._on_teleop(_twist(lx=1.0, ly=1.0, wz=1.0)) + (published,) = rig.captured.cmd_vel + assert published.linear.x == pytest.approx(0.5) + assert published.linear.y == pytest.approx(2.0) + assert published.angular.z == pytest.approx(0.25) + + +def test_teleop_zero_is_estop_after_motion_and_stray_when_idle(rig: Rig) -> None: + # Stray zero while nothing moves: dropped entirely. + rig.module._on_teleop(_twist()) + assert rig.captured.cmd_vel == [] + assert rig.nav.cancels == 0 + + # Release after motion: one zero + cancel. + rig.module._on_teleop(_twist(lx=0.3)) + rig.captured.clear() + rig.module._on_teleop(_twist()) + assert rig.nav.cancels == 1 + (zero,) = rig.captured.cmd_vel + assert _is_zero(zero) + + # The pad repeats the zero; those are stray again. + rig.captured.clear() + rig.module._on_teleop(_twist()) + rig.module._on_teleop(_twist()) + assert rig.captured.cmd_vel == [] + assert rig.nav.cancels == 1 + + +def test_teleop_zero_with_active_goal_cancels_it(rig: Rig) -> None: + rig.module._on_goal_request(_goal()) + rig.captured.clear() + rig.module._on_teleop(_twist()) # Space / e-stop while navigating + assert rig.nav.cancels == 1 + (zero,) = rig.captured.cmd_vel + assert _is_zero(zero) + assert rig.captured.nav_state[-1]["state"] == "cancelled" + + +def test_agent_mode_ignores_teleop(rig: Rig) -> None: + rig.module._on_ui_command(_command("set_mode", mode="agent")) + rig.module._on_goal_request(_goal()) + rig.captured.clear() + + rig.module._on_teleop(_twist(lx=0.5)) + rig.module._on_teleop(_twist()) + + assert rig.captured.cmd_vel == [] + assert rig.nav.cancels == 0 + # Navigation keeps flowing in agent mode. + rig.module._on_nav(_twist(lx=0.2)) + (nav,) = rig.captured.cmd_vel + assert nav.linear.x == pytest.approx(0.2) + + +# ---------------------------------------------------------------- nav mux + + +def test_nav_twists_dropped_inside_cooldown_then_resume(rig: Rig) -> None: + rig.module._on_teleop(_twist(lx=0.3)) + rig.captured.clear() + + rig.clock.advance(0.5) + rig.module._on_nav(_twist(lx=0.9)) + assert rig.captured.cmd_vel == [] + + rig.clock.advance(0.6) # past tele_cooldown_sec=1.0 + rig.module._on_nav(_twist(lx=0.9)) + (nav,) = rig.captured.cmd_vel + assert nav.linear.x == pytest.approx(0.9) + assert rig.module._teleop_active is False + + +def test_nav_twists_dropped_in_cooldown_after_estop_zero(rig: Rig) -> None: + rig.module._on_teleop(_twist(lx=0.3)) + rig.clock.advance(0.9) + rig.module._on_teleop(_twist()) # release restamps the cooldown + rig.captured.clear() + rig.clock.advance(0.5) + rig.module._on_nav(_twist(lx=0.9)) + assert rig.captured.cmd_vel == [] + rig.clock.advance(0.6) + rig.module._on_nav(_twist(lx=0.9)) + assert len(rig.captured.cmd_vel) == 1 + + +def test_nav_twists_pass_when_no_teleop_happened(rig: Rig) -> None: + rig.module._on_nav(_twist(lx=0.9)) + (nav,) = rig.captured.cmd_vel + assert nav.linear.x == pytest.approx(0.9) + + +def test_nav_twists_dropped_while_locked(rig: Rig) -> None: + rig.module._on_policy_state(_policy_state(locked=True)) + rig.captured.clear() + rig.module._on_nav(_twist(lx=0.9)) + assert rig.captured.cmd_vel == [] + + rig.module._on_policy_state(_policy_state(locked=False)) + rig.module._on_nav(_twist(lx=0.9)) + assert len(rig.captured.cmd_vel) == 1 + + +# ---------------------------------------------------------------- policy lock + + +def test_locked_rising_edge_cancels_nav_and_zeroes_once(rig: Rig) -> None: + rig.module._on_goal_request(_goal()) + rig.captured.clear() + + rig.module._on_policy_state(_policy_state(locked=True)) + assert rig.nav.cancels == 1 + (zero,) = rig.captured.cmd_vel + assert _is_zero(zero) + assert rig.captured.nav_state[-1]["state"] == "cancelled" + + # Still locked: no repeated cancel / zero. + rig.captured.clear() + rig.module._on_policy_state(_policy_state(locked=True)) + assert rig.nav.cancels == 1 + assert rig.captured.cmd_vel == [] + + +def test_teleop_while_locked_publishes_one_zero_then_nothing(rig: Rig) -> None: + rig.module._on_policy_state(_policy_state(locked=True)) + rig.captured.clear() + # The rising edge already zeroed; held keys produce nothing more. + rig.module._on_teleop(_twist(lx=0.5)) + rig.module._on_teleop(_twist(lx=0.5)) + assert rig.captured.cmd_vel == [] + + rig.module._on_policy_state(_policy_state(locked=False)) + rig.module._on_teleop(_twist(lx=0.5)) + (forwarded,) = rig.captured.cmd_vel + assert forwarded.linear.x == pytest.approx(0.5) + + # Lock again: a zero goes out once more (the edge), and the next pad twist adds none. + rig.captured.clear() + rig.module._on_policy_state(_policy_state(locked=True)) + rig.module._on_teleop(_twist(lx=0.5)) + assert len(rig.captured.cmd_vel) == 1 and _is_zero(rig.captured.cmd_vel[0]) + + +def test_locked_from_the_start_without_edge_zero_sends_one_zero_on_teleop(rig: Rig) -> None: + # Lock flag flips on with the module's own edge handling disabled by a + # malformed first message: cover the 'publish zero once' path directly. + rig.module._on_policy_state("garbage") # ignored + rig.module._locked = True # as if the edge had been missed + rig.module._on_teleop(_twist(lx=0.5)) + rig.module._on_teleop(_twist(lx=0.5)) + assert len(rig.captured.cmd_vel) == 1 and _is_zero(rig.captured.cmd_vel[0]) + + +# ---------------------------------------------------------------- nav_state + + +def _tick(rig: Rig, state: NavigationState, reached: bool) -> dict[str, Any]: + rig.nav.state = state + rig.nav.reached = reached + rig.module._tick() + return rig.captured.nav_state[-1] + + +def test_nav_state_json_shape_and_lifecycle(rig: Rig) -> None: + initial = _tick(rig, NavigationState.IDLE, False) + assert set(initial) == {"state", "goal", "since", "t"} + assert initial["state"] == "idle" and initial["goal"] is None + assert rig.captured.mode[-1]["mode"] == "teleop" # mode rides every tick + + rig.module._on_goal_request(_goal(1.0, 2.0, 0.5)) + pending = rig.captured.nav_state[-1] # published immediately + assert pending["state"] == "idle" + assert pending["goal"] == {"x": 1.0, "y": 2.0, "yaw": pytest.approx(0.5)} + + following = _tick(rig, NavigationState.FOLLOWING_PATH, False) + assert following["state"] == "following_path" + assert following["goal"]["x"] == 1.0 + since_following = following["since"] + + still = _tick(rig, NavigationState.FOLLOWING_PATH, False) + assert still["since"] == since_following # since only moves on state change + + recovery = _tick(rig, NavigationState.RECOVERY, False) + assert recovery["state"] == "recovery" + + reached = _tick(rig, NavigationState.IDLE, True) + assert reached["state"] == "reached" + assert reached["goal"] is None + # Sticky until the next goal. + assert _tick(rig, NavigationState.IDLE, True)["state"] == "reached" + + rig.module._on_goal_request(_goal(-1.0, 0.0, 0.0)) + assert rig.captured.nav_state[-1]["state"] == "idle" + assert rig.captured.nav_state[-1]["goal"]["x"] == -1.0 + assert all(s in NAV_STATES for s in {m["state"] for m in rig.captured.nav_state}) + + +def test_nav_state_ignores_reached_latched_from_previous_goal(rig: Rig) -> None: + rig.module._on_goal_request(_goal()) + _tick(rig, NavigationState.FOLLOWING_PATH, False) + assert _tick(rig, NavigationState.IDLE, True)["state"] == "reached" + + # New goal while the planner still reports the old 'reached' flag. + rig.module._on_goal_request(_goal(0.5, 0.5)) + assert _tick(rig, NavigationState.IDLE, True)["state"] == "idle" + # The planner processes the goal (flag drops), then really arrives. + assert _tick(rig, NavigationState.IDLE, False)["state"] == "idle" + assert _tick(rig, NavigationState.IDLE, True)["state"] == "reached" + + +def test_nav_state_no_path_after_timeout(rig: Rig) -> None: + rig.module._on_goal_request(_goal()) + assert _tick(rig, NavigationState.IDLE, False)["state"] == "idle" + rig.clock.advance(NO_PATH_TIMEOUT_SEC - 0.5) + assert _tick(rig, NavigationState.IDLE, False)["state"] == "idle" + rig.clock.advance(1.0) + no_path = _tick(rig, NavigationState.IDLE, False) + assert no_path["state"] == "no_path" + assert no_path["goal"] is None + + +def test_nav_state_cancelled_when_planner_stops_early(rig: Rig) -> None: + rig.module._on_goal_request(_goal()) + _tick(rig, NavigationState.FOLLOWING_PATH, False) + # Brief replan gap: not cancelled yet, goal still shown. + gap = _tick(rig, NavigationState.IDLE, False) + assert gap["state"] == "idle" and gap["goal"] is not None + rig.clock.advance(STOP_SETTLE_SEC / 2) + assert _tick(rig, NavigationState.FOLLOWING_PATH, False)["state"] == "following_path" + + # Stopped from elsewhere (skill stop_moving via RPC): idle for the settle window. + _tick(rig, NavigationState.IDLE, False) + rig.clock.advance(STOP_SETTLE_SEC) + cancelled = _tick(rig, NavigationState.IDLE, False) + assert cancelled["state"] == "cancelled" + assert cancelled["goal"] is None + + +def test_nav_state_tracks_goal_set_behind_our_back(rig: Rig) -> None: + # A skill calling set_goal over RPC never publishes goal_request. + following = _tick(rig, NavigationState.FOLLOWING_PATH, False) + assert following["state"] == "following_path" + assert following["goal"] is None + assert _tick(rig, NavigationState.IDLE, True)["state"] == "reached" + + +def test_nav_state_cancelled_after_teleop_then_new_goal_starts_fresh(rig: Rig) -> None: + rig.module._on_goal_request(_goal()) + _tick(rig, NavigationState.FOLLOWING_PATH, False) + rig.module._on_teleop(_twist(lx=0.2)) + assert rig.captured.nav_state[-1]["state"] == "cancelled" + # Planner reports idle/not reached afterwards: stays cancelled. + assert _tick(rig, NavigationState.IDLE, False)["state"] == "cancelled" + + +def test_planner_rpc_failures_keep_the_loop_alive(rig: Rig) -> None: + rig.module._on_goal_request(_goal()) + _tick(rig, NavigationState.FOLLOWING_PATH, False) + rig.nav.raise_on = {"get_state", "is_goal_reached", "cancel_goal"} + rig.captured.clear() + + rig.module._tick() # must not raise + # A failed poll changes nothing but still publishes the last snapshot. + assert rig.captured.nav_state[-1]["state"] == "following_path" + assert rig.captured.mode[-1]["mode"] == "teleop" + + rig.module._on_ui_command(_command("cancel_nav")) # cancel_goal raising + assert rig.captured.nav_state[-1]["state"] == "cancelled" + assert _is_zero(rig.captured.cmd_vel[-1]) + + +def test_missing_navigation_ref_is_tolerated(rig: Rig) -> None: + del rig.module._navigation + rig.module._tick() + rig.module._on_ui_command(_command("cancel_nav")) + assert rig.captured.nav_state[-1]["state"] == "idle" + assert _is_zero(rig.captured.cmd_vel[-1]) + + +# ---------------------------------------------------------------- lifecycle + + +def test_start_publishes_mode_runs_state_thread_and_stops_cleanly() -> None: + module = DuckControlModule(state_hz=50.0) + nav = FakeNavigation() + module._navigation = nav + transports = { + name: FakeTransport() + for name in ("nav_cmd_vel", "tele_cmd_vel", "ui_command", "policy_state", "goal_request") + } + for name, transport in transports.items(): + getattr(module, name).transport = transport + captured, unsubs = _attach(module) + try: + module.start() + assert captured.mode and captured.mode[0]["mode"] == "teleop" + deadline = time.monotonic() + 5.0 + while len(captured.nav_state) < 3 and time.monotonic() < deadline: + time.sleep(0.01) + assert len(captured.nav_state) >= 3 + assert "get_state" in nav.calls and "is_goal_reached" in nav.calls + + # Inputs are wired through the transports. + transports["ui_command"].publish( + json.dumps({"name": "set_mode", "args": {"mode": "agent"}}) + ) + assert captured.mode[-1]["mode"] == "agent" + transports["goal_request"].publish(_goal(0.3, 0.4)) + assert captured.nav_state[-1]["goal"]["x"] == pytest.approx(0.3) + transports["tele_cmd_vel"].publish(_twist(lx=0.5)) # agent mode: dropped + assert all(_is_zero(m) or m.linear.x != 0.5 for m in captured.cmd_vel) + transports["nav_cmd_vel"].publish(_twist(lx=0.7)) + assert captured.cmd_vel[-1].linear.x == pytest.approx(0.7) + + thread = module._state_thread + assert thread is not None and thread.is_alive() + finally: + for unsub in unsubs: + unsub() + module.stop() + assert not thread.is_alive() + assert module._state_thread is None + assert all(t.subscribers == [] for t in transports.values()) + + +def test_stop_without_start_is_safe() -> None: + module = DuckControlModule() + module.stop() + + +def test_unwired_inputs_are_skipped_on_start() -> None: + module = DuckControlModule(state_hz=50.0) + module._navigation = FakeNavigation() + try: + module.start() # no transports at all: subscriptions skipped, thread runs + assert module._state_thread is not None and module._state_thread.is_alive() + finally: + module.stop() + + +def test_config_rejects_unknown_default_mode() -> None: + with pytest.raises(ValidationError): + DuckControlModule(default_mode="autopilot") + + +def test_default_mode_agent_is_honoured() -> None: + module = DuckControlModule(default_mode="agent") + captured, unsubs = _attach(module) + try: + module._on_teleop(_twist(lx=0.5)) + assert captured.cmd_vel == [] + finally: + for unsub in unsubs: + unsub() + module._close_module() diff --git a/dimos/robot/pollen/microduck/test_gait.py b/dimos/robot/pollen/microduck/test_gait.py new file mode 100644 index 0000000000..68b6e14654 --- /dev/null +++ b/dimos/robot/pollen/microduck/test_gait.py @@ -0,0 +1,224 @@ +# Copyright 2025-2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Observation contract (``MicroduckObserver``), ONNX session loading and the +single-policy ``MicroduckGaitPolicy`` wrapper the plain microduck-sim +blueprint runs. Tests needing the asset cache skip when it is absent.""" + +from __future__ import annotations + +import math +from pathlib import Path +from typing import Any + +import numpy as np +import pytest + +from dimos.robot.pollen.microduck import assets_fetch +from dimos.robot.pollen.microduck.gait import ( + COMMAND_LEN, + CONTROL_DT, + OBS_LEN, + VX_RANGE, + VY_RANGE, + WZ_RANGE, + MicroduckGaitPolicy, + MicroduckObserver, + load_policy_session, +) + +ROOM_SCENE = Path(__file__).with_name("assets") / "room_scene.xml" +EXPECTED_JOINTS = ( + "left_hip_yaw", + "left_hip_roll", + "left_hip_pitch", + "left_knee", + "left_ankle", + "neck_pitch", + "head_pitch", + "head_yaw", + "head_roll", + "right_hip_yaw", + "right_hip_roll", + "right_hip_pitch", + "right_knee", + "right_ankle", +) + + +def _cache_or_skip() -> None: + if ( + not assets_fetch.robot_mjcf_path().exists() + or not assets_fetch.walking_policy_path().exists() + ): + pytest.skip(f"Microduck asset cache not present ({assets_fetch.assets_root()})") + + +def _model(robot_xml: Path) -> Any: + import mujoco + + scene = mujoco.MjSpec.from_file(str(ROOM_SCENE)) + robot = mujoco.MjSpec.from_file(str(robot_xml)) + scene.option.timestep = 0.005 + robot.option.timestep = 0.005 + scene.attach(robot, prefix="", frame=scene.worldbody.add_frame(pos=[0.0, 0.0, 0.0])) + return scene.compile() + + +def test_contract_constants() -> None: + assert OBS_LEN == 61 and COMMAND_LEN == 13 + assert CONTROL_DT == pytest.approx(0.02) + assert 3 + 3 + 14 + 14 + 14 + COMMAND_LEN == OBS_LEN + assert VX_RANGE == (-0.25, 0.3) and VY_RANGE == (-0.2, 0.2) and WZ_RANGE == (-1.5, 1.5) + + +def test_load_policy_session_reads_metadata() -> None: + pytest.importorskip("onnxruntime") + _cache_or_skip() + session = load_policy_session(assets_fetch.walking_policy_path()) + assert session.path == assets_fetch.walking_policy_path() + assert session.joint_names == EXPECTED_JOINTS + assert session.default_pose.shape == (14,) and session.default_pose.dtype == np.float32 + assert session.default_pose[2] == pytest.approx(-0.458, abs=1e-3) + assert session.action_scale == 1.0 + assert session.input_name == "obs" and session.output_name == "actions" + action = session.run(np.zeros(OBS_LEN, dtype=np.float32)) + assert action.shape == (14,) and action.dtype == np.float32 and np.isfinite(action).all() + + +def test_load_policy_session_validates_metadata(monkeypatch: pytest.MonkeyPatch) -> None: + ort = pytest.importorskip("onnxruntime") + + class Meta: + def __init__(self, custom: dict[str, str]) -> None: + self.custom_metadata_map = custom + + class IO: + def __init__(self, name: str, shape: list[Any]) -> None: + self.name = name + self.shape = shape + + class FakeSession: + meta: dict[str, str] = {} + obs_dim = OBS_LEN + + def __init__(self, path: str) -> None: + self.path = path + + def get_modelmeta(self) -> Meta: + return Meta(dict(self.meta)) + + def get_inputs(self) -> list[IO]: + return [IO("obs", [1, self.obs_dim])] + + def get_outputs(self) -> list[IO]: + return [IO("actions", [1, 14])] + + monkeypatch.setattr(ort, "InferenceSession", FakeSession) + names = ",".join(EXPECTED_JOINTS) + pose = ",".join(["0.1"] * 14) + + FakeSession.meta = {"default_joint_pos": pose} + with pytest.raises(RuntimeError, match="metadata"): + load_policy_session("x.onnx") + FakeSession.meta = {"joint_names": names, "default_joint_pos": ",".join(["0.1"] * 13)} + with pytest.raises(RuntimeError, match="13-long default pose"): + load_policy_session("x.onnx") + FakeSession.meta = {"joint_names": names, "default_joint_pos": pose} + FakeSession.obs_dim = 57 + with pytest.raises(RuntimeError, match="obs dim 57"): + load_policy_session("x.onnx") + FakeSession.obs_dim = OBS_LEN + FakeSession.meta = {"joint_names": names, "default_joint_pos": pose, "action_scale": "0.5"} + session = load_policy_session("x.onnx") + assert session.action_scale == 0.5 and session.joint_names == EXPECTED_JOINTS + assert session.default_pose.tolist() == pytest.approx([0.1] * 14) + + +@pytest.mark.mujoco +def test_observer_builds_the_61_float_observation() -> None: + mujoco = pytest.importorskip("mujoco") + _cache_or_skip() + model = _model(assets_fetch.variant_mjcf_path("default")) + home = np.linspace(-0.3, 0.3, 14, dtype=np.float32) + observer = MicroduckObserver(model, EXPECTED_JOINTS, home) + assert observer.num_joints == 14 and observer.joint_names == list(EXPECTED_JOINTS) + data = mujoco.MjData(model) + data.qpos[observer.root_qpos_adr : observer.root_qpos_adr + 2] = (0.4, -0.2) + observer.initial_qpos(data) + mujoco.mj_forward(model, data) + assert data.qpos[observer.root_qpos_adr : observer.root_qpos_adr + 3].tolist() == pytest.approx( + [0.4, -0.2, 0.125] + ) + last_action = np.full(14, 0.25, dtype=np.float32) + command = np.arange(COMMAND_LEN, dtype=np.float32) + obs = observer.build(data, last_action, command) + assert obs.shape == (OBS_LEN,) and obs.dtype == np.float32 + assert obs[0:3].tolist() == pytest.approx([0.0, 0.0, 0.0]) # gyro at rest + assert obs[3:6].tolist() == pytest.approx([0.0, 0.0, -1.0]) # upright gravity + assert obs[6:20].tolist() == pytest.approx([0.0] * 14, abs=1e-6) # at home pose + assert obs[20:34].tolist() == pytest.approx([0.0] * 14) + assert obs[34:48].tolist() == pytest.approx([0.25] * 14) + assert obs[48:61].tolist() == pytest.approx(list(range(COMMAND_LEN))) + + # Yaw and tilt come from the root quaternion. + half = math.pi / 4 + data.qpos[observer.root_qpos_adr + 3 : observer.root_qpos_adr + 7] = ( + math.cos(half), + 0, + 0, + math.sin(half), + ) + assert observer.root_yaw(data) == pytest.approx(math.pi / 2) + assert observer.projected_gravity(data).tolist() == pytest.approx([0.0, 0.0, -1.0], abs=1e-6) + data.qpos[observer.root_qpos_adr + 3 : observer.root_qpos_adr + 7] = ( + 0.0, + 1.0, + 0.0, + 0.0, + ) # upside down + assert observer.projected_gravity(data)[2] == pytest.approx(1.0) + + with pytest.raises(ValueError): + MicroduckObserver(model, EXPECTED_JOINTS, home[:3]) + with pytest.raises(RuntimeError, match="not found"): + MicroduckObserver(model, ("left_hip_yaw", "no_such_joint"), home[:2]) + + +@pytest.mark.mujoco +def test_gait_policy_wrapper_keeps_its_api() -> None: + pytest.importorskip("onnxruntime") + mujoco = pytest.importorskip("mujoco") + _cache_or_skip() + model = _model(assets_fetch.robot_mjcf_path()) + gait = MicroduckGaitPolicy(assets_fetch.walking_policy_path(), model) + assert gait.joint_names == list(EXPECTED_JOINTS) and gait.num_joints == 14 + assert gait.default_pose.shape == (14,) and gait.action_scale == 1.0 + assert isinstance(gait.root_qpos_adr, int) + + data = mujoco.MjData(model) + gait.initial_qpos(data) + mujoco.mj_forward(model, data) + assert gait.projected_gravity(data).tolist() == pytest.approx([0.0, 0.0, -1.0]) + gait.set_twist(1.0, -1.0, 9.0) + obs = gait.build_observation(data) + assert obs.shape == (OBS_LEN,) + assert obs[48:51].tolist() == pytest.approx([VX_RANGE[1], VY_RANGE[0], WZ_RANGE[1]]) + targets = gait.step(data) + assert targets.shape == (14,) and targets.dtype == np.float32 and np.isfinite(targets).all() + assert gait.last_action.any() + assert np.allclose(targets, gait.default_pose + gait.last_action) + gait.reset() + assert not gait.last_action.any() + assert not gait.build_observation(data)[48:61].any() diff --git a/dimos/robot/pollen/microduck/test_places.py b/dimos/robot/pollen/microduck/test_places.py new file mode 100644 index 0000000000..332b559db3 --- /dev/null +++ b/dimos/robot/pollen/microduck/test_places.py @@ -0,0 +1,648 @@ +# Copyright 2025-2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Four-room scene constants, scene XML, and the sqlite-backed PlacesMemory.""" + +from __future__ import annotations + +from collections.abc import Iterator +import json +import math +import os +from pathlib import Path +import xml.etree.ElementTree as ET + +import pytest + +from dimos.robot.pollen.microduck.places import ( + ARENA_HALF_EXTENT, + BALL_BODY, + BALL_FREEJOINT, + BALL_GEOM, + BALL_GEOM_GROUP, + BALL_MASS, + BALL_RADIUS, + BALL_START, + DEFAULT_SCENE, + FOUR_ROOM_XML, + HUB_HALF_EXTENT, + MICRODUCK_OBJECTS, + MICRODUCK_ROOMS, + PlaceRecord, + PlacesMemory, + RoomSpec, + add_ball_body, + normalize_place_query, + place_key, + scene_id, +) + +# What the pre-existing agentic-sim blueprint seeds: two objects, no rooms. +LEGACY_OBJECTS = {"red_box": (1.5, 0.8), "blue_box": (-1.5, -0.8)} + +_ROBOT_MJCF = Path("~/.cache/dimos/microduck/robot/robot_allcollisions.xml").expanduser() + + +def _floats(text: str | None) -> tuple[float, ...]: + return tuple(float(v) for v in (text or "").split()) + + +def _geoms() -> dict[str, ET.Element]: + root = ET.parse(FOUR_ROOM_XML).getroot() + return {g.get("name", ""): g for g in root.iter("geom") if g.get("name")} + + +# Scene XML <-> constants + + +def test_scene_file_exists() -> None: + assert FOUR_ROOM_XML.is_file() + assert FOUR_ROOM_XML.name == "four_room_scene.xml" + + +def test_objects_match_scene_positions() -> None: + geoms = _geoms() + for name, (x, y) in MICRODUCK_OBJECTS.items(): + assert name in geoms, f"{name} missing from {FOUR_ROOM_XML.name}" + pos = _floats(geoms[name].get("pos")) + assert pos[:2] == pytest.approx((x, y)), name + assert geoms[name].get("group") == "0", f"{name} must be lidar-visible (group 0)" + size = _floats(geoms[name].get("size")) + # Sized 6..14 cm (half-extents 0.03..0.14 depending on the shape). + assert 0.03 <= min(size) and max(size) <= 0.14, (name, size) + + +def test_scene_lists_no_unknown_landmarks() -> None: + """Every coloured object geom in the scene is in MICRODUCK_OBJECTS.""" + geoms = _geoms() + structural = {"floor"} + landmarks = { + n + for n, g in geoms.items() + if g.get("rgba") is not None and n not in structural and not n.startswith("floor_") + } + assert landmarks == set(MICRODUCK_OBJECTS) + + +def test_walls_and_stubs_match_design() -> None: + geoms = _geoms() + stubs = { + "stub_east": ((1.4, 0.0, 0.25), (0.6, 0.05, 0.25)), + "stub_west": ((-1.4, 0.0, 0.25), (0.6, 0.05, 0.25)), + "stub_north": ((0.0, 1.4, 0.25), (0.05, 0.6, 0.25)), + "stub_south": ((0.0, -1.4, 0.25), (0.05, 0.6, 0.25)), + } + for name, (pos, size) in stubs.items(): + assert _floats(geoms[name].get("pos")) == pytest.approx(pos), name + assert _floats(geoms[name].get("size")) == pytest.approx(size), name + for name in ("wall_north", "wall_south", "wall_east", "wall_west"): + size = _floats(geoms[name].get("size")) + assert size[2] == pytest.approx(0.25) + assert min(size[:2]) == pytest.approx(0.05) + # The hub opening between stub tips is 1.6 m wide; the inner wall faces + # are at +/- ARENA_HALF_EXTENT. + assert (1.4 - 0.6) * 2 == pytest.approx(2 * HUB_HALF_EXTENT) + for name, sign, axis in ( + ("wall_north", 1, 1), + ("wall_south", -1, 1), + ("wall_east", 1, 0), + ("wall_west", -1, 0), + ): + pos = _floats(geoms[name].get("pos")) + half = _floats(geoms[name].get("size")) + assert pos[axis] - sign * half[axis] == pytest.approx(sign * ARENA_HALF_EXTENT), name + for name, g in geoms.items(): + assert g.get("group") == "0", name + + +def test_room_floor_patches_are_visual_only() -> None: + geoms = _geoms() + for room in MICRODUCK_ROOMS.values(): + patch = geoms[f"floor_{room.name}"] + assert patch.get("contype") == "0" and patch.get("conaffinity") == "0" + cx, cy = room.center + assert _floats(patch.get("pos"))[:2] == pytest.approx((cx, cy)) + + +def test_scene_declares_no_bodies_or_joints() -> None: + """The engine takes joint 0 as the robot's root free joint and MuJoCo + numbers joints in body order, so nothing jointed (the ball included) may + be declared in the scene ahead of the attached robot.""" + root = ET.parse(FOUR_ROOM_XML).getroot() + worldbody = root.find("worldbody") + assert worldbody is not None + assert worldbody.findall("body") == [] + assert list(root.iter("freejoint")) == [] and list(root.iter("joint")) == [] + assert not any(b.get("name") == BALL_BODY for b in root.iter("body")) + + +def test_ball_constants_match_design() -> None: + assert BALL_BODY == "ball" + assert BALL_FREEJOINT == f"{BALL_BODY}_freejoint" + assert BALL_GEOM == f"{BALL_BODY}_geom" + assert BALL_START == pytest.approx((1.2, -0.6, 0.035)) + assert BALL_START[2] == pytest.approx(BALL_RADIUS) # resting on the floor + assert BALL_RADIUS == pytest.approx(0.035) + assert BALL_MASS == pytest.approx(0.05) + assert BALL_GEOM_GROUP == 1 # not group 0: the raycast lidar must not see it + # It starts inside the office, well clear of the stubs and walls. + assert MICRODUCK_ROOMS["office"].contains(BALL_START[0], BALL_START[1]) + + +def test_visual_offscreen_size() -> None: + root = ET.parse(FOUR_ROOM_XML).getroot() + glob = root.find("visual/global") + assert glob is not None + assert glob.get("offwidth") == "1280" and glob.get("offheight") == "720" + + +def test_room_table_matches_design() -> None: + assert list(MICRODUCK_ROOMS) == ["kitchen", "living", "bedroom", "office"] + expected = { + "kitchen": (("space A",), (0, 2, 0, 2), (1.2, 1.0, 0.0)), + "living": (("space B", "living room", "lounge"), (-2, 0, 0, 2), (-1.2, 1.0, 3.14159)), + "bedroom": (("space C",), (-2, 0, -2, 0), (-1.2, -1.0, 3.14159)), + "office": (("space D", "study"), (0, 2, -2, 0), (1.2, -1.0, 0.0)), + } + for name, (aliases, bounds, target) in expected.items(): + room = MICRODUCK_ROOMS[name] + assert isinstance(room, RoomSpec) + assert room.name == name + assert room.aliases == aliases + assert room.bounds == pytest.approx(bounds) + assert room.target == pytest.approx(target) + # Target lies inside the room, past the hub, and its yaw looks into + # the room (away from the hub at the origin) so the head camera sees + # the room's landmark after arriving. + assert room.contains(target[0], target[1]) + assert abs(target[0]) > HUB_HALF_EXTENT or abs(target[1]) > HUB_HALF_EXTENT + outward = math.atan2(target[1], target[0]) + assert math.cos(outward - target[2]) > 0.7 + + +def test_every_object_sits_in_a_room() -> None: + for name, (x, y) in MICRODUCK_OBJECTS.items(): + assert any(r.contains(x, y) for r in MICRODUCK_ROOMS.values()), name + + +# MuJoCo compile / compose (explicit: pytest -m mujoco) + + +@pytest.mark.mujoco +def test_scene_compiles_standalone() -> None: + mujoco = pytest.importorskip("mujoco") + model = mujoco.MjModel.from_xml_path(str(FOUR_ROOM_XML)) + assert model.njnt == 0 and model.nbody == 1 # world only: nothing ahead of the robot + for name in MICRODUCK_OBJECTS: + gid = model.geom(name).id + assert model.geom_pos[gid][:2] == pytest.approx(MICRODUCK_OBJECTS[name]) + assert model.geom_group[gid] == 0 + assert (model.vis.global_.offwidth, model.vis.global_.offheight) == (1280, 720) + + +def _attach_robot(mujoco, spec_scene): # type: ignore[no-untyped-def] + """Same recipe as ``MicroduckSimModule._compose_model``: MjSpec attach.""" + if not _ROBOT_MJCF.is_file(): + pytest.skip(f"robot MJCF not cached at {_ROBOT_MJCF}") + spec_robot = mujoco.MjSpec.from_file(str(_ROBOT_MJCF)) + if not spec_robot.meshdir or not os.path.isabs(spec_robot.meshdir): + spec_robot.meshdir = str(_ROBOT_MJCF.parent / (spec_robot.meshdir or "assets")) + assert spec_robot.body("trunk_base") is not None + spec_scene.option.timestep = 0.005 + spec_robot.option.timestep = 0.005 + frame = spec_scene.worldbody.add_frame(pos=[0.0, 0.0, 0.0], quat=[1.0, 0.0, 0.0, 0.0]) + spec_scene.attach(spec_robot, prefix="", frame=frame) + + +@pytest.mark.mujoco +def test_scene_composes_with_robot() -> None: + """The robot's root free joint must be joint 0 of the composed model: + ``MujocoEngine._find_first_freejoint_adrs`` takes ``jnt_qposadr[0]`` as + the robot root (odom, spawn, IMU fallback, lidar exclusion).""" + mujoco = pytest.importorskip("mujoco") + spec_scene = mujoco.MjSpec.from_file(str(FOUR_ROOM_XML)) + _attach_robot(mujoco, spec_scene) + model = spec_scene.compile() + data = mujoco.MjData(model) + for _ in range(50): + mujoco.mj_step(model, data) + assert model.body("trunk_base").id > 0 + assert model.nu == 14 + assert model.joint(0).name == "trunk_base_freejoint" + assert model.jnt_type[0] == mujoco.mjtJoint.mjJNT_FREE + assert model.jnt_qposadr[0] == model.jnt_qposadr[model.joint("trunk_base_freejoint").id] == 0 + + +@pytest.mark.mujoco +def test_ball_added_after_robot_keeps_trunk_as_joint_zero() -> None: + """``add_ball_body`` (called by the sim module after ``attach``) yields + the design's ball without displacing the robot from joint 0.""" + mujoco = pytest.importorskip("mujoco") + spec_scene = mujoco.MjSpec.from_file(str(FOUR_ROOM_XML)) + _attach_robot(mujoco, spec_scene) + body = add_ball_body(spec_scene) + assert body.name == BALL_BODY + model = spec_scene.compile() + data = mujoco.MjData(model) + for _ in range(50): + mujoco.mj_step(model, data) + + assert model.joint(0).name == "trunk_base_freejoint" + ball_joint = model.joint(BALL_FREEJOINT) + assert ball_joint.id > 0 + assert ball_joint.type == mujoco.mjtJoint.mjJNT_FREE + adr = model.jnt_qposadr[ball_joint.id] + assert adr >= 7 + assert data.qpos[adr : adr + 3] == pytest.approx(BALL_START, abs=2e-3) # rests in place + gid = model.geom(BALL_GEOM).id + assert model.geom_group[gid] == BALL_GEOM_GROUP + assert model.geom_size[gid][0] == pytest.approx(BALL_RADIUS) + assert model.body_mass[model.body(BALL_BODY).id] == pytest.approx(BALL_MASS) + + +@pytest.mark.mujoco +def test_add_ball_body_custom_name_and_pos() -> None: + mujoco = pytest.importorskip("mujoco") + spec = mujoco.MjSpec.from_file(str(FOUR_ROOM_XML)) + add_ball_body(spec, name="football", pos=(-0.5, 0.5, 0.035)) + model = spec.compile() + assert model.joint("football_freejoint").type == mujoco.mjtJoint.mjJNT_FREE + assert model.geom("football_geom").id >= 0 + assert model.body_pos[model.body("football").id] == pytest.approx((-0.5, 0.5, 0.035)) + + +# PlacesMemory + + +@pytest.fixture +def memory(tmp_path: Path) -> Iterator[PlacesMemory]: + mem = PlacesMemory(tmp_path / "places.db") + mem.seed(MICRODUCK_ROOMS, MICRODUCK_OBJECTS) + try: + yield mem + finally: + mem.close() + + +def test_seed_is_idempotent(tmp_path: Path) -> None: + mem = PlacesMemory(tmp_path / "places.db") + try: + assert mem.seed(MICRODUCK_ROOMS, MICRODUCK_OBJECTS) == 9 + assert mem.seed(MICRODUCK_ROOMS, MICRODUCK_OBJECTS) == 0 + records = mem.all() + assert len(records) == 9 + assert len({(r.kind, r.name) for r in records}) == 9 + assert [r.name for r in records if r.kind == "room"] == list(MICRODUCK_ROOMS) + assert [r.name for r in records if r.kind == "object"] == list(MICRODUCK_OBJECTS) + finally: + mem.close() + + +def test_seed_rewrites_moved_object(tmp_path: Path) -> None: + mem = PlacesMemory(tmp_path / "places.db") + try: + mem.seed({}, {"red_box": (1.0, 1.0)}) + assert mem.seed({}, {"red_box": (1.5, 1.5)}) == 1 + rec = mem.find("red_box") + assert rec is not None and (rec.x, rec.y) == (1.5, 1.5) + assert len(mem.all()) == 1 + finally: + mem.close() + + +def test_persists_across_reopen(tmp_path: Path) -> None: + db = tmp_path / "places.db" + mem = PlacesMemory(db) + mem.seed(MICRODUCK_ROOMS, MICRODUCK_OBJECTS) + mem.add("charger", 0.3, -0.2, 0.0) + mem.close() + + reopened = PlacesMemory(db) + try: + assert reopened.seed(MICRODUCK_ROOMS, MICRODUCK_OBJECTS) == 0 + records = reopened.all() + assert len(records) == 10 + charger = reopened.find("charger") + assert charger == PlaceRecord("tagged", "charger", (), 0.3, -0.2, 0.0, None) + finally: + reopened.close() + + +def test_scene_id_is_stable_and_table_sensitive() -> None: + cockpit = scene_id(MICRODUCK_ROOMS, MICRODUCK_OBJECTS) + assert cockpit == scene_id(MICRODUCK_ROOMS, MICRODUCK_OBJECTS) + assert cockpit == scene_id(dict(reversed(list(MICRODUCK_ROOMS.items()))), MICRODUCK_OBJECTS) + assert cockpit.startswith("scene-") and len(cockpit) == len("scene-") + 12 + assert scene_id({}, LEGACY_OBJECTS) != cockpit + assert scene_id({}, {}) != cockpit + assert scene_id(MICRODUCK_ROOMS, {**MICRODUCK_OBJECTS, "red_box": (1.5, 0.8)}) != cockpit + assert scene_id() == scene_id({}, {}) == scene_id(None, None) + assert scene_id() != DEFAULT_SCENE + + +def test_default_scene_and_explicit_scene(tmp_path: Path) -> None: + mem = PlacesMemory(tmp_path / "places.db") + try: + assert mem.scene == DEFAULT_SCENE == "default" + finally: + mem.close() + mem = PlacesMemory(tmp_path / "places.db", scene="") + try: + assert mem.scene == DEFAULT_SCENE + finally: + mem.close() + mem = PlacesMemory(tmp_path / "places.db", scene="cockpit-a") + try: + assert mem.scene == "cockpit-a" + loc = mem.add("charger", 0.0, 0.0) + assert loc.metadata["scene"] == "cockpit-a" + finally: + mem.close() + + +def test_scenes_sharing_one_db_do_not_leak(tmp_path: Path) -> None: + """Cockpit (4 rooms, 5 objects) and the old agentic sim (2 objects) share + ``~/.cache/dimos/microduck/places.db``; each must only see its own world.""" + db = tmp_path / "places.db" + cockpit = PlacesMemory(db, scene=scene_id(MICRODUCK_ROOMS, MICRODUCK_OBJECTS)) + legacy = PlacesMemory(db, scene=scene_id({}, LEGACY_OBJECTS)) + try: + assert cockpit.seed(MICRODUCK_ROOMS, MICRODUCK_OBJECTS) == 9 + assert legacy.seed({}, LEGACY_OBJECTS) == 2 + cockpit.add("charger", 0.3, -0.2) + legacy.add("nest", -0.3, 0.2) + + assert len(cockpit.all()) == 10 + assert legacy.rooms() == [] + assert [r.name for r in legacy.all()] == ["red_box", "blue_box", "nest"] + + # Same object name, different position per scene. + red_cockpit = cockpit.find("red_box") + red_legacy = legacy.find("red_box") + assert red_cockpit is not None and (red_cockpit.x, red_cockpit.y) == (1.5, 1.5) + assert red_legacy is not None and (red_legacy.x, red_legacy.y) == (1.5, 0.8) + + # Rooms, tagged spots and the spatial index are scoped too. + assert cockpit.room_at(1.5, 0.8) is not None + assert legacy.room_at(1.5, 0.8) is None + assert legacy.find("kitchen") is None and legacy.find("space A") is None + assert legacy.find("charger") is None and cockpit.find("nest") is None + assert [r.name for r in legacy.near(1.5, 0.8, 0.2)] == ["red_box"] + assert [r.name for r in cockpit.near(1.5, 0.8, 0.2)] == [] + assert [r.name for r in cockpit.near(0.3, -0.2, 0.05)] == ["charger"] + assert legacy.near(0.3, -0.2, 0.05) == [] + + legacy_json = json.loads(legacy.to_json()) + assert legacy_json["rooms"] == [] and legacy_json["tagged"] == [ + {"name": "nest", "x": -0.3, "y": 0.2, "yaw": 0.0} + ] + assert [o["name"] for o in json.loads(cockpit.to_json())["objects"]] == list( + MICRODUCK_OBJECTS + ) + + # Re-seeding either scene writes nothing (idempotent per scene). + assert cockpit.seed(MICRODUCK_ROOMS, MICRODUCK_OBJECTS) == 0 + assert legacy.seed({}, LEGACY_OBJECTS) == 0 + finally: + cockpit.close() + legacy.close() + + # Reopening sees the same partition; the default scene sees nothing. + reopened = PlacesMemory(db, scene=scene_id({}, LEGACY_OBJECTS)) + default = PlacesMemory(db) + try: + assert [r.name for r in reopened.all()] == ["red_box", "blue_box", "nest"] + assert default.all() == [] + finally: + reopened.close() + default.close() + + +def test_expands_home_and_creates_parents(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("HOME", str(tmp_path)) + mem = PlacesMemory("~/nested/dir/places.db") + try: + assert mem.db_path == tmp_path / "nested" / "dir" / "places.db" + assert mem.db_path.parent.is_dir() + finally: + mem.close() + + +@pytest.mark.parametrize( + ("query", "expected"), + [ + ("kitchen", "kitchen"), + ("space A", "kitchen"), + ("Space a", "kitchen"), + ("SPACE-A", "kitchen"), + ("the kitchen", "kitchen"), + ("kitchen please", "kitchen"), + ("go to the kitchen", "kitchen"), + ("room A", "kitchen"), + ("space B", "living"), + ("living room", "living"), + ("the living room", "living"), + ("lounge", "living"), + ("living", "living"), + ("space C", "bedroom"), + ("bedroom", "bedroom"), + ("space D", "office"), + ("study", "office"), + ("the office please", "office"), + ], +) +def test_find_rooms_by_name_and_alias(memory: PlacesMemory, query: str, expected: str) -> None: + rec = memory.find(query) + assert rec is not None and rec.kind == "room" and rec.name == expected + + +@pytest.mark.parametrize( + ("query", "expected"), + [ + ("red_box", "red_box"), + ("red box", "red_box"), + ("Red Box", "red_box"), + ("the red box", "red_box"), + ("blue box", "blue_box"), + ("green cylinder", "green_cylinder"), + ("yellow pillar", "yellow_pillar"), + ("yellow", "yellow_pillar"), + ("orange crate", "orange_crate"), + ], +) +def test_find_objects(memory: PlacesMemory, query: str, expected: str) -> None: + rec = memory.find(query) + assert rec is not None and rec.kind == "object" and rec.name == expected + + +def test_find_unknown_and_kind_filter(memory: PlacesMemory) -> None: + assert memory.find("garage") is None + assert memory.find("") is None + assert memory.find(" ") is None + assert memory.find("red box", kind="room") is None + assert memory.find("space A", kind="object") is None + kitchen = memory.find("space A", kind="room") + assert kitchen is not None and kitchen.name == "kitchen" + + +def test_find_prefers_exact_name_over_substring(memory: PlacesMemory) -> None: + memory.add("box", 0.1, 0.2) + rec = memory.find("box") + assert rec is not None and rec.kind == "tagged" and rec.name == "box" + assert memory.matches("box") == [rec] + + +@pytest.mark.parametrize("query", ["box", "the box", "space", "room", "the room please"]) +def test_find_is_none_when_ambiguous_or_filler_only(memory: PlacesMemory, query: str) -> None: + assert memory.find(query) is None + + +def test_matches_lists_candidates_in_table_order(memory: PlacesMemory) -> None: + assert [r.name for r in memory.matches("box")] == ["red_box", "blue_box"] + assert [r.name for r in memory.matches("box", kind="object")] == ["red_box", "blue_box"] + assert memory.matches("box", kind="room") == [] + assert [r.name for r in memory.matches("kitchen")] == ["kitchen"] + assert [r.name for r in memory.matches("space A")] == ["kitchen"] + assert [r.name for r in memory.matches("yellow")] == ["yellow_pillar"] + # Filler-only queries hit nothing by containment... + assert memory.matches("space") == [] + assert memory.matches("room") == [] + assert memory.matches("") == [] + # ...but still resolve a place that is literally called that. + memory.add("spot", 0.0, 0.5) + assert [r.name for r in memory.matches("spot")] == ["spot"] + + +def test_normalize_place_query() -> None: + assert normalize_place_query("Space a") == ("space a", "a") + assert normalize_place_query("the kitchen please") == ("the kitchen please", "kitchen") + assert normalize_place_query("go to space B") == ("go to space b", "b") + assert normalize_place_query("red_box") == ("red box",) + assert normalize_place_query("") == () + assert normalize_place_query("the") == ("the",) + + +def test_room_records_carry_target_and_bounds(memory: PlacesMemory) -> None: + living = memory.find("living") + assert living is not None + assert living.bounds == pytest.approx((-2, 0, 0, 2)) + assert (living.x, living.y, living.yaw) == pytest.approx((-1.2, 1.0, 3.14159)) + assert living.aliases == ("space B", "living room", "lounge") + + +@pytest.mark.parametrize( + ("x", "y", "expected"), + [ + (1.5, 1.5, "kitchen"), + (0.9, 0.1, "kitchen"), + (-1.0, 1.9, "living"), + (-1.5, -1.5, "bedroom"), + (1.2, -1.0, "office"), + (0.0, 0.0, None), + (0.5, -0.5, None), + (0.79, 0.79, None), + (2.5, 0.0, None), + (0.0, -2.5, None), + ], +) +def test_room_at(memory: PlacesMemory, x: float, y: float, expected: str | None) -> None: + rec = memory.room_at(x, y) + assert (rec.name if rec is not None else None) == expected + + +def test_near_uses_spatial_index(memory: PlacesMemory) -> None: + near = memory.near(1.4, 1.4, 0.6) + names = [r.name for r in near] + assert names[0] == "red_box" + assert "kitchen" in names # its target (1.2, 1.0) is 0.45 m away + assert "blue_box" not in names + assert memory.near(0.0, 0.0, 0.1) == [] + memory.add("charger", 0.05, 0.0) + assert [r.name for r in memory.near(0.0, 0.0, 0.1)] == ["charger"] + + +def test_add_updates_existing_tagged_place(memory: PlacesMemory) -> None: + memory.add("charger", 0.3, -0.2, 0.0) + memory.add("charger", 0.4, -0.2, 1.0) + tagged = [r for r in memory.all() if r.kind == "tagged"] + assert tagged == [PlaceRecord("tagged", "charger", (), 0.4, -0.2, 1.0, None)] + + +def test_place_key_folds_case_punctuation_and_spacing() -> None: + assert place_key("Charger") == place_key("charger") == place_key(" charger! ") == "charger" + assert place_key("Front Door") == place_key("front_door") == "front door" + assert place_key("Space A") == "space a" + assert place_key("kitchen") != place_key("the kitchen") # different names, both kept + assert place_key("") == "" + + +def test_add_is_case_insensitive_and_keeps_latest_spelling(memory: PlacesMemory) -> None: + """'Charger' and 'charger' are one place: re-tagging overwrites instead of + duplicating (find() would otherwise report the pair as ambiguous).""" + memory.add("charger", 0.3, -0.2, 0.0) + memory.add("Charger", 0.5, -0.1, 1.0) + memory.add("CHARGER!", 0.6, -0.1, 2.0) + tagged = [r for r in memory.all() if r.kind == "tagged"] + assert tagged == [PlaceRecord("tagged", "CHARGER!", (), 0.6, -0.1, 2.0, None)] + found = memory.find("charger") + assert found is not None and (found.x, found.yaw) == (0.6, 2.0) + assert len(memory.matches("Charger")) == 1 + payload = json.loads(memory.to_json()) + assert [t["name"] for t in payload["tagged"]] == ["CHARGER!"] + + +def test_seed_treats_case_variants_as_one_place(tmp_path: Path) -> None: + """A respelled table entry renames the stored place (the blueprint's + table is ground truth) instead of leaving two records behind.""" + mem = PlacesMemory(tmp_path / "places.db") + try: + assert mem.seed({}, {"Red Box": (1.0, 1.0)}) == 1 + assert mem.seed({}, {"red_box": (1.0, 1.0)}) == 1 # respelled: rewritten once... + assert mem.seed({}, {"red_box": (1.0, 1.0)}) == 0 # ...then idempotent again + objects = [r for r in mem.all() if r.kind == "object"] + assert [(r.name, r.x) for r in objects] == [("red_box", 1.0)] + assert len(mem.matches("red box")) == 1 + finally: + mem.close() + + +def test_add_returns_robot_location(memory: PlacesMemory) -> None: + loc = memory.add("charger", 0.3, -0.2, 0.5, metadata={"note": "usb-c"}) + assert loc.name == "charger" + assert loc.position == (0.3, -0.2, 0.0) + assert loc.rotation == (0.0, 0.0, 0.5) + assert loc.frame_id == "world" + assert loc.metadata["kind"] == "tagged" + assert loc.metadata["note"] == "usb-c" + + +def test_to_json_shape(memory: PlacesMemory) -> None: + memory.add("charger", 0.3, -0.2, 0.0) + payload = json.loads(memory.to_json(t=123.0)) + assert set(payload) == {"frame", "rooms", "objects", "tagged", "t"} + assert payload["frame"] == "world" + assert payload["t"] == 123.0 + assert payload["rooms"][0] == { + "name": "kitchen", + "aliases": ["space A"], + "bounds": [0.0, 2.0, 0.0, 2.0], + "target": [1.2, 1.0, 0.0], + } + assert [r["name"] for r in payload["rooms"]] == list(MICRODUCK_ROOMS) + assert payload["objects"][0] == {"name": "red_box", "x": 1.5, "y": 1.5} + assert [o["name"] for o in payload["objects"]] == list(MICRODUCK_OBJECTS) + assert payload["tagged"] == [{"name": "charger", "x": 0.3, "y": -0.2, "yaw": 0.0}] + # Compact separators, as every JSON stream in the cockpit. + assert ", " not in memory.to_json() and ": " not in memory.to_json() + assert "t" in json.loads(memory.to_json()) diff --git a/dimos/robot/pollen/microduck/test_policies.py b/dimos/robot/pollen/microduck/test_policies.py new file mode 100644 index 0000000000..c3aa2794d4 --- /dev/null +++ b/dimos/robot/pollen/microduck/test_policies.py @@ -0,0 +1,1226 @@ +# Copyright 2025-2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Microduck policy catalogue, ``PolicyScheduler`` state machine (sim-time +ticks, fake wall clock), ``PolicyBank`` / asset helpers, and an opt-in +headless MuJoCo run of every policy. + +The MuJoCo-backed tests carry the ``mujoco`` marker, which the repo's pytest +``addopts`` deselect by default, so they need ``-m mujoco``:: + + pytest dimos/robot/pollen/microduck/test_policies.py -m mujoco + +The per-policy headless matrix (``test_slow_policy_runs_in_headless_sim``, +a few seconds per variant on Apple Silicon) is additionally gated on an env +flag and needs the cached ONNX/MJCF assets under ``~/.cache/dimos/microduck``:: + + MICRODUCK_SLOW_TESTS=1 pytest dimos/robot/pollen/microduck/test_policies.py \\ + -m mujoco -k headless -s # default variant, prints outcomes + MICRODUCK_SLOW_VARIANTS=rollers MICRODUCK_SLOW_TESTS=1 pytest ... -m mujoco -k headless +""" + +from __future__ import annotations + +from collections.abc import Callable, Iterable +import json +import math +import os +from pathlib import Path +import threading +from typing import Any +import urllib.error + +import numpy as np +import pytest + +from dimos.robot.pollen.microduck import assets_fetch +from dimos.robot.pollen.microduck.policies import ( + ACTIVE_BRAKING, + ACTIVE_STANDING_UP, + ASSET_MISSING_REASON, + BASE_POLICIES, + BRAKE_DURATION_S, + FALL_GRAVITY_Z, + GROUND_PICK_END_PHASE, + GROUND_PICK_PERIOD_S, + KICK_BALL_OFFSETS, + KICK_DURATION_S, + LAST_ERROR_TTL_S, + ONESHOT_POLICIES, + POLICY_NAMES, + POLICY_SPECS, + ROLLER_BRAKE_THROTTLE, + ROLLER_THROTTLE_RANGE, + ROULADE_DURATION_S, + ROULADE_GRACE_S, + STAND_UP_DURATION_S, + PolicyBank, + PolicyKind, + PolicyName, + PolicyScheduler, + policy_availability, +) + +DT = 0.02 # the policies' control period +ROOM_SCENE = Path(__file__).with_name("assets") / "room_scene.xml" +DEFAULT_POLICIES = ( + "walk", + "stand", + "sitstand", + "kick_left", + "kick_right", + "roulade", + "ground_pick", +) +ROLLERS_POLICIES = ( + "walk", + "stand", + "roller", + "roller_crouch", + "kick_left", + "kick_right", + "ground_pick", +) + + +class FakeClock: + def __init__(self) -> None: + self.now = 100.0 + + def __call__(self) -> float: + return self.now + + def advance(self, seconds: float) -> None: + self.now += seconds + + +class Harness: + """A scheduler plus the recorded ball spawns and a helper to run sim ticks.""" + + def __init__(self, variant: str = "default", missing: Iterable[str] = ()) -> None: + self.clock = FakeClock() + self.spawns: list[tuple[float, float]] = [] + self.sched = PolicyScheduler( + policy_availability(variant, missing), + variant, + spawn_ball=lambda dx, dy: self.spawns.append((dx, dy)), + clock=self.clock, + ) + + def run(self, seconds: float) -> list[tuple[str, np.ndarray]]: + n = round(seconds / DT) + return [self.sched.tick(DT) for _ in range(n)] + + def names(self, seconds: float) -> list[str]: + return [name for name, _ in self.run(seconds)] + + +@pytest.fixture +def h() -> Harness: + return Harness() + + +@pytest.fixture +def rollers() -> Harness: + return Harness("rollers") + + +# --------------------------------------------------------------- catalogue + + +def test_policy_specs_display_order_kinds_and_files() -> None: + assert POLICY_NAMES == ( + "walk", + "stand", + "roller", + "roller_crouch", + "sitstand", + "kick_left", + "kick_right", + "roulade", + "ground_pick", + ) + assert [str(n) for n in POLICY_SPECS] == list(POLICY_NAMES) + kinds = {str(n): s.kind for n, s in POLICY_SPECS.items()} + assert BASE_POLICIES == {"walk", "stand", "roller", "roller_crouch"} + assert kinds["sitstand"] is PolicyKind.POSTURE + assert ONESHOT_POLICIES == {"kick_left", "kick_right", "roulade", "ground_pick"} + files = {str(n): s.onnx for n, s in POLICY_SPECS.items()} + assert files == { + "walk": "alpha_walking.onnx", + "stand": "alpha_stand.onnx", + "roller": "roller.onnx", + "roller_crouch": "roller_crouch.onnx", + "sitstand": "alpha_sitstand.onnx", + "kick_left": "ball_kick_left.onnx", + "kick_right": "ball_kick_right.onnx", + "roulade": "roulade.onnx", + "ground_pick": "alpha_ground_pick.onnx", + } + assert assets_fetch.POLICY_FILES == {PolicyName(n): f for n, f in files.items()} + for spec in POLICY_SPECS.values(): + assert (spec.duration is not None) == (spec.kind is PolicyKind.ONESHOT) + assert str(PolicyName.WALK) == "walk" and PolicyName("kick_left") is PolicyName.KICK_LEFT + assert json.dumps({"p": PolicyName.ROULADE}) == '{"p": "roulade"}' + assert FALL_GRAVITY_Z == -0.55 # the sim module's existing threshold + + +def test_policy_availability_per_variant() -> None: + default = policy_availability("default") + assert [n for n, r in default.items() if r is None] == list(DEFAULT_POLICIES) + assert "rollers" in default["roller"] and "MICRODUCK_VARIANT=rollers" in default["roller"] + rollers = policy_availability("rollers") + assert [n for n, r in rollers.items() if r is None] == list(ROLLERS_POLICIES) + assert rollers["sitstand"] == "not supported on the rollers variant" + assert rollers["roulade"] == "not supported on the rollers variant" + with pytest.raises(ValueError): + policy_availability("wheels") + missing = policy_availability("default", missing=("roulade", PolicyName.STAND)) + assert missing["roulade"] == ASSET_MISSING_REASON + assert missing["stand"] == ASSET_MISSING_REASON + assert missing["walk"] is None + + +# --------------------------------------------------------------- scheduler + + +def test_initial_state_and_snapshot_shape(h: Harness) -> None: + snap = h.sched.snapshot() + assert list(snap) == [ + "variant", + "active", + "base", + "seated", + "fallen", + "locked", + "oneshot", + "policies", + "last_error", + "t", + ] + assert snap["variant"] == "default" + assert snap["active"] == "walk" and type(snap["active"]) is str + assert snap["base"] == "walk" + assert snap["seated"] is False and snap["fallen"] is False and snap["locked"] is False + assert snap["oneshot"] is None and snap["last_error"] is None + assert isinstance(snap["t"], float) + assert [p["name"] for p in snap["policies"]] == list(POLICY_NAMES) + assert all(set(p) == {"name", "kind", "available", "reason"} for p in snap["policies"]) + by_name = {p["name"]: p for p in snap["policies"]} + assert by_name["walk"] == {"name": "walk", "kind": "base", "available": True, "reason": None} + assert by_name["sitstand"]["kind"] == "posture" and by_name["kick_left"]["kind"] == "oneshot" + assert by_name["roller"]["available"] is False and by_name["roller"]["reason"] + json.dumps(snap) # plain JSON, no enum members + name, cmd = h.sched.tick(DT) + assert name == "walk" and type(name) is str + assert cmd.shape == (13,) and cmd.dtype == np.float32 and not cmd.any() + + +def test_request_is_applied_on_tick_and_latest_pending_wins(h: Harness) -> None: + assert h.sched.request("stand", "start") == (True, "") + assert h.sched.active == "walk" # nothing changes off the sim thread + assert h.sched.request("kick_left", "start") == (True, "") # replaces the pending stand + name, _ = h.sched.tick(DT) + assert name == "kick_left" and h.sched.base == "walk" + + +def test_walk_stand_switch_is_immediate(h: Harness) -> None: + h.sched.request("stand", "start") + assert h.names(DT) == ["stand"] + assert h.sched.base == "stand" and not h.sched.locked + h.sched.request("stand", "stop") # stop on a base = back to walk + assert h.names(DT) == ["walk"] + h.sched.request("stand", "toggle") + assert h.names(DT) == ["stand"] + h.sched.request("walk", "toggle") + assert h.names(DT) == ["walk"] + assert h.sched.request("walk", "start") == (True, "") # no-op, still accepted + assert h.names(DT) == ["walk"] + + +def test_walk_follows_clipped_twist_and_stand_ignores_it(h: Harness) -> None: + h.sched.set_twist(1.0, -1.0, 3.0) + _, cmd = h.sched.tick(DT) + assert cmd[:3].tolist() == pytest.approx([0.3, -0.2, 1.5]) + assert not cmd[3:].any() + h.sched.set_twist(-1.0, 0.1, -0.5) + _, cmd = h.sched.tick(DT) + assert cmd[:3].tolist() == pytest.approx([-0.25, 0.1, -0.5]) + h.sched.request("stand", "start") + name, cmd = h.sched.tick(DT) + assert name == "stand" and not cmd.any() + + +def test_roller_to_walk_brakes_with_zero_throttle(rollers: Harness) -> None: + s = rollers.sched + s.request("roller", "start") + s.set_twist(0.48, 0.0, 0.3) + name, cmd = s.tick(DT) + assert name == "roller" and cmd[0] == pytest.approx(0.48) and cmd[2] == pytest.approx(0.3) + assert cmd[1] == 0.0 + s.set_twist(2.0, 0.5, -3.0) + _, cmd = s.tick(DT) + assert cmd[0] == pytest.approx(ROLLER_THROTTLE_RANGE[1]) and cmd[2] == pytest.approx(-1.0) + + assert s.request("walk", "start") == (True, "") + name, cmd = s.tick(DT) + # Braking keeps the roller policy at zero throttle (never reverse thrust, + # which would drive a resting duck backwards) and ignores the joystick. + assert name == "roller" and cmd[0] == ROLLER_BRAKE_THROTTLE == 0.0 + assert not cmd.any() + assert s.active == ACTIVE_BRAKING and s.base == "roller" and s.locked + assert s.snapshot()["active"] == "braking" + assert s.request("stand", "start") == (False, "locked: braking") + names = rollers.names(BRAKE_DURATION_S + 5 * DT) + braking = 1 + names.count("roller") + assert braking * DT >= BRAKE_DURATION_S - 1e-9 + assert names[-1] == "walk" and s.base == "walk" and not s.locked + + +def test_roller_crouch_brakes_on_its_own_loop_when_roller_is_missing() -> None: + # Regression: braking used to hand the bank ``roller`` unconditionally, + # which raised KeyError on the sim thread when only roller_crouch loaded. + crouch_only = Harness("rollers", missing=("roller",)) + s = crouch_only.sched + assert s.request("roller", "start") == (False, f"roller unavailable: {ASSET_MISSING_REASON}") + assert s.request("roller_crouch", "start") == (True, "") + crouch_only.run(0.5) + assert s.request("walk", "start") == (True, "") + name, cmd = s.tick(DT) + assert name == "roller_crouch" and s.active == ACTIVE_BRAKING + phi = 0.5 / 3.0 # the crouch phase keeps advancing while braking + assert cmd[0] == pytest.approx(math.cos(2 * math.pi * phi), abs=1e-5) + assert cmd[1] == pytest.approx(math.sin(2 * math.pi * phi), abs=1e-5) + _, cmd2 = s.tick(DT) + assert not np.allclose(cmd2[:2], cmd[:2]) + names = crouch_only.names(BRAKE_DURATION_S) + assert "roller" not in names and names[-1] == "walk" + assert s.snapshot()["last_error"] is None + + +def test_braking_keeps_the_roller_family_base_being_left(rollers: Harness) -> None: + s = rollers.sched + s.request("roller_crouch", "start") + rollers.run(0.3) + s.request("walk", "start") + assert rollers.names(BRAKE_DURATION_S) == ["roller_crouch"] * round(BRAKE_DURATION_S / DT) + assert rollers.names(DT) == ["walk"] + + +def test_roller_family_switches_do_not_brake_but_leaving_does(rollers: Harness) -> None: + s = rollers.sched + s.request("roller", "start") + s.tick(DT) + s.request("roller_crouch", "start") + name, cmd = s.tick(DT) + assert name == "roller_crouch" and s.base == "roller_crouch" + assert cmd[0] == pytest.approx(1.0) and cmd[1] == pytest.approx(0.0) # phase 0 + _, cmd = s.tick(DT) + phi = DT / 3.0 + assert cmd[0] == pytest.approx(math.cos(2 * math.pi * phi), abs=1e-5) + assert cmd[1] == pytest.approx(math.sin(2 * math.pi * phi), abs=1e-5) + s.request("roller", "toggle") + assert rollers.names(DT) == ["roller"] + s.request("stand", "start") + assert rollers.names(DT) == ["roller"] and s.active == ACTIVE_BRAKING + rollers.run(BRAKE_DURATION_S) + assert rollers.names(DT) == ["stand"] + + +def test_roller_stop_means_walk_via_braking(rollers: Harness) -> None: + s = rollers.sched + s.request("roller_crouch", "start") + s.tick(DT) + s.request("roller_crouch", "stop") + s.tick(DT) + assert s.active == ACTIVE_BRAKING + rollers.run(BRAKE_DURATION_S) + assert rollers.names(DT) == ["walk"] + + +def test_sitstand_lifecycle(h: Harness) -> None: + s = h.sched + assert s.request("sitstand", "stop") == (False, "not seated") + assert s.snapshot()["last_error"] == "not seated" + assert s.request("sitstand", "start") == (True, "") + assert s.snapshot()["last_error"] is None # an accepted request clears it + h.sched.set_twist(0.3, 0.0, 0.0) + name, cmd = s.tick(DT) + assert name == "sitstand" and cmd[0] == 1.0 and not cmd[1:].any() + snap = s.snapshot() + assert snap["seated"] is True and snap["locked"] is True and snap["active"] == "sitstand" + assert snap["oneshot"] is None + assert s.request("stand", "start") == (False, "locked: seated") + assert s.request("kick_left", "start") == (False, "locked: seated") + assert s.request("sitstand", "start") == (True, "already seated") + assert h.names(0.5) == ["sitstand"] * 25 + + assert s.request("sitstand", "stop") == (True, "") + name, cmd = s.tick(DT) + assert name == "sitstand" and not cmd.any() # flag 0 = stand up + snap = s.snapshot() + assert snap["active"] == ACTIVE_STANDING_UP and snap["seated"] is False + assert snap["locked"] is True + assert s.request("walk", "start") == (False, "locked: standing up") + assert s.request("sitstand", "stop") == (True, "already standing up") + names = h.names(STAND_UP_DURATION_S) + assert names[:-1] == ["sitstand"] * (round(STAND_UP_DURATION_S / DT) - 1) + assert names[-1] == "walk" + assert not s.locked and s.base == "walk" + _, cmd = s.tick(DT) + assert cmd[0] == pytest.approx(0.3) # twist honoured again + + +def test_sitstand_toggle_and_policy_less_stop(h: Harness) -> None: + s = h.sched + s.request("sitstand", "toggle") + s.tick(DT) + assert s.seated + assert s.request(None, "stop") == (True, "") + s.tick(DT) + assert s.active == ACTIVE_STANDING_UP + h.run(STAND_UP_DURATION_S) + assert s.active == "walk" + s.request("sitstand", "toggle") + s.tick(DT) + assert s.seated + s.request("sitstand", "toggle") + s.tick(DT) + assert s.active == ACTIVE_STANDING_UP + assert s.request(None, "start") == (False, "start needs a policy name") + h.run(STAND_UP_DURATION_S) + assert s.request(None, "stop") == (True, "") # idle: harmless + assert h.names(DT) == ["walk"] + + +@pytest.mark.parametrize( + ("name", "expected_ticks"), + [ + ("kick_left", round(KICK_DURATION_S / DT)), + ("kick_right", round(KICK_DURATION_S / DT)), + ("roulade", round(ROULADE_DURATION_S / DT)), + ("ground_pick", round(GROUND_PICK_END_PHASE * GROUND_PICK_PERIOD_S / DT)), + ], +) +def test_oneshot_runs_its_window_then_hands_back( + h: Harness, name: str, expected_ticks: int +) -> None: + s = h.sched + s.request("stand", "start") + s.tick(DT) + assert s.request(name, "start") == (True, "") + progress: list[float] = [] + ran = 0 + for _ in range(expected_ticks + 10): + snap = s.snapshot() + if snap["oneshot"] is not None: + assert snap["oneshot"]["name"] == name + progress.append(snap["oneshot"]["progress"]) + policy, _ = s.tick(DT) + if policy == name: + ran += 1 + assert s.locked + assert ran == expected_ticks + assert progress[0] == pytest.approx(DT / (expected_ticks * DT), abs=0.01) + assert progress[-1] == 1.0 + assert progress == sorted(progress) + snap = s.snapshot() + assert snap["oneshot"] is None and snap["active"] == "stand" and snap["locked"] is False + assert h.names(DT) == ["stand"] + + +def test_ground_pick_command_is_phase_encoded(h: Harness) -> None: + s = h.sched + s.request("ground_pick", "start") + cmds = [cmd for _, cmd in h.run(5 * DT)] + for k, cmd in enumerate(cmds): + phi = k * DT / GROUND_PICK_PERIOD_S + assert cmd[0] == pytest.approx(math.cos(2 * math.pi * phi), abs=1e-5) + assert cmd[1] == pytest.approx(math.sin(2 * math.pi * phi), abs=1e-5) + assert not cmd[2:].any() + + +def test_kick_command_is_zero_and_ignores_twist(h: Harness) -> None: + h.sched.set_twist(0.3, 0.1, 0.5) + h.sched.request("kick_right", "start") + name, cmd = h.sched.tick(DT) + assert name == "kick_right" and not cmd.any() + + +@pytest.mark.parametrize("action", ["stop", "toggle"]) +def test_oneshot_abort(h: Harness, action: str) -> None: + s = h.sched + s.request("roulade", "start") + h.run(0.5) + assert s.active == "roulade" + assert s.request("roulade", action) == (True, "") + name, _ = s.tick(DT) + assert name == "walk" and s.active == "walk" and not s.locked + assert s.snapshot()["oneshot"] is None + # policy-less stop aborts too + s.request("kick_left", "start") + s.tick(DT) + assert s.request(None, "stop") == (True, "") + assert h.names(DT) == ["walk"] + assert s.request("kick_left", "stop") == (False, "kick_left is not running") + assert s.request("kick_left", "start") == (True, "") + s.tick(DT) + assert s.request("kick_left", "start") == (True, "already running") + assert s.request("roulade", "start") == (False, "locked: kick_left running") + + +@pytest.mark.parametrize( + ("name", "action"), + [("kick_left", "toggle"), ("kick_left", "stop"), ("roulade", "stop"), ("roulade", "toggle")], +) +def test_abort_landing_on_the_final_tick_does_not_restart_or_error( + h: Harness, name: str, action: str +) -> None: + """A stop/toggle accepted while the oneshot runs must keep meaning "abort" + even if the window expires before the sim thread consumes it.""" + s = h.sched + s.request(name, "start") + window = POLICY_SPECS[PolicyName(name)].duration or 0.0 + assert set(h.names(window)) == {name} # the whole window, expiry not yet ticked + snap = s.snapshot() + assert snap["locked"] and snap["oneshot"] == {"name": name, "progress": 1.0} + assert s.request(name, action) == (True, "") + policy, _ = s.tick(DT) + assert policy == "walk" and s.active == "walk" and not s.locked + assert s.snapshot()["last_error"] is None + assert len(h.spawns) == (1 if name == "kick_left" else 0) + assert h.names(0.5) == ["walk"] * 25 # nothing restarted later either + assert len(h.spawns) == (1 if name == "kick_left" else 0) + + +def test_stop_right_behind_a_pending_start_cancels_it(h: Harness) -> None: + s = h.sched + assert s.request("kick_left", "start") == (True, "") + assert s.request("kick_left", "stop") == (True, "") # cancels the unconsumed start + assert h.names(DT) == ["walk"] and h.spawns == [] and s.snapshot()["last_error"] is None + assert s.request("kick_right", "start") == (True, "") + assert s.request("kick_right", "toggle") == (True, "") # toggle behind a start = cancel + assert h.names(DT) == ["walk"] and h.spawns == [] + assert s.request("sitstand", "start") == (True, "") + assert s.request("sitstand", "stop") == (True, "") + assert h.names(DT) == ["walk"] and not s.seated + assert s.request("sitstand", "toggle") == (True, "") + assert s.request("sitstand", "toggle") == (True, "") + assert h.names(DT) == ["walk"] and not s.seated + # A policy-less stop behind a pending start cancels it too (no "locked:" + # error later) - and, with nothing to abort, keeps a parked base select. + assert s.request("ground_pick", "start") == (True, "") + assert s.request(None, "stop") == (True, "") + assert h.names(DT) == ["walk"] and s.snapshot()["last_error"] is None + assert s.request("stand", "start") == (True, "") + assert s.request(None, "stop") == (True, "") + assert h.names(DT) == ["stand"] + + +def test_start_right_behind_a_pending_abort_cancels_it(h: Harness) -> None: + """The reverse of the test above: the newest intent wins in both + directions, so an unconsumed abort/stand-up is dropped by a start/sit.""" + s = h.sched + s.request("kick_left", "start") + s.tick(DT) + assert s.request("kick_left", "stop") == (True, "") # abort pending + assert s.request("kick_left", "start") == (True, "already running") # ... cancelled + assert h.names(DT) == ["kick_left"] and s.snapshot()["last_error"] is None + assert s.request("kick_left", "toggle") == (True, "") + assert s.request("kick_left", "toggle") == (True, "") # toggle behind an abort = cancel + assert h.names(DT) == ["kick_left"] + assert s.request(None, "stop") == (True, "") + assert s.request("kick_left", "start") == (True, "already running") + assert h.names(DT) == ["kick_left"] + assert s.request("kick_left", "stop") == (True, "") + assert s.request("kick_right", "start") == (False, "locked: kick_left running") # not a cancel + assert h.names(DT) == ["walk"] + assert len(h.spawns) == 1 # the kick ran exactly once + + s.request("sitstand", "start") + s.tick(DT) + assert s.seated + assert s.request("sitstand", "stop") == (True, "") # stand-up pending + assert s.request("sitstand", "start") == (True, "already seated") # ... cancelled + assert h.names(DT) == ["sitstand"] and s.seated + assert s.request(None, "stop") == (True, "") + assert s.request("sitstand", "toggle") == (True, "") # toggle behind a stand-up = cancel + assert h.names(DT) == ["sitstand"] and s.seated + assert s.request("sitstand", "toggle") == (True, "") + assert s.request("sitstand", "toggle") == (True, "") + assert h.names(DT) == ["sitstand"] and s.seated + assert s.request("sitstand", "stop") == (True, "") + assert s.request("sitstand", "stop") == (True, "") # a repeat is still a stand-up + s.tick(DT) + assert s.active == ACTIVE_STANDING_UP + + +def test_stale_stand_up_after_a_fall_is_dropped_silently(h: Harness) -> None: + s = h.sched + s.request("sitstand", "start") + s.tick(DT) + assert s.request("sitstand", "stop") == (True, "") + s.notify_fall(True) # the sim thread beats the request; the duck is no longer seated + assert h.names(DT) == ["walk"] + assert s.active == "walk" and s.snapshot()["last_error"] is None + # ... whereas a pending *start* that a fall overtook is reported. + s.notify_fall(False) + assert s.request("kick_left", "start") == (True, "") + s.notify_fall(True) + assert h.names(DT) == ["walk"] and h.spawns == [] + assert s.snapshot()["last_error"] == "locked: fallen" + + +def test_spawn_ball_failure_does_not_break_the_tick() -> None: + calls: list[tuple[float, float]] = [] + + def broken(dx: float, dy: float) -> None: + calls.append((dx, dy)) + raise RuntimeError("no ball body in this scene") + + s = PolicyScheduler(policy_availability("default"), "default", spawn_ball=broken) + s.request("kick_right", "start") + name, cmd = s.tick(DT) + assert name == "kick_right" and not cmd.any() and s.locked + assert calls == [(0.09, -0.042)] + assert s.snapshot()["oneshot"] == {"name": "kick_right", "progress": pytest.approx(0.04)} + + +def test_fall_aborts_kick_and_locks_until_recovered(h: Harness) -> None: + s = h.sched + s.set_twist(0.3, 0.0, 0.0) + s.request("kick_left", "start") + s.tick(DT) + s.notify_fall(True) + snap = s.snapshot() + assert snap["fallen"] is True and snap["locked"] is True + assert snap["active"] == "walk" and snap["oneshot"] is None + assert s.request("kick_right", "start") == (False, "locked: fallen") + assert s.request("stand", "start") == (False, "locked: fallen") + name, cmd = s.tick(DT) + assert name == "walk" and not cmd.any() # twist ignored while fallen + s.notify_fall(False) + assert not s.locked + _, cmd = s.tick(DT) + assert cmd[0] == pytest.approx(0.3) + + +def test_fall_does_not_abort_roulade_in_window_and_grace(h: Harness) -> None: + s = h.sched + assert not s.suspend_fall_detector + s.request("roulade", "start") + s.tick(DT) + assert s.suspend_fall_detector + h.run(1.0) + s.notify_fall(True) # mid-roll: the duck is upside down by design + assert s.active == "roulade" and not s.fallen + remaining = round(ROULADE_DURATION_S / DT) - 1 - round(1.0 / DT) + names = h.names(remaining * DT) + assert set(names) == {"roulade"} + grace_ticks = 0 + while s.suspend_fall_detector: # window over, then the grace period + s.notify_fall(True) + assert not s.fallen + name, _ = s.tick(DT) + if name == "walk": + grace_ticks += 1 + assert grace_ticks < 1000 + assert grace_ticks * DT == pytest.approx(ROULADE_GRACE_S) + s.notify_fall(True) + assert s.fallen and s.locked + + +def test_fall_while_seated_standing_up_or_braking(h: Harness, rollers: Harness) -> None: + s = h.sched + s.request("sitstand", "start") + s.tick(DT) + s.notify_fall(True) + assert s.active == "walk" and not s.seated and s.fallen + s.notify_fall(False) + s.request("sitstand", "start") + s.tick(DT) + s.request("sitstand", "stop") + s.tick(DT) + assert s.active == ACTIVE_STANDING_UP + s.notify_fall(True) + assert s.active == "walk" + + r = rollers.sched + r.request("roller", "start") + r.tick(DT) + r.request("walk", "start") + r.tick(DT) + assert r.active == ACTIVE_BRAKING + r.notify_fall(True) + assert r.active == "walk" and r.base == "walk" and r.fallen + + +def test_rejections_set_last_error_which_expires(h: Harness) -> None: + s = h.sched + s.request("kick_left", "start") + s.tick(DT) + assert s.request("stand", "start") == (False, "locked: kick_left running") + assert s.snapshot()["last_error"] == "locked: kick_left running" + h.clock.advance(LAST_ERROR_TTL_S - 0.5) + assert s.snapshot()["last_error"] == "locked: kick_left running" + h.clock.advance(1.0) + assert s.snapshot()["last_error"] is None + # A request that was fine when queued but stale by tick time is reported too. + h.run(1.0) # kick over + assert s.request("stand", "start") == (True, "") + s.notify_fall(True) # sim thread beats the pending request + assert h.names(DT) == ["walk"] + assert s.snapshot()["last_error"] == "locked: fallen" + + +def test_unavailable_unknown_and_bad_requests(h: Harness, rollers: Harness) -> None: + ok, reason = h.sched.request("roller", "start") + assert not ok and reason.startswith("roller unavailable: requires the rollers variant") + assert h.sched.snapshot()["last_error"] == reason + ok, reason = rollers.sched.request("sitstand", "toggle") + assert not ok and reason == "sitstand unavailable: not supported on the rollers variant" + ok, reason = rollers.sched.request("roulade", "start") + assert not ok and "not supported on the rollers variant" in reason + assert h.sched.request("moonwalk", "start") == (False, "unknown policy 'moonwalk'") + assert h.sched.request("walk", "pause") == (False, "unknown action 'pause'") + assert h.names(DT) == ["walk"] # nothing leaked into the pending slot + + +def test_kick_spawns_ball_with_signed_offsets(h: Harness) -> None: + s = h.sched + s.request("kick_left", "start") + assert h.spawns == [] # spawned on the sim thread, not at request time + s.tick(DT) + assert h.spawns == [(0.09, 0.042)] + s.request("kick_left", "stop") + s.tick(DT) + s.request("kick_right", "toggle") + s.tick(DT) + assert h.spawns == [(0.09, 0.042), (0.09, -0.042)] + assert KICK_BALL_OFFSETS == {"kick_left": (0.09, 0.042), "kick_right": (0.09, -0.042)} + s.request("kick_right", "stop") + s.tick(DT) + s.request("roulade", "start") + s.tick(DT) + assert len(h.spawns) == 2 # only kicks spawn + + +def test_missing_assets_and_base_fallback() -> None: + hh = Harness(missing=("walk", "kick_left")) + s = hh.sched + assert s.base == "stand" and hh.names(DT) == ["stand"] + by_name = {p["name"]: p for p in s.snapshot()["policies"]} + assert by_name["walk"] == { + "name": "walk", + "kind": "base", + "available": False, + "reason": ASSET_MISSING_REASON, + } + assert s.request("kick_left", "start") == (False, "kick_left unavailable: asset missing") + assert s.request("stand", "stop") == (False, "walk unavailable: asset missing") + with pytest.raises(ValueError): + PolicyScheduler(policy_availability("default", missing=BASE_POLICIES), "default") + with pytest.raises(ValueError): + PolicyScheduler(policy_availability("default"), "wheels") + # Policies the bank never reported are treated as unavailable, not crashed on. + s2 = PolicyScheduler({"walk": None}, "default") + assert s2.request("stand", "start") == (False, "stand unavailable: not loaded") + + +def test_requests_from_other_threads_are_serialised(h: Harness) -> None: + s = h.sched + stop = threading.Event() + errors: list[BaseException] = [] + + def hammer(seed: int) -> None: + rng = np.random.default_rng(seed) + try: + while not stop.is_set(): + s.request( + str(rng.choice(POLICY_NAMES)), str(rng.choice(["start", "stop", "toggle"])) + ) + s.set_twist(float(rng.normal()), 0.0, 0.0) + s.snapshot() + except BaseException as exc: # pragma: no cover - failure path + errors.append(exc) + + threads = [threading.Thread(target=hammer, args=(i,)) for i in range(4)] + for t in threads: + t.start() + for _ in range(2000): + name, cmd = s.tick(DT) + assert name in POLICY_NAMES and cmd.shape == (13,) + if s.fallen: + s.notify_fall(False) + stop.set() + for t in threads: + t.join(timeout=5) + assert not errors + json.dumps(s.snapshot()) + + +# ------------------------------------------------------------------ assets + + +def test_assets_variant_helpers(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("DIMOS_MICRODUCK_ASSETS", str(tmp_path)) + assert assets_fetch.assets_root() == tmp_path + assert assets_fetch.ROBOT_MJCF_BY_VARIANT == { + "default": "robot_allcollisions.xml", + "rollers": "robot_allcollisions_rollers.xml", + } + assert ( + assets_fetch.variant_mjcf_path("rollers") + == tmp_path / "robot" / "robot_allcollisions_rollers.xml" + ) + assert assets_fetch.robot_mjcf_path() == tmp_path / "robot" / "robot_walk.xml" + assert assets_fetch.walking_policy_path() == tmp_path / "policies" / "alpha_walking.onnx" + assert assets_fetch.policy_path("roulade") == tmp_path / "policies" / "roulade.onnx" + with pytest.raises(ValueError): + assets_fetch.ensure_assets("wheels") + assets = assets_fetch.MicroduckAssets(tmp_path / "robot", tmp_path / "policies", ()) + assert assets.robot_mjcf() == tmp_path / "robot" / "robot_allcollisions.xml" + assert assets.policy_path(PolicyName.KICK_LEFT) == tmp_path / "policies" / "ball_kick_left.onnx" + + +def _seed_cache(root: Path, policies: Iterable[str]) -> None: + (root / "robot").mkdir(parents=True) + for name in ("robot_walk.xml", "robot_allcollisions.xml", "robot_allcollisions_rollers.xml"): + (root / "robot" / name).write_text("") + (root / "policies").mkdir() + for name in policies: + (root / "policies" / name).write_bytes(b"x" * 10) + + +def test_ensure_assets_fetches_only_missing_and_never_deletes( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("DIMOS_MICRODUCK_ASSETS", str(tmp_path)) + _seed_cache(tmp_path, ["alpha_walking.onnx", "alpha_stand.onnx"]) + (tmp_path / "policies" / "extra.onnx").write_bytes(b"keep me") + fetched: list[str] = [] + + def fake_fetch(dest: Path) -> None: + fetched.append(dest.name) + if dest.name == "roller.onnx": + raise RuntimeError("pinned source does not serve roller.onnx") + dest.write_bytes(b"y" * 10) + + monkeypatch.setattr(assets_fetch, "_fetch_policy", fake_fetch) + monkeypatch.setattr( + assets_fetch, "_fetch_robot_dir", lambda *a, **k: pytest.fail("robot dir was cached") + ) + assets = assets_fetch.ensure_assets("default") + assert assets.robot_dir == tmp_path / "robot" and assets.policy_dir == tmp_path / "policies" + assert assets.missing == (PolicyName.ROLLER,) + assert "alpha_walking.onnx" not in fetched and "alpha_stand.onnx" not in fetched + assert len(fetched) == 7 + assert (tmp_path / "policies" / "extra.onnx").read_bytes() == b"keep me" + assert (tmp_path / "policies" / "alpha_stand.onnx").read_bytes() == b"x" * 10 + assert (tmp_path / ".complete-1").exists() + # Second call: everything but roller is cached; only roller is retried. + fetched.clear() + assets = assets_fetch.ensure_assets("rollers") + assert fetched == ["roller.onnx"] and assets.missing == (PolicyName.ROLLER,) + + +def test_ensure_assets_policy_subset_skips_probes_for_the_rest( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("DIMOS_MICRODUCK_ASSETS", str(tmp_path)) + _seed_cache(tmp_path, ["alpha_stand.onnx"]) + fetched: list[str] = [] + + def fake_fetch(dest: Path) -> None: + fetched.append(dest.name) + dest.write_bytes(b"y" * 10) + + monkeypatch.setattr(assets_fetch, "_fetch_policy", fake_fetch) + assets = assets_fetch.ensure_assets(policies=("stand",)) # walk is always wanted + assert fetched == ["alpha_walking.onnx"] + assert assets.missing == tuple( + n for n in PolicyName if n not in (PolicyName.WALK, PolicyName.STAND) + ) + fetched.clear() + assets = assets_fetch.ensure_assets(policies=[PolicyName.KICK_LEFT, "roulade"]) + assert fetched == ["ball_kick_left.onnx", "roulade.onnx"] + assert PolicyName.KICK_LEFT not in assets.missing and PolicyName.GROUND_PICK in assets.missing + + +def test_ensure_assets_offline_marks_rest_missing_but_needs_walk( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("DIMOS_MICRODUCK_ASSETS", str(tmp_path)) + _seed_cache(tmp_path, ["alpha_walking.onnx"]) + attempts: list[str] = [] + + def offline(dest: Path) -> None: + attempts.append(dest.name) + raise assets_fetch._SourceUnreachableError("no network") + + monkeypatch.setattr(assets_fetch, "_fetch_policy", offline) + assets = assets_fetch.ensure_assets() + assert attempts == ["alpha_stand.onnx"] # one probe, then no more network calls + assert assets.missing == tuple(n for n in PolicyName if n is not PolicyName.WALK) + (tmp_path / "policies" / "alpha_walking.onnx").unlink() + with pytest.raises(RuntimeError, match="walking policy"): + assets_fetch.ensure_assets() + + +def test_ensure_assets_fetches_robot_dir_when_variant_mjcf_missing( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("DIMOS_MICRODUCK_ASSETS", str(tmp_path)) + _seed_cache(tmp_path, [f for f in assets_fetch.POLICY_FILES.values()]) + (tmp_path / "robot" / "robot_allcollisions_rollers.xml").unlink() + calls: list[tuple[Path, set[str]]] = [] + + def fake_robot(dest: Path, required: Iterable[str]) -> None: + calls.append((dest, set(required))) + (dest / "robot_allcollisions_rollers.xml").write_text("") + + monkeypatch.setattr(assets_fetch, "_fetch_robot_dir", fake_robot) + monkeypatch.setattr(assets_fetch, "_fetch_policy", lambda d: pytest.fail("policies cached")) + assets_fetch.ensure_assets("default") + assert calls == [] + assets_fetch.ensure_assets("rollers") + assert calls == [(tmp_path / "robot", {"robot_walk.xml", "robot_allcollisions_rollers.xml"})] + assert (tmp_path / "robot" / "robot_walk.xml").exists() # nothing removed + + def down(dest: Path, required: Iterable[str]) -> None: + raise OSError("down") + + monkeypatch.setattr(assets_fetch, "_fetch_robot_dir", down) + (tmp_path / "robot" / "robot_allcollisions.xml").unlink() + with pytest.raises(RuntimeError, match="robot models"): + assets_fetch.ensure_assets("default") + + +def test_source_serves_classifies_head_results(monkeypatch: pytest.MonkeyPatch) -> None: + class Resp: + status = 200 + + def __enter__(self) -> Resp: + return self + + def __exit__(self, *exc: object) -> None: + return None + + seen: list[str] = [] + + def fake_urlopen(request: Any, timeout: float) -> Resp: + seen.append(request.get_method()) + url = request.full_url + if url.endswith("missing.onnx"): + raise urllib.error.HTTPError(url, 404, "nope", {}, None) # type: ignore[arg-type] + if url.endswith("offline.onnx"): + raise urllib.error.URLError("no route") + return Resp() + + monkeypatch.setattr(assets_fetch.urllib.request, "urlopen", fake_urlopen) + assert assets_fetch._source_serves("https://x/ok.onnx") is True + assert assets_fetch._source_serves("https://x/missing.onnx") is False + with pytest.raises(assets_fetch._SourceUnreachableError): + assets_fetch._source_serves("https://x/offline.onnx") + assert seen == ["HEAD"] * 3 + with pytest.raises(RuntimeError, match="does not serve"): + assets_fetch._fetch_policy(Path("/nonexistent/missing.onnx")) + + +# --------------------------------------------------------- with the cache + + +def _cache_or_skip(variant: str = "default") -> None: + if not assets_fetch.variant_mjcf_path(variant).exists(): + pytest.skip(f"Microduck robot cache not present ({assets_fetch.robot_dir()})") + if not assets_fetch.walking_policy_path().exists(): + pytest.skip(f"Microduck policy cache not present ({assets_fetch.policy_dir()})") + + +def test_ensure_assets_is_offline_when_cache_complete(monkeypatch: pytest.MonkeyPatch) -> None: + _cache_or_skip() + if not all(p.exists() for p in map(assets_fetch.policy_path, PolicyName)): + pytest.skip("policy cache incomplete") + before = sorted(p.name for p in assets_fetch.policy_dir().iterdir()) + + def no_network(*a: Any, **k: Any) -> None: + pytest.fail("ensure_assets touched the network with a complete cache") + + monkeypatch.setattr(assets_fetch.urllib.request, "urlopen", no_network) + assets = assets_fetch.ensure_assets("default") + assert assets.missing == () + assert assets.robot_mjcf("default").exists() and assets.policy_path("walk").exists() + assert sorted(p.name for p in assets_fetch.policy_dir().iterdir()) == before + + +def _compose(variant: str, *, with_ball: bool) -> Any: + """room_scene + robot MJCF (+ ball) at 200 Hz, like the reference harness.""" + import mujoco + + scene = mujoco.MjSpec.from_file(str(ROOM_SCENE)) + robot = mujoco.MjSpec.from_file(str(assets_fetch.variant_mjcf_path(variant))) + scene.option.timestep = 0.005 + robot.option.timestep = 0.005 + scene.attach(robot, prefix="", frame=scene.worldbody.add_frame(pos=[0.0, 0.0, 0.0])) + if with_ball: + ball = mujoco.MjSpec.from_file(str(assets_fetch.robot_dir() / "ball.xml")) + scene.attach(ball, prefix="", frame=scene.worldbody.add_frame(pos=[-1.0, 1.0, 0.0])) + return scene.compile() + + +def _actuator_perm(model: Any, joint_names: list[str]) -> list[int]: + import mujoco + + act_joint = [ + mujoco.mj_id2name(model, mujoco.mjtObj.mjOBJ_JOINT, int(model.actuator_trnid[i, 0])) + for i in range(model.nu) + ] + return [act_joint.index(n) for n in joint_names] + + +@pytest.mark.mujoco +def test_policy_bank_loads_variant_policies_and_steps() -> None: + pytest.importorskip("onnxruntime") + mujoco = pytest.importorskip("mujoco") + _cache_or_skip() + model = _compose("default", with_ball=False) + bank = PolicyBank(assets_fetch.policy_dir(), model, variant="default", missing=("roulade",)) + assert bank.names == tuple(n for n in DEFAULT_POLICIES if n != "roulade") + assert bank.missing == (PolicyName.ROULADE,) + assert bank.availability["roulade"] == ASSET_MISSING_REASON + assert bank.availability["roller"] and bank.availability["walk"] is None + assert bank.num_joints == 14 and len(bank.joint_names) == 14 + assert bank.default_pose.dtype == np.float32 and bank.default_pose.shape == (14,) + data = mujoco.MjData(model) + bank.initial_qpos(data) + mujoco.mj_forward(model, data) + assert data.qpos[bank.root_qpos_adr + 2] == pytest.approx(0.125) + assert bank.projected_gravity(data)[2] == pytest.approx(-1.0) + assert bank.root_yaw(data) == pytest.approx(0.0) + assert not bank.last_action.any() + for name in bank.names: + targets = bank.step(name, np.zeros(13), data) + assert targets.shape == (14,) and targets.dtype == np.float32 + assert np.isfinite(targets).all() + assert bank.last_action.any() + bank.reset() + assert not bank.last_action.any() + with pytest.raises(KeyError): + bank.step("roller", np.zeros(13), data) + with pytest.raises(ValueError): + bank.step("walk", np.zeros(3), data) + sched = PolicyScheduler(bank.availability, bank.variant) + assert sched.request("roulade", "start") == (False, "roulade unavailable: asset missing") + with pytest.raises(RuntimeError, match="no Microduck policy"): + PolicyBank(Path("/nonexistent"), model) + + +# ------------------------------------------------ opt-in headless sim run + + +def _slow_variants() -> list[str]: + raw = os.environ.get("MICRODUCK_SLOW_VARIANTS", "default") + return [v for v in raw.split(",") if v] + + +def _slow_cases() -> list[tuple[str, str]]: + if os.environ.get("MICRODUCK_SLOW_TESTS") != "1": + return [("default", "walk")] # placeholder; skipped below + cases = [] + for variant in _slow_variants(): + for name in DEFAULT_POLICIES if variant == "default" else ROLLERS_POLICIES: + cases.append((variant, name)) + return cases + + +class _SimRun: + """PolicyBank + PolicyScheduler driving a headless MuJoCo model at 50 Hz.""" + + FALL_Z = FALL_GRAVITY_Z + + def __init__(self, variant: str, *, spawn_x: float = 0.0) -> None: + import mujoco + + self.mujoco = mujoco + self.model = _compose(variant, with_ball=True) + self.data = mujoco.MjData(self.model) + self.bank = PolicyBank(assets_fetch.policy_dir(), self.model, variant=variant) + self.sched = PolicyScheduler(self.bank.availability, variant, spawn_ball=self.spawn_ball) + self.perm = _actuator_perm(self.model, self.bank.joint_names) + self.ball_adr = int( + self.model.jnt_qposadr[ + mujoco.mj_name2id(self.model, mujoco.mjtObj.mjOBJ_JOINT, "ball_free") + ] + ) + self.ball_body = mujoco.mj_name2id(self.model, mujoco.mjtObj.mjOBJ_BODY, "ball") + self.bank.initial_qpos(self.data) + self.data.qpos[self.bank.root_qpos_adr] = spawn_x + mujoco.mj_forward(self.model, self.data) + self.t = 0.0 + self.fell_unprotected = False + self.max_grav_z: dict[str, float] = {} # closest to falling (> -0.55) per policy + self.log: list[str] = [] + + def spawn_ball(self, dx: float, dy: float) -> None: + yaw = self.bank.root_yaw(self.data) + px, py = self.data.qpos[self.bank.root_qpos_adr : self.bank.root_qpos_adr + 2] + x = px + dx * math.cos(yaw) - dy * math.sin(yaw) + y = py + dx * math.sin(yaw) + dy * math.cos(yaw) + self.data.qpos[self.ball_adr : self.ball_adr + 7] = [x, y, 0.035, 1.0, 0.0, 0.0, 0.0] + dof = int(self.model.jnt_dofadr[self.model.body_jntadr[self.ball_body]]) + self.data.qvel[dof : dof + 6] = 0.0 + + @property + def pos(self) -> np.ndarray: + adr = self.bank.root_qpos_adr + return np.array(self.data.qpos[adr : adr + 3]) + + @property + def ball_pos(self) -> np.ndarray: + return np.array(self.data.xpos[self.ball_body]) + + def run(self, seconds: float, until: Callable[[], bool] | None = None) -> None: + steps = round(seconds / 0.005) + for i in range(steps): + if i % 4 == 0: + name, cmd = self.sched.tick(DT) + targets = self.bank.step(name, cmd, self.data) + self.data.ctrl[self.perm] = targets + gz = float(self.bank.projected_gravity(self.data)[2]) + self.max_grav_z[name] = max(self.max_grav_z.get(name, -1.0), gz) + fallen = gz > self.FALL_Z + if fallen and not self.sched.suspend_fall_detector: + self.fell_unprotected = True + self.log.append( + f"t={self.t:.2f} FELL during {self.sched.active} grav_z={gz:+.2f}" + ) + self.sched.notify_fall(fallen) + if until is not None and until(): + return + self.mujoco.mj_step(self.model, self.data) + self.t += 0.005 + + def upright(self) -> bool: + return float(self.bank.projected_gravity(self.data)[2]) <= self.FALL_Z + + +@pytest.mark.mujoco +@pytest.mark.parametrize(("variant", "name"), _slow_cases()) +def test_slow_policy_runs_in_headless_sim(variant: str, name: str) -> None: + if os.environ.get("MICRODUCK_SLOW_TESTS") != "1": + pytest.skip("set MICRODUCK_SLOW_TESTS=1 to run the headless policy matrix") + pytest.importorskip("onnxruntime") + pytest.importorskip("mujoco") + _cache_or_skip(variant) + # The wheeled runs cover up to ~2.5 m (push + open-loop glide); the room's + # east wall is at x=2.05, so start those a metre back. + sim = _SimRun(variant, spawn_x=-1.0 if name.startswith("roller") else 0.0) + sched = sim.sched + sim.run(2.0) # settle on walk with zero twist + assert sim.upright(), "did not settle upright" + start = sim.pos.copy() + outcome: dict[str, Any] = {"variant": variant, "policy": name} + + if name == "walk": + sched.set_twist(0.3, 0.0, 0.0) + sim.run(4.0) + sched.set_twist(0.0, 0.0, 0.0) + sim.run(2.0) + outcome["dx"] = float(sim.pos[0] - start[0]) + assert outcome["dx"] > 0.3, outcome + elif name == "stand": + sched.request("stand", "start") + sim.run(4.0) + assert sched.active == "stand" + sched.request("walk", "start") + sim.run(2.0) + outcome["dx"] = float(sim.pos[0] - start[0]) + elif name == "sitstand": + sched.request("sitstand", "start") + sim.run(4.0) + assert sched.seated and sched.active == "sitstand" + outcome["seated_z"] = float(sim.pos[2]) + sched.request("sitstand", "stop") + sim.run(STAND_UP_DURATION_S + 0.1) + assert sched.active == "walk" and not sched.locked, sched.snapshot() + sim.run(3.0) + outcome["final_z"] = float(sim.pos[2]) + assert outcome["final_z"] > 0.10, outcome + elif name in ("kick_left", "kick_right"): + sched.request(name, "start") + sim.run(DT) + ball0 = sim.ball_pos.copy() + outcome["ball_spawn"] = [round(float(v), 3) for v in ball0] + sim.run(KICK_DURATION_S + 3.0) + assert sched.active == "walk" + outcome["ball_dx"] = float(sim.ball_pos[0] - ball0[0]) + assert outcome["ball_dx"] > 0.15, outcome + elif name == "roulade": + sched.request("roulade", "start") + sim.run(ROULADE_DURATION_S + 0.1) + assert sched.active == "walk" and sched.suspend_fall_detector + outcome["max_grav_z_roulade"] = sim.max_grav_z.get("roulade") + sim.run(ROULADE_GRACE_S + 3.0) + assert not sched.suspend_fall_detector + outcome["dx"] = float(sim.pos[0] - start[0]) + assert outcome["dx"] > 0.2, outcome + elif name == "ground_pick": + sched.request("ground_pick", "start") + sim.run(GROUND_PICK_END_PHASE * GROUND_PICK_PERIOD_S + 0.1) + assert sched.active == "walk", sched.snapshot() + sim.run(3.0) + outcome["dx"] = float(sim.pos[0] - start[0]) + elif name == "roller": + # 2 s at throttle 0.48 is ~0.85 m, then zero throttle for the brake + # window and a little residual glide once walk takes over. + sched.request("roller", "start") + sched.set_twist(0.48, 0.0, 0.0) + sim.run(2.0) + outcome["rolled_dx"] = float(sim.pos[0] - start[0]) + assert outcome["rolled_dx"] > 0.5, outcome + sched.request("walk", "start") + sched.set_twist(0.0, 0.0, 0.0) + sim.run(DT) + assert sched.active == ACTIVE_BRAKING + rolled = sim.pos.copy() + sim.run(BRAKE_DURATION_S + 0.1) + assert sched.active == "walk", sched.snapshot() + braked = sim.pos.copy() + outcome["brake_dx"] = float(braked[0] - rolled[0]) + sim.run(3.0) + outcome["walk_glide_dx"] = float(sim.pos[0] - braked[0]) + # Open-loop brake: measured 0.7-1.1 m of forward creep in the window + # from this entry speed (see BRAKE_DURATION_S), never reverse. + assert -0.1 < outcome["brake_dx"] < 1.5, outcome + elif name == "roller_crouch": + sched.request("roller_crouch", "start") + sim.run(6.0) + assert sched.active == "roller_crouch" + outcome["crouch_dx"] = float(sim.pos[0] - start[0]) + sched.request("walk", "start") + sim.run(DT) + assert sched.active == ACTIVE_BRAKING # brakes on its own loop, no roller needed + sim.run(BRAKE_DURATION_S + 3.0) + assert sched.active == "walk" + else: # pragma: no cover + pytest.fail(f"no scenario for {name}") + + outcome["max_grav_z"] = {k: round(v, 2) for k, v in sim.max_grav_z.items()} + outcome["final_pos"] = [round(float(v), 3) for v in sim.pos] + outcome["fell_unprotected"] = sim.fell_unprotected + outcome["upright_at_end"] = sim.upright() + print(f"\nSLOW_RESULT {json.dumps(outcome)}") + assert not sim.fell_unprotected, "\n".join(sim.log) + assert sim.upright(), outcome + assert not sched.fallen diff --git a/dimos/robot/pollen/microduck/test_sim_module.py b/dimos/robot/pollen/microduck/test_sim_module.py new file mode 100644 index 0000000000..1aa973c465 --- /dev/null +++ b/dimos/robot/pollen/microduck/test_sim_module.py @@ -0,0 +1,666 @@ +# Copyright 2025-2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""``MicroduckSimModule``: policy_request parsing, policy_state publishing, +chase-camera plumbing, ball spawning and model composition. Tests that need +MuJoCo + the asset cache are marked ``mujoco`` (``pytest -m mujoco``).""" + +from __future__ import annotations + +import json +import math +from pathlib import Path +from typing import Any + +import numpy as np +import pytest + +from dimos.msgs.sensor_msgs.Image import ImageFormat +from dimos.robot.pollen.microduck import assets_fetch +from dimos.robot.pollen.microduck.places import BALL_BODY, BALL_RADIUS, FOUR_ROOM_XML +from dimos.robot.pollen.microduck.sim_module import ( + CHASE_CAMERA_NAME, + LIDAR_CAMERA_SPECS, + PHYSICS_TIMESTEP, + POV_CAMERA_NAME, + MicroduckSimModule, + MicroduckSimModuleConfig, + _ball_spawn_xy, + _camera_quat_wxyz, + _shape_twist, + _state_key, +) +from dimos.simulation.engines import mujoco_sim_module as msm +from dimos.simulation.engines.mujoco_engine import CameraFrame +from dimos.simulation.engines.mujoco_sim_module import MujocoSimModule + + +def _cache_or_skip() -> Path: + robot_xml = assets_fetch.variant_mjcf_path("default") + if not robot_xml.exists(): + pytest.skip(f"Microduck asset cache not present ({assets_fetch.assets_root()})") + return robot_xml + + +class _FakeScheduler: + """Records ``request`` calls; stands in for PolicyScheduler in parsing tests.""" + + def __init__(self, accept: bool = True) -> None: + self.requests: list[tuple[str | None, str]] = [] + self.accept = accept + self.suspend_fall_detector = False + self.falls: list[bool] = [] + + def request(self, policy: str | None, action: str) -> tuple[bool, str]: + self.requests.append((policy, action)) + return (self.accept, "" if self.accept else "nope") + + def notify_fall(self, fallen: bool) -> None: + self.falls.append(fallen) + + +class _FakeBank: + def __init__(self, gravity_z: float = -1.0, root_qpos_adr: int = 0) -> None: + self.gravity_z = gravity_z + self.root_qpos_adr = root_qpos_adr + + def projected_gravity(self, data: Any) -> np.ndarray: + return np.array([0.0, 0.0, self.gravity_z]) + + def root_yaw(self, data: Any) -> float: + w, x, y, z = data.qpos[self.root_qpos_adr + 3 : self.root_qpos_adr + 7] + return math.atan2(2.0 * (w * z + x * y), 1.0 - 2.0 * (y * y + z * z)) + + +def _frame(ts: float, shape: tuple[int, int] = (360, 640)) -> CameraFrame: + return CameraFrame( + rgb=np.full((*shape, 3), 120, dtype=np.uint8), + depth=np.ones(shape, dtype=np.float32), + cam_pos=np.zeros(3), + cam_mat=np.eye(3), + fovy=60.0, + timestamp=ts, + ) + + +# ---------------------------------------------------------------- pure helpers + + +def test_config_defaults_match_design() -> None: + cfg = MicroduckSimModuleConfig() + assert cfg.dof == 14 + assert cfg.camera_name == "head_camera" and cfg.base_frame_id == "trunk_base" + assert cfg.variant == "default" and cfg.policy_dir is None + assert cfg.chase_cam is True + assert cfg.chase_cam_size == (640, 360) and cfg.chase_cam_fps == 12.0 + assert cfg.chase_cam_offset == (-0.8, 0.0, 0.45) + assert cfg.chase_cam_pitch_deg == -20.0 and cfg.chase_cam_fovy == 60.0 + assert cfg.ball_body == BALL_BODY + assert cfg.state_hz == 5.0 + assert cfg.cast_shadows is False + assert cfg.extra_cameras == {} + assert (cfg.cmd_gain_linear, cfg.cmd_gain_angular) == (2.4, 2.6) + # The gait's WZ_RANGE maximum: below it pure turns barely rotate. + assert cfg.min_effective_wz == 1.5 and cfg.cmd_timeout == 1.0 + assert MicroduckSimModuleConfig(chase_cam=False, ball_body="").ball_body == "" + + +@pytest.mark.parametrize( + ("twist", "expected"), + [ + # Gains only; a forward request stays a forward request. + ((0.125, 0.0, 0.0), (0.3, 0.0, 0.0)), + ((0.0, -0.1, 0.0), (0.0, -0.24, 0.0)), + # The planner's rotate-in-place twist (0.275 * 2.6 = 0.715) is + # bumped to the effective minimum, keeping its sign. + ((0.0, 0.0, 0.275), (0.0, 0.0, 1.5)), + ((0.0, 0.0, -0.275), (0.0, 0.0, -1.5)), + # ... also while walking (the bump is on |wz| alone). + ((0.125, 0.0, -0.275), (0.3, 0.0, -1.5)), + # Requests already past the minimum are left alone (the gait clips). + ((0.0, 0.0, 0.6), (0.0, 0.0, 1.56)), + # Noise-level yaw is not a turn request. + ((0.0, 0.0, 0.01), (0.0, 0.0, 0.026)), + ((0.0, 0.0, 0.0), (0.0, 0.0, 0.0)), + ], +) +def test_shape_twist_applies_gains_and_yaw_deadband_bump( + twist: tuple[float, float, float], expected: tuple[float, float, float] +) -> None: + got = _shape_twist(MicroduckSimModuleConfig(), *twist) + assert got == pytest.approx(expected) + + +def test_shape_twist_honours_configured_gains_and_minimum() -> None: + cfg = MicroduckSimModuleConfig(cmd_gain_linear=1.0, cmd_gain_angular=1.0, min_effective_wz=0.8) + assert _shape_twist(cfg, 0.2, 0.0, 0.3) == pytest.approx((0.2, 0.0, 0.8)) + assert _shape_twist(cfg, 0.0, 0.0, -0.9) == pytest.approx((0.0, 0.0, -0.9)) + + +@pytest.mark.parametrize( + ("yaw", "expected"), + [ + (0.0, (1.09, 2.042)), + (math.pi / 2, (1.0 - 0.042, 2.0 + 0.09)), + (math.pi, (1.0 - 0.09, 2.0 - 0.042)), + (-math.pi / 2, (1.0 + 0.042, 2.0 - 0.09)), + ], +) +def test_ball_spawn_xy_rotates_offset_into_trunk_yaw_frame( + yaw: float, expected: tuple[float, float] +) -> None: + x, y = _ball_spawn_xy(1.0, 2.0, yaw, 0.09, 0.042) + assert (x, y) == pytest.approx(expected, abs=1e-9) + + +@pytest.mark.parametrize( + ("yaw", "pitch_deg"), + [(0.0, 0.0), (math.radians(120.0), 0.0), (math.radians(-120.0), 0.0), (0.0, -20.0)], +) +def test_camera_quat_points_camera_minus_z_along_yaw_pitch(yaw: float, pitch_deg: float) -> None: + from scipy.spatial.transform import Rotation as R + + w, x, y, z = _camera_quat_wxyz(yaw, math.radians(pitch_deg)) + rot = R.from_quat([x, y, z, w]) + forward = rot.apply([0.0, 0.0, -1.0]) # MuJoCo cameras look along -z + up = rot.apply([0.0, 1.0, 0.0]) + pitch = math.radians(pitch_deg) + assert forward == pytest.approx( + [math.cos(pitch) * math.cos(yaw), math.cos(pitch) * math.sin(yaw), math.sin(pitch)], + abs=1e-9, + ) + assert up[2] > 0.0 # image "up" is world up, not mirrored + + +def test_state_key_drops_timestamp_only() -> None: + snap = {"active": "walk", "locked": False, "t": 12.5, "oneshot": None} + assert _state_key(snap) == {"active": "walk", "locked": False, "oneshot": None} + assert "t" in snap # not mutated + + +# ------------------------------------------------------------ policy_state out + + +def test_policy_state_publishes_on_change_or_when_period_elapsed() -> None: + module = MicroduckSimModule(state_hz=5.0) + try: + published: list[str] = [] + module.policy_state.subscribe(published.append) + walk = {"active": "walk", "locked": False, "t": 1.0} + + assert module._publish_policy_state(walk, now=100.0) is True # first ever + assert module._publish_policy_state({**walk, "t": 1.05}, now=100.05) is False + assert module._publish_policy_state({**walk, "t": 1.1}, now=100.1) is False + kick = {"active": "kick_left", "locked": True, "t": 1.15} + assert module._publish_policy_state(kick, now=100.15) is True # changed + assert module._publish_policy_state({**kick, "t": 1.3}, now=100.3) is False + assert module._publish_policy_state({**kick, "t": 1.36}, now=100.36) is True # 1/5 s due + assert module._publish_policy_state({**kick, "t": 1.4}, now=100.4) is False + + assert [json.loads(raw)["active"] for raw in published] == [ + "walk", + "kick_left", + "kick_left", + ] + assert json.loads(published[-1])["t"] == 1.36 + assert published[0] == '{"active":"walk","locked":false,"t":1.0}' # compact JSON + finally: + module.stop() + + +# ------------------------------------------------------------ policy_request in + + +def test_on_policy_request_forwards_valid_requests_to_scheduler() -> None: + module = MicroduckSimModule() + try: + scheduler = _FakeScheduler() + module._scheduler = scheduler # type: ignore[assignment] + module._on_policy_request(json.dumps({"action": "start", "policy": "kick_left", "t": 1.0})) + module._on_policy_request(json.dumps({"action": "toggle", "policy": "sitstand"})) + module._on_policy_request(json.dumps({"action": "stop"})) # bare stop: policy absent + module._on_policy_request(json.dumps({"action": "stop", "policy": None})) + assert scheduler.requests == [ + ("kick_left", "start"), + ("sitstand", "toggle"), + (None, "stop"), + (None, "stop"), + ] + finally: + module.stop() + + +@pytest.mark.parametrize( + "raw", + [ + "{not json", + "", + json.dumps([1, 2, 3]), + json.dumps("start"), + json.dumps({"policy": "walk"}), # no action + json.dumps({"action": 7, "policy": "walk"}), + json.dumps({"action": "start", "policy": ["walk"]}), + ], +) +def test_on_policy_request_ignores_malformed_payloads(raw: str) -> None: + module = MicroduckSimModule() + try: + scheduler = _FakeScheduler() + module._scheduler = scheduler # type: ignore[assignment] + module._on_policy_request(raw) # must not raise + assert scheduler.requests == [] + finally: + module.stop() + + +def test_on_policy_request_passes_unknown_names_through_and_survives_rejection() -> None: + # Validation of names/actions is the scheduler's job (it reports via + # policy_state.last_error); the module only guards the JSON shape. + module = MicroduckSimModule() + try: + scheduler = _FakeScheduler(accept=False) + module._scheduler = scheduler # type: ignore[assignment] + module._on_policy_request(json.dumps({"action": "jump", "policy": "moonwalk"})) + assert scheduler.requests == [("moonwalk", "jump")] + finally: + module.stop() + + +def test_on_policy_request_before_start_is_a_noop() -> None: + module = MicroduckSimModule() + try: + module._on_policy_request(json.dumps({"action": "start", "policy": "walk"})) + finally: + module.stop() + + +# --------------------------------------------------------------- fall detector + + +class _FakeEngineData: + def __init__(self) -> None: + self.qpos = np.array([1.0, 0.0, 0.12, 1.0, 0.0, 0.0, 0.0]) + + +class _FakeEngine: + def __init__(self) -> None: + self.data = _FakeEngineData() + self.model = None + + +@pytest.mark.parametrize( + ("spawn_xy", "expected_x"), + [(None, 0.75), ((2.0, 0.0), 1.25), ((1.0, 0.0), 1.0)], +) +def test_check_fall_is_debounced_and_nudges_toward_spawn( + monkeypatch: pytest.MonkeyPatch, spawn_xy: tuple[float, float] | None, expected_x: float +) -> None: + module = MicroduckSimModule(auto_stand=True, auto_stand_after=2.0, spawn_xy=spawn_xy) + try: + bank = _FakeBank(gravity_z=0.9) # lying on its back + scheduler = _FakeScheduler() + module._bank = bank # type: ignore[assignment] + module._scheduler = scheduler # type: ignore[assignment] + stood: list[Any] = [] + monkeypatch.setattr(module, "_stand_in_place", stood.append) + engine = _FakeEngine() + + assert module._check_fall(engine, now=10.0) is False # timer starts + assert module._check_fall(engine, now=11.9) is False # within grace + assert scheduler.falls == [] and stood == [] + assert module._check_fall(engine, now=12.1) is True # stood back up + assert scheduler.falls == [True] + assert stood == [engine] + assert engine.data.qpos[0] == pytest.approx(expected_x) + assert module._fallen_since is None + + bank.gravity_z = -1.0 # upright again + assert module._check_fall(engine, now=12.2) is False + assert scheduler.falls == [True, False] + finally: + module.stop() + + +def test_check_fall_without_auto_stand_only_notifies() -> None: + module = MicroduckSimModule(auto_stand=False, auto_stand_after=0.5) + try: + scheduler = _FakeScheduler() + module._bank = _FakeBank(gravity_z=0.2) # type: ignore[assignment] + module._scheduler = scheduler # type: ignore[assignment] + stood: list[Any] = [] + module._stand_in_place = stood.append # type: ignore[method-assign] + engine = _FakeEngine() + assert module._check_fall(engine, now=1.0) is False + assert module._check_fall(engine, now=2.0) is False + assert scheduler.falls == [True] and stood == [] + assert engine.data.qpos[0] == 1.0 + finally: + module.stop() + + +def test_check_fall_is_skipped_while_scheduler_suspends_it() -> None: + module = MicroduckSimModule(auto_stand_after=0.0) + try: + scheduler = _FakeScheduler() + scheduler.suspend_fall_detector = True # e.g. the roulade is running + module._bank = _FakeBank(gravity_z=1.0) # type: ignore[assignment] + module._scheduler = scheduler # type: ignore[assignment] + engine = _FakeEngine() + assert module._check_fall(engine, now=1.0) is False + assert module._check_fall(engine, now=5.0) is False + assert scheduler.falls == [] and module._fallen_since is None + finally: + module.stop() + + +# ---------------------------------------------------------------- chase camera + + +def test_publish_chase_emits_once_per_rendered_frame() -> None: + module = MicroduckSimModule(chase_cam=True) + try: + frames: dict[str, CameraFrame | None] = {CHASE_CAMERA_NAME: None} + + class _Engine: + def read_camera(self, name: str) -> CameraFrame | None: + return frames.get(name) + + engine = _Engine() + images: list[Any] = [] + module.chase_image.subscribe(images.append) + + module._publish_chase(engine) # nothing rendered yet + assert images == [] + frames[CHASE_CAMERA_NAME] = _frame(5.0) + module._publish_chase(engine) + module._publish_chase(engine) # same frame again: no republish + assert len(images) == 1 + frames[CHASE_CAMERA_NAME] = _frame(5.1) + module._publish_chase(engine) + assert len(images) == 2 + image = images[-1] + assert image.ts == 5.1 + assert image.format == ImageFormat.RGB + assert image.frame_id == f"{CHASE_CAMERA_NAME}_optical_frame" + assert np.asarray(image.data).shape == (360, 640, 3) + finally: + module.stop() + + +def test_publish_chase_is_disabled_with_chase_cam_false() -> None: + module = MicroduckSimModule(chase_cam=False) + try: + + class _Engine: + def read_camera(self, name: str) -> CameraFrame: + raise AssertionError("must not read the chase camera when disabled") + + images: list[Any] = [] + module.chase_image.subscribe(images.append) + module._publish_chase(_Engine()) + assert images == [] + finally: + module.stop() + + +# ------------------------------------------- extra_cameras on MujocoSimModule + + +class _EngineCapturedError(Exception): + def __init__(self, kwargs: dict[str, Any]) -> None: + super().__init__("captured") + self.kwargs = kwargs + + +class _NoShm: + def __init__(self, key: str) -> None: + self.key = key + + def __getattr__(self, name: str) -> Any: + return lambda *args, **kwargs: None + + +def _capture_engine_cameras( + monkeypatch: pytest.MonkeyPatch, module: MujocoSimModule +) -> list[msm.CameraConfig]: + """Run ``start()`` up to engine construction and return the camera list.""" + + def _fake_engine(**kwargs: Any) -> None: + raise _EngineCapturedError(kwargs) + + monkeypatch.setattr(msm, "ManipShmWriter", _NoShm) + monkeypatch.setattr(msm, "MujocoEngine", _fake_engine) + monkeypatch.setattr(module, "_compose_model", lambda: None) + with pytest.raises(_EngineCapturedError) as info: + module.start() + return list(info.value.kwargs["cameras"]) + + +def test_extra_cameras_default_leaves_camera_list_unchanged( + monkeypatch: pytest.MonkeyPatch, +) -> None: + kwargs = dict(robot_mjcf="/nonexistent/robot.xml", width=320, height=240, fps=5) + module = MujocoSimModule(enable_color=True, **kwargs) + try: + assert module.config.extra_cameras == {} + cameras = _capture_engine_cameras(monkeypatch, module) + assert [(c.name, c.width, c.height, c.fps) for c in cameras] == [ + (module.config.camera_name, 320, 240, 5.0) + ] + finally: + module.stop() + + +def test_extra_cameras_append_after_the_primary_camera(monkeypatch: pytest.MonkeyPatch) -> None: + module = MujocoSimModule( + robot_mjcf="/nonexistent/robot.xml", + width=320, + height=240, + fps=5, + enable_color=True, + base_frame_id="trunk_base", + extra_cameras={ + "chase_camera": (640, 360, 12.0), + "": (10, 10, 1.0), # ignored + "wrist_camera": (1, 1, 1.0), # same name as the primary: ignored + }, + ) + try: + cameras = _capture_engine_cameras(monkeypatch, module) + assert [(c.name, c.width, c.height, c.fps) for c in cameras] == [ + ("wrist_camera", 320, 240, 5.0), + ("chase_camera", 640, 360, 12.0), + ] + chase = cameras[1] + assert chase.base_body_name == "trunk_base" + assert chase.geom_groups is None and chase.max_geom == 10000 + finally: + module.stop() + + +def test_extra_cameras_work_without_a_primary_camera(monkeypatch: pytest.MonkeyPatch) -> None: + module = MujocoSimModule( + robot_mjcf="/nonexistent/robot.xml", + enable_color=False, + enable_depth=False, + enable_pointcloud=False, + extra_cameras={"chase_camera": (640, 360, 12.0)}, + ) + try: + cameras = _capture_engine_cameras(monkeypatch, module) + assert [c.name for c in cameras] == ["chase_camera"] + finally: + module.stop() + + +def test_microduck_extra_cameras_default_is_empty_until_start() -> None: + module = MicroduckSimModule() + try: + # start() merges the chase camera in; the config itself stays generic. + assert module.config.extra_cameras == {} + assert module.config.chase_cam is True + finally: + module.stop() + + +# ------------------------------------------------ MuJoCo composition (-m mujoco) + + +def _module(**overrides: Any) -> MicroduckSimModule: + robot_xml = _cache_or_skip() + kwargs: dict[str, Any] = dict( + scene_xml=FOUR_ROOM_XML, robot_mjcf=str(robot_xml), headless=True, spawn_xy=(0.0, 0.0) + ) + kwargs.update(overrides) + return MicroduckSimModule(**kwargs) + + +@pytest.mark.mujoco +def test_compose_model_adds_chase_camera_and_ball_to_four_room_scene() -> None: + mujoco = pytest.importorskip("mujoco") + module = _module() + try: + model = module._compose_model() + # Trunk free joint stays joint 0 (engine's robot root); ball comes after. + assert model.jnt_type[0] == mujoco.mjtJoint.mjJNT_FREE + assert mujoco.mj_id2name(model, mujoco.mjtObj.mjOBJ_JOINT, 0) == "trunk_base_freejoint" + ball = model.joint(f"{BALL_BODY}_freejoint") + assert int(ball.type[0]) == int(mujoco.mjtJoint.mjJNT_FREE) + assert model.nu == 14 and model.njnt == 16 + assert model.opt.timestep == pytest.approx(PHYSICS_TIMESTEP) + + cam = model.camera(CHASE_CAMERA_NAME) + assert int(cam.mode[0]) == int(mujoco.mjtCamLight.mjCAMLIGHT_TRACK) + assert ( + mujoco.mj_id2name(model, mujoco.mjtObj.mjOBJ_BODY, int(cam.bodyid[0])) == "trunk_base" + ) + assert cam.pos == pytest.approx(module.config.chase_cam_offset) + assert float(cam.fovy[0]) == pytest.approx(module.config.chase_cam_fovy) + for name, _ in LIDAR_CAMERA_SPECS: + assert model.camera(name).id >= 0 + assert model.camera("head_camera").id >= 0 + + # The POV camera is the one worth asserting on: the MJCF's stock + # head_camera looks along body -x, i.e. backwards into the duck's own + # jaw, so the first-person view (and the observe skill) uses ours. + data = mujoco.MjData(model) + mujoco.mj_forward(model, data) + pov = mujoco.mj_name2id(model, mujoco.mjtObj.mjOBJ_CAMERA, POV_CAMERA_NAME) + head = mujoco.mj_name2id(model, mujoco.mjtObj.mjOBJ_CAMERA, "head_camera") + assert pov >= 0 + # MuJoCo cameras look along their own -z. + forward = -data.cam_xmat[pov].reshape(3, 3)[:, 2] + assert forward[0] == pytest.approx(1.0, abs=1e-6) # body +x, where it walks + assert -data.cam_xmat[head].reshape(3, 3)[:, 2][0] == pytest.approx(-1.0, abs=1e-6) + # Same eye position as the stock camera, just facing the other way. + assert data.cam_xpos[pov] == pytest.approx(data.cam_xpos[head], abs=2e-3) + + assert model.vis.global_.offwidth >= 1280 + assert model.vis.global_.offheight >= 720 + assert model.vis.global_.offwidth >= module.config.chase_cam_size[0] + assert model.vis.global_.offheight >= module.config.chase_cam_size[1] + + # The scene's lights are kept but stop casting shadows by default. + assert model.nlight == 2 + assert not model.light_castshadow.any() + finally: + module.stop() + + +@pytest.mark.mujoco +def test_compose_model_omits_chase_camera_and_ball_when_disabled() -> None: + mujoco = pytest.importorskip("mujoco") + module = _module(chase_cam=False, ball_body="", cast_shadows=True) + try: + model = module._compose_model() + assert mujoco.mj_name2id(model, mujoco.mjtObj.mjOBJ_CAMERA, CHASE_CAMERA_NAME) == -1 + assert mujoco.mj_name2id(model, mujoco.mjtObj.mjOBJ_JOINT, f"{BALL_BODY}_freejoint") == -1 + assert model.njnt == 15 and model.nu == 14 + # cast_shadows=True leaves the scene's shadow-casting lights alone. + assert model.nlight == 2 and model.light_castshadow.all() + finally: + module.stop() + + +@pytest.mark.mujoco +def test_compose_model_raises_offscreen_buffer_to_chase_cam_size() -> None: + mujoco = pytest.importorskip("mujoco") + # No scene file: MjSpec's offscreen buffer defaults to 640x480. + module = _module(scene_xml=None, chase_cam_size=(1024, 576), chase_cam_fovy=45.0) + try: + model = module._compose_model() + assert model.vis.global_.offwidth >= 1024 + assert model.vis.global_.offheight >= 576 + assert float(model.camera(CHASE_CAMERA_NAME).fovy[0]) == pytest.approx(45.0) + assert mujoco.mj_name2id(model, mujoco.mjtObj.mjOBJ_JOINT, f"{BALL_BODY}_freejoint") >= 0 + finally: + module.stop() + + +class _ModelEngine: + """Just enough engine for ``_spawn_ball``: a model, its data, and stop().""" + + def __init__(self, model: Any) -> None: + import mujoco + + self.model = model + self.data = mujoco.MjData(model) + + def disconnect(self) -> None: + pass + + +@pytest.mark.mujoco +def test_spawn_ball_places_ball_ahead_of_the_trunk_in_its_yaw_frame() -> None: + pytest.importorskip("mujoco") + module = _module() + try: + model = module._compose_model() + engine = _ModelEngine(model) + data = engine.data + ball = model.joint(f"{BALL_BODY}_freejoint") + module._ball_qpos_adr = int(ball.qposadr[0]) + module._ball_qvel_adr = int(ball.dofadr[0]) + module._bank = _FakeBank(root_qpos_adr=0) # type: ignore[assignment] + module._engine = engine # type: ignore[assignment] + + # Trunk at (1, 2) facing +y (yaw 90 deg); ball moving before the spawn. + data.qpos[0:3] = (1.0, 2.0, 0.12) + data.qpos[3:7] = (math.cos(math.pi / 4), 0.0, 0.0, math.sin(math.pi / 4)) + data.qvel[module._ball_qvel_adr : module._ball_qvel_adr + 6] = 3.0 + module._spawn_ball(0.09, 0.042) + + adr = module._ball_qpos_adr + assert data.qpos[adr : adr + 3] == pytest.approx([1.0 - 0.042, 2.0 + 0.09, BALL_RADIUS]) + assert data.qpos[adr + 3 : adr + 7] == pytest.approx([1.0, 0.0, 0.0, 0.0]) + assert np.all(data.qvel[module._ball_qvel_adr : module._ball_qvel_adr + 6] == 0.0) + # mj_forward ran: the ball body's world position reflects the new qpos. + assert data.xpos[model.body(BALL_BODY).id][:2] == pytest.approx([0.958, 2.09]) + finally: + module.stop() + + +@pytest.mark.mujoco +def test_spawn_ball_is_a_noop_without_a_ball_joint() -> None: + pytest.importorskip("mujoco") + module = _module(ball_body="") + try: + engine = _ModelEngine(module._compose_model()) + before = engine.data.qpos.copy() + module._bank = _FakeBank(root_qpos_adr=0) # type: ignore[assignment] + module._engine = engine # type: ignore[assignment] + module._spawn_ball(0.09, 0.042) + assert np.array_equal(engine.data.qpos, before) + finally: + module.stop() diff --git a/dimos/robot/pollen/microduck/test_skills.py b/dimos/robot/pollen/microduck/test_skills.py new file mode 100644 index 0000000000..f91e68f992 --- /dev/null +++ b/dimos/robot/pollen/microduck/test_skills.py @@ -0,0 +1,1199 @@ +# Copyright 2025-2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""MicroduckSkillContainer with a scripted navigation stub and captured outputs.""" + +from __future__ import annotations + +from collections.abc import Callable, Iterator +from dataclasses import dataclass, field +import json +import math +from pathlib import Path +import pickle +import threading +import time +from typing import Any + +import pytest + +from dimos.core.stream import Stream, Transport +from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped +from dimos.msgs.geometry_msgs.Quaternion import Quaternion +from dimos.msgs.geometry_msgs.Vector3 import Vector3 +from dimos.navigation.base import NavigationState +from dimos.robot.pollen.microduck import skills +from dimos.robot.pollen.microduck.places import ( + MICRODUCK_OBJECTS, + MICRODUCK_ROOMS, + PlacesMemory, + RoomSpec, + scene_id, +) +from dimos.robot.pollen.microduck.skills import MicroduckSkillContainer + +FOLLOWING = NavigationState.FOLLOWING_PATH +IDLE = NavigationState.IDLE + + +class _DirectTransport(Transport): # type: ignore[type-arg] + """Synchronous in-process transport so ``start()`` can wire the inputs.""" + + def __init__(self) -> None: + self._subscribers: list[Callable[[Any], Any]] = [] + + def broadcast(self, _selfstream: Any, value: Any) -> None: + for callback in list(self._subscribers): + callback(value) + + def subscribe( + self, callback: Callable[[Any], Any], _selfstream: Stream[Any] | None = None + ) -> Callable[[], None]: + self._subscribers.append(callback) + + def _unsubscribe() -> None: + if callback in self._subscribers: + self._subscribers.remove(callback) + + return _unsubscribe + + def start(self) -> None: ... + + def stop(self) -> None: + self._subscribers.clear() + + +class FakeNav: + """Scripted ``NavigationInterfaceSpec`` stand-in. + + ``get_state`` walks through ``states`` (the last one repeats); + ``is_goal_reached`` turns true once the script is exhausted when + ``reached`` is set. + """ + + def __init__( + self, states: tuple[NavigationState, ...] = (FOLLOWING, FOLLOWING), reached: bool = True + ) -> None: + self.states = list(states) + self.reached = reached + self.calls = 0 + self.goals: list[PoseStamped] = [] + self.cancel_calls = 0 + self.cancel_result = True + self.set_goal_result = True + + def script(self, *states: NavigationState, reached: bool) -> None: + self.states = list(states) + self.reached = reached + self.calls = 0 + + def set_goal(self, goal: PoseStamped) -> bool: + self.goals.append(goal) + return self.set_goal_result + + def get_state(self) -> NavigationState: + state = self.states[min(self.calls, len(self.states) - 1)] + self.calls += 1 + return state + + def is_goal_reached(self) -> bool: + return self.reached and self.calls >= len(self.states) + + def cancel_goal(self) -> bool: + self.cancel_calls += 1 + return self.cancel_result + + +def _policy_state(**overrides: Any) -> dict[str, Any]: + policies = [ + {"name": "walk", "kind": "base", "available": True, "reason": None}, + {"name": "stand", "kind": "base", "available": True, "reason": None}, + {"name": "roller", "kind": "base", "available": False, "reason": "rollers variant only"}, + { + "name": "roller_crouch", + "kind": "base", + "available": False, + "reason": "rollers variant only", + }, + {"name": "sitstand", "kind": "posture", "available": True, "reason": None}, + {"name": "kick_left", "kind": "oneshot", "available": True, "reason": None}, + {"name": "kick_right", "kind": "oneshot", "available": True, "reason": None}, + {"name": "roulade", "kind": "oneshot", "available": True, "reason": None}, + {"name": "ground_pick", "kind": "oneshot", "available": True, "reason": None}, + ] + state: dict[str, Any] = { + "variant": "default", + "active": "walk", + "base": "walk", + "seated": False, + "fallen": False, + "locked": False, + "oneshot": None, + "policies": policies, + "last_error": None, + "t": time.time(), + } + state.update(overrides) + return state + + +@dataclass +class Captured: + goals: list[PoseStamped] = field(default_factory=list) + policy_requests: list[str] = field(default_factory=list) + places: list[str] = field(default_factory=list) + + @property + def last_request(self) -> dict[str, Any]: + return json.loads(self.policy_requests[-1]) + + @property + def last_places(self) -> dict[str, Any]: + return json.loads(self.places[-1]) + + +@dataclass +class Harness: + module: MicroduckSkillContainer + nav: FakeNav + captured: Captured + timers: list[threading.Timer] = field(default_factory=list) + + def feed_odom(self, x: float, y: float, yaw: float = 0.0) -> None: + pose = PoseStamped( + ts=time.time(), + frame_id="world", + position=Vector3(x, y, 0.0), + orientation=Quaternion(0.0, 0.0, math.sin(yaw / 2), math.cos(yaw / 2)), + ) + self.module.odom.transport.broadcast(None, pose) + + def feed_state(self, **overrides: Any) -> None: + self.module.policy_state.transport.broadcast( + None, json.dumps(_policy_state(**overrides), separators=(",", ":")) + ) + + def feed_state_later(self, delay: float, **overrides: Any) -> None: + timer = threading.Timer(delay, lambda: self.feed_state(**overrides)) + timer.daemon = True + self.timers.append(timer) + timer.start() + + def on_request(self, callback: Callable[[dict[str, Any]], None]) -> Callable[[], None]: + """Run ``callback`` synchronously whenever a policy_request is published.""" + return self.module.policy_request.subscribe(lambda msg: callback(json.loads(msg))) + + def close(self) -> None: + for timer in self.timers: + timer.cancel() + timer.join(timeout=2.0) + + +@pytest.fixture +def fast_polling(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(skills, "_NAV_POLL_S", 0.005) + monkeypatch.setattr(skills, "_NAV_START_TIMEOUT_S", 0.2) + monkeypatch.setattr(skills, "_NAV_SETTLE_S", 0.05) + monkeypatch.setattr(skills, "_POLICY_START_GRACE_S", 0.3) + monkeypatch.setattr(skills, "_POLICY_FIRST_STATE_WAIT_S", 0.2) + + +def _make_module(tmp_path: Path, **overrides: Any) -> MicroduckSkillContainer: + config: dict[str, Any] = { + "rooms": MICRODUCK_ROOMS, + "objects": MICRODUCK_OBJECTS, + "places_db": str(tmp_path / "places.db"), + "nav_timeout_s": 1.0, + "policy_timeout_s": 1.0, + "places_republish_s": 0.0, + } + config.update(overrides) + return MicroduckSkillContainer(**config) + + +@pytest.fixture +def harness(tmp_path: Path, fast_polling: None) -> Iterator[Harness]: + module = _make_module(tmp_path) + module.odom.transport = _DirectTransport() + module.policy_state.transport = _DirectTransport() + nav = FakeNav() + module._navigation = nav # type: ignore[assignment] + captured = Captured() + unsubs = [ + module.goal_request.subscribe(captured.goals.append), + module.policy_request.subscribe(captured.policy_requests.append), + module.places.subscribe(captured.places.append), + ] + module.start() + h = Harness(module, nav, captured) + try: + yield h + finally: + h.close() + for unsub in unsubs: + unsub() + module.stop() + + +# lifecycle / config + + +def test_blueprint_keeps_objects_kwarg_and_config_pickles(tmp_path: Path) -> None: + bp = MicroduckSkillContainer.blueprint(objects={"red_box": (1.5, 0.8)}) + assert bp is not None + module = _make_module(tmp_path) + try: + assert module.config.rooms["kitchen"].aliases == ("space A",) + assert module.config.objects["red_box"] == (1.5, 1.5) + assert module.config.places_db.endswith("places.db") + assert module.config.nav_timeout_s == 1.0 + assert module.config.scene == "" + assert module._scene() == scene_id(MICRODUCK_ROOMS, MICRODUCK_OBJECTS) + restored = pickle.loads(pickle.dumps(module.config)) + assert restored.rooms == MICRODUCK_ROOMS + finally: + module.stop() + + +def test_blueprints_with_different_tables_share_db_without_leaking( + tmp_path: Path, fast_polling: None +) -> None: + """The cockpit (4 rooms) and the old agentic sim (2 objects) use one db file.""" + db = str(tmp_path / "shared.db") + cockpit = _make_module(tmp_path, places_db=db) + cockpit.odom.transport = _DirectTransport() + cockpit._navigation = FakeNav() # type: ignore[assignment] + cockpit.start() + try: + cockpit.odom.transport.broadcast( + None, + PoseStamped( + ts=time.time(), + frame_id="world", + position=Vector3(0.3, -0.2, 0.0), + orientation=Quaternion(0.0, 0.0, 0.0, 1.0), + ), + ) + assert cockpit.remember_place("charger").startswith("Remembered 'charger'") + finally: + cockpit.stop() + + legacy_objects = {"red_box": (1.5, 0.8), "blue_box": (-1.5, -0.8)} + legacy = _make_module(tmp_path, places_db=db, rooms={}, objects=legacy_objects) + legacy.odom.transport = _DirectTransport() + legacy._navigation = FakeNav() # type: ignore[assignment] + received: list[str] = [] + unsub = legacy.places.subscribe(received.append) + legacy.start() + try: + assert legacy._scene() != scene_id(MICRODUCK_ROOMS, MICRODUCK_OBJECTS) + payload = json.loads(received[-1]) + assert payload["rooms"] == [] + assert [(o["name"], o["x"], o["y"]) for o in payload["objects"]] == [ + ("red_box", 1.5, 0.8), + ("blue_box", -1.5, -0.8), + ] + assert payload["tagged"] == [] + listing = legacy.list_places() + assert "Rooms:" not in listing and "charger" not in listing + assert "- red_box at (1.50, 0.80)" in listing + assert legacy.go_to_place("charger").startswith("Unknown place 'charger'") + finally: + unsub() + legacy.stop() + + # An explicit scene id overrides the derived one. + pinned = _make_module(tmp_path, places_db=db, scene="cockpit-a") + try: + assert pinned._scene() == "cockpit-a" + finally: + pinned.stop() + + +def test_stream_declarations() -> None: + module = MicroduckSkillContainer() + try: + assert module.odom.type is PoseStamped + assert module.policy_state.type is str + assert module.goal_request.type is PoseStamped + assert module.policy_request.type is str + assert module.places.type is str + finally: + module.stop() + + +def test_publishes_places_on_start(harness: Harness) -> None: + assert len(harness.captured.places) == 1 + payload = harness.captured.last_places + assert payload["frame"] == "world" + assert [r["name"] for r in payload["rooms"]] == list(MICRODUCK_ROOMS) + assert [o["name"] for o in payload["objects"]] == list(MICRODUCK_OBJECTS) + assert payload["tagged"] == [] + assert isinstance(payload["t"], float) + db = Path(harness.module.config.places_db) + assert db.is_file() + + +def test_places_heartbeat_republishes(tmp_path: Path, fast_polling: None) -> None: + module = _make_module(tmp_path, places_republish_s=0.02) + module.odom.transport = _DirectTransport() + module.policy_state.transport = _DirectTransport() + module._navigation = FakeNav() # type: ignore[assignment] + received: list[str] = [] + unsub = module.places.subscribe(received.append) + module.start() + try: + deadline = time.monotonic() + 2.0 + while len(received) < 3 and time.monotonic() < deadline: + time.sleep(0.01) + assert len(received) >= 3 + thread = module._republish_thread + assert thread is not None and thread.is_alive() + finally: + unsub() + module.stop() + assert module._republish_thread is None + assert not thread.is_alive() + + +def test_start_without_policy_state_transport(tmp_path: Path, fast_polling: None) -> None: + """Old blueprints have no policy_state producer; start must still work.""" + module = _make_module(tmp_path, rooms={}, objects={"red_box": (1.5, 0.8)}) + module.odom.transport = _DirectTransport() + module._navigation = FakeNav() # type: ignore[assignment] + requests: list[str] = [] + unsub = module.policy_request.subscribe(requests.append) + module.start() + try: + assert "red_box" in module.list_places() + assert "Rooms:" not in module.list_places() + out = module.perform("kick_left") + assert out.startswith("Sent 'kick_left' start request") + assert json.loads(requests[-1])["policy"] == "kick_left" + finally: + unsub() + module.stop() + + +def test_places_memory_is_not_reopened_after_stop(tmp_path: Path, fast_polling: None) -> None: + """A skill thread still unwinding after stop() must not reopen the + database stop() just closed (that handle would leak).""" + module = _make_module(tmp_path) + module.odom.transport = _DirectTransport() + module._navigation = FakeNav() # type: ignore[assignment] + module.start() + assert module._memory is not None + module.stop() + assert module._memory is None + with pytest.raises(RuntimeError, match="stopped"): + module._places_memory() + assert module._memory is None + # Publishing is a no-op once stopped, and the arrival summary degrades + # instead of raising into the skill's caller. + module._publish_places() + assert module._memory is None + pose = PoseStamped(ts=time.time(), frame_id="world", position=Vector3(0.3, -0.2, 0.0)) + module._on_odom(pose) + assert module._where_summary() == "Robot was at (0.30, -0.20) (module stopping)." + assert module._memory is None + # start() re-arms it. + module.start() + try: + assert module._memory is not None + assert module._where_summary() == "Robot is now at (0.30, -0.20) in the central hub." + finally: + module.stop() + + +def test_scene_without_rooms_reports_no_hub(tmp_path: Path, fast_polling: None) -> None: + """The legacy agentic-sim scene has objects but no rooms: no 'central + hub' phrasing, and move_to's bounds follow the objects.""" + module = _make_module(tmp_path, rooms={}, objects={"red_box": (1.5, 0.8)}) + module.odom.transport = _DirectTransport() + module._navigation = FakeNav() # type: ignore[assignment] + module.start() + try: + pose = PoseStamped(ts=time.time(), frame_id="world", position=Vector3(0.1, -0.1, 0.0)) + module._on_odom(pose) + out = module.where_am_i() + assert out.startswith("Robot is at (0.10, -0.10), heading 0 deg.") + assert "hub" not in out and "room" not in out + assert module.remember_place("charger") == "Remembered 'charger' at (0.10, -0.10)." + assert module.move_to(0.5, 0.5).endswith("Robot is now at (0.10, -0.10).") + assert module.go_to_room("kitchen") == "Unknown room 'kitchen'. Known rooms: (none)" + assert module._arena_half_extent() == pytest.approx(2.0) + finally: + module.stop() + + +def test_move_to_extent_follows_larger_scenes(tmp_path: Path, fast_polling: None) -> None: + hall = RoomSpec("hall", ("space Z",), (-6.0, 3.0, -1.0, 1.0), (0.0, 0.0, 0.0)) + module = _make_module(tmp_path, rooms={"hall": hall}, objects={"far_box": (2.0, 4.5)}) + module.odom.transport = _DirectTransport() + module._navigation = FakeNav() # type: ignore[assignment] + module.start() + try: + assert module._arena_half_extent() == pytest.approx(6.0) + assert module.move_to(-5.5, 0.0).startswith("Arrived.") + assert module.move_to(0.0, 4.4).startswith("Arrived.") + out = module.move_to(6.5, 0.0) + assert out == "(6.50, 0.00) is outside the arena walls (x and y must be within +/-6 m)." + finally: + module.stop() + + +def test_remember_place_respelled_updates_instead_of_duplicating(harness: Harness) -> None: + harness.feed_odom(0.3, -0.2) + assert harness.module.remember_place("charger").startswith("Remembered 'charger'") + harness.feed_odom(0.5, -0.1) + assert harness.module.remember_place("Charger").startswith("Remembered 'Charger'") + tagged = harness.captured.last_places["tagged"] + assert [(t["name"], t["x"]) for t in tagged] == [("Charger", 0.5)] + out = harness.module.go_to_place("charger") + assert out.startswith("Arrived while walking to Charger.") + assert (harness.captured.goals[-1].position.x, harness.captured.goals[-1].position.y) == ( + pytest.approx(0.5), + pytest.approx(-0.1), + ) + + +# navigation skills + + +def test_go_to_room_by_alias_publishes_goal_and_arrives(harness: Harness) -> None: + harness.feed_odom(0.0, 0.0) + out = harness.module.go_to_room("space A") + assert out.startswith("Arrived while walking to the kitchen.") + assert "central hub" in out + assert len(harness.captured.goals) == 1 + goal = harness.captured.goals[0] + assert goal.frame_id == "world" + assert (goal.position.x, goal.position.y) == pytest.approx((1.2, 1.0)) + assert goal.yaw == pytest.approx(0.0, abs=1e-6) + # No transport on goal_request in this harness -> RPC fallback delivers it too. + assert len(harness.nav.goals) == 1 + + +def test_goal_request_transport_disables_rpc_fallback(harness: Harness) -> None: + transport = _DirectTransport() + seen: list[PoseStamped] = [] + transport.subscribe(seen.append) + harness.module.goal_request.transport = transport + harness.feed_odom(0.0, 0.0) + out = harness.module.go_to_room("living room") + assert out.startswith("Arrived") + assert len(seen) == 1 and seen[0].position.x == pytest.approx(-1.2) + assert seen[0].yaw == pytest.approx(3.14159, abs=1e-4) + assert harness.nav.goals == [] + + +def test_go_to_room_unknown_lists_rooms(harness: Harness) -> None: + out = harness.module.go_to_room("garage") + assert out.startswith("Unknown room 'garage'") + assert "kitchen (space A)" in out + assert "living (space B, living room, lounge)" in out + assert harness.captured.goals == [] + + +def test_go_to_room_already_at_entry_point(harness: Harness) -> None: + harness.feed_odom(1.15, 1.0) + out = harness.module.go_to_room("kitchen") + assert out.startswith("Already at the kitchen entry point.") + assert harness.captured.goals == [] + + +def test_go_to_object_keeps_approach_offset(harness: Harness) -> None: + harness.feed_odom(0.0, 0.0) + out = harness.module.go_to_object("red box") + assert out.startswith("Arrived while walking to red_box.") + goal = harness.captured.goals[-1] + heading = math.atan2(1.5, 1.5) + assert goal.position.x == pytest.approx(1.5 - math.cos(heading) * 0.35) + assert goal.position.y == pytest.approx(1.5 - math.sin(heading) * 0.35) + assert goal.yaw == pytest.approx(heading, abs=1e-6) + + +def test_go_to_object_already_next_to(harness: Harness) -> None: + harness.feed_odom(1.3, 1.5) + out = harness.module.go_to_object("red_box") + assert out.startswith("Already next to red_box (0.20 m away).") + assert harness.captured.goals == [] + + +def test_go_to_object_needs_odom_and_known_name(harness: Harness) -> None: + assert harness.module.go_to_object("red_box").startswith("Robot position unknown") + out = harness.module.go_to_object("purple_sphere") + assert out.startswith("Unknown object 'purple_sphere'") + assert "red_box" in out and "orange_crate" in out + assert harness.captured.goals == [] + + +def test_ambiguous_names_ask_instead_of_guessing(harness: Harness) -> None: + harness.feed_odom(0.0, 0.0) + out = harness.module.go_to_object("box") + assert out == "Ambiguous object 'box': did you mean red_box, blue_box? Use the exact name." + out = harness.module.go_to_place("box") + assert out == "Ambiguous place 'box': did you mean red_box, blue_box? Use the exact name." + # "space" / "room" are filler words: on their own they match nothing. + assert harness.module.go_to_room("space").startswith("Unknown room 'space'") + assert harness.module.go_to_room("room").startswith("Unknown room 'room'") + assert harness.module.go_to_place("space").startswith("Unknown place 'space'") + assert harness.captured.goals == [] + # A remembered spot literally called "box" is exact and wins. + harness.feed_odom(0.2, 0.1) + assert harness.module.remember_place("box").startswith("Remembered 'box'") + harness.feed_odom(0.0, 0.0) + assert harness.module.go_to_place("box").startswith("Arrived while walking to box.") + assert len(harness.captured.goals) == 1 + + +def test_go_to_place_routes_by_kind(harness: Harness) -> None: + harness.feed_odom(0.0, 0.0) + assert harness.module.go_to_place("lounge").startswith("Arrived while walking to the living.") + assert harness.captured.goals[-1].position.x == pytest.approx(-1.2) + + harness.nav.script(FOLLOWING, FOLLOWING, reached=True) + assert harness.module.go_to_place("blue box").startswith("Arrived while walking to blue_box.") + goal = harness.captured.goals[-1] + assert math.hypot(goal.position.x + 1.5, goal.position.y - 1.5) == pytest.approx(0.35) + + harness.feed_odom(0.3, -0.2, 0.7) + assert harness.module.remember_place("charger").startswith("Remembered 'charger'") + harness.feed_odom(0.0, 0.0) + harness.nav.script(FOLLOWING, FOLLOWING, reached=True) + out = harness.module.go_to_place("charger") + assert out.startswith("Arrived while walking to charger.") + goal = harness.captured.goals[-1] + assert (goal.position.x, goal.position.y) == pytest.approx((0.3, -0.2)) + assert goal.yaw == pytest.approx(0.7, abs=1e-6) + + assert harness.module.go_to_place("nowhere").startswith("Unknown place 'nowhere'") + + +def test_move_to_validates_and_faces_goal(harness: Harness) -> None: + harness.feed_odom(0.0, 0.0) + out = harness.module.move_to(0.5, -0.5) + assert out.startswith("Arrived.") + goal = harness.captured.goals[-1] + assert (goal.position.x, goal.position.y) == pytest.approx((0.5, -0.5)) + assert goal.yaw == pytest.approx(-math.pi / 4, abs=1e-6) + + assert "outside the arena walls" in harness.module.move_to(3.0, 0.0) + assert "Invalid coordinates" in harness.module.move_to("abc", 0.0) # type: ignore[arg-type] + assert "Invalid coordinates" in harness.module.move_to(float("nan"), 0.0) + assert len(harness.captured.goals) == 1 + + +def test_move_to_without_odom_uses_identity_heading(harness: Harness) -> None: + out = harness.module.move_to(1.0, 1.0) + assert out.startswith("Arrived.") + assert "Robot position unknown." in out + assert harness.captured.goals[-1].yaw == pytest.approx(0.0, abs=1e-6) + + +def test_navigation_failure_outcomes(harness: Harness) -> None: + harness.feed_odom(0.0, 0.0) + harness.nav.script(IDLE, reached=False) + assert harness.module.move_to(1.0, 1.0).startswith( + "Navigation never started following a path (no route found?)" + ) + + harness.nav.script(FOLLOWING, IDLE, reached=False) + assert harness.module.move_to(1.0, 1.0).startswith( + "Navigation stopped early (cancelled or no path)" + ) + + harness.nav.script(FOLLOWING, reached=False) + harness.module.config.nav_timeout_s = 0.15 + t0 = time.monotonic() + assert harness.module.move_to(1.0, 1.0).startswith("Navigation timed out") + assert time.monotonic() - t0 < 1.0 + + +def test_rpc_fallback_rejection(harness: Harness) -> None: + harness.nav.set_goal_result = False + out = harness.module.move_to(1.0, 1.0) + assert out.startswith("Navigation rejected the goal") + assert len(harness.captured.goals) == 1 + + +def test_stop_moving(harness: Harness) -> None: + assert harness.module.stop_moving() == "Stopped." + harness.nav.cancel_result = False + assert harness.module.stop_moving() == "There was no active navigation goal." + assert harness.nav.cancel_calls == 2 + + +def test_where_am_i(harness: Harness) -> None: + assert harness.module.where_am_i().startswith("Robot position unknown") + harness.feed_odom(1.4, 1.4, 0.5) + out = harness.module.where_am_i() + assert out.startswith("Robot is at (1.40, 1.40), heading 29 deg, in the kitchen (aka space A).") + assert "Nearby: red_box (0.14 m)" in out + assert "kitchen (" not in out.split("Nearby:")[1] + harness.feed_odom(0.1, -0.1) + out = harness.module.where_am_i() + assert "in the central hub (no room)" in out + assert "Nearby" not in out + + +def test_remember_place_persists_and_republishes(harness: Harness, tmp_path: Path) -> None: + assert harness.module.remember_place("charger").startswith("Robot position unknown") + assert ( + harness.module.remember_place(" ") + == "Give the place a name, e.g. remember_place('charger')." + ) + harness.feed_odom(0.3, -0.2, 0.25) + published_before = len(harness.captured.places) + out = harness.module.remember_place("charger") + assert out == "Remembered 'charger' at (0.30, -0.20) in the central hub." + assert len(harness.captured.places) == published_before + 1 + tagged = harness.captured.last_places["tagged"] + assert len(tagged) == 1 + assert tagged[0]["name"] == "charger" + assert (tagged[0]["x"], tagged[0]["y"]) == pytest.approx((0.3, -0.2)) + assert tagged[0]["yaw"] == pytest.approx(0.25, abs=1e-6) + assert "- charger at (0.30, -0.20)" in harness.module.list_places() + + # Reserved names are refused (compared on the normalised forms find() + # matches on, so the spot could never shadow a room or object); + # re-remembering moves the spot. + assert harness.module.remember_place("kitchen").startswith( + "'kitchen' is already the name of a room" + ) + assert harness.module.remember_place("red_box").startswith( + "'red_box' is already the name of a object" + ) + assert harness.module.remember_place("red box").startswith( + "'red box' is already the name of a object" + ) + assert harness.module.remember_place("Space A").startswith( + "'Space A' is already the name of a room" + ) + assert harness.module.remember_place("the lounge").startswith( + "'the lounge' is already the name of a room" + ) + assert len(harness.captured.last_places["tagged"]) == 1 + harness.feed_odom(1.6, 1.0) + assert harness.module.remember_place("charger").endswith("in the kitchen.") + assert len(harness.captured.last_places["tagged"]) == 1 + + # Survives a restart on the same database (same scene) and is invisible + # to a memory opened on another scene of that database. + other = PlacesMemory(tmp_path / "places.db", scene=scene_id(MICRODUCK_ROOMS, MICRODUCK_OBJECTS)) + try: + rec = other.find("charger") + assert rec is not None and (rec.x, rec.y) == pytest.approx((1.6, 1.0)) + finally: + other.close() + foreign = PlacesMemory(tmp_path / "places.db") + try: + assert foreign.find("charger") is None + assert foreign.all() == [] + finally: + foreign.close() + + +def test_list_places(harness: Harness) -> None: + harness.feed_odom(0.0, 0.0) + out = harness.module.list_places() + assert out.startswith("Rooms:") + assert "- kitchen (aka space A): x 0..2, y 0..2, entry point (1.20, 1.00), 1.56 m away" in out + assert "- living (aka space B, living room, lounge)" in out + assert "Objects:" in out + assert "- red_box at (1.50, 1.50) in the kitchen, 2.12 m away" in out + assert "- orange_crate at (0.60, 1.70) in the kitchen" in out + assert "Remembered places:" in out + assert "(none yet" in out + assert out.endswith("Robot is now at (0.00, 0.00) in the central hub.") + + +def test_list_objects(harness: Harness) -> None: + """Kept for the agentic-sim system prompt, which still tells the agent to call it.""" + out = harness.module.list_objects() + assert out.startswith("Objects in the room:\n") + assert "- red_box at (1.50, 1.50) in the kitchen" in out + assert "Rooms:" not in out and "Remembered" not in out + assert " m away" not in out + harness.feed_odom(0.0, 0.0) + assert "- red_box at (1.50, 1.50) in the kitchen, 2.12 m away" in harness.module.list_objects() + + +def test_list_objects_without_objects(tmp_path: Path, fast_polling: None) -> None: + module = _make_module(tmp_path, rooms={}, objects={}) + try: + assert module.list_objects() == "No objects are registered in this scene." + assert module.list_places() == "No places are registered in this scene." + finally: + module.stop() + + +def test_wait_clamps(harness: Harness) -> None: + t0 = time.monotonic() + assert harness.module.wait(-5) == "Waited 0 s." + assert harness.module.wait(0.05) == "Waited 0 s." + assert time.monotonic() - t0 < 1.0 + assert "Invalid duration" in harness.module.wait("soon") # type: ignore[arg-type] + + +# policy skills + + +def test_list_policies_without_and_with_state(harness: Harness) -> None: + out = harness.module.list_policies() + assert "- kick_left [oneshot]: kick the ball with the left foot" in out + assert "has not reported its policy state yet" in out + + harness.feed_state(active="stand", base="stand") + out = harness.module.list_policies() + assert "- roller [base]" in out and "NOT available: rollers variant only" in out + assert "- sitstand [posture]" in out + assert "Now: active policy 'stand' (base 'stand')." in out + + +def test_perform_oneshot_success(harness: Harness) -> None: + harness.feed_state() + + def on_request(req: dict[str, Any]) -> None: + assert req == {"policy": "kick_left", "action": "start", "t": req["t"]} + harness.feed_state( + active="kick_left", locked=True, oneshot={"name": "kick_left", "progress": 0.1} + ) + harness.feed_state_later(0.05, oneshot=None) + + unsub = harness.on_request(on_request) + try: + out = harness.module.perform("kick_left") + finally: + unsub() + assert out.startswith("Finished kick_left.") + assert "Now: active policy 'walk'" in out + assert len(harness.captured.policy_requests) == 1 + + +def test_perform_reports_fall_and_aliases(harness: Harness) -> None: + harness.feed_state() + + def on_request(req: dict[str, Any]) -> None: + assert req["policy"] == "roulade" + harness.feed_state( + active="roulade", locked=True, oneshot={"name": "roulade", "progress": 0.5} + ) + harness.feed_state_later(0.05, oneshot=None, fallen=True, locked=True) + + unsub = harness.on_request(on_request) + try: + out = harness.module.perform("somersault") + finally: + unsub() + assert out.startswith("Finished roulade. The duck fell over and is getting back up.") + assert "FALLEN" in out + + +def test_perform_rejected_reports_last_error(harness: Harness) -> None: + harness.feed_state() + + def on_request(req: dict[str, Any]) -> None: + harness.feed_state(last_error="rejected: robot is locked") + + unsub = harness.on_request(on_request) + try: + out = harness.module.perform("ground_pick") + finally: + unsub() + assert out.startswith("Could not start ground_pick: rejected: robot is locked.") + + +def test_perform_without_ack_times_out_quickly(harness: Harness) -> None: + harness.feed_state() + t0 = time.monotonic() + out = harness.module.perform("kick_right") + assert out.startswith("Could not start kick_right: no acknowledgement.") + assert time.monotonic() - t0 < 1.5 + + +def test_perform_precheck_from_state(harness: Harness) -> None: + harness.feed_state( + oneshot={"name": "kick_left", "progress": 0.5}, active="kick_left", locked=True + ) + assert harness.module.perform("roulade").startswith("The duck is busy with kick_left") + assert harness.module.perform("roulade", "stop") == ( + "roulade is not running. " + + harness.module._policy_summary(harness.module._current_policy_state()) # type: ignore[arg-type] + ) + harness.feed_state(fallen=True, locked=True) + assert harness.module.perform("kick_left").startswith("The duck has fallen") + harness.feed_state(seated=True, active="sitstand", locked=True) + assert harness.module.perform("kick_left") == "The duck is sitting; call stand_up first." + assert harness.module.perform("roller").startswith( + "'roller' is not available on this robot (rollers variant only)" + ) + assert harness.captured.policy_requests == [] + + +def test_perform_stop_oneshot(harness: Harness) -> None: + harness.feed_state( + oneshot={"name": "kick_left", "progress": 0.5}, active="kick_left", locked=True + ) + + def on_request(req: dict[str, Any]) -> None: + assert req["policy"] == "kick_left" and req["action"] == "stop" + harness.feed_state() + + unsub = harness.on_request(on_request) + try: + out = harness.module.perform("kick_left", "stop") + finally: + unsub() + assert out.startswith("Stopped kick_left.") + + +def test_perform_toggle_oneshot_stops_when_running_and_starts_otherwise( + harness: Harness, +) -> None: + harness.feed_state( + oneshot={"name": "kick_left", "progress": 0.5}, active="kick_left", locked=True + ) + + def on_stop(req: dict[str, Any]) -> None: + assert req["policy"] == "kick_left" and req["action"] == "stop" + harness.feed_state() + + unsub = harness.on_request(on_stop) + try: + out = harness.module.perform("kick_left", "toggle") + finally: + unsub() + assert out.startswith("Stopped kick_left.") + + def on_start(req: dict[str, Any]) -> None: + assert req["policy"] == "kick_left" and req["action"] == "start" + harness.feed_state( + active="kick_left", locked=True, oneshot={"name": "kick_left", "progress": 0.1} + ) + harness.feed_state_later(0.05, oneshot=None) + + unsub = harness.on_request(on_start) + try: + out = harness.module.perform("kick_left", "toggle") + finally: + unsub() + assert out.startswith("Finished kick_left.") + assert [json.loads(r)["action"] for r in harness.captured.policy_requests] == ["stop", "start"] + + # Toggling a trick that is not running while another one is stays a start, + # so the busy pre-check applies. + harness.feed_state(oneshot={"name": "roulade", "progress": 0.5}, active="roulade", locked=True) + assert harness.module.perform("kick_left", "toggle").startswith("The duck is busy with roulade") + assert len(harness.captured.policy_requests) == 2 + + +def test_repeated_identical_rejection_fails_fast(harness: Harness) -> None: + """The scheduler re-issues the same last_error string on every rejected + request (only ``t`` changes); that must not look like silence.""" + harness.feed_state(seated=False, locked=True, last_error="rejected: robot is locked") + + def reject(req: dict[str, Any]) -> None: + harness.feed_state(seated=False, locked=True, last_error="rejected: robot is locked") + + unsub = harness.on_request(reject) + try: + t0 = time.monotonic() + out = harness.module.sit() + sit_elapsed = time.monotonic() - t0 + t0 = time.monotonic() + base = harness.module.perform("stand") + base_elapsed = time.monotonic() - t0 + finally: + unsub() + assert out == "Could not sit down: rejected: robot is locked." + assert base == "Could not switch to stand: rejected: robot is locked." + # Grace period (0.3 s under fast_polling), not policy_timeout_s (1 s). + assert sit_elapsed < 0.8 and base_elapsed < 0.8 + assert [json.loads(r)["policy"] for r in harness.captured.policy_requests] == [ + "sitstand", + "stand", + ] + + +def test_stand_up_identical_rejection_fails_fast(harness: Harness) -> None: + harness.feed_state(seated=True, active="sitstand", locked=True, last_error="locked: seated") + + def reject(req: dict[str, Any]) -> None: + assert req["action"] == "stop" + harness.feed_state(seated=True, active="sitstand", locked=True, last_error="locked: seated") + + unsub = harness.on_request(reject) + try: + t0 = time.monotonic() + out = harness.module.stand_up() + elapsed = time.monotonic() - t0 + finally: + unsub() + assert out == "Could not stand up: locked: seated." + assert elapsed < 0.8 + + +def test_posture_heartbeat_then_settles_within_grace(harness: Harness) -> None: + """A heartbeat with last_error cleared is not by itself an acknowledgement; + the posture change that follows within the grace period is.""" + harness.feed_state(last_error="rejected: robot is locked") + + def on_sit(req: dict[str, Any]) -> None: + harness.feed_state(last_error=None) # heartbeat, not yet seated + harness.feed_state_later(0.1, seated=True, active="sitstand", locked=True) + + unsub = harness.on_request(on_sit) + try: + out = harness.module.sit() + finally: + unsub() + assert out.startswith("The duck is now sitting.") + + +def test_posture_heartbeats_without_transition_fail_fast(harness: Harness) -> None: + """Fresh heartbeats with last_error None (a request the scheduler never + saw) do not count as acknowledgement: 'no acknowledgement' after the + grace period instead of waiting the full policy_timeout_s.""" + harness.feed_state() + stop = threading.Event() + + def heartbeat() -> None: + while not stop.wait(0.02): + harness.feed_state(last_error=None) + + thread = threading.Thread(target=heartbeat, daemon=True) + thread.start() + t0 = time.monotonic() + try: + out = harness.module.sit() + finally: + stop.set() + thread.join(timeout=2.0) + elapsed = time.monotonic() - t0 + assert out == "Could not sit down: no acknowledgement." + assert 0.25 <= elapsed < 0.8, elapsed + + +def test_base_switch_heartbeats_without_transition_fail_fast(harness: Harness) -> None: + harness.feed_state() + + def on_request(req: dict[str, Any]) -> None: + harness.feed_state(last_error=None) # heartbeat only, base unchanged + + unsub = harness.on_request(on_request) + t0 = time.monotonic() + try: + out = harness.module.perform("stand") + finally: + unsub() + assert out == "Could not switch to stand: no acknowledgement." + assert time.monotonic() - t0 < 0.8 + + +def test_short_oneshot_started_and_finished_between_polls(harness: Harness) -> None: + """Start and end states arriving back-to-back (before the skill polls once) + still count as the trick having run.""" + harness.feed_state() + + def on_request(req: dict[str, Any]) -> None: + harness.feed_state( + active="kick_left", locked=True, oneshot={"name": "kick_left", "progress": 0.5} + ) + harness.feed_state(oneshot=None) # already over + + unsub = harness.on_request(on_request) + t0 = time.monotonic() + try: + out = harness.module.perform("kick_left") + finally: + unsub() + assert out.startswith("Finished kick_left.") + assert "Now: active policy 'walk'" in out + assert time.monotonic() - t0 < 0.25 + + +def test_posture_settled_between_polls(harness: Harness) -> None: + """Sit begun and settled in states that arrive before the first poll.""" + harness.feed_state(seated=True, active="sitstand", locked=True) + + def on_stand(req: dict[str, Any]) -> None: + harness.feed_state(seated=False, active="standing_up", locked=True) + harness.feed_state(seated=False, active="walk") + + unsub = harness.on_request(on_stand) + try: + out = harness.module.stand_up() + finally: + unsub() + assert out.startswith("The duck is now standing.") + + +def test_policy_history_is_bounded(harness: Harness) -> None: + for _ in range(skills._POLICY_HISTORY + 20): + harness.feed_state() + assert len(harness.module._policy_history) == skills._POLICY_HISTORY + seqs = [seq for seq, _ in harness.module._policy_history] + assert seqs == sorted(seqs) and seqs[-1] == harness.module._policy_state_seq + + +def test_posture_error_after_acknowledgement(harness: Harness) -> None: + harness.feed_state() + + def on_sit(req: dict[str, Any]) -> None: + harness.feed_state(active="sitstand", locked=True) # begun + harness.feed_state_later(0.05, active="walk", locked=True, last_error="fell during sit") + + unsub = harness.on_request(on_sit) + try: + out = harness.module.sit() + finally: + unsub() + assert out == "Could not sit down: fell during sit." + + +def test_posture_stale_state_only_times_out(harness: Harness) -> None: + """No state newer than the request at all: the grace period expires with + 'no acknowledgement' rather than the full policy timeout.""" + harness.feed_state() + t0 = time.monotonic() + out = harness.module.sit() + assert out == "Could not sit down: no acknowledgement." + assert time.monotonic() - t0 < 0.8 + + +def test_base_switch_acknowledged_via_braking(harness: Harness) -> None: + harness.feed_state() + + def on_request(req: dict[str, Any]) -> None: + harness.feed_state(active="braking", base="walk", locked=True) + harness.feed_state_later(0.05, active="stand", base="stand") + + unsub = harness.on_request(on_request) + try: + out = harness.module.perform("stand") + finally: + unsub() + assert out.startswith("Base policy is now stand.") + + +def test_sit_and_stand_up(harness: Harness) -> None: + harness.feed_state() + + def on_sit(req: dict[str, Any]) -> None: + assert req == {"policy": "sitstand", "action": "start", "t": req["t"]} + harness.feed_state(seated=True, active="sitstand", locked=True) + + unsub = harness.on_request(on_sit) + try: + out = harness.module.sit() + finally: + unsub() + assert out.startswith("The duck is now sitting.") + assert "seated" in out + assert harness.module.sit().startswith("Already sitting.") + + def on_stand(req: dict[str, Any]) -> None: + assert req == {"policy": "sitstand", "action": "stop", "t": req["t"]} + harness.feed_state(seated=True, active="standing_up", locked=True) + harness.feed_state_later(0.05, seated=False, active="walk") + + unsub = harness.on_request(on_stand) + try: + out = harness.module.stand_up() + finally: + unsub() + assert out.startswith("The duck is now standing.") + assert harness.module.stand_up().startswith("Already standing.") + assert [json.loads(r)["action"] for r in harness.captured.policy_requests] == ["start", "stop"] + + +def test_perform_aliases_force_posture_action(harness: Harness) -> None: + harness.feed_state(seated=True, active="sitstand", locked=True) + + def on_request(req: dict[str, Any]) -> None: + assert req["policy"] == "sitstand" and req["action"] == "stop" + harness.feed_state(seated=False, active="walk") + + unsub = harness.on_request(on_request) + try: + assert harness.module.perform("stand up").startswith("The duck is now standing.") + finally: + unsub() + + +def test_perform_base_switch(harness: Harness) -> None: + harness.feed_state() + + def on_request(req: dict[str, Any]) -> None: + assert req == {"policy": "stand", "action": "start", "t": req["t"]} + harness.feed_state(active="stand", base="stand") + + unsub = harness.on_request(on_request) + try: + out = harness.module.perform("stand") + finally: + unsub() + assert out.startswith("Base policy is now stand.") + assert harness.module.perform("stand").startswith("Already using the stand policy.") + + def on_stop(req: dict[str, Any]) -> None: + assert req == {"policy": "stand", "action": "stop", "t": req["t"]} + harness.feed_state() + + unsub = harness.on_request(on_stop) + try: + assert harness.module.perform("stand", "stop").startswith("Base policy is now walk.") + finally: + unsub() + + +def test_perform_base_switch_error(harness: Harness) -> None: + harness.feed_state() + + def on_request(req: dict[str, Any]) -> None: + harness.feed_state(last_error="cannot switch while locked", locked=True) + + unsub = harness.on_request(on_request) + try: + out = harness.module.perform("stand") + finally: + unsub() + assert out == "Could not switch to stand: cannot switch while locked." + + +def test_perform_bad_inputs(harness: Harness) -> None: + harness.feed_state() + out = harness.module.perform("moonwalk") + assert out.startswith("Unknown policy 'moonwalk'. Known policies: walk, stand, roller") + assert harness.module.perform("kick_left", "pause").startswith("Unknown action 'pause'") + assert harness.captured.policy_requests == [] + + +def test_perform_waits_for_first_state(harness: Harness) -> None: + """policy_state is wired but nothing arrived yet: wait briefly, then give up.""" + t0 = time.monotonic() + out = harness.module.perform("kick_left") + assert "has not reported any policy state" in out + assert time.monotonic() - t0 < 1.0 + assert harness.captured.last_request["policy"] == "kick_left" + + +def test_malformed_policy_state_is_ignored(harness: Harness) -> None: + harness.module.policy_state.transport.broadcast(None, "not json") + harness.module.policy_state.transport.broadcast(None, "[1,2]") + assert harness.module._current_policy_state() is None + harness.feed_state(active="stand") + state = harness.module._current_policy_state() + assert state is not None and state["active"] == "stand" diff --git a/dimos/robot/pollen/microduck/test_web_codecs.py b/dimos/robot/pollen/microduck/test_web_codecs.py new file mode 100644 index 0000000000..082c97242b --- /dev/null +++ b/dimos/robot/pollen/microduck/test_web_codecs.py @@ -0,0 +1,181 @@ +# Copyright 2025-2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Microduck cockpit encoders: message in, compact JSON frame bytes out. + +No bridge here - these are the plain registry functions the cockpit blueprint +resolves by encoding id. The transcript entry number the viewer dedupes on is +deliberately absent from every payload: it belongs to the bridge's replay log +and rides the frame meta instead (RelayBridgeModule._with_n). +""" + +from __future__ import annotations + +import json +import time +from types import SimpleNamespace +from typing import Any + +from langchain_core.messages import ( + AIMessage, + AIMessageChunk, + ChatMessage, + FunctionMessage, + HumanMessage, + HumanMessageChunk, + SystemMessage, + SystemMessageChunk, + ToolMessage, + ToolMessageChunk, +) +import pytest + +from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped +from dimos.msgs.geometry_msgs.Vector3 import Vector3 +from dimos.msgs.nav_msgs.Path import Path as NavPath +from dimos.robot.pollen.microduck.web_codecs import ( + _CHAT_CONTENT_MAX_CHARS, + _PATH_MAX_POINTS, + encode_chat, + encode_flag, + encode_path, + encode_state_json, +) +from dimos.web.codecs import encoder_definition + +_STATE_ENCODINGS = ["navstate.json.v1", "mode.json.v1", "places.json.v1", "policy.json.v1"] + + +def chat(msg: Any) -> dict[str, Any]: + payload = encode_chat(msg) + assert b": " not in payload and b", " not in payload # compact + return json.loads(payload) + + +def test_encode_chat_shapes() -> None: + t0 = time.time() + human = chat(HumanMessage(content="hi there")) + assert human == { + "role": "human", + "content": "hi there", + "name": None, + "tool_calls": [], + "tool_call_id": None, + "id": None, + "t": human["t"], + } + assert t0 <= human["t"] <= time.time() + + ai = chat( + AIMessage( + content=[ + {"type": "text", "text": "Looking."}, + {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,..."}}, + {"type": "text", "text": "Found it."}, + ], + tool_calls=[{"name": "navigate_to", "args": {"place": "kitchen"}, "id": "c1"}], + id="run-1", + ), + ) + assert ai["role"] == "ai" + assert ai["content"] == "Looking.\n[image]\nFound it." + assert ai["tool_calls"] == [{"id": "c1", "name": "navigate_to", "args": {"place": "kitchen"}}] + assert ai["tool_call_id"] is None and ai["id"] == "run-1" + + tool = chat(ToolMessage(content="ok", tool_call_id="c1", name="navigate_to")) + assert (tool["role"], tool["content"], tool["tool_call_id"], tool["name"]) == ( + "tool", + "ok", + "c1", + "navigate_to", + ) + assert tool["tool_calls"] == [] + + system = chat(SystemMessage(content="You are a duck.")) + assert (system["role"], system["content"]) == ("system", "You are a duck.") + + +def test_encode_chat_caps_content() -> None: + cap = _CHAT_CONTENT_MAX_CHARS + entry = chat(ToolMessage(content="x" * (cap + 5000), tool_call_id="c1")) + assert entry["content"] == "x" * cap + "\n[truncated]" + exact = chat(HumanMessage(content="y" * cap)) + assert exact["content"] == "y" * cap + + +@pytest.mark.parametrize( + ("msg", "role"), + [ + (HumanMessageChunk(content="h"), "human"), + (AIMessageChunk(content="a"), "ai"), + (ToolMessageChunk(content="t", tool_call_id="c1"), "tool"), + (SystemMessageChunk(content="s"), "system"), + (ChatMessage(content="c", role="narrator"), "system"), + (FunctionMessage(content="f", name="fn"), "system"), + # The encoder reads the message's own `type` tag, nothing langchain + # specific: any object shaped like a message encodes. + (SimpleNamespace(type="human", content="duck"), "human"), + (SimpleNamespace(type="mystery", content=["a", "b"]), "system"), + ], + ids=lambda v: v if isinstance(v, str) else type(v).__name__, +) +def test_encode_chat_role_from_message_type(msg: Any, role: str) -> None: + entry = chat(msg) + assert entry["role"] == role + assert entry["tool_calls"] == [] and entry["tool_call_id"] == getattr(msg, "tool_call_id", None) + assert isinstance(entry["t"], float) + + +def test_encode_flag_and_path() -> None: + flag = json.loads(encode_flag(True)) + assert flag["value"] is True and isinstance(flag["t"], float) + + def pose(i: int) -> PoseStamped: + return PoseStamped(ts=1.0, frame_id="map", position=Vector3(float(i), -0.5 * i, 0.0)) + + short = NavPath(ts=1.0, frame_id="map", poses=[pose(i) for i in range(3)]) + path = json.loads(encode_path(short)) + assert path["frame"] == "map" + assert path["points"] == [[0.0, 0.0], [1.0, -0.5], [2.0, -1.0]] + assert isinstance(path["t"], float) + + long = NavPath(ts=1.0, frame_id="map", poses=[pose(i) for i in range(1000)]) + points = json.loads(encode_path(long))["points"] + # Uniform decimation keeps both endpoints and the order. + assert len(points) == _PATH_MAX_POINTS + assert points[0] == [0.0, 0.0] and points[-1] == [999.0, -499.5] + assert points == sorted(points) + + +@pytest.mark.parametrize("encoding", _STATE_ENCODINGS) +def test_passthrough_encoders_validate_json(encoding: str) -> None: + # One behaviour behind four encoding ids; the id is only what tells the + # cockpit which panel slot the channel fills. + definition = encoder_definition(encoding) + assert definition is not None and definition.encode is encode_state_json + # Re-dumped compactly, the producer's own "t" kept. + payload = definition.encode('{"state": "idle", "goal": null, "t": 12.5}') + assert payload == b'{"state":"idle","goal":null,"t":12.5}' + # A producer that forgot "t" gets the bridge clock. + record = json.loads(definition.encode('{"mode": "agent"}')) + assert record["mode"] == "agent" and isinstance(record["t"], float) + + +@pytest.mark.parametrize("payload", ["not json", "[1, 2]", '"a string"', ""]) +def test_passthrough_rejects_non_objects(payload: str) -> None: + # Raising is how a bad sample is dropped now: the bridge's _run_encoder + # catches it, logs at most once per channel per window, and sends no + # frame - rather than forwarding garbage to every viewer. + with pytest.raises(ValueError): + encode_state_json(payload) diff --git a/dimos/robot/pollen/microduck/web_codecs.py b/dimos/robot/pollen/microduck/web_codecs.py new file mode 100644 index 0000000000..9616b29504 --- /dev/null +++ b/dimos/robot/pollen/microduck/web_codecs.py @@ -0,0 +1,148 @@ +# Copyright 2025-2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Web encoders for the microduck cockpit's own channels. + +The cockpit blueprint declares these streams with `Channel(...)` and the +bridge resolves the encoder by encoding id (dimos/web/codecs.py), so none of +this lives in the generic relay bridge: the transcript encoder can import +langchain outright, and no microduck stream welds a port onto +RelayBridgeModule. + +Encoders are module level because the registry ships them into the worker by +pickle reference. Each returns compact JSON bytes; every record carries "t", +the encode-time wall clock (a passthrough keeps the producer's own). +""" + +from __future__ import annotations + +import json +import time +from typing import Any + +from langchain_core.messages.base import BaseMessage + +from dimos.msgs.nav_msgs.Path import Path as NavPath +from dimos.web.codecs import web_encoder + +_JSON_SEPARATORS = (",", ":") + +# Transcript content cap (characters): a pathological tool result must not +# turn one reliable frame into a megabyte for every viewer. +_CHAT_CONTENT_MAX_CHARS = 16 * 1024 + +# Path points per frame: enough for any room-scale plan; longer plans are +# decimated uniformly (endpoints kept) so the viewer's polyline stays cheap. +_PATH_MAX_POINTS = 256 + + +def _json_frame(record: dict[str, Any]) -> bytes: + record.setdefault("t", time.time()) + return json.dumps(record, separators=_JSON_SEPARATORS, default=str).encode() + + +def _chat_text(content: Any) -> str: + """Flatten langchain message content - a string or a list of provider + content blocks - into transcript text (images become "[image]").""" + if isinstance(content, str): + return content + if not isinstance(content, list): + return str(content) + parts: list[str] = [] + for block in content: + if isinstance(block, str): + parts.append(block) + elif isinstance(block, dict): + kind = block.get("type") + if kind == "text" and isinstance(block.get("text"), str): + parts.append(block["text"]) + elif kind in ("image", "image_url"): + parts.append("[image]") + elif kind: + parts.append(f"[{kind}]") + else: + parts.append(str(block)) + return "\n".join(parts) + + +def _chat_role(msg: BaseMessage) -> str: + """Transcript role from the message's own `type` tag ("human", "ai", + "tool", "system"; the streaming variants say "AIMessageChunk" and so on). + The exotic kinds the agent loop never emits (chat, function) read as + system notes.""" + kind = str(getattr(msg, "type", "")).lower().removesuffix("messagechunk") + return kind if kind in ("human", "ai", "tool") else "system" + + +@web_encoder("chat.json.v1") +def encode_chat(msg: BaseMessage) -> bytes: + """One transcript entry per langchain message. + + The entry number the viewer dedupes on is NOT here: it belongs to the + bridge's replay log, which ships it as the frame's `n` meta so a replayed + frame and its live original agree (see RelayBridgeModule._with_n). + """ + tool_calls = [ + {"id": call.get("id"), "name": call.get("name"), "args": call.get("args")} + for call in (getattr(msg, "tool_calls", None) or []) + ] + content = _chat_text(getattr(msg, "content", "")) + if len(content) > _CHAT_CONTENT_MAX_CHARS: + content = content[:_CHAT_CONTENT_MAX_CHARS] + "\n[truncated]" + return _json_frame( + { + "role": _chat_role(msg), + "content": content, + "name": getattr(msg, "name", None), + "tool_calls": tool_calls, + "tool_call_id": getattr(msg, "tool_call_id", None), + "id": getattr(msg, "id", None), + } + ) + + +@web_encoder("flag.json.v1") +def encode_flag(msg: bool) -> bytes: + return _json_frame({"value": bool(msg)}) + + +@web_encoder("path.json.v1") +def encode_path(msg: NavPath) -> bytes: + poses = msg.poses + if len(poses) > _PATH_MAX_POINTS: + step = (len(poses) - 1) / (_PATH_MAX_POINTS - 1) + poses = [poses[round(i * step)] for i in range(_PATH_MAX_POINTS)] + return _json_frame( + { + "frame": msg.frame_id, + "points": [[p.position.x, p.position.y] for p in poses], + } + ) + + +# One behaviour, four encoding ids: DuckControl and PlacesMemory all publish +# JSON strings, and the id is what tells the cockpit which panel slot a +# channel fills (dimos/web/relay_bridge/manifest.py, _SLOT_KINDS). +@web_encoder("navstate.json.v1") +@web_encoder("mode.json.v1") +@web_encoder("places.json.v1") +@web_encoder("policy.json.v1") +def encode_state_json(msg: str) -> bytes: + """Re-dump a JSON-string producer's payload compactly. A payload that is + not a JSON object raises, so the bridge drops the frame (throttled log) + rather than forwarding garbage to every viewer.""" + record = json.loads(msg) + if not isinstance(record, dict): + raise ValueError(f"expected a JSON object, got {type(record).__name__}") + return _json_frame(record) diff --git a/dimos/simulation/engines/mujoco_engine.py b/dimos/simulation/engines/mujoco_engine.py index feb65fe79e..7c0c431a7e 100644 --- a/dimos/simulation/engines/mujoco_engine.py +++ b/dimos/simulation/engines/mujoco_engine.py @@ -50,6 +50,19 @@ _MJJNT_FREE = int(mujoco.mjtJoint.mjJNT_FREE) # type: ignore[attr-defined] _RESET_WAIT_TIMEOUT_S = 5.0 +# How far behind its wall-clock schedule the sim loop may fall and still spend +# time on camera rendering. One render legitimately costs several step budgets +# (a 640x360 camera is ~10 ms against a 5 ms step), so the loop is briefly in +# debt after every frame and repays it on the next near-empty steps; the +# threshold has to sit above that normal sawtooth and below the sustained debt +# that means rendering no longer fits in real time. +RENDER_LAG_BUDGET = 0.020 +# Debt past this is treated as unrecoverable (a host suspend, a long GC pause) +# and abandoned, so it cannot suppress rendering forever afterwards. +MAX_LAG_CATCHUP = 0.25 +# Rate limit for the "skipping renders" warning. +SKIP_REPORT_INTERVAL = 10.0 + def _camera_name_candidates(camera_name: str) -> tuple[str, ...]: stripped = camera_name.strip("/") @@ -80,6 +93,9 @@ def _camera_ray_directions(width: int, height: int, fovy_degrees: float) -> NDAr return cast("NDArray[np.float64]", directions / norms) +camera_ray_directions = _camera_ray_directions + + @dataclass class CameraConfig: name: str @@ -89,12 +105,16 @@ class CameraConfig: max_geom: int | None = 10000 geom_groups: tuple[int, ...] | None = None base_body_name: str | None = None + # Depth is a second full render of the same scene, so it roughly doubles + # the camera's cost on the sim thread. Cameras whose depth nobody reads + # (a video feed) turn it off; CameraFrame.depth is then None. + render_depth: bool = True @dataclass class CameraFrame: rgb: NDArray[np.uint8] - depth: NDArray[np.float32] + depth: NDArray[np.float32] | None cam_pos: NDArray[np.float64] cam_mat: NDArray[np.float64] fovy: float @@ -127,7 +147,7 @@ class _CameraRendererState: cfg: CameraConfig cam_id: int rgb_renderer: mujoco.Renderer - depth_renderer: mujoco.Renderer + depth_renderer: mujoco.Renderer | None scene_option: mujoco.MjvOption | None interval: float base_body_id: int | None = None @@ -378,13 +398,15 @@ def _init_cameras(self) -> dict[str, _CameraRendererState]: width=cfg.width, max_geom=max_geom, ) # type: ignore[call-arg] - depth_renderer = mujoco.Renderer( - self._model, - height=cfg.height, - width=cfg.width, - max_geom=max_geom, - ) # type: ignore[call-arg] - depth_renderer.enable_depth_rendering() + depth_renderer = None + if cfg.render_depth: + depth_renderer = mujoco.Renderer( + self._model, + height=cfg.height, + width=cfg.width, + max_geom=max_geom, + ) # type: ignore[call-arg] + depth_renderer.enable_depth_rendering() scene_option = None if cfg.geom_groups is not None: scene_option = mujoco.MjvOption() @@ -456,14 +478,16 @@ def _render_cameras(self, now: float, cam_renderers: dict[str, _CameraRendererSt ) rgb = state.rgb_renderer.render().copy() - state.depth_renderer.update_scene( - self._data, camera=state.cam_id, scene_option=state.scene_option - ) - depth = state.depth_renderer.render().copy() + depth = None + if state.depth_renderer is not None: + state.depth_renderer.update_scene( + self._data, camera=state.cam_id, scene_option=state.scene_option + ) + depth = state.depth_renderer.render().copy().astype(np.float32) frame = CameraFrame( rgb=rgb, - depth=depth.astype(np.float32), + depth=depth, cam_pos=self._data.cam_xpos[state.cam_id].copy(), cam_mat=self._data.cam_xmat[state.cam_id].copy(), fovy=float(self._model.cam_fovy[state.cam_id]), @@ -538,7 +562,8 @@ def _raycast_lidars( def _close_cam_renderers(cam_renderers: dict[str, _CameraRendererState]) -> None: for state in cam_renderers.values(): state.rgb_renderer.close() - state.depth_renderer.close() + if state.depth_renderer is not None: + state.depth_renderer.close() def _reset_unlocked(self) -> None: if self._model.nkey > 0: @@ -605,8 +630,23 @@ def _sim_loop(self) -> None: cam_renderers = self._init_cameras() lidar_states = self._init_raycast_lidars() + # When this iteration was due to start, on the MONOTONIC clock: pacing + # must not be steerable by NTP or a manual clock change, which on + # time.time() would show up as the loop sleeping for minutes or + # sprinting through its debt. Frame/sensor timestamps stay wall-clock; + # only the schedule is monotonic. Cameras render inline on this thread + # (see _render_cameras), so without a schedule to measure against a + # slow render silently steals from the simulation clock instead of + # from the frame rate. + next_step_at = time.monotonic() + skipped_renders = 0 + last_skip_report = next_step_at + def _step_once(sync_viewer: bool) -> None: - loop_start = time.time() + nonlocal next_step_at, skipped_renders, last_skip_report + loop_start = time.time() # wall clock: stamps frames and sensors + loop_started_at = time.monotonic() # monotonic: paces the loop + lag = loop_started_at - next_step_at reset_done_events: list[threading.Event] = [] with self._lock: if self._reset_requested: @@ -631,13 +671,43 @@ def _step_once(sync_viewer: bool) -> None: self._on_after_step(self) except Exception as exc: logger.error("on_after_step failed", error=str(exc)) - self._render_cameras(loop_start, cam_renderers) + # Cameras render inline on this thread and one render costs + # several times the whole per-step budget, so they have to yield + # to the simulation clock: when the loop is already behind real + # time - GPU contention from a screen recorder, another encoder, + # the compositor - skip this cycle's renders. Contention then + # costs frame rate, which recovers on its own, instead of slowing + # simulated time, which does not. Rendering resumes by itself as + # soon as the loop catches up; _render_cameras leaves + # last_render_time untouched when it does not run, so the next + # opportunity is taken immediately. + if lag <= RENDER_LAG_BUDGET: + self._render_cameras(loop_start, cam_renderers) + else: + skipped_renders += 1 + # Lidar is deliberately NOT skipped: it feeds navigation, runs at + # ~1 Hz, and is a rounding error next to the cameras. self._raycast_lidars(loop_start, lidar_states) - elapsed = time.time() - loop_start - sleep_time = dt - elapsed + next_step_at += dt + now = time.monotonic() + sleep_time = next_step_at - now if sleep_time > 0: time.sleep(sleep_time) + elif now - next_step_at > MAX_LAG_CATCHUP: + # Unrecoverable debt (a long stall, the host suspending). + # Abandon it rather than sprint - and rather than let it + # suppress every render from here on. + next_step_at = now + if skipped_renders and now - last_skip_report >= SKIP_REPORT_INTERVAL: + logger.warning( + "sim loop behind real time; camera renders skipped to protect the " + "simulation clock (GPU contention?)", + skipped=skipped_renders, + window_s=round(now - last_skip_report, 1), + ) + skipped_renders = 0 + last_skip_report = now if self._headless: while not self._stop_event.is_set(): diff --git a/dimos/simulation/engines/mujoco_sim_module.py b/dimos/simulation/engines/mujoco_sim_module.py index 87ff5a0a21..e17c262a9f 100644 --- a/dimos/simulation/engines/mujoco_sim_module.py +++ b/dimos/simulation/engines/mujoco_sim_module.py @@ -267,6 +267,11 @@ class MujocoSimModuleConfig(ModuleConfig, DepthCameraConfig): enable_pointcloud: bool = False pointcloud_fps: float = 5.0 camera_info_fps: float = 1.0 + # Additional rendered MJCF cameras, name -> (width, height, fps). They + # are rendered by the engine alongside the primary camera (each blocks + # the sim thread while rendering); subclasses read them back with + # ``engine.read_camera(name)`` and publish them however they like. + extra_cameras: dict[str, tuple[int, int, float]] = Field(default_factory=dict) # Optional MuJoCo-native lidar: cast rays from one or more named cameras # and publish world-frame PointCloud2 points on ``pointcloud``. enable_mujoco_lidar: bool = False @@ -434,6 +439,16 @@ def start(self) -> None: cameras_by_name: dict[str, CameraConfig] = {} raycast_lidars: list[RaycastLidarConfig] = [] + # The only readers of a rendered depth image: the depth_image publisher + # and the RGB-D pointcloud back-projection (which the raycast lidar + # replaces when enabled). With neither, the second render per frame is + # pure cost on the sim thread. Bound before add_camera rather than + # after: the closure reads it, so a future caller placed above the old + # assignment would have raised NameError. + depth_needed = self.config.enable_depth or ( + self.config.enable_pointcloud and not self.config.enable_mujoco_lidar + ) + def add_camera( name: str, *, @@ -457,16 +472,28 @@ def add_camera( max_geom=max_geom, geom_groups=groups, base_body_name=self.config.base_frame_id, + render_depth=depth_needed, ) - primary_needed = ( - self.config.enable_color - or self.config.enable_depth - or (self.config.enable_pointcloud and not self.config.enable_mujoco_lidar) - ) + # Same two readers, plus colour: depth_needed already spells the rest. + primary_needed = self.config.enable_color or depth_needed if primary_needed: add_camera(self.config.camera_name) + for extra_name, (extra_w, extra_h, extra_fps) in self.config.extra_cameras.items(): + if not extra_name or extra_name in cameras_by_name: + continue + cameras_by_name[extra_name] = CameraConfig( + name=extra_name, + width=int(extra_w), + height=int(extra_h), + fps=float(extra_fps), + base_body_name=self.config.base_frame_id, + # Extra cameras are video feeds (e.g. the microduck chase + # cam); nothing reads their depth. + render_depth=False, + ) + if self.config.enable_pointcloud and self.config.enable_mujoco_lidar: for camera_name in self._mujoco_lidar_camera_names(): raycast_lidars.append( @@ -497,6 +524,7 @@ def add_camera( if self.config.robot_mjcf is not None: engine_kwargs["config_path"] = Path(self.config.robot_mjcf) engine_kwargs["model"] = self._compose_model() + engine_kwargs["robot_sim_spec"] = self.config.robot_sim_spec else: engine_kwargs["config_path"] = Path(self.config.address) engine_kwargs["assets"] = engine_assets @@ -923,6 +951,16 @@ def _publish_loop(self) -> None: self.color_image.publish(color_img) if self.config.enable_depth: + # depth_needed in start() must have registered this camera with + # render_depth=True. Say so out loud: if that derivation ever + # drifts, publishing Image(data=None) would fail somewhere far + # from the cause, or not at all. + if frame.depth is None: + raise RuntimeError( + f"camera {self.config.camera_name!r} rendered no depth while " + "enable_depth is set; CameraConfig.render_depth and the " + "depth_needed derivation in start() have diverged" + ) depth_img = Image( data=frame.depth, format=ImageFormat.DEPTH, @@ -938,7 +976,7 @@ def _publish_loop(self) -> None: logger.info( "MujocoSimModule first frame published", rgb_shape=frame.rgb.shape, - depth_shape=frame.depth.shape, + depth_shape=None if frame.depth is None else frame.depth.shape, ) elapsed = time.monotonic() - loop_start @@ -1024,6 +1062,15 @@ def _generate_pointcloud(self) -> None: frame_id=self._color_optical_frame, ts=frame.timestamp, ) + # Same invariant as the publish path: this branch only runs when + # the raycast lidar is off, which is exactly when depth_needed is + # true, so a None here means that derivation drifted. + if frame.depth is None: + raise RuntimeError( + f"camera {self.config.camera_name!r} rendered no depth while the " + "RGB-D pointcloud path is active; CameraConfig.render_depth and " + "the depth_needed derivation in start() have diverged" + ) depth_img = Image( data=frame.depth, format=ImageFormat.DEPTH, diff --git a/dimos/simulation/engines/test_mujoco_sim_module.py b/dimos/simulation/engines/test_mujoco_sim_module.py index 3fa20048b5..63a361a033 100644 --- a/dimos/simulation/engines/test_mujoco_sim_module.py +++ b/dimos/simulation/engines/test_mujoco_sim_module.py @@ -25,7 +25,7 @@ import pytest from dimos.msgs.sensor_msgs.CameraInfo import CameraInfo -from dimos.simulation.engines.mujoco_engine import CameraFrame, MujocoEngine +from dimos.simulation.engines.mujoco_engine import CameraConfig, CameraFrame, MujocoEngine from dimos.simulation.engines.mujoco_sim_module import MujocoSimModule, MujocoSimModuleConfig @@ -452,6 +452,141 @@ def freejoint_engine(tmp_path: Path) -> Iterator[MujocoEngine]: engine.disconnect() +@pytest.mark.mujoco +def test_sim_loop_holds_real_time_with_no_cameras(tmp_path: Path) -> None: + """The loop must pace against an absolute deadline, not per iteration. + + time.sleep() overshoots - asking for 5 ms typically returns after ~6 - so a + pacer that sleeps `dt - elapsed` and starts the next iteration from + scratch throws that overshoot away every step and tops out near RTF 0.83 + with nothing rendering at all. Only the accumulated `next_step_at` + schedule repays it. + + Note this asserts against engine.data.time, MuJoCo's own clock. It cannot + be written against the odom message timestamp: that is stamped with + time.time(), so odom-vs-wall is wall-vs-wall and reads 1.000 no matter how + far behind the simulation actually is. + """ + robot_xml = tmp_path / "freejoint.xml" + _write_freejoint_xml(robot_xml) + engine = MujocoEngine(config_path=robot_xml, headless=True) + assert engine.connect() is True + try: + time.sleep(0.5) + t0, sim0 = time.monotonic(), float(engine.data.time) + time.sleep(3.0) + rtf = (float(engine.data.time) - sim0) / (time.monotonic() - t0) + finally: + engine.disconnect() + assert rtf > 0.95, f"sim loop cannot hold real time even when idle (RTF={rtf:.3f})" + + +@pytest.mark.mujoco +def test_slow_renders_cost_frame_rate_not_simulated_time(tmp_path: Path) -> None: + """Cameras render inline on the sim thread, so a slow renderer (GPU + contention: a screen recorder, another encoder) must be allowed to cost + frames, never simulated time. Before the lag governor, a render that + overran the step budget silently pushed the simulation clock behind the + wall clock - the user-visible symptom being that time ran in slow motion.""" + robot_xml = tmp_path / "camera.xml" + robot_xml.write_text( + """ + + +""".strip() + ) + engine = MujocoEngine( + config_path=robot_xml, + headless=True, + cameras=[CameraConfig(name="cam", width=32, height=32, fps=60.0, render_depth=False)], + ) + assert engine.connect() is True + try: + # Each render sleeps far longer than the 5 ms step budget: a renderer + # roughly 8x slower than real, which is past anything the loop could + # absorb by rendering every frame. + renders = 0 + real_render = engine._render_cameras + + def _slow_render(now: float, states: dict[str, Any]) -> None: + nonlocal renders + renders += 1 + time.sleep(0.040) + real_render(now, states) + + engine._render_cameras = _slow_render # type: ignore[method-assign] + + time.sleep(0.5) # let the loop settle before sampling + t0, sim0 = time.monotonic(), float(engine.data.time) + time.sleep(4.0) + wall = time.monotonic() - t0 + sim = float(engine.data.time) - sim0 + finally: + engine.disconnect() + + rtf = sim / wall + # The clock is protected: simulated time keeps up with the wall clock. + assert rtf > 0.9, f"simulated time fell behind the wall clock (RTF={rtf:.3f})" + # And the cost was paid in frames instead - the governor skipped renders + # rather than letting them run every cycle (60 fps over ~4 s would be + # ~240; a renderer this slow cannot sustain that and must be throttled). + assert 0 < renders < 200, f"expected renders to be throttled, got {renders}" + + +@pytest.mark.mujoco +@pytest.mark.parametrize("render_depth", [True, False]) +def test_camera_render_depth_controls_the_second_render(tmp_path: Path, render_depth: bool) -> None: + """Depth is a whole second render of the scene, so a camera whose depth + nobody reads (a video feed) must be able to skip it and still deliver rgb.""" + robot_xml = tmp_path / "camera.xml" + robot_xml.write_text( + """ + + +""".strip() + ) + engine = MujocoEngine( + config_path=robot_xml, + headless=True, + cameras=[ + CameraConfig(name="cam", width=16, height=16, fps=1000.0, render_depth=render_depth) + ], + ) + assert engine.connect() is True + try: + deadline = time.monotonic() + 5.0 + frame = None + while frame is None and time.monotonic() < deadline: + frame = engine.read_camera("cam") + if frame is None: + time.sleep(0.01) + assert frame is not None, "camera never produced a frame" + assert frame.rgb.shape == (16, 16, 3) # rgb is unaffected either way + if render_depth: + assert frame.depth is not None + assert frame.depth.shape == (16, 16) + else: + assert frame.depth is None + finally: + engine.disconnect() + + @pytest.mark.mujoco def test_engine_request_reset_to_applies_pose_in_sim_loop(freejoint_engine: MujocoEngine) -> None: assert freejoint_engine.request_reset_to( diff --git a/dimos/web/cockpit.py b/dimos/web/cockpit.py index 136294fa57..ce2a50c359 100644 --- a/dimos/web/cockpit.py +++ b/dimos/web/cockpit.py @@ -173,6 +173,17 @@ class Channel: params: Mapping[str, Any] | None = field(default=None, kw_only=True) publish: Literal["none", "shared", "exclusive"] = field(default="none", kw_only=True) required_scope: str | None = field(default=None, kw_only=True) + # Replay the newest message to a channel's first viewer, so a page opened + # mid-run shows current state instead of waiting for the next publish. + resend_on_subscribe: bool = field(default=False, kw_only=True) + # False = every message is encoded, maxHz notwithstanding: for an event + # stream (a chat turn, a mode flip) a skipped sample is lost data, not a + # dropped frame. + rate_gate: bool = field(default=True, kw_only=True) + # > 1 replays that many past messages to a first viewer (a transcript must + # survive a page reload) instead of only the newest. The bridge numbers + # the entries and ships the number as the frame's `n` meta. + replay_depth: int = field(default=1, kw_only=True) def __post_init__(self) -> None: _check_stream("stream", self.stream) @@ -205,6 +216,12 @@ def __post_init__(self) -> None: "publish='exclusive' channels are enabled by the exclusive publisher " "lease ticket (W8); use publish='shared' for interleavable input" ) + if not isinstance(self.replay_depth, int) or isinstance(self.replay_depth, bool): + raise ValueError(f"replay_depth must be an int, got {self.replay_depth!r}") + if self.replay_depth < 1: + raise ValueError(f"replay_depth must be >= 1, got {self.replay_depth}") + if self.replay_depth > 1 and not self.resend_on_subscribe: + raise ValueError("replay_depth > 1 requires resend_on_subscribe=True") if self.dir == "rx" and self.publish != "none": raise ValueError("rx channels must use publish='none'") if self.dir == "tx" and self.publish == "none": @@ -259,12 +276,19 @@ def _panel_params(self) -> dict[str, Any]: @dataclass(frozen=True) class Video(Panel): - """JPEG video feed of one image stream.""" + """JPEG video feed of one image stream, optionally with a second drawn + inset over the first (picture-in-picture): `Video("chase_image", + inset="color_image")` puts the close view in the corner of the wide one. + The inset keeps its own rate and quality, so a cheap thumbnail can ride + along with an expensive main feed.""" kind: ClassVar[str] = "video" stream: str = "color_image" max_hz: float = field(default=30.0, kw_only=True) quality: int = field(default=75, kw_only=True) + inset: str | None = field(default=None, kw_only=True) + inset_max_hz: float = field(default=10.0, kw_only=True) + inset_quality: int = field(default=60, kw_only=True) title: str = field(default="", kw_only=True) def __post_init__(self) -> None: @@ -276,9 +300,24 @@ def __post_init__(self) -> None: or not 0 <= self.quality <= 100 ): raise ValueError(f"quality must be an int in 0..100, got {self.quality!r}") + if self.inset is not None: + _check_stream("inset", self.inset) + if self.inset == self.stream: + raise ValueError(f"inset must differ from stream, both are {self.stream!r}") + _check_rate("inset_max_hz", self.inset_max_hz) + if ( + isinstance(self.inset_quality, bool) + or not isinstance(self.inset_quality, int) + or not 0 <= self.inset_quality <= 100 + ): + raise ValueError( + f"inset_quality must be an int in 0..100, got {self.inset_quality!r}" + ) def _channel_requests(self) -> tuple[ChannelRequest, ...]: - return ( + # Main feed first: the web panel draws channels[0] full-bleed and + # channels[1], when present, inset over it. + requests = [ ChannelRequest( self.stream, "rx", @@ -286,8 +325,20 @@ def _channel_requests(self) -> tuple[ChannelRequest, ...]: self.max_hz, {"quality": self.quality}, delivery="latest", - ), - ) + ) + ] + if self.inset is not None: + requests.append( + ChannelRequest( + self.inset, + "rx", + "jpeg.v1", + self.inset_max_hz, + {"quality": self.inset_quality}, + delivery="latest", + ) + ) + return tuple(requests) @dataclass(frozen=True) @@ -327,6 +378,14 @@ class Teleop(Panel): All params ride the tx channel in the manifest: the Cockpit machine reads speeds and cadence from there, the bridge reads watchdog_ms (its deadman window) and clamps incoming twists to max * boost. + + `mode` names a mode stream some other panel in the cockpit already + subscribes (the store is keyed by channel, so the pad can read it without + requesting it). Naming it lets the pad refuse to arm on a robot that + ignores teleop in agent mode, instead of going green while the robot + discards every key. Unset - every robot but the duck - keeps the pad + exactly as it was: one tx channel, no extra slot, so the "teleop binds + exactly one channel" manifest rule is untouched. """ kind: ClassVar[str] = "teleop" @@ -336,6 +395,7 @@ class Teleop(Panel): boost: float = field(default=2.0, kw_only=True) publish_hz: float = field(default=15.0, kw_only=True) watchdog_ms: float = field(default=300.0, kw_only=True) + mode: str | None = field(default=None, kw_only=True) title: str = field(default="", kw_only=True) def __post_init__(self) -> None: @@ -345,6 +405,8 @@ def __post_init__(self) -> None: _check_rate("boost", self.boost) _check_rate("publish_hz", self.publish_hz) _check_rate("watchdog_ms", self.watchdog_ms) + if self.mode is not None: + _check_stream("mode", self.mode) def _channel_requests(self) -> tuple[ChannelRequest, ...]: return ( @@ -363,6 +425,144 @@ def _channel_requests(self) -> tuple[ChannelRequest, ...]: ), ) + def _panel_params(self) -> dict[str, Any]: + return {} if self.mode is None else {"mode": self.mode} + + +# The three panels below bind several streams each. Their role -> channel +# mapping rides `params` (keyed by role) so the web panel looks channels up +# by role instead of index-guessing; the channel slot order (request order) +# is what manifest.py / manifest.ts validate per kind. Rates are fixed: the +# bridge's channel table caps them anyway and none of these are tunable +# from the cockpit today. + + +@dataclass(frozen=True, kw_only=True) +class Chat(Panel): + """Agent chat: transcript, idle flag and mode in; typed human input out. + + read_only disables the panel's composer; it is not a transport access policy. + """ + + kind: ClassVar[str] = "chat" + chat: str = "agent" + idle: str = "agent_idle" + mode: str = "mode" + input: str = "human_input" + read_only: bool = False + title: str = "Agent" + + def __post_init__(self) -> None: + _check_stream("chat", self.chat) + _check_stream("idle", self.idle) + _check_stream("mode", self.mode) + _check_stream("input", self.input) + + def _channel_requests(self) -> tuple[ChannelRequest, ...]: + return ( + ChannelRequest(self.chat, "rx", "chat.json.v1", 30.0), + ChannelRequest(self.idle, "rx", "flag.json.v1", 10.0), + ChannelRequest(self.mode, "rx", "mode.json.v1", 10.0), + ChannelRequest(self.input, "tx", "text.json.v1", 2.0), + ) + + def _panel_params(self) -> dict[str, Any]: + params = {"chat": self.chat, "idle": self.idle, "mode": self.mode, "input": self.input} + if self.read_only: + return {**params, "readOnly": True} + return params + + +@dataclass(frozen=True, kw_only=True) +class NavMap(Panel): + """Costmap with pose, planned path, named places and nav state overlays; + click-to-goal and cancel go out as goal poses / UI commands.""" + + kind: ClassVar[str] = "navmap" + costmap: str = "global_costmap" + pose: str = "odom" + path: str = "path" + places: str = "places" + nav_state: str = "nav_state" + goal: str = "goal_request" + command: str = "ui_command" + title: str = "Nav map" + fit_places: bool = False + + def __post_init__(self) -> None: + _check_stream("costmap", self.costmap) + _check_stream("pose", self.pose) + _check_stream("path", self.path) + _check_stream("places", self.places) + _check_stream("nav_state", self.nav_state) + _check_stream("goal", self.goal) + _check_stream("command", self.command) + + def _channel_requests(self) -> tuple[ChannelRequest, ...]: + return ( + ChannelRequest(self.costmap, "rx", "costmap.zlib.v1", 2.0, delivery="latest"), + ChannelRequest(self.pose, "rx", "pose.json.v1", 10.0), + ChannelRequest(self.path, "rx", "path.json.v1", 5.0, delivery="latest"), + ChannelRequest(self.places, "rx", "places.json.v1", 2.0), + # State, not a frame stream: reliable. A latest channel costs one + # relay->viewer stream per frame out of a budget shared with the + # cameras (web/README.md bug 12), and a dropped nav_state would + # leave the chip showing a state the robot has already left. + ChannelRequest(self.nav_state, "rx", "navstate.json.v1", 10.0), + ChannelRequest(self.goal, "tx", "pose_goal.json.v1", 5.0), + ChannelRequest(self.command, "tx", "command.json.v1", 10.0), + ) + + def _panel_params(self) -> dict[str, Any]: + params: dict[str, Any] = { + "costmap": self.costmap, + "pose": self.pose, + "path": self.path, + "places": self.places, + "navState": self.nav_state, + "goal": self.goal, + "command": self.command, + } + if self.fit_places: + params["fitPlaces"] = True + return params + + +@dataclass(frozen=True, kw_only=True) +class Control(Panel): + """Top bar: teleop/agent mode switch, policy buttons and nav status in; + UI commands (set_mode / policy / cancel_nav) out.""" + + kind: ClassVar[str] = "control" + mode: str = "mode" + policies: str = "policy_state" + nav_state: str = "nav_state" + command: str = "ui_command" + title: str = "Control" + + def __post_init__(self) -> None: + _check_stream("mode", self.mode) + _check_stream("policies", self.policies) + _check_stream("nav_state", self.nav_state) + _check_stream("command", self.command) + + def _channel_requests(self) -> tuple[ChannelRequest, ...]: + return ( + ChannelRequest(self.mode, "rx", "mode.json.v1", 10.0), + # Both are state, not frames: see the note in NavMap. + ChannelRequest(self.policies, "rx", "policy.json.v1", 10.0), + ChannelRequest(self.nav_state, "rx", "navstate.json.v1", 10.0), + ChannelRequest(self.command, "tx", "command.json.v1", 10.0), + ) + + def _panel_params(self) -> dict[str, Any]: + return { + "mode": self.mode, + "policies": self.policies, + "navState": self.nav_state, + "command": self.command, + } + class _Split: """Base for Row/Col: children plus optional flex shares.""" @@ -651,7 +851,7 @@ def cockpit( tuple(pages), registry={b.ch: (b.encoding, b.delivery) for b in BUILTIN_CHANNELS}, tx_streams={s.name for s in atom.streams if s.direction == "out"}, - tx_registry={ch: (encoding, delivery) for ch, encoding, delivery in TX_CHANNELS}, + tx_registry={tx.ch: (tx.encoding, tx.delivery) for tx in TX_CHANNELS}, channels=tuple( ChannelRequest( c.stream, @@ -731,7 +931,13 @@ def cockpit( params=dict(wire["params"]), encoder=codec.encode, encoder_takes_params=codec.takes_params, - resend_on_subscribe=builtin.resend_on_subscribe if builtin is not None else False, + resend_on_subscribe=( + builtin.resend_on_subscribe + if builtin is not None + else explicit is not None and explicit.resend_on_subscribe + ), + rate_gate=explicit.rate_gate if explicit is not None else True, + replay_depth=explicit.replay_depth if explicit is not None else 1, ) ) # Generated classes carry only the custom ports; the built-ins are diff --git a/dimos/web/relay_bridge/_wt_session.py b/dimos/web/relay_bridge/_wt_session.py index 0437e07804..d551d659df 100644 --- a/dimos/web/relay_bridge/_wt_session.py +++ b/dimos/web/relay_bridge/_wt_session.py @@ -283,9 +283,15 @@ def _queue_control_msg(self, msg: Msg | DataFrame) -> None: drain-and-requeue. """ if isinstance(msg, Subs): + replay = set(msg.replay or []) for queued in self._drain_control_msgs(): - if not isinstance(queued, Subs): + if isinstance(queued, Subs): + replay.update(queued.replay or []) + else: self.control_msgs.put_nowait(queued) + # Replay is an event carried alongside the full subscription state. + # Retain pending requests across a burst, bounded by active channels. + msg = msg.model_copy(update={"replay": sorted(replay.intersection(msg.chs)) or None}) if self.control_msgs.full(): msgs = self._drain_control_msgs() # A victim always exists: at most one queued Subs (coalesced diff --git a/dimos/web/relay_bridge/dynamic.py b/dimos/web/relay_bridge/dynamic.py index 50b818e4b3..bc8483f6ed 100644 --- a/dimos/web/relay_bridge/dynamic.py +++ b/dimos/web/relay_bridge/dynamic.py @@ -37,7 +37,7 @@ class special-case, serializes the canonical port specs, and reconstructs from dimos.core.stream import In, Out from dimos.web.relay_bridge.manifest import MAX_MANIFEST_ID_LEN, RESERVED_CHANNEL_PREFIX, Dir -from dimos.web.relay_bridge.relay_bridge_module import RelayBridgeModule +from dimos.web.relay_bridge.relay_bridge_module import TX_CHANNELS, RelayBridgeModule @dataclass(frozen=True) @@ -78,6 +78,10 @@ class _DynamicRelayBridgeMeta(ABCMeta): | {"ref", "rpc", "encoded"} ) +# Legacy command ports may be explicitly retyped for acknowledged publish. +# Motion retains its dedicated lease path, and methods/state remain reserved. +_PUBLISHABLE_LEGACY_PORTS = frozenset(tx.ch for tx in TX_CHANNELS if tx.model is not None) + # Process-global on purpose: repeated unpickles of the same class must return # the identical class object, because blueprints and the coordinator compare # module classes with `is`. @@ -116,7 +120,9 @@ def _validate_specs(specs: tuple[DynamicPortSpec, ...]) -> None: ) if len(stream) > MAX_MANIFEST_ID_LEN: raise ValueError(f"stream id {stream!r} is longer than {MAX_MANIFEST_ID_LEN} chars") - if stream in _RESERVED_NAMES: + if stream in _RESERVED_NAMES and not ( + spec.direction == "tx" and stream in _PUBLISHABLE_LEGACY_PORTS + ): raise ValueError( f"stream id {stream!r} collides with an existing RelayBridgeModule attribute" ) diff --git a/dimos/web/relay_bridge/locate.py b/dimos/web/relay_bridge/locate.py index c5b71a2e0d..0e029c51ac 100644 --- a/dimos/web/relay_bridge/locate.py +++ b/dimos/web/relay_bridge/locate.py @@ -87,6 +87,7 @@ def relay_run_cmd( cockpit_dir: Path | None = None, sdk_dir: Path | None = None, serve_dir: Path | None = None, + entrypoint: Path | None = None, ) -> list[str]: """Build the argv that runs the relay with the pinned config and least permissions.""" # Canonical paths: the relay realpath-checks served files against its @@ -106,6 +107,8 @@ def relay_run_cmd( # the cockpit build tooling), which would make this run materialize # node_modules next to the config -- inside site-packages under a wheel. allow_read = ",".join([str(web_dir), *(str(path) for _, path in dirs)]) + if entrypoint is not None: + allow_read += "," + str(entrypoint.resolve().parent) cmd = [ deno, "run", @@ -115,7 +118,7 @@ def relay_run_cmd( "--allow-net", "--config", str(web_dir / "deno.json"), - str(web_dir / "relay" / "main.ts"), + str(entrypoint.resolve() if entrypoint else web_dir / "relay" / "main.ts"), ] for flag, path in dirs: cmd += [flag, str(path)] diff --git a/dimos/web/relay_bridge/manifest.py b/dimos/web/relay_bridge/manifest.py index dbe0821260..205ffb0c6b 100644 --- a/dimos/web/relay_bridge/manifest.py +++ b/dimos/web/relay_bridge/manifest.py @@ -19,7 +19,7 @@ manifest as one opaque dict; this module is the single owner of its structure and domain rules: version gate, bounded unique ids, positive rates, panel/layout/pages references that resolve, and kind-specific panel -rules (video, map2d, teleop). +rules (video, map2d, teleop, chat, navmap, control). Manifest v1 is frozen. Additive changes (new panel kinds, new params) ride the existing shape: unknown keys and kinds pass through validation. @@ -123,6 +123,48 @@ class Manifest(_ManifestModel): _MANIFEST_TA: TypeAdapter[Manifest] = TypeAdapter(Manifest) +# Panel kinds with a fixed channel slot table (the cockpit panels authored by +# dimos/web/cockpit.py Chat/NavMap/Control): the panel must bind exactly +# these slots, in this order, each with the listed dir + encoding. Delivery +# is deliberately not checked (it is the bridge's per-channel choice) and the +# role->channel mapping the panel also carries in `params` is informational. +# Mirrored by SLOT_KINDS in manifest.ts (same codes, same slot order). +_SLOT_KINDS: dict[str, tuple[str, tuple[tuple[Dir, str], ...]]] = { + # chat, idle flag, mode, then the text input. + "chat": ( + "invalid_chat_panel", + ( + ("rx", "chat.json.v1"), + ("rx", "flag.json.v1"), + ("rx", "mode.json.v1"), + ("tx", "text.json.v1"), + ), + ), + # costmap, pose, path, places, nav state, then goal + command outputs. + "navmap": ( + "invalid_navmap_panel", + ( + ("rx", "costmap.zlib.v1"), + ("rx", "pose.json.v1"), + ("rx", "path.json.v1"), + ("rx", "places.json.v1"), + ("rx", "navstate.json.v1"), + ("tx", "pose_goal.json.v1"), + ("tx", "command.json.v1"), + ), + ), + # mode, policy state, nav state, then the command output. + "control": ( + "invalid_control_panel", + ( + ("rx", "mode.json.v1"), + ("rx", "policy.json.v1"), + ("rx", "navstate.json.v1"), + ("tx", "command.json.v1"), + ), + ), +} + def _bounded_id(s: str) -> bool: return 1 <= len(s) <= MAX_MANIFEST_ID_LEN @@ -274,16 +316,20 @@ def parse_manifest(data: Any) -> Manifest: # unknown kinds stay unvalidated (forward compatibility with newer # bridges). if panel.kind == "video": - if len(panel.channels) != 1: - raise ManifestError( - "invalid_video_panel", f"video panel {panel.id} must bind exactly one channel" - ) - bound = ch_ids[panel.channels[0]] - if bound.encoding != "jpeg.v1" or bound.delivery != "latest" or bound.dir != "rx": + # One feed, or two: the second is drawn inset over the first + # (picture-in-picture). Both must be real video. + if not 1 <= len(panel.channels) <= 2: raise ManifestError( "invalid_video_panel", - f"video panel {panel.id} needs a jpeg.v1 latest rx channel", + f"video panel {panel.id} must bind one channel, or two for an inset", ) + for ch in panel.channels: + bound = ch_ids[ch] + if bound.encoding != "jpeg.v1" or bound.delivery != "latest" or bound.dir != "rx": + raise ManifestError( + "invalid_video_panel", + f"video panel {panel.id} needs a jpeg.v1 latest rx channel", + ) if panel.kind == "map2d": # channels[0] is the costmap; channels[1] (optional) the pose overlay. if len(panel.channels) not in (1, 2): @@ -320,6 +366,20 @@ def parse_manifest(data: Any) -> Manifest: "invalid_teleop_panel", f"teleop panel {panel.id} needs a twist.json.v1 latest tx channel", ) + if panel.kind in _SLOT_KINDS: + code, slots = _SLOT_KINDS[panel.kind] + if len(panel.channels) != len(slots): + raise ManifestError( + code, f"{panel.kind} panel {panel.id} must bind exactly {len(slots)} channels" + ) + for i, (dir_, encoding) in enumerate(slots): + bound = ch_ids[panel.channels[i]] + if bound.encoding != encoding or bound.dir != dir_: + raise ManifestError( + code, + f"{panel.kind} panel {panel.id} channel {i} must be a {encoding} " + f"{dir_} channel", + ) seen: set[str] = set() if manifest.layout is not None: diff --git a/dimos/web/relay_bridge/protocol.py b/dimos/web/relay_bridge/protocol.py index d57f05db0d..624f049876 100644 --- a/dimos/web/relay_bridge/protocol.py +++ b/dimos/web/relay_bridge/protocol.py @@ -76,6 +76,7 @@ # hello is rejected. Generic publish (amended into v5 pre-release): # pub/pub_ack/pub_nack and the error requestId correlation; an older v5 peer # drops the unknown messages, so a publish times out instead of misparsing. +# Legacy tx commands also remain supported within v5. # v4: the twist datagram gains vy (strafe) and the teleop # lease messages (teleop_start/teleop_started/teleop_stop) enter the control # plane; robot-bound twist/stop/teleop_start/teleop_stop carry the @@ -115,6 +116,22 @@ # hostile/corrupt payloadLen (same constant as the relay's ingress cap). MAX_DATA_FRAME_BYTES = 64 * 1024 * 1024 +# Generic tx command (Tx) budget, mirrored in protocol.ts. `data` encodes to +# at most TX_DATA_MAX_BYTES of compact JSON (UTF-8 bytes: the same figure +# JS measures with TextEncoder), `ch` is bounded like a manifest channel id +# and shaped like a bridge stream name, and no other keys are allowed, so a +# whole Tx datagram never exceeds MAX_TX_MSG_BYTES (the ~1100 B datagram +# budget: 900 data + 64 ch + a 16-digit seq + framing). +TX_DATA_MAX_BYTES = 900 +TX_CH_MAX_LEN = MAX_MANIFEST_ID_LEN +TX_CH_PATTERN = r"^[a-z][a-z0-9_]*$" +MAX_TX_MSG_BYTES = 1100 + +# seq is a JS safe integer on both sides: Python parses larger JSON integers +# exactly while JSON.parse rounds them, so the mirrors could otherwise +# disagree on the value. +_MAX_SAFE_INTEGER = 2**53 - 1 + Role = Literal["robot", "viewer"] @@ -265,6 +282,18 @@ class Subs(_WireModel): t: Literal["subs"] = "subs" chs: list[str] n: int | float + replay: list[str] | None = None + + @field_validator("replay", mode="before") + @classmethod + def _replay_is_subscribed(cls, value: Any, info: ValidationInfo) -> list[str] | None: + if value is None and info.context is not _WIRE_CTX: + return None + if not isinstance(value, list) or any( + not isinstance(ch, str) or ch not in info.data.get("chs", []) for ch in value + ): + raise ValueError("replay must name subscribed channels") + return value # Teleop (T6). twist/stop ride datagrams viewer->relay->robot (loss-tolerant: @@ -357,6 +386,47 @@ class PubNack(_WireModel): message: str +def tx_data_bytes(data: dict[str, Any]) -> int: + """UTF-8 byte length of the compact JSON encoding of a Tx.data record. + + ensure_ascii=False so the count matches TextEncoder(JSON.stringify(data)) + byte for byte; allow_nan=False refuses NaN/Infinity (JSON.parse never + produces them, and a mirror must not encode them). + """ + return len( + json.dumps(data, separators=(",", ":"), ensure_ascii=False, allow_nan=False).encode() + ) + + +class Tx(_WireModel): + """Viewer->relay->robot generic command on a manifest tx channel. + + The viewer sends it on its control stream; the relay forwards it to the + robot on the twist/stop leg (datagrams, hence the byte budget) once the + watched manifest declares `ch` as a non-twist tx channel. The bridge + routes `data` onto the stream bound to `ch` (chat text, nav goals, UI + commands). `seq` is the viewer's per-channel counter. Unlike the other + messages, extra keys are rejected instead of ignored: the data cap must + bound the whole message. + """ + + # Merged with _WireModel's strict/allow_inf_nan config. + model_config = ConfigDict(extra="forbid") + + t: Literal["tx"] = "tx" + ch: str = Field(min_length=1, max_length=TX_CH_MAX_LEN, pattern=TX_CH_PATTERN) + seq: int = Field(ge=0, le=_MAX_SAFE_INTEGER) + data: dict[str, Any] + + @field_validator("data") + @classmethod + def _bounded_data(cls, value: dict[str, Any]) -> dict[str, Any]: + n = tx_data_bytes(value) + if n > TX_DATA_MAX_BYTES: + raise ValueError(f"data encodes to {n} bytes (max {TX_DATA_MAX_BYTES})") + return value + + Msg = ( Hello | Welcome @@ -374,6 +444,7 @@ class PubNack(_WireModel): | TeleopStart | TeleopStarted | TeleopStop + | Tx | Pub | PubAck | PubNack diff --git a/dimos/web/relay_bridge/relay_bridge_module.py b/dimos/web/relay_bridge/relay_bridge_module.py index a605a51674..7e926cd4bc 100644 --- a/dimos/web/relay_bridge/relay_bridge_module.py +++ b/dimos/web/relay_bridge/relay_bridge_module.py @@ -24,7 +24,9 @@ open cockpit does no encode work. Channels with resend_on_subscribe additionally keep one always-on raw subscription (decode only, never encode) so the newest message can be replayed the moment a channel gains its first -viewer, even when the producer went quiet before that. +viewer, even when the producer went quiet before that; a replay_depth > 1 +channel keeps a bounded log instead of a single message and replays it +oldest-first, so a transcript survives a page reload. Threading: input callbacks fire on the transport (LCM) thread, which gates on maxHz and encodes there (RerunBridge precedent, ~3 ms per JPEG), then hands @@ -36,9 +38,11 @@ from __future__ import annotations import asyncio -from collections.abc import AsyncIterator, Callable, Collection +from collections import deque +from collections.abc import AsyncIterator, Callable, Collection, Iterator from dataclasses import dataclass, field, replace import functools +import itertools import json import math from pathlib import Path @@ -48,13 +52,14 @@ from typing import Any, Literal, TypeVar import webbrowser -from pydantic import Field +from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator from reactivex.disposable import Disposable from dimos.core.coordination.blueprints import Blueprint, autoconnect from dimos.core.module import Module, ModuleConfig from dimos.core.stream import In, Out from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped +from dimos.msgs.geometry_msgs.Quaternion import Quaternion from dimos.msgs.geometry_msgs.Twist import Twist from dimos.msgs.geometry_msgs.Vector3 import Vector3 from dimos.msgs.nav_msgs.OccupancyGrid import OccupancyGrid @@ -95,6 +100,7 @@ TeleopStart as WireTeleopStart, TeleopStop as WireTeleopStop, Twist as WireTwist, + Tx, ) from dimos.web.relay_bridge.relay_process import RelayProcess, ensure_web_dist from dimos.web.relay_bridge.wt_client import ( @@ -130,6 +136,20 @@ # 30 Hz stream must not flood the log). _ENCODE_ERROR_LOG_S = 5.0 +# Generic tx (non-twist) commands carry no lease generation, so the per-channel +# seq high-water mark cannot tell a reordered datagram from a fresh viewer +# whose counter restarted at 1 (page reload, second tab). Reordering and +# duplication happen within milliseconds of the neighbouring datagrams, so a +# lower/equal seq is stale only while the channel is busy; after this much +# silence it rebaselines. +_TX_SEQ_WINDOW_S = 1.0 + +# Minimum spacing of drop-warnings per key (unknown tx channel, invalid data): +# a misbehaving peer must not flood the log. +_LOG_THROTTLE_S = 5.0 + +_JSON_SEPARATORS = (",", ":") + @dataclass(frozen=True) class _TeleopParams: @@ -231,6 +251,22 @@ class RuntimeChannelSpec: # viewer: a new session must not wait for the next publish (the producer # may have gone quiet, possibly before the first viewer ever attached). resend_on_subscribe: bool = False + # False = every message reaches the encoder, maxHz notwithstanding. For an + # event stream (a chat turn, a mode flip) a skipped sample is lost data, + # not a dropped frame; maxHz stays advertised as the expected ceiling. + rate_gate: bool = True + # > 1 turns the newest-message cache into a bounded log of that many + # entries, replayed oldest-first to a channel's first viewer (a chat + # transcript must survive a page reload). The bridge numbers each entry + # and ships the number as frame meta `n`, so a replayed frame and its + # live original carry the same number and the viewer can dedupe. + replay_depth: int = 1 + + def __post_init__(self) -> None: + if self.replay_depth < 1: + raise ValueError(f"channel {self.ch!r}: replay_depth must be >= 1") + if self.replay_depth > 1 and not self.resend_on_subscribe: + raise ValueError(f"channel {self.ch!r}: replay_depth > 1 requires resend_on_subscribe") class RelayBridgeConfig(ModuleConfig): @@ -275,6 +311,15 @@ class RelayBridgeConfig(ModuleConfig): from the manifest against BUILTIN_CHANNELS.""" +def _with_n(meta: _FrameMeta, n: int | None) -> _FrameMeta: + """Fold a log channel's entry number into the frame meta. The encoder + never sees `n` (it is the bridge's, not the message's), and the viewer + dedupes a replayed frame against its live original on it.""" + if n is None: + return meta + return {"n": n} if meta is None else {**meta, "n": n} + + def _passes_rate_gate( last_input: dict[str, float], ch: str, @@ -288,6 +333,19 @@ def _passes_rate_gate( return True +@dataclass(frozen=True, slots=True) +class _LogEntry: + """One numbered message in a replay_depth > 1 channel's bounded log. + + `n` is assigned once, on append, from one process-wide counter, so an + entry replayed to a late viewer carries the same number the live frame + did and the viewer's dedupe sees them as one message.""" + + msg: Any + recv_ts: float + n: int + + def _matches_message_type(value: Any, message_type: type[Any]) -> bool: """Decoded-result check with JSON's number/bool subtleties: bool is exact (Python bool subclasses int), int excludes bool, float accepts int (JSON @@ -404,11 +462,97 @@ class BuiltinChannel: ), ) -# The tx (viewer->robot) counterpart of BUILTIN_CHANNELS: stream -> -# (encoding, delivery). Every entry needs a matching `Out` on the module and -# a handler in _supervise; it is also the delivery source for tx channels in -# authored manifests (dimos/web/cockpit.py). -TX_CHANNELS: tuple[tuple[str, str, Delivery], ...] = (("tele_cmd_vel", "twist.json.v1", "latest"),) + +class _TxIn(BaseModel): + """Base for generic tx data records: strict types (no coercion from the + wire), finite floats; unknown keys are ignored like the other wire + models (the Tx envelope already bounds the whole record).""" + + model_config = ConfigDict(strict=True, allow_inf_nan=False) + + +# Tx.data is capped at 900 bytes, so a longer text could never arrive anyway; +# the cap here keeps the published string bounded by contract, not by accident. +_CHAT_IN_MAX_CHARS = 900 + + +class _ChatIn(_TxIn): + text: str = Field(min_length=1, max_length=_CHAT_IN_MAX_CHARS) + + @field_validator("text") + @classmethod + def _stripped_nonempty(cls, value: str) -> str: + value = value.strip() + if not value: + raise ValueError("text must not be blank") + return value + + +# Goal bound (m, either axis): room-scale scenes; a click far outside the map +# is a viewer bug, not a plan. +_GOAL_MAX_ABS_M = 50.0 + + +class _GoalIn(_TxIn): + x: float = Field(ge=-_GOAL_MAX_ABS_M, le=_GOAL_MAX_ABS_M) + y: float = Field(ge=-_GOAL_MAX_ABS_M, le=_GOAL_MAX_ABS_M) + yaw: float = 0.0 + frame: str = Field(default="world", min_length=1, max_length=64) + + +class _CommandIn(_TxIn): + name: Literal["set_mode", "policy", "cancel_nav"] + args: dict[str, Any] = Field(default_factory=dict) + + +def _build_chat(module: RelayBridgeModule, data: _ChatIn) -> str: + return data.text + + +def _build_goal(module: RelayBridgeModule, data: _GoalIn) -> PoseStamped: + return PoseStamped( + ts=time.time(), + frame_id=data.frame, + position=Vector3(data.x, data.y, 0.0), + orientation=Quaternion.from_euler(Vector3(0.0, 0.0, data.yaw)), + ) + + +def _build_command(module: RelayBridgeModule, data: _CommandIn) -> str: + # The consumer (DuckControl.ui_command) parses {"name", "args"}. + return json.dumps({"name": data.name, "args": data.args}, separators=_JSON_SEPARATORS) + + +@dataclass(frozen=True) +class TxChannelDef: + """A viewer->robot channel: wire identity plus, for generic Tx commands, + the data record model, the per-channel rate floor and the builder that + turns a validated record into the value published on the Out named + `ch`. Teleop twists keep their dedicated lease-guarded path (no model).""" + + ch: str + encoding: str + delivery: Delivery + model: type[BaseModel] | None = None + min_interval_s: float = 0.0 + build: Callable[[RelayBridgeModule, Any], Any] | None = None + + def __iter__(self) -> Iterator[str]: + # Consumers that unpack rows as `(ch, encoding, delivery)` tuples - + # cockpit()'s tx_registry, main()'s manifest check - keep working. + yield from (self.ch, self.encoding, self.delivery) + + +# The tx (viewer->robot) counterpart of BUILTIN_CHANNELS. Every entry needs a +# matching `Out` on the module; twist has its lease-guarded handler in +# _supervise, the rest go through _on_wire_tx. It is also the delivery source +# for tx channels in authored manifests (dimos/web/cockpit.py). +TX_CHANNELS: tuple[TxChannelDef, ...] = ( + TxChannelDef("tele_cmd_vel", "twist.json.v1", "latest"), + TxChannelDef("human_input", "text.json.v1", "reliable", _ChatIn, 0.2, _build_chat), + TxChannelDef("goal_request", "pose_goal.json.v1", "reliable", _GoalIn, 0.2, _build_goal), + TxChannelDef("ui_command", "command.json.v1", "reliable", _CommandIn, 0.05, _build_command), +) def default_manifest(config: RelayBridgeConfig, available: Collection[str]) -> dict[str, Any]: @@ -482,6 +626,13 @@ class RelayBridgeModule(Module): # tx: cockpit teleop twists, autoconnected by name+type to # MovementManager.tele_cmd_vel (publish is a no-op while unwired). tele_cmd_vel: Out[Twist] + # tx: generic Tx commands, one Out per non-twist TX_CHANNELS entry. These + # stay static (unlike rx ports, which cockpit() generates) because + # Channel still refuses dir="tx" - generic browser-to-robot publish + # arrives with the publish ticket (W7), and these move onto it then. + human_input: Out[str] + goal_request: Out[PoseStamped] + ui_command: Out[str] # NEVER add handle_color_image/handle_odom methods here: _auto_bind_handlers # subscribes any handle_ eagerly at start(), defeating lazy encode. @@ -510,6 +661,22 @@ def __init__(self, **kwargs: Any) -> None: # reconnect replays too. Pins the full grid (MBs, one per channel); # encoding stays lazy. self._last_msg: dict[str, tuple[Any, float]] = {} + # replay_depth > 1 channels: the bounded log behind _last_msg (oldest + # first) plus the sessions being fed live from it. The log is written + # on the transport thread and snapshotted on the loop, so it takes a + # real lock; _log_live is swapped copy-on-write instead. + self._replay_log: dict[str, deque[_LogEntry]] = {} + self._log_live: dict[str, tuple[tuple[_Session, _Sender], ...]] = {} + self._log_lock = threading.Lock() + self._log_counter = itertools.count(1) + # Generic tx: the handlers the manifest actually advertised, and the + # per-channel seq high-water mark / last-accepted time behind the + # replay and rate-floor guards. Loop-thread only. + self._tx_defs: dict[str, TxChannelDef] = {} + self._tx_last_seq: dict[str, int] = {} + self._tx_last_rx: dict[str, float] = {} + # Last warn time per throttle key (peer-driven drops). + self._warn_last: dict[str, float] = {} # Teleop state, all touched on the module loop only. None params = # the manifest advertises no teleop channel; every teleop message is # then ignored. `driving` implements the release-edge rule: publish @@ -568,20 +735,28 @@ async def main(self) -> AsyncIterator[None]: self._channel_specs = self._resolve_builtin_specs(rx_wire) self._min_interval = {s.ch: 1.0 / s.max_hz for s in self._channel_specs} self.encoded = {s.ch: 0 for s in self._channel_specs} - by_tx = {ch: (encoding, delivery) for ch, encoding, delivery in TX_CHANNELS} + by_tx = {tx.ch: tx for tx in TX_CHANNELS} for spec in manifest.channels: if spec.dir != "tx" or spec.publish != "none": # Publish tx channels are declaration-driven (adopted # above); the static tx table covers only the # specialized protocol paths. continue - if by_tx.get(spec.ch) != (spec.encoding, spec.delivery): + handler = by_tx.get(spec.ch) + if handler is None or (handler.encoding, handler.delivery) != ( + spec.encoding, + spec.delivery, + ): raise RuntimeError( f"manifest tx channel {spec.ch!r} ({spec.encoding}/{spec.delivery}) has " f"no matching handler; this bridge supports: {sorted(by_tx)}" ) if spec.ch == "tele_cmd_vel": + # The teleop path: lease/gen/seq guarded, params-driven. self._teleop_params = self._resolve_teleop_params(spec) + else: + assert handler.model is not None and handler.build is not None + self._tx_defs[spec.ch] = handler # No runtime stream probing: an authored channel whose input got # no transport stays advertised (its panel shows "waiting for # data"); _reconcile just never subscribes it. @@ -914,7 +1089,7 @@ async def _supervise(self, session: _Session) -> None: async for msg in session.client.control_messages(): if isinstance(msg, Subs) and msg.n > session.last_n: session.last_n = msg.n - self._reconcile(session, set(msg.chs)) + self._reconcile(session, set(msg.chs), replay=msg.replay) elif isinstance(msg, DataFrame): # A forwarded viewer publish (tx channel data on # the carrier); never raises out of the loop. @@ -927,6 +1102,8 @@ async def _supervise(self, session: _Session) -> None: self._on_wire_teleop_start(msg) elif isinstance(msg, WireTeleopStop): self._on_wire_teleop_stop(msg) + elif isinstance(msg, Tx): + self._on_wire_tx(msg) # The iterator only ends when the session closed. except Exception: # An unguarded error here would silently end supervision while @@ -1140,6 +1317,67 @@ def _on_wire_teleop_stop(self, msg: WireTeleopStop) -> None: self._teleop_last_seq = -math.inf self._teleop_last_rx = 0.0 + def _log_throttled(self, key: str, message: str) -> None: + """Warn at most once per _LOG_THROTTLE_S per key (peer-driven drops).""" + now = time.monotonic() + if now - self._warn_last.get(key, -math.inf) < _LOG_THROTTLE_S: + return + self._warn_last[key] = now + logger.warning(message) + + def _on_wire_tx(self, msg: Tx) -> None: + """A generic viewer command: route `data` to the Out named `ch`. + + No lease: these are discrete requests (a chat line, a nav goal, a + UI command), not a motion stream. Drops, all silent to the viewer + (the SDK infers delivery from the robot's own echo): an unknown or + twist channel; a seq at or below the channel's high-water mark while + the channel is busy (a reordered/duplicated datagram - after + _TX_SEQ_WINDOW_S of silence any seq rebaselines, so a reloaded page + restarting at 1 is not locked out); a record inside the channel's + rate floor; a record the channel model rejects. + """ + handler = self._tx_defs.get(msg.ch) + if handler is None: + self._log_throttled( + f"tx:{msg.ch}", + f"relay bridge: dropping tx on unhandled channel {msg.ch!r} " + f"(handled: {sorted(self._tx_defs)})", + ) + return + now = time.monotonic() + last_rx = self._tx_last_rx.get(msg.ch, -math.inf) + if msg.seq <= self._tx_last_seq.get(msg.ch, -1) and now - last_rx < _TX_SEQ_WINDOW_S: + return + if now - last_rx < handler.min_interval_s: + return + assert handler.model is not None and handler.build is not None + try: + record = handler.model.model_validate(msg.data) + except ValidationError as e: + problems = "; ".join( + f"{'.'.join(str(part) for part in err['loc'])}: {err['msg']}" for err in e.errors() + ) + self._log_throttled( + f"tx-invalid:{msg.ch}", + f"relay bridge: dropping invalid {msg.ch} record ({problems})", + ) + return + self._tx_last_seq[msg.ch] = msg.seq + self._tx_last_rx[msg.ch] = now + try: + value = handler.build(self, record) + except Exception: + logger.exception(f"relay bridge: building {msg.ch} value failed") + return + getattr(self, msg.ch).publish(value) + + def _tx_reset(self) -> None: + # Session teardown: datagrams are QUIC-session-scoped, and the next + # session's viewers start their counters afresh. + self._tx_last_seq.clear() + self._tx_last_rx.clear() + def _teleop_zero(self, reason: str, *, force: bool = False) -> None: """Publish one zero twist; edge-gated unless `force` (e-stop).""" if not force and not self._teleop_driving: @@ -1194,7 +1432,9 @@ async def _reconnect(self) -> _Session | None: logger.warning(f"relay reconnect failed ({e}); retrying") await asyncio.sleep(_RECONNECT_PAUSE_S) - def _reconcile(self, session: _Session, want: set[str]) -> None: + def _reconcile( + self, session: _Session, want: set[str], *, replay: list[str] | None = None + ) -> None: """Subscribe/unsubscribe inputs so exactly `want` is being encoded.""" for spec in self._channel_specs: active = spec.ch in session.unsubs @@ -1204,25 +1444,29 @@ def _reconcile(self, session: _Session, want: set[str]) -> None: # Advertised but unwired (manifest-authored): nothing to # subscribe; the panel shows "waiting for data". continue - cached = self._last_msg.get(spec.ch) - if cached is not None: + sender = session.senders[spec.ch] + if spec.replay_depth > 1: + # Log channel: live frames come from the always-on log + # subscription, attached before the replay so no entry + # slips between the two (an entry seen by both carries + # the same n, and the viewer dedupes on n). + session.unsubs[spec.ch] = self._attach_log_sender(spec, session, sender) + self._replay(session, spec, sender) + else: # Replay precedes the subscribe: this offer runs # synchronously on the loop, so a live frame - possible # only once subscribed - always queues behind it and wins - # the 1-slot mailbox. Fires on 0->1 transitions only: the - # relay reports sub-set changes and stays cache-free, so - # an extra viewer on an already-active channel waits for - # the next publish (review issue 2, deferred). - msg, recv_ts = cached - encoded = self._run_encoder(spec, msg) - if encoded is not None: - # self.encoded counts live-path encodes only; the - # arrival ts keeps a stale replay honest about its age. - self._offer(session, session.senders[spec.ch], *encoded, recv_ts) - session.unsubs[spec.ch] = self.inputs[spec.ch].subscribe( - functools.partial(self._on_input, session, spec, session.senders[spec.ch]) - ) + # the 1-slot mailbox. Late reliable viewers request a + # replay explicitly without detaching active encoders. + self._replay(session, spec, sender) + session.unsubs[spec.ch] = self.inputs[spec.ch].subscribe( + functools.partial(self._on_input, session, spec, sender) + ) logger.info(f"relay bridge: viewer subscribed to {spec.ch}; encoding started") + elif active and should and spec.ch in (replay or []) and spec.resend_on_subscribe: + # Existing viewers dedupe log entries by their stable message n. + # Only requested channels replay; other live encoders stay attached. + self._replay(session, spec, session.senders[spec.ch]) elif active and not should: unsubscribe = session.unsubs[spec.ch] unsubscribe() @@ -1232,19 +1476,62 @@ def _reconcile(self, session: _Session, want: set[str]) -> None: if unknown: logger.debug(f"relay bridge: ignoring unknown channels {sorted(unknown)}") + def _replay(self, session: _Session, spec: RuntimeChannelSpec, sender: _Sender) -> None: + """Offer the channel's cached message(s) - the log oldest first - to a + session whose channel just gained its first viewer. self.encoded + counts live-path encodes only; the arrival ts keeps a stale replay + honest about its age.""" + items: list[tuple[Any, float, int | None]] + if spec.replay_depth > 1: + # Snapshot under the lock: an entry evicted between here and its + # encode still replays, and under its original number. + with self._log_lock: + items = [(e.msg, e.recv_ts, e.n) for e in self._replay_log.get(spec.ch, ())] + else: + cached = self._last_msg.get(spec.ch) + items = [] if cached is None else [(cached[0], cached[1], None)] + for msg, recv_ts, n in items: + encoded = self._run_encoder(spec, msg) + if encoded is not None: + self._offer(session, sender, encoded[0], _with_n(encoded[1], n), recv_ts) + + def _attach_log_sender( + self, spec: RuntimeChannelSpec, session: _Session, sender: _Sender + ) -> Callable[[], None]: + """Feed `session` live from the channel's log subscription; returns + the detach (the session.unsubs entry). Copy-on-write tuples: the + loop swaps, the transport thread iterates a snapshot.""" + entry = (session, sender) + self._log_live[spec.ch] = (*self._log_live.get(spec.ch, ()), entry) + + def detach() -> None: + self._log_live[spec.ch] = tuple( + live for live in self._log_live.get(spec.ch, ()) if live is not entry + ) + + return detach + def _on_input( - self, session: _Session, spec: RuntimeChannelSpec, sender: _Sender, msg: Any + self, + session: _Session, + spec: RuntimeChannelSpec, + sender: _Sender, + msg: Any, + n: int | None = None, ) -> None: """Transport-thread callback: maxHz gate, encode, hand to the loop.""" if session.retired.is_set(): return now = time.monotonic() - if not _passes_rate_gate(self._last_input, spec.ch, now, self._min_interval[spec.ch]): + if spec.rate_gate and not _passes_rate_gate( + self._last_input, spec.ch, now, self._min_interval[spec.ch] + ): return encoded = self._run_encoder(spec, msg) if encoded is None: return payload, meta = encoded + meta = _with_n(meta, n) self.encoded[spec.ch] += 1 loop = self._loop if loop is not None and loop.is_running(): @@ -1252,8 +1539,21 @@ def _on_input( def _cache_input(self, spec: RuntimeChannelSpec, msg: Any) -> None: """Transport-thread callback: remember the newest raw message so a - 0->1 subscribe can replay it (its arrival time becomes the frame ts).""" - self._last_msg[spec.ch] = (msg, time.time()) + 0->1 subscribe can replay it (its arrival time becomes the frame ts). + A log channel also appends a numbered record to its bounded log and + feeds that record to the sessions attached to the log.""" + recv_ts = time.time() + self._last_msg[spec.ch] = (msg, recv_ts) + if spec.replay_depth <= 1: + return + with self._log_lock: + log = self._replay_log.get(spec.ch) + if log is None: + log = self._replay_log[spec.ch] = deque(maxlen=spec.replay_depth) + entry = _LogEntry(msg, recv_ts, next(self._log_counter)) + log.append(entry) + for session, sender in self._log_live.get(spec.ch, ()): + self._on_input(session, spec, sender, entry.msg, entry.n) def _offer( self, @@ -1285,6 +1585,7 @@ async def _disconnect(self, session: _Session | None = None) -> None: if self._teleop_params is not None: self._teleop_zero("relay session ended") self._teleop_reset() + self._tx_reset() for ch, unsubscribe in tuple(target.unsubs.items()): try: unsubscribe() diff --git a/dimos/web/relay_bridge/relay_process.py b/dimos/web/relay_bridge/relay_process.py index 7cb9fba944..d725664c37 100644 --- a/dimos/web/relay_bridge/relay_process.py +++ b/dimos/web/relay_bridge/relay_process.py @@ -296,7 +296,9 @@ def __init__( sdk_dir: Path | None = None, serve_dir: Path | None = None, timeout: float = 20.0, + entrypoint: Path | None = None, ) -> None: + self._entrypoint = entrypoint self._port = port self._host = host self._web_dir = web_dir @@ -328,6 +330,7 @@ def start(self) -> RelayReadyInfo: cockpit_dir=cockpit_dir, sdk_dir=sdk_dir, serve_dir=self._serve_dir, + entrypoint=self._entrypoint, ) logger.info(f"starting relay: {' '.join(cmd)}") env = os.environ | {"NO_COLOR": "1"} diff --git a/dimos/web/relay_bridge/test_dynamic.py b/dimos/web/relay_bridge/test_dynamic.py index da71d36bac..f4bd100566 100644 --- a/dimos/web/relay_bridge/test_dynamic.py +++ b/dimos/web/relay_bridge/test_dynamic.py @@ -249,6 +249,11 @@ def test_blueprint_atom_discovers_exact_stream_refs(): StreamRef(name="odom", type=PoseStamped, direction="in"), StreamRef(name="global_costmap", type=OccupancyGrid, direction="in"), StreamRef(name="tele_cmd_vel", type=Twist, direction="out"), + # The static generic-tx ports (TX_CHANNELS beyond the twist); they + # move onto generated ports when Channel accepts dir="tx" (W7). + StreamRef(name="human_input", type=str, direction="out"), + StreamRef(name="goal_request", type=PoseStamped, direction="out"), + StreamRef(name="ui_command", type=str, direction="out"), StreamRef(name="operator_note", type=str, direction="out"), StreamRef(name="sensor_ping", type=Vector3, direction="in"), ) @@ -291,7 +296,13 @@ def test_module_init_constructs_runtime_streams(local_modules): module = generated() local_modules.append(module) assert sorted(module.inputs) == ["color_image", "global_costmap", "odom", "sensor_ping"] - assert sorted(module.outputs) == ["operator_note", "tele_cmd_vel"] + assert sorted(module.outputs) == [ + "goal_request", + "human_input", + "operator_note", + "tele_cmd_vel", + "ui_command", + ] assert isinstance(module.sensor_ping, In) assert module.sensor_ping.type is Vector3 assert isinstance(module.operator_note, Out) diff --git a/dimos/web/relay_bridge/test_manifest.py b/dimos/web/relay_bridge/test_manifest.py index 061291785f..76b3c91512 100644 --- a/dimos/web/relay_bridge/test_manifest.py +++ b/dimos/web/relay_bridge/test_manifest.py @@ -83,3 +83,156 @@ def test_integral_float_version_accepted_like_ts(): # the way JS `1.0 === 1` does. manifest = parse_manifest({"version": 1.0, "channels": []}) assert manifest.version == 1 + + +# Slot-table kinds (chat / navmap / control): the shapes dimos/web/cockpit.py +# authors for the microduck cockpit. Golden vectors pin the cross-language +# codes; these spell the rules out per kind so a slot-table edit here cannot +# silently drift from what cockpit.py emits. + +SLOT_SHAPES = { + "chat": ( + "invalid_chat_panel", + [ + ("agent", "rx", "chat.json.v1", "reliable"), + ("agent_idle", "rx", "flag.json.v1", "reliable"), + ("mode", "rx", "mode.json.v1", "reliable"), + ("human_input", "tx", "text.json.v1", "reliable"), + ], + ), + "navmap": ( + "invalid_navmap_panel", + [ + ("global_costmap", "rx", "costmap.zlib.v1", "latest"), + ("odom", "rx", "pose.json.v1", "reliable"), + ("path", "rx", "path.json.v1", "latest"), + ("places", "rx", "places.json.v1", "reliable"), + ("nav_state", "rx", "navstate.json.v1", "latest"), + ("goal_request", "tx", "pose_goal.json.v1", "reliable"), + ("ui_command", "tx", "command.json.v1", "reliable"), + ], + ), + "control": ( + "invalid_control_panel", + [ + ("mode", "rx", "mode.json.v1", "reliable"), + ("policy_state", "rx", "policy.json.v1", "latest"), + ("nav_state", "rx", "navstate.json.v1", "latest"), + ("ui_command", "tx", "command.json.v1", "reliable"), + ], + ), +} + + +def slot_manifest(kind: str, **overrides) -> dict: + """A one-panel manifest of `kind` whose channels follow SLOT_SHAPES; + `overrides` patch one channel by index: {2: {"dir": "tx"}}.""" + _, shape = SLOT_SHAPES[kind] + channels = [ + {"ch": ch, "dir": direction, "encoding": encoding, "delivery": delivery, "maxHz": 10.0} + for ch, direction, encoding, delivery in shape + ] + for index, patch in overrides.items(): + channels[int(index)].update(patch) + return { + "version": 1, + "channels": channels, + "panels": [{"id": "p0", "kind": kind, "channels": [c["ch"] for c in channels]}], + } + + +@pytest.mark.parametrize("kind", sorted(SLOT_SHAPES)) +def test_slot_kind_accepts_its_shape(kind): + manifest = parse_manifest(slot_manifest(kind)) + (panel,) = manifest.panels + assert panel.kind == kind + assert len(panel.channels) == len(SLOT_SHAPES[kind][1]) + + +@pytest.mark.parametrize("kind", sorted(SLOT_SHAPES)) +def test_slot_kind_ignores_delivery(kind): + # Delivery is the bridge's per-channel choice, not part of the slot rule. + n = len(SLOT_SHAPES[kind][1]) + flipped = { + i: {"delivery": "latest" if SLOT_SHAPES[kind][1][i][3] == "reliable" else "reliable"} + for i in range(n) + } + parse_manifest(slot_manifest(kind, **{str(k): v for k, v in flipped.items()})) + + +@pytest.mark.parametrize("kind", sorted(SLOT_SHAPES)) +def test_slot_kind_rejects_wrong_channel_count(kind): + code, shape = SLOT_SHAPES[kind] + data = slot_manifest(kind) + panel = data["panels"][0] + for channels in ([], panel["channels"][:-1], [*panel["channels"], panel["channels"][0]]): + panel["channels"] = channels + with pytest.raises(ManifestError) as exc_info: + parse_manifest(data) + assert exc_info.value.code == code, channels + + +@pytest.mark.parametrize("kind", sorted(SLOT_SHAPES)) +def test_slot_kind_rejects_wrong_encoding_in_every_slot(kind): + code, shape = SLOT_SHAPES[kind] + for i in range(len(shape)): + with pytest.raises(ManifestError) as exc_info: + parse_manifest(slot_manifest(kind, **{str(i): {"encoding": "jpeg.v1"}})) + assert exc_info.value.code == code, i + + +@pytest.mark.parametrize("kind", sorted(SLOT_SHAPES)) +def test_slot_kind_rejects_wrong_dir_in_every_slot(kind): + code, shape = SLOT_SHAPES[kind] + for i, (_, direction, _, _) in enumerate(shape): + flipped = "tx" if direction == "rx" else "rx" + with pytest.raises(ManifestError) as exc_info: + parse_manifest(slot_manifest(kind, **{str(i): {"dir": flipped}})) + assert exc_info.value.code == code, i + + +@pytest.mark.parametrize("kind", sorted(SLOT_SHAPES)) +def test_slot_kind_rejects_swapped_slots(kind): + code, _ = SLOT_SHAPES[kind] + data = slot_manifest(kind) + channels = data["panels"][0]["channels"] + channels[0], channels[1] = channels[1], channels[0] + with pytest.raises(ManifestError) as exc_info: + parse_manifest(data) + assert exc_info.value.code == code + + +def test_slot_kind_unknown_channel_reported_before_slot_rule(): + data = slot_manifest("control") + data["panels"][0]["channels"][0] = "missing" + with pytest.raises(ManifestError) as exc_info: + parse_manifest(data) + assert exc_info.value.code == "unknown_panel_channel" + + +def test_slot_kinds_share_channels_across_panels(): + # The microduck cockpit binds mode/nav_state/ui_command from both the + # control bar and the nav map; each panel is validated independently. + control = slot_manifest("control") + navmap = slot_manifest("navmap") + by_ch = {c["ch"]: c for c in [*control["channels"], *navmap["channels"]]} + data = { + "version": 1, + "channels": list(by_ch.values()), + "panels": [ + {**control["panels"][0], "id": "p0"}, + {**navmap["panels"][0], "id": "p1"}, + ], + "layout": {"col": ["p0", "p1"]}, + } + manifest = parse_manifest(data) + assert [p.kind for p in manifest.panels] == ["control", "navmap"] + + +@pytest.mark.parametrize("kind", sorted(SLOT_SHAPES)) +def test_golden_vectors_cover_slot_kind(kind): + # The cross-language contract for these kinds lives in the fixtures; + # both an accepted and a rejected vector must exist per kind. + code, _ = SLOT_SHAPES[kind] + assert any(any(p.get("kind") == kind for p in v["data"].get("panels", [])) for v in VALID), kind + assert any(v["error"] == code for v in INVALID), code diff --git a/dimos/web/relay_bridge/test_protocol.py b/dimos/web/relay_bridge/test_protocol.py index eb109a520f..d1d3b5c62b 100644 --- a/dimos/web/relay_bridge/test_protocol.py +++ b/dimos/web/relay_bridge/test_protocol.py @@ -21,6 +21,7 @@ import pytest from dimos.web.relay_bridge.locate import find_web_dir +from dimos.web.relay_bridge.manifest import MAX_MANIFEST_ID_LEN from dimos.web.relay_bridge.protocol import ( CONTROL_CHANNEL, MAX_CONTROL_PAYLOAD_BYTES, @@ -28,8 +29,12 @@ MAX_HEADER_LEN, MAX_PUB_DATA_BYTES, MAX_REQUEST_ID_LEN, + MAX_TX_MSG_BYTES, PROTOCOL_VERSION, RESERVED_CHANNEL_PREFIX, + TX_CH_MAX_LEN, + TX_CH_PATTERN, + TX_DATA_MAX_BYTES, ControlFrameReader, DataFrameStreamError, DataFrameStreamReader, @@ -42,7 +47,9 @@ Pub, RobotInfo, Robots, + Subs, TeleopStop, + Tx, decode_data_frame, decode_datagram, encode_control_frame, @@ -50,6 +57,7 @@ encode_datagram, msg_from_dict, peek_data_frame_lengths, + tx_data_bytes, ) FIXTURES = find_web_dir() / "shared" / "fixtures" @@ -63,6 +71,7 @@ def _vectors(name): CONTROL = _vectors("control_frames.json") DATAGRAMS = _vectors("datagrams.json") DATA = _vectors("data_frames.json") +TX = _vectors("tx_messages.json") def _header(d): @@ -473,6 +482,156 @@ def test_huge_int_is_a_valid_number(): assert isinstance(msg, Ping) and msg.n == 10**400 +def _tx(**overrides): + base = {"t": "tx", "ch": "ui_command", "seq": 4, "data": {"name": "stop"}} + return {**base, **overrides} + + +@pytest.mark.parametrize("vector", TX, ids=[v["name"] for v in TX]) +def test_tx_vector_parity(vector): + # `valid` pins acceptance on both sides; valid vectors also pin the + # byte-exact datagram encoding (and it always fits the tx budget). + raw_in = json.dumps(vector["message"], ensure_ascii=False).encode() + if not vector["valid"]: + with pytest.raises(ProtocolError): + msg_from_dict(vector["message"]) + assert decode_datagram(raw_in) is None + return + msg = msg_from_dict(vector["message"]) + assert isinstance(msg, Tx) + raw = base64.b64decode(vector["b64"]) + assert encode_datagram(msg) == raw + assert decode_datagram(raw) == msg + assert decode_datagram(raw_in) == msg + assert len(raw) <= MAX_TX_MSG_BYTES + + +def test_tx_vectors_cover_the_design_cases(): + names = {v["name"] for v in TX} + assert {"tx_chat", "tx_goal", "tx_command"} <= names # valid: chat text, goal, command + assert {"tx_bad_data_oversize", "tx_bad_ch_too_long", "tx_bad_extra_gen"} <= names + # The valid tx_* vectors also ride the shared datagram/control sets, so + # the generic golden tests exercise them too. + datagram_names = {v["name"] for v in DATAGRAMS} + assert {"tx_chat", "tx_goal", "tx_command"} <= datagram_names + + +def test_tx_constants_pin_the_wire_budget(): + assert TX_DATA_MAX_BYTES == 900 + assert TX_CH_MAX_LEN == 64 == MAX_MANIFEST_ID_LEN + assert TX_CH_PATTERN == r"^[a-z][a-z0-9_]*$" + assert MAX_TX_MSG_BYTES == 1100 + # Worst case: longest ch, largest seq, data at the cap. It must still fit + # MAX_TX_MSG_BYTES (the relay sizes datagram buffers off it). + data = {"k": "x" * (TX_DATA_MAX_BYTES - len('{"k":""}'))} + assert tx_data_bytes(data) == TX_DATA_MAX_BYTES + worst = Tx(ch="c" * TX_CH_MAX_LEN, seq=2**53 - 1, data=data) + assert len(encode_datagram(worst)) <= MAX_TX_MSG_BYTES + assert msg_from_dict(worst.model_dump()) == worst + + +def test_tx_round_trips_and_keeps_nulls_inside_data(): + # exclude_none strips absent optional *fields*; the opaque data record + # travels untouched (a null inside it is the command's business). + msg = Tx(ch="human_input", seq=7, data={"text": "salut", "reply_to": None, "tags": [None]}) + raw = encode_datagram(msg) + expected = b'{"t":"tx","ch":"human_input","seq":7,"data":{"text":"salut","reply_to":null,"tags":[null]}}' + assert raw == expected + assert decode_datagram(raw) == msg + assert encode_control_frame(msg)[4:] == raw + assert msg_from_dict(_tx()) == Tx(ch="ui_command", seq=4, data={"name": "stop"}) + + +def test_tx_data_bytes_counts_compact_utf8(): + # Same figure as TextEncoder(JSON.stringify(data)).length in protocol.ts: + # compact separators, raw (unescaped) non-ASCII. + assert tx_data_bytes({}) == 2 + assert tx_data_bytes({"text": "é"}) == len('{"text":""}') + 2 + assert tx_data_bytes({"a": None, "b": [1.5, "x"]}) == len('{"a":null,"b":[1.5,"x"]}') + + +def test_tx_rejects_oversize_data(): + at_cap = {"text": "x" * (TX_DATA_MAX_BYTES - len('{"text":""}'))} + assert tx_data_bytes(at_cap) == TX_DATA_MAX_BYTES + assert msg_from_dict(_tx(data=at_cap)).data == at_cap + over = {"text": at_cap["text"] + "x"} + with pytest.raises(ProtocolError): + msg_from_dict(_tx(data=over)) + with pytest.raises(ValueError): + Tx(ch="a", seq=1, data=over) # local construction fails fast too + # Bytes, not characters: 446 x "é" is 900 B and passes, 447 is 902 B. + assert msg_from_dict(_tx(data={"t": "é" * 446})).data == {"t": "é" * 446} + with pytest.raises(ProtocolError): + msg_from_dict(_tx(data={"t": "é" * 447})) + + +@pytest.mark.parametrize( + "ch", + ["", "Human", "2cam", "_cam", "cam-left", "cam.left", "@control", "cam\n", "ĉam", "c" * 65], +) +def test_tx_rejects_bad_ch(ch): + with pytest.raises(ProtocolError): + msg_from_dict(_tx(ch=ch)) + with pytest.raises(ValueError): + Tx(ch=ch, seq=1, data={}) + + +@pytest.mark.parametrize("ch", ["a", "human_input", "cam2_left", "c" * 64]) +def test_tx_accepts_stream_shaped_ch(ch): + assert msg_from_dict(_tx(ch=ch)).ch == ch + + +def test_tx_rejects_extra_fields(): + # Unlike the other messages (see test_nested_roundtrip_returns_models), + # tx rejects unknown keys: the data cap must bound the whole message. + with pytest.raises(ProtocolError): + msg_from_dict(_tx(gen=1)) + with pytest.raises(ProtocolError): + msg_from_dict(_tx(later=1.5)) + with pytest.raises(ValueError): + Tx(ch="a", seq=1, data={}, gen=1) + # ...and the strict/allow_inf_nan config is still inherited. + with pytest.raises(ProtocolError): + msg_from_dict(_tx(seq=1.0)) + + +@pytest.mark.parametrize("seq", [-1, 1.5, True, "1", 2**53, 10**400, None]) +def test_tx_rejects_bad_seq(seq): + with pytest.raises(ProtocolError): + msg_from_dict(_tx(seq=seq)) + + +def test_tx_seq_bounds_are_js_safe_integers(): + assert msg_from_dict(_tx(seq=0)).seq == 0 + assert msg_from_dict(_tx(seq=2**53 - 1)).seq == 2**53 - 1 + with pytest.raises(ProtocolError): + msg_from_dict(_tx(seq=2**53)) + + +@pytest.mark.parametrize("data", [[], None, "x", 5]) +def test_tx_rejects_non_record_data(data): + with pytest.raises(ProtocolError): + msg_from_dict(_tx(data=data)) + + +def test_tx_rejects_missing_fields(): + for key in ("ch", "seq", "data"): + d = _tx() + del d[key] + with pytest.raises(ProtocolError): + msg_from_dict(d) + + +def test_tx_data_non_finite_rejected(): + # Python's JSON parser accepts NaN inside the opaque data record where + # JSON.parse errors; the byte counter refuses it (allow_nan=False), and a + # local encode fails fast instead of emitting wire JSON the relay drops. + assert decode_datagram(b'{"t":"tx","ch":"a","seq":1,"data":{"x":NaN}}') is None + assert decode_datagram(b'{"t":"tx","ch":"a","seq":1,"data":{"x":1e999}}') is None + with pytest.raises(ValueError): + Tx(ch="a", seq=1, data={"x": float("nan")}) + + def test_control_reader_drops_invalid_keeps_valid_neighbors(): hello = encode_control_frame( msg_from_dict({"t": "hello", "v": PROTOCOL_VERSION, "role": "viewer"}) @@ -484,3 +643,14 @@ def test_control_reader_drops_invalid_keeps_valid_neighbors(): msg_from_dict({"t": "hello", "v": PROTOCOL_VERSION, "role": "viewer"}), msg_from_dict({"t": "ping", "n": 3, "ts": 4.5}), ] + + +def test_subscription_replay_hint_names_an_active_channel() -> None: + message = {"t": "subs", "chs": ["agent"], "n": 3, "replay": ["agent"]} + assert msg_from_dict(message) == Subs(chs=["agent"], n=3, replay=["agent"]) + + +@pytest.mark.parametrize("replay", [None, 1, "agent", [1], ["unknown"]]) +def test_subscription_rejects_invalid_replay_hint(replay) -> None: + with pytest.raises(ProtocolError): + msg_from_dict({"t": "subs", "chs": ["agent"], "n": 3, "replay": replay}) diff --git a/dimos/web/relay_bridge/test_relay_bridge_authoring.py b/dimos/web/relay_bridge/test_relay_bridge_authoring.py index 90d06e79d8..3c7dd967d7 100644 --- a/dimos/web/relay_bridge/test_relay_bridge_authoring.py +++ b/dimos/web/relay_bridge/test_relay_bridge_authoring.py @@ -55,6 +55,7 @@ PubAck, PubNack, Subs, + Tx, ) from dimos.web.relay_bridge.relay_bridge_module import ( RelayBridgeConfig, @@ -460,6 +461,21 @@ def test_publish_frame_decodes_publishes_then_acks(monkeypatch) -> None: stop_module(module) +def test_legacy_tx_cannot_bypass_acknowledged_publish(monkeypatch) -> None: + module, clients = _start_pub_bridge(monkeypatch) + try: + seen: list[str] = [] + module.human_input.subscribe(seen.append) + push(module, clients[0], Tx(ch="human_input", seq=1, data={"text": "bypass"})) + flush_loop(module) + assert seen == [] + push(module, clients[0], _pub_frame(b'"allowed"')) + assert wait_until(lambda: clients[0].control_frames) + assert seen == ["allowed"] + finally: + stop_module(module) + + def test_publish_decoder_context_and_no_context_paths(monkeypatch) -> None: module, clients = _start_pub_bridge( monkeypatch, diff --git a/dimos/web/relay_bridge/test_relay_bridge_microduck.py b/dimos/web/relay_bridge/test_relay_bridge_microduck.py new file mode 100644 index 0000000000..d8fc74ca3b --- /dev/null +++ b/dimos/web/relay_bridge/test_relay_bridge_microduck.py @@ -0,0 +1,715 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Microduck cockpit streams, legacy commands, and bounded chat replay.""" + +from __future__ import annotations + +from collections import deque +from dataclasses import replace +import json +import time +from typing import Any + +from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, ToolMessage +import numpy as np +import pytest + +from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped +from dimos.msgs.geometry_msgs.Quaternion import Quaternion +from dimos.msgs.geometry_msgs.Twist import Twist +from dimos.msgs.geometry_msgs.Vector3 import Vector3 +from dimos.msgs.sensor_msgs.Image import Image +from dimos.robot.pollen.microduck import web_codecs +from dimos.robot.pollen.microduck.blueprints.microduck_cockpit_sim import ( + MICRODUCK_COCKPIT_CHANNELS, + MICRODUCK_COCKPIT_LAYOUT, +) +from dimos.web.cockpit import Channel, Chat, Video, cockpit +from dimos.web.relay_bridge import relay_bridge_module +from dimos.web.relay_bridge.e2e_support import stop_module +from dimos.web.relay_bridge.manifest import parse_manifest +from dimos.web.relay_bridge.module_test_support import ( + FakeClient, + FakeTransport, + flush_loop, + kill_session, + make_bridge, + push, + start_authored, + wait_until, +) +from dimos.web.relay_bridge.protocol import ( + Subs, + Tx, +) +from dimos.web.relay_bridge.relay_bridge_module import ( + TX_CHANNELS, + RelayBridgeConfig, + RelayBridgeModule, + RuntimeChannelSpec, + default_manifest, +) +from dimos.web.relay_bridge.test_relay_bridge_module import teleop_manifest, wire_twist + + +def settle(module: RelayBridgeModule) -> None: + """Give queued teleop handling time to run before a negative assert.""" + flush_loop(module) + time.sleep(0.03) + flush_loop(module) + + +# The authored cockpit's rx channels in manifest order: the bridge built-ins +# first, then the MICRODUCK_COCKPIT_CHANNELS declarations in panel order. +MICRODUCK_RX = ( + "color_image", + "odom", + "global_costmap", + "mode", + "policy_state", + "nav_state", + # Built-ins first (BUILTIN_CHANNELS order), then the authored ones in + # first-declaration order - which is panel order, so chase_image leads: + # it is the main feed of the first video panel. + "chase_image", + "path", + "places", + "agent", + "agent_idle", +) +# The tx channels that go through the generic Tx path; the twist keeps its own +# lease-guarded one. +MICRODUCK_TX = ("human_input", "goal_request", "ui_command") + + +def microduck_channels(*streams: str) -> tuple[Channel, ...]: + """The named MICRODUCK_COCKPIT_CHANNELS declarations, so a test can build + a slice of the duck's cockpit without repeating its authoring.""" + declared = {c.stream: c for c in MICRODUCK_COCKPIT_CHANNELS} + return tuple(declared[stream] for stream in streams) + + +def microduck_tx_manifest(tx: tuple[str, ...] = MICRODUCK_TX) -> dict[str, Any]: + """Channel-only manifest for the Microduck tx commands (no panels: the + bridge validates tx channels against TX_CHANNELS, not panel bindings).""" + by_tx = {td.ch: td for td in TX_CHANNELS} + return { + "version": 1, + "channels": [ + { + "ch": ch, + "dir": "tx", + "encoding": by_tx[ch].encoding, + "delivery": by_tx[ch].delivery, + "maxHz": 5.0, + } + for ch in tx + ], + } + + +def transport_of(module: RelayBridgeModule, ch: str) -> FakeTransport: + transport = getattr(module, ch).transport + assert isinstance(transport, FakeTransport) + return transport + + +def spec_of(module: RelayBridgeModule, ch: str) -> RuntimeChannelSpec: + return next(spec for spec in module._channel_specs if spec.ch == ch) + + +def frames_on(client: FakeClient, ch: str) -> list[dict[str, Any]]: + """Decoded reliable frame payloads sent on `ch`, in order.""" + return [json.loads(payload) for c, payload, _delivery, _meta in client.frames if c == ch] + + +def ns_on(client: FakeClient, ch: str) -> list[int | None]: + """The `n` frame meta of those frames: the bridge's log entry numbers, + which is where the viewer's dedupe key lives (never in the payload).""" + return [None if meta is None else meta.get("n") for c, _p, _d, meta in client.frames if c == ch] + + +@pytest.fixture +def microduck_bridge(monkeypatch): + """The full authored Microduck cockpit, every rx input wired.""" + module, clients = start_authored( + monkeypatch, + cockpit(layout=MICRODUCK_COCKPIT_LAYOUT, channels=MICRODUCK_COCKPIT_CHANNELS), + wire=MICRODUCK_RX, + ) + try: + yield module, clients + finally: + stop_module(module) + + +def test_microduck_manifest_starts_with_every_channel(microduck_bridge) -> None: + module, clients = microduck_bridge + _, hello_manifest = clients[0].hello_args + assert [c["ch"] for c in hello_manifest["channels"]] == [ + *MICRODUCK_RX, + "tele_cmd_vel", + *MICRODUCK_TX, + ] + # The twist keeps the lease-guarded path; every other tx channel goes + # through the generic Tx handler. + assert module._teleop_params is not None + assert set(module._tx_defs) == set(MICRODUCK_TX) + # Every declared channel replays on subscribe: one always-on raw cache + # subscription each, nothing encoding yet. + for channel in MICRODUCK_COCKPIT_CHANNELS: + assert len(transport_of(module, channel.stream).subscribers) == 1, channel.stream + assert all(module.encoded[ch] == 0 for ch in MICRODUCK_RX) + assert module._min_interval["agent"] == pytest.approx(1.0 / 30.0) + + +def test_cockpit_microduck_layout_starts(microduck_bridge) -> None: + # The authored cockpit (Control bar, chase camera, Chat, NavMap) compiles + # to runtime specs this bridge runs: a typed port per declaration, the + # encoder resolved from web_codecs by id, and the authoring flags carried + # through - the event streams skip the rate gate, the transcript keeps a + # replay log. + module, _clients = microduck_bridge + for channel in MICRODUCK_COCKPIT_CHANNELS: + spec = spec_of(module, channel.stream) + assert module.inputs[channel.stream].type is channel.message_type + assert (spec.encoding, spec.delivery, spec.max_hz) == ( + channel.encoding, + channel.delivery, + channel.max_hz, + ) + assert (spec.resend_on_subscribe, spec.rate_gate, spec.replay_depth) == ( + channel.resend_on_subscribe, + channel.rate_gate, + channel.replay_depth, + ) + assert spec_of(module, "agent").encoder is web_codecs.encode_chat + assert spec_of(module, "places").encoder is web_codecs.encode_state_json + + +def test_microduck_manifest_rejects_mismatches(monkeypatch) -> None: + # Authoring time: a declaration whose encoding does not match the stream's + # message type, or that contradicts the panel binding the same stream, + # fails at blueprint definition. + with pytest.raises(ValueError, match="encodes BaseMessage, not bool"): + cockpit(channels=[Channel("agent", bool, encoding="chat.json.v1")]) + with pytest.raises(ValueError, match="conflicting requirements for stream 'agent'"): + cockpit( + layout=Chat(), + channels=[Channel("agent", BaseMessage, encoding="chat.json.v1", delivery="latest")], + ) + + # Start time: the tx side is still the bridge's own table. + async def fake_connect(url: str, role: str, **kwargs: Any) -> FakeClient: + raise AssertionError("must not reach the relay with an invalid manifest") + + monkeypatch.setattr(relay_bridge_module, "connect_with_backoff", fake_connect) + + def start_with(manifest: dict[str, Any]) -> None: + module = RelayBridgeModule( + relay_url="https://127.0.0.1:1", robot_id="unit-bot", manifest=manifest + ) + try: + module.start() + finally: + stop_module(module) + + def with_channel(ch: str, **overrides: Any) -> dict[str, Any]: + base = microduck_tx_manifest(tx=("human_input",)) + for channel in base["channels"]: + if channel["ch"] == ch: + channel.update(overrides) + return base + + with pytest.raises(RuntimeError, match="no matching handler"): + start_with(with_channel("human_input", encoding="pose_goal.json.v1")) + with pytest.raises(RuntimeError, match="no matching handler"): + start_with(with_channel("human_input", delivery="latest")) + unknown = microduck_tx_manifest(tx=()) + unknown["channels"].append( + { + "ch": "mystery", + "dir": "tx", + "encoding": "text.json.v1", + "delivery": "reliable", + "maxHz": 1.0, + } + ) + with pytest.raises(RuntimeError, match="no matching handler"): + start_with(unknown) + + +def test_microduck_channels_are_advertised_only_when_authored() -> None: + # The auto (no-manifest) mode knows BUILTIN_CHANNELS and nothing else, so + # the duck's streams are simply not available there any more... + auto = default_manifest( + RelayBridgeConfig(image_max_hz=25.0), ("chase_image", "agent", "policy_state") + ) + assert auto["channels"] == [] and auto["panels"] == [] + # ... they reach a cockpit through the blueprint's own declarations, which + # advertise them channel-only when no panel binds them. + (atom,) = cockpit(channels=MICRODUCK_COCKPIT_CHANNELS).blueprints + manifest = atom.kwargs["manifest"] + assert [(c["ch"], c["dir"], c["encoding"], c["maxHz"]) for c in manifest["channels"]] == [ + (c.stream, "rx", c.encoding, c.max_hz) for c in MICRODUCK_COCKPIT_CHANNELS + ] + assert manifest["panels"] == [] and manifest["layout"] is None + # And the parser accepts the channel-only output too. + parse_manifest(manifest) + + +def tx(ch: str, seq: int, **data: Any) -> Tx: + return Tx(ch=ch, seq=seq, data=data) + + +@pytest.fixture +def tx_bridge(monkeypatch): + module, clients = make_bridge(monkeypatch, wire=(), manifest=microduck_tx_manifest()) + texts: list[str] = [] + goals: list[PoseStamped] = [] + commands: list[str] = [] + module.human_input.subscribe(texts.append) + module.goal_request.subscribe(goals.append) + module.ui_command.subscribe(commands.append) + try: + yield module, clients, texts, goals, commands + finally: + stop_module(module) + + +def unthrottle(module: RelayBridgeModule, ch: str) -> None: + """Lift a channel's rate floor so a test can exercise the other gates + with back-to-back sends.""" + module._tx_defs[ch] = replace(module._tx_defs[ch], min_interval_s=0.0) + + +def test_tx_human_input_publishes_stripped_text(tx_bridge) -> None: + module, clients, texts, goals, commands = tx_bridge + push(module, clients[0], tx("human_input", 1, text=" go to the kitchen \n")) + assert wait_until(lambda: texts == ["go to the kitchen"]) + assert goals == [] and commands == [] + + +def test_tx_goal_request_publishes_pose_stamped(tx_bridge) -> None: + module, clients, texts, goals, commands = tx_bridge + t0 = time.time() + push(module, clients[0], tx("goal_request", 1, x=1.5, y=-2, yaw=1.2, frame="map")) + assert wait_until(lambda: len(goals) == 1) + goal = goals[0] + assert isinstance(goal, PoseStamped) + assert goal.frame_id == "map" + assert (goal.position.x, goal.position.y, goal.position.z) == (1.5, -2.0, 0.0) + assert goal.yaw == pytest.approx(1.2) + assert goal.orientation == Quaternion.from_euler(Vector3(0.0, 0.0, 1.2)) + assert t0 <= goal.ts <= time.time() + # yaw and frame default; an int coordinate is a number too. + time.sleep(0.25) # the goal channel's rate floor + push(module, clients[0], tx("goal_request", 2, x=0, y=3)) + assert wait_until(lambda: len(goals) == 2) + assert goals[1].frame_id == "world" + assert goals[1].yaw == pytest.approx(0.0) + assert (goals[1].position.x, goals[1].position.y) == (0.0, 3.0) + + +def test_tx_ui_command_publishes_compact_json(tx_bridge) -> None: + module, clients, texts, goals, commands = tx_bridge + push(module, clients[0], tx("ui_command", 1, name="set_mode", args={"mode": "agent"})) + assert wait_until(lambda: len(commands) == 1) + assert commands[0] == '{"name":"set_mode","args":{"mode":"agent"}}' + time.sleep(0.06) # the command channel's rate floor + push(module, clients[0], tx("ui_command", 2, name="cancel_nav")) + assert wait_until(lambda: len(commands) == 2) + assert json.loads(commands[1]) == {"name": "cancel_nav", "args": {}} + + +def test_tx_stale_seq_dropped_while_channel_busy(tx_bridge) -> None: + module, clients, texts, goals, commands = tx_bridge + unthrottle(module, "human_input") + push(module, clients[0], tx("human_input", 5, text="five")) + push(module, clients[0], tx("human_input", 4, text="four")) # reordered: dropped + push(module, clients[0], tx("human_input", 5, text="five again")) # duplicated: dropped + push(module, clients[0], tx("human_input", 6, text="six")) + assert wait_until(lambda: len(texts) == 2) + settle(module) + assert texts == ["five", "six"] + + +def test_tx_seq_rebaselines_after_silence(tx_bridge, monkeypatch) -> None: + # A reloaded page (or a second tab) restarts its counter at 1; without + # a lease generation the only tell is time, so a quiet channel accepts + # any seq again. + module, clients, texts, goals, commands = tx_bridge + unthrottle(module, "human_input") + monkeypatch.setattr(relay_bridge_module, "_TX_SEQ_WINDOW_S", 0.05) + push(module, clients[0], tx("human_input", 50, text="old tab")) + assert wait_until(lambda: texts == ["old tab"]) + time.sleep(0.1) + push(module, clients[0], tx("human_input", 1, text="new tab")) + assert wait_until(lambda: texts == ["old tab", "new tab"]) + # The high-water mark followed the rebaseline: 2 is fresh now. + push(module, clients[0], tx("human_input", 2, text="next")) + assert wait_until(lambda: texts == ["old tab", "new tab", "next"]) + + +def test_tx_rate_floor_drops_bursts(tx_bridge) -> None: + module, clients, texts, goals, commands = tx_bridge + push(module, clients[0], tx("ui_command", 1, name="policy", args={"policy": "kick_left"})) + push(module, clients[0], tx("ui_command", 2, name="policy", args={"policy": "kick_right"})) + assert wait_until(lambda: len(commands) == 1) + settle(module) + assert json.loads(commands[0])["args"] == {"policy": "kick_left"} + time.sleep(0.06) + push(module, clients[0], tx("ui_command", 3, name="cancel_nav")) + assert wait_until(lambda: len(commands) == 2) + + +@pytest.mark.parametrize( + ("ch", "data"), + [ + ("human_input", {"text": " "}), + ("human_input", {"text": ""}), + ("human_input", {"text": 5}), + ("human_input", {}), + ("goal_request", {"x": 100.0, "y": 0.0}), + ("goal_request", {"x": 0.0, "y": -50.5}), + ("goal_request", {"x": "1", "y": 0.0}), + ("goal_request", {"x": 1.0}), + ("goal_request", {"x": 1.0, "y": 1.0, "yaw": "north"}), + ("goal_request", {"x": 1.0, "y": 1.0, "frame": ""}), + ("ui_command", {"name": "explode"}), + ("ui_command", {"name": "policy", "args": []}), + ("ui_command", {"args": {}}), + ], +) +def test_tx_invalid_record_dropped(tx_bridge, ch: str, data: dict[str, Any]) -> None: + module, clients, texts, goals, commands = tx_bridge + unthrottle(module, ch) + push(module, clients[0], Tx(ch=ch, seq=1, data=data)) + settle(module) + assert (texts, goals, commands) == ([], [], []) + # An invalid record consumes neither the seq nor the rate slot: the + # viewer's corrected resend at the same seq goes through. + valid = { + "human_input": {"text": "ok"}, + "goal_request": {"x": 1.0, "y": 1.0}, + "ui_command": {"name": "cancel_nav"}, + }[ch] + push(module, clients[0], Tx(ch=ch, seq=1, data=valid)) + assert wait_until(lambda: len(texts) + len(goals) + len(commands) == 1) + assert len(clients) == 1 # supervisor alive + + +def test_tx_unhandled_channel_dropped(tx_bridge) -> None: + module, clients, texts, goals, commands = tx_bridge + twists: list[Twist] = [] + module.tele_cmd_vel.subscribe(twists.append) + # A Tx on the twist channel is not a twist (no lease, no params) and an + # unknown channel has no Out: both dropped, the supervisor unharmed. + push(module, clients[0], tx("tele_cmd_vel", 1, vx=1.0, vy=0.0, wz=0.0)) + push(module, clients[0], tx("mystery", 1, text="hi")) + push(module, clients[0], tx("human_input", 1, text="still works")) + assert wait_until(lambda: texts == ["still works"]) + settle(module) + assert twists == [] and goals == [] and commands == [] + assert len(clients) == 1 + + +def test_tx_ignored_without_tx_channels(bridge) -> None: + module, clients = bridge + texts: list[str] = [] + module.human_input.subscribe(texts.append) + push(module, clients[0], tx("human_input", 1, text="hi")) + settle(module) + assert texts == [] + assert len(clients) == 1 + + +def test_tx_seq_state_resets_with_the_session(tx_bridge) -> None: + module, clients, texts, goals, commands = tx_bridge + unthrottle(module, "human_input") + push(module, clients[0], tx("human_input", 9, text="first session")) + assert wait_until(lambda: texts == ["first session"]) + kill_session(module, clients[0]) + assert wait_until(lambda: len(clients) == 2) + # New session, new viewers, counters from 1 - immediately, no window wait. + push(module, clients[1], tx("human_input", 1, text="second session")) + assert wait_until(lambda: texts == ["first session", "second session"]) + + +def test_tx_twist_path_untouched_alongside_generic_tx(monkeypatch) -> None: + manifest = teleop_manifest() + manifest["channels"] += microduck_tx_manifest()["channels"] + module, clients = make_bridge(monkeypatch, manifest=manifest) + twists: list[Twist] = [] + texts: list[str] = [] + module.tele_cmd_vel.subscribe(twists.append) + module.human_input.subscribe(texts.append) + try: + assert module._teleop_params is not None + assert set(module._tx_defs) == set(MICRODUCK_TX) + push(module, clients[0], wire_twist(0.4, 0.0, 0.0, seq=1)) + push(module, clients[0], tx("human_input", 1, text="hello")) + assert wait_until(lambda: len(twists) == 1 and texts == ["hello"]) + assert twists[0].linear.x == pytest.approx(0.4) + finally: + stop_module(module) + + +@pytest.fixture +def agent_bridge(monkeypatch): + """Just the transcript pair, no panels: these exercise the bridge's log + and rate-gate machinery, not the duck's layout.""" + module, clients = start_authored( + monkeypatch, + cockpit(channels=microduck_channels("agent", "agent_idle")), + wire=("agent", "agent_idle"), + ) + try: + yield module, clients + finally: + stop_module(module) + + +def test_agent_log_replays_in_order_and_caps(agent_bridge) -> None: + module, clients = agent_bridge + client = clients[0] + depth = spec_of(module, "agent").replay_depth + assert depth == 200 + # A transcript that grew before any viewer attached (cold start): only + # the newest `depth` entries are kept. + for i in range(depth + 5): + transport_of(module, "agent").publish(HumanMessage(content=f"m{i}")) + assert module.encoded["agent"] == 0 + assert len(transport_of(module, "agent").subscribers) == 1 + + push(module, client, Subs(chs=["agent"], n=1)) + assert wait_until(lambda: len(frames_on(client, "agent")) == depth) + flush_loop(module) + entries = frames_on(client, "agent") + assert [e["content"] for e in entries] == [f"m{i}" for i in range(5, depth + 5)] + assert all(e["role"] == "human" for e in entries) + ns = ns_on(client, "agent") + assert ns == list(range(ns[0], ns[0] + depth)) # in order, gapless + assert module.encoded["agent"] == 0 # replays are not live encodes + assert all(delivery == "reliable" for _, _, delivery, _ in client.frames) + # Live frames are fed from the log subscription: no second subscription. + assert len(transport_of(module, "agent").subscribers) == 1 + + +def test_agent_live_and_replayed_entries_share_n(agent_bridge) -> None: + module, clients = agent_bridge + client = clients[0] + push(module, client, Subs(chs=["agent"], n=1)) + assert wait_until(lambda: "agent" in (module._session.unsubs if module._session else {})) + transport_of(module, "agent").publish(HumanMessage(content="hello")) + transport_of(module, "agent").publish(AIMessage(content="hi!")) + assert wait_until(lambda: len(frames_on(client, "agent")) == 2) + live, live_ns = frames_on(client, "agent"), ns_on(client, "agent") + assert module.encoded["agent"] == 2 + + push(module, client, Subs(chs=[], n=2)) + assert wait_until(lambda: module._session is not None and "agent" not in module._session.unsubs) + transport_of(module, "agent").publish(ToolMessage(content="done", tool_call_id="c1")) + flush_loop(module) + assert len(frames_on(client, "agent")) == 2 # nobody watching: no encode + assert module.encoded["agent"] == 2 + + push(module, client, Subs(chs=["agent"], n=3)) + assert wait_until(lambda: len(frames_on(client, "agent")) == 5) + replayed, replayed_ns = frames_on(client, "agent")[2:], ns_on(client, "agent")[2:] + # The two entries seen live come back with the same n (the viewer's + # dedupe key), followed by the one published while unwatched. + assert replayed_ns[:2] == live_ns + assert [e["content"] for e in replayed[:2]] == [e["content"] for e in live] + assert replayed[2]["content"] == "done" and replayed_ns[2] == live_ns[1] + 1 + assert module.encoded["agent"] == 2 + + +def test_chat_entry_number_is_frame_meta_not_payload(agent_bridge) -> None: + # The number belongs to the bridge's log, not to the message: the encoder + # never sees it, so it rides the frame meta and the payload stays clean. + module, clients = agent_bridge + client = clients[0] + push(module, client, Subs(chs=["agent"], n=1)) + assert wait_until(lambda: "agent" in (module._session.unsubs if module._session else {})) + transport_of(module, "agent").publish(HumanMessage(content="x")) + transport_of(module, "agent").publish(HumanMessage(content="y")) + assert wait_until(lambda: len(frames_on(client, "agent")) == 2) + first, second = ns_on(client, "agent") + assert isinstance(first, int) and second == first + 1 + assert all("n" not in entry for entry in frames_on(client, "agent")) + # A replay_depth 1 channel numbers nothing: there is no log to dedupe. + push(module, client, Subs(chs=["agent", "agent_idle"], n=2)) + assert wait_until(lambda: len(transport_of(module, "agent_idle").subscribers) == 2) + transport_of(module, "agent_idle").publish(True) + assert wait_until(lambda: frames_on(client, "agent_idle")) + assert ns_on(client, "agent_idle") == [None] + + +def test_agent_log_entry_keeps_its_number_after_eviction(agent_bridge) -> None: + module, _clients = agent_bridge + spec = spec_of(module, "agent") + depth = spec.replay_depth + transport_of(module, "agent").publish(HumanMessage(content="first")) + (first,) = module._replay_log["agent"] + for i in range(depth): + transport_of(module, "agent").publish(HumanMessage(content=f"m{i}")) + log = module._replay_log["agent"] + assert len(log) == depth and first not in log # the transcript rolled past it + assert [entry.n for entry in log] == list(range(first.n + 1, first.n + depth + 1)) + # _replay snapshots (msg, recv_ts, n) triples under the lock and encodes + # afterwards, so an entry evicted in between still goes out under the + # number the live viewers saw, at its own arrival time - never as a + # fresh, newer-looking one. + sent: list[tuple[bytes, dict[str, Any] | None, float | None]] = [] + with module._log_lock: + module._replay_log["agent"] = deque([first], maxlen=depth) + assert module._session is not None + module._replay( + module._session, spec, lambda payload, meta, ts: sent.append((payload, meta, ts)) + ) + ((payload, meta, ts),) = sent + assert (meta, ts) == ({"n": first.n}, first.recv_ts) + assert json.loads(payload)["content"] == "first" + + +def test_runtime_spec_rejects_unreplayable_logs() -> None: + # A log is fed by the always-on cache subscription, which only + # resend_on_subscribe channels get: a spec that would never deliver a + # live frame is refused. + (atom,) = cockpit(channels=microduck_channels("agent")).blueprints + (agent,) = atom.kwargs["channels"] + assert (agent.replay_depth, agent.resend_on_subscribe) == (200, True) + with pytest.raises(ValueError, match="requires resend_on_subscribe"): + replace(agent, resend_on_subscribe=False) + with pytest.raises(ValueError, match="replay_depth must be >= 1"): + replace(agent, replay_depth=0) + replace(agent, replay_depth=1, resend_on_subscribe=False) # a plain channel is fine + + +def test_tx_channel_def_unpacks_as_a_triple() -> None: + # TX_CHANNELS rows used to be (ch, encoding, delivery) tuples; consumers + # that still unpack them that way (dimos/web/cockpit.py before its + # _tx_registry) must keep working. + rows = [(ch, encoding, delivery) for ch, encoding, delivery in TX_CHANNELS] + assert rows == [(td.ch, td.encoding, td.delivery) for td in TX_CHANNELS] + assert rows[0] == ("tele_cmd_vel", "twist.json.v1", "latest") + + +def test_agent_no_rate_gate(agent_bridge) -> None: + module, clients = agent_bridge + client = clients[0] + push(module, client, Subs(chs=["agent", "agent_idle"], n=1)) + assert wait_until(lambda: len(transport_of(module, "agent_idle").subscribers) == 2) + # Back-to-back publishes, far inside the advertised maxHz intervals + # (30 Hz / 10 Hz): every one is a frame - these are events, not samples. + for i in range(5): + transport_of(module, "agent").publish(HumanMessage(content=f"burst{i}")) + transport_of(module, "agent_idle").publish(False) + transport_of(module, "agent_idle").publish(True) + assert wait_until(lambda: len(frames_on(client, "agent")) == 5) + assert wait_until(lambda: len(frames_on(client, "agent_idle")) == 2) + assert [e["content"] for e in frames_on(client, "agent")] == [f"burst{i}" for i in range(5)] + assert [e["value"] for e in frames_on(client, "agent_idle")] == [False, True] + assert not spec_of(module, "agent").rate_gate + + +def test_agent_idle_replays_newest_flag(agent_bridge) -> None: + module, clients = agent_bridge + client = clients[0] + transport_of(module, "agent_idle").publish(False) + transport_of(module, "agent_idle").publish(True) + push(module, client, Subs(chs=["agent_idle"], n=1)) + assert wait_until(lambda: frames_on(client, "agent_idle")) + flush_loop(module) + assert [e["value"] for e in frames_on(client, "agent_idle")] == [True] + assert module.encoded["agent_idle"] == 0 + + +def test_agent_log_survives_reconnect(agent_bridge) -> None: + module, clients = agent_bridge + push(module, clients[0], Subs(chs=["agent"], n=1)) + assert wait_until(lambda: "agent" in (module._session.unsubs if module._session else {})) + transport_of(module, "agent").publish(HumanMessage(content="before")) + assert wait_until(lambda: len(frames_on(clients[0], "agent")) == 1) + kill_session(module, clients[0]) + assert wait_until(lambda: len(clients) == 2) + # The retired session is detached from the log; the new one replays it. + assert module._log_live["agent"] == () + push(module, clients[1], Subs(chs=["agent"], n=1)) + assert wait_until(lambda: len(frames_on(clients[1], "agent")) == 1) + assert ns_on(clients[1], "agent") == ns_on(clients[0], "agent") + + +def test_chase_image_uses_its_own_quality(microduck_bridge, monkeypatch) -> None: + # The chase cam's rate and quality are authored twice - on its Video panel + # and on the Channel that gives the stream its port - and cockpit()'s + # merge would have rejected any disagreement, so the compiled spec is + # both halves at once, not the Video panel's 75 default. + module, clients = microduck_bridge + (chase,) = microduck_channels("chase_image") + spec = spec_of(module, "chase_image") + assert spec.params == dict(chase.params) + assert spec.params["quality"] != Video().quality + # The head cam is a built-in port with no Channel, so it is authored by + # its panel alone - and the two cameras are free to differ. + assert spec_of(module, "color_image").params["quality"] != Video().quality + assert module._min_interval["chase_image"] == pytest.approx(1.0 / chase.max_hz) + + qualities: list[int] = [] + real = Image.to_jpeg_bytes + + def spy(self: Image, quality: int = 75) -> bytes: + qualities.append(quality) + return real(self, quality=quality) + + monkeypatch.setattr(Image, "to_jpeg_bytes", spy) + client = clients[0] + push(module, client, Subs(chs=["chase_image"], n=1)) + assert wait_until(lambda: len(transport_of(module, "chase_image").subscribers) == 2) + transport_of(module, "chase_image").publish( + Image.from_numpy(np.zeros((8, 12, 3), dtype=np.uint8)) + ) + assert wait_until(lambda: qualities == [chase.params["quality"]]) + assert wait_until(lambda: client.writers["chase_image"].offers) + assert client.writers["chase_image"].offers[0][1] == {"w": 12, "h": 8} + + +def test_late_viewer_replays_live_chat_without_resubscribing(agent_bridge) -> None: + module, clients = agent_bridge + client = clients[0] + transport_of(module, "agent").publish(HumanMessage(content="hello")) + push(module, client, Subs(chs=["agent"], n=1)) + assert wait_until(lambda: len(frames_on(client, "agent")) == 1) + first_number = ns_on(client, "agent")[0] + push(module, client, Subs(chs=["agent"], n=2, replay=["agent"])) + assert wait_until(lambda: len(frames_on(client, "agent")) == 2) + assert [entry["content"] for entry in frames_on(client, "agent")] == ["hello", "hello"] + assert ns_on(client, "agent") == [first_number, first_number] + assert len(transport_of(module, "agent").subscribers) == 1 + push(module, client, Subs(chs=["agent"], n=2, replay=["agent"])) + flush_loop(module) + assert len(frames_on(client, "agent")) == 2 + + +def test_agent_stream_is_typed_by_langchain_base_message() -> None: + # The transcript port is generated from the blueprint's declaration, and + # its type must be langchain's own class: autoconnect keys transports on + # (name, type), so anything else would silently never wire + # McpClient.agent (Out[BaseMessage]). + (atom,) = cockpit(channels=microduck_channels("agent")).blueprints + assert atom.module is not RelayBridgeModule # a generated subclass + agent = next(s for s in atom.streams if s.name == "agent") + assert (agent.direction, agent.type) == ("in", BaseMessage) diff --git a/dimos/web/relay_bridge/test_relay_bridge_module.py b/dimos/web/relay_bridge/test_relay_bridge_module.py index 1cf39ec6e4..ada603eba0 100644 --- a/dimos/web/relay_bridge/test_relay_bridge_module.py +++ b/dimos/web/relay_bridge/test_relay_bridge_module.py @@ -30,6 +30,7 @@ import socket import subprocess import sys +import textwrap import threading import time from typing import Any @@ -46,6 +47,7 @@ from dimos.msgs.nav_msgs.OccupancyGrid import OccupancyGrid from dimos.msgs.sensor_msgs.Image import Image from dimos.simulation.mujoco.constants import VIDEO_FPS +from dimos.web.cockpit import cockpit from dimos.web.relay_bridge import builtin_codecs, relay_bridge_module from dimos.web.relay_bridge.e2e_support import stop_module from dimos.web.relay_bridge.manifest import ManifestError, parse_manifest @@ -462,6 +464,32 @@ def test_bridge_import_does_not_pull_matplotlib() -> None: subprocess.run([sys.executable, "-c", code], check=True) +def test_bridge_imports_without_langchain() -> None: + # langchain-core is the `agents` extra, not a bridge dependency: a web-only + # install (`uv sync --extra web`) must still build every built-in cockpit + # and run `--local-relay`. A None entry in sys.modules makes the import + # raise exactly like a missing package; nothing under the bridge reaches + # for it, because the duck's transcript channel and its encoder are + # authored on the microduck side (dimos/robot/pollen/microduck/web_codecs.py). + code = textwrap.dedent( + """ + import sys + sys.modules["langchain_core"] = None + from dimos.web.relay_bridge import relay_bridge_module as m + from dimos.web.cockpit import cockpit + assert not any( + name.split(".")[0].startswith("langchain") and mod is not None + for name, mod in sys.modules.items() + ) + cockpit() # the default preset (go2-style cockpits) still compiles + atom = m.RelayBridgeModule.blueprint().blueprints[0] + names = [s.name for s in atom.streams] + assert "agent" not in names, names + """ + ) + subprocess.run([sys.executable, "-c", code], check=True) + + def test_manifest_omits_pose_binding_when_odom_unwired(monkeypatch) -> None: module, clients = make_bridge(monkeypatch, wire=("color_image", "global_costmap")) try: @@ -854,7 +882,6 @@ def test_advertised_unwired_channel_is_not_probed(monkeypatch) -> None: def test_default_manifest_matches_cockpit_default_preset() -> None: # Drift guard: the auto-mode manifest for a fully-wired go2 must equal # what the authoring API's default preset produces. - from dimos.web.cockpit import cockpit (atom,) = cockpit().blueprints assert atom.kwargs["manifest"] == default_manifest( @@ -885,11 +912,11 @@ def test_supervisor_survives_reconcile_error(bridge, monkeypatch) -> None: real = module._reconcile calls: list[int] = [] - def flaky(session: Any, want: set[str]) -> None: + def flaky(session: Any, want: set[str], *, replay: list[str] | None = None) -> None: calls.append(1) if len(calls) == 1: raise RuntimeError("boom") - real(session, want) + real(session, want, replay=replay) monkeypatch.setattr(module, "_reconcile", flaky) push(module, clients[0], Subs(chs=["odom"], n=1)) diff --git a/dimos/web/relay_bridge/test_wt_session.py b/dimos/web/relay_bridge/test_wt_session.py index 24e83de7c6..33b7fdcc20 100644 --- a/dimos/web/relay_bridge/test_wt_session.py +++ b/dimos/web/relay_bridge/test_wt_session.py @@ -354,6 +354,23 @@ async def test_newer_subs_snapshot_supersedes_the_queued_one(): assert session.control_dropped == 0 +async def test_coalesced_subscriptions_preserve_pending_replays(): + session = _session() + session.incoming_is_carrier = True + snapshots = [ + Subs(chs=["agent", "mode", "places"], n=1, replay=["agent"]), + Subs(chs=["agent", "mode", "places"], n=2, replay=["places"]), + Subs(chs=["agent", "mode", "places"], n=3, replay=["agent"]), + Subs(chs=["agent", "mode"], n=4), + ] + # One transport read delivers the whole subscription burst before consumption. + wire = b"".join(_control_bytes(encode_datagram(msg), seq=i) for i, msg in enumerate(snapshots)) + session._stream_data_received(3, wire, False) + assert session.control_msgs.qsize() == 1 + assert session.control_msgs.get_nowait() == Subs(chs=["agent", "mode"], n=4, replay=["agent"]) + assert session.control_dropped == 0 + + async def test_carrier_reset_fails_the_robot_session(): # The relay never replaces a carrier, so a reset while the connection # lives (e.g. the relay's carrier dispose racing its delayed session diff --git a/dimos/web/relay_bridge/wt_client.py b/dimos/web/relay_bridge/wt_client.py index 103fe3bf20..2a0489897f 100644 --- a/dimos/web/relay_bridge/wt_client.py +++ b/dimos/web/relay_bridge/wt_client.py @@ -120,6 +120,9 @@ async def connect( f"relay URL path must be {expected_path!r} for role={role}, got {parsed.path!r}" ) + if parsed.query: + path += "?" + parsed.query + ctx = aioquic_connect( host, port, @@ -138,7 +141,9 @@ async def connect( except BaseException: await ctx.__aexit__(None, None, None) raise - logger.info(f"WebTransport session established: {url} path={path}") + logger.info( + f"WebTransport session established: {parsed.scheme}://{host}:{port}{expected_path}" + ) return cls(url, role, session, ctx) async def __aenter__(self) -> RelayClient: diff --git a/dimos/web/test_cockpit.py b/dimos/web/test_cockpit.py index 3efc74caa1..93e20ea91b 100644 --- a/dimos/web/test_cockpit.py +++ b/dimos/web/test_cockpit.py @@ -15,6 +15,7 @@ """Authoring-API tests: cockpit()/panels/layout compile to pinned manifests.""" from dataclasses import dataclass +import json import pickle import struct import subprocess @@ -29,16 +30,21 @@ from dimos.web.cockpit import ( Channel, ChannelRequest, + Chat, Col, + Control, Map2D, + NavMap, Panel, Row, Teleop, Video, + build_manifest_data, cockpit, ) from dimos.web.codecs import EncodedPayload, decode_json_v1, encode_json_v1, web_encoder from dimos.web.relay_bridge.builtin_codecs import decode_text +from dimos.web.relay_bridge.locate import find_web_dir from dimos.web.relay_bridge.manifest import ManifestError, parse_manifest from dimos.web.relay_bridge.protocol import ( MAX_CONTROL_PAYLOAD_BYTES, @@ -47,7 +53,7 @@ RobotInfo, encode_datagram, ) -from dimos.web.relay_bridge.relay_bridge_module import RelayBridgeModule +from dimos.web.relay_bridge.relay_bridge_module import TX_CHANNELS, RelayBridgeModule # The frozen-contract example (see the plan/spec): what the go2 cockpit # blueprint authors. Golden below is exact; edits here are manifest changes @@ -178,6 +184,25 @@ def test_unknown_tx_stream_raises() -> None: cockpit(layout=Teleop(stream="cmd_vel")) +def test_teleop_mode_is_a_param_not_a_channel() -> None: + # The pad reads a mode stream some other panel already subscribes, so it + # can refuse to arm on a robot that ignores teleop in agent mode. It must + # stay a params-only role: "a teleop panel binds exactly one channel" is + # enforced by manifest.py and mirrored in manifest.ts. + plain = manifest_of(cockpit(layout=Teleop())) + (panel,) = plain["panels"] + assert panel["channels"] == ["tele_cmd_vel"] and panel["params"] == {} + + bound = manifest_of(cockpit(layout=Row(Teleop(mode="odom"), Video("color_image")))) + teleop_panel = next(p for p in bound["panels"] if p["kind"] == "teleop") + assert teleop_panel["channels"] == ["tele_cmd_vel"] + assert teleop_panel["params"] == {"mode": "odom"} + assert parse_manifest(bound).model_dump() == bound + + with pytest.raises(ValueError, match="mode"): + Teleop(mode="") + + def test_wrong_tx_encoding_raises() -> None: class Sender(Panel): kind = "sender" @@ -190,6 +215,51 @@ def _channel_requests(self) -> tuple[ChannelRequest, ...]: cockpit(layout=Sender()) +def test_video_inset_binds_a_second_feed() -> None: + # Picture-in-picture: channels[0] is the main feed, channels[1] the + # inset. Each keeps its own rate and quality, so a cheap thumbnail can + # ride along with an expensive main view. + # chase_image is an authored channel, so cockpit() generates a bridge + # subclass for its port; read the manifest off the atom directly. + (atom,) = cockpit( + layout=Video( + "chase_image", + max_hz=30.0, + quality=70, + inset="color_image", + inset_max_hz=12.0, + inset_quality=55, + ), + channels=[ + Channel( + "chase_image", + Image, + encoding="jpeg.v1", + delivery="latest", + max_hz=30.0, + params={"quality": 70}, + ) + ], + ).blueprints + manifest = atom.kwargs["manifest"] + (panel,) = manifest["panels"] + assert panel["kind"] == "video" + assert panel["channels"] == ["chase_image", "color_image"] + by_ch = {c["ch"]: c for c in manifest["channels"]} + assert by_ch["chase_image"]["params"] == {"quality": 70} + assert by_ch["color_image"]["params"] == {"quality": 55} + assert by_ch["color_image"]["maxHz"] == 12.0 + # The domain parser accepts two feeds on one video panel. + assert parse_manifest(manifest).model_dump() == manifest + + # A single-feed video panel is unchanged, and an inset cannot alias the + # main stream (it would draw the same picture over itself). + (plain,) = manifest_of(cockpit(layout=Video("color_image")))["panels"] + assert plain["channels"] == ["color_image"] + with pytest.raises(ValueError, match="inset must differ"): + Video("color_image", inset="color_image") + + def test_pages_get_ids_after_the_grid() -> None: manifest = manifest_of(cockpit(layout=Video("color_image"), pages=[Map2D(pose=None)])) assert [p["id"] for p in manifest["panels"]] == ["p0", "p1"] @@ -654,3 +724,360 @@ def test_import_stays_light() -> None: "assert 'aioquic' not in sys.modules" ) subprocess.run([sys.executable, "-c", code], check=True) + + +# +# Compiled through build_manifest_data with stand-in channel tables shaped +# like the microduck bridge's (rx table, then tx table), so these pins hold +# regardless of which streams the go2 bridge happens to advertise. + +MICRODUCK_RX_REGISTRY = { + "color_image": ("jpeg.v1", "latest"), + "chase_image": ("jpeg.v1", "latest"), + "global_costmap": ("costmap.zlib.v1", "latest"), + "odom": ("pose.json.v1", "reliable"), + "agent": ("chat.json.v1", "reliable"), + "agent_idle": ("flag.json.v1", "reliable"), + "path": ("path.json.v1", "latest"), + # State, not frames: reliable, so they cost no per-frame relay->viewer + # stream out of the budget the cameras need (web/README.md bug 12). + "nav_state": ("navstate.json.v1", "reliable"), + "mode": ("mode.json.v1", "reliable"), + "places": ("places.json.v1", "reliable"), + "policy_state": ("policy.json.v1", "reliable"), +} +MICRODUCK_TX_REGISTRY = { + "tele_cmd_vel": ("twist.json.v1", "latest"), + "human_input": ("text.json.v1", "reliable"), + "goal_request": ("pose_goal.json.v1", "reliable"), + "ui_command": ("command.json.v1", "reliable"), +} + +MICRODUCK_LAYOUT = Col( + Control(), + Row( + Video("chase_image", title="Chase cam"), + Col( + NavMap(), + Row(Video("color_image", title="Head cam"), Teleop(), shares=[1, 1]), + shares=[3, 2], + ), + Chat(), + shares=[5, 4, 3], + ), + shares=[1, 11], +) + + +def _rx(ch: str, encoding: str, delivery: str, max_hz: float, params: dict | None = None) -> dict: + return { + "ch": ch, + "dir": "rx", + "encoding": encoding, + "delivery": delivery, + "maxHz": max_hz, + "params": params or {}, + "publish": "none", + "requiredScope": None, + } + + +def _tx(ch: str, encoding: str, delivery: str, max_hz: float, params: dict | None = None) -> dict: + return {**_rx(ch, encoding, delivery, max_hz, params), "dir": "tx"} + + +MICRODUCK_MANIFEST = { + "version": 1, + "channels": [ + _rx("color_image", "jpeg.v1", "latest", 30.0, {"quality": 75}), + _rx("chase_image", "jpeg.v1", "latest", 30.0, {"quality": 75}), + _rx("global_costmap", "costmap.zlib.v1", "latest", 2.0), + _rx("odom", "pose.json.v1", "reliable", 10.0), + _rx("agent", "chat.json.v1", "reliable", 30.0), + _rx("agent_idle", "flag.json.v1", "reliable", 10.0), + _rx("path", "path.json.v1", "latest", 5.0), + _rx("nav_state", "navstate.json.v1", "reliable", 10.0), + _rx("mode", "mode.json.v1", "reliable", 10.0), + _rx("places", "places.json.v1", "reliable", 2.0), + _rx("policy_state", "policy.json.v1", "reliable", 10.0), + _tx( + "tele_cmd_vel", + "twist.json.v1", + "latest", + 15.0, + {"maxLinear": 0.8, "maxAngular": 1.0, "boost": 2.0, "watchdogMs": 300.0}, + ), + _tx("human_input", "text.json.v1", "reliable", 2.0), + _tx("goal_request", "pose_goal.json.v1", "reliable", 5.0), + _tx("ui_command", "command.json.v1", "reliable", 10.0), + ], + "panels": [ + { + "id": "p0", + "kind": "control", + "title": "Control", + "channels": ["mode", "policy_state", "nav_state", "ui_command"], + "params": { + "mode": "mode", + "policies": "policy_state", + "navState": "nav_state", + "command": "ui_command", + }, + }, + { + "id": "p1", + "kind": "video", + "title": "Chase cam", + "channels": ["chase_image"], + "params": {}, + }, + { + "id": "p2", + "kind": "navmap", + "title": "Nav map", + "channels": [ + "global_costmap", + "odom", + "path", + "places", + "nav_state", + "goal_request", + "ui_command", + ], + "params": { + "costmap": "global_costmap", + "pose": "odom", + "path": "path", + "places": "places", + "navState": "nav_state", + "goal": "goal_request", + "command": "ui_command", + }, + }, + { + "id": "p3", + "kind": "video", + "title": "Head cam", + "channels": ["color_image"], + "params": {}, + }, + {"id": "p4", "kind": "teleop", "title": "", "channels": ["tele_cmd_vel"], "params": {}}, + { + "id": "p5", + "kind": "chat", + "title": "Agent", + "channels": ["agent", "agent_idle", "mode", "human_input"], + "params": { + "chat": "agent", + "idle": "agent_idle", + "mode": "mode", + "input": "human_input", + }, + }, + ], + "layout": { + "col": [ + "p0", + { + "row": [ + "p1", + { + "col": ["p2", {"row": ["p3", "p4"], "shares": [1, 1]}], + "shares": [3, 2], + }, + "p5", + ], + "shares": [5, 4, 3], + }, + ], + "shares": [1, 11], + }, + "pages": [], +} + + +def build_microduck(layout, pages=(), **overrides) -> dict: + kwargs = dict( + registry=MICRODUCK_RX_REGISTRY, + tx_streams=set(MICRODUCK_TX_REGISTRY), + tx_registry=MICRODUCK_TX_REGISTRY, + ) + kwargs.update(overrides) + return build_manifest_data(layout, tuple(pages), **kwargs) + + +def test_microduck_layout_manifest_golden() -> None: + manifest = build_microduck(MICRODUCK_LAYOUT) + assert manifest == MICRODUCK_MANIFEST + # The domain parser (what cockpit() runs) accepts it and normalization + # is idempotent, so the kind rules and the authoring side agree. + assert parse_manifest(manifest).model_dump() == manifest + + +def test_microduck_panel_params_name_bound_channels() -> None: + # Every role -> channel entry points at a channel the panel binds, so the + # web panel can look channels up by role instead of by slot index. + manifest = build_microduck(MICRODUCK_LAYOUT) + for panel in manifest["panels"]: + if panel["kind"] in ("chat", "navmap", "control"): + assert panel["params"] + assert set(panel["params"].values()) == set(panel["channels"]) + + +def test_microduck_layout_matches_golden_fixture_panels() -> None: + # web/shared/fixtures/manifests.json carries the same cockpit as the TS + # side sees it (cockpit_microduck). Panel identity (kind, title, slots, + # role params) and the layout tree must agree; rates/shares/deliveries + # in the fixture are generator-shaped and not compared. + with open(find_web_dir() / "shared" / "fixtures" / "manifests.json") as f: + vectors = json.load(f)["vectors"] + (vector,) = [v for v in vectors if v["name"] == "cockpit_microduck"] + fixture = vector["manifest"] + manifest = build_microduck(MICRODUCK_LAYOUT) + keys = ("id", "kind", "title", "channels", "params") + assert [{k: p[k] for k in keys} for p in manifest["panels"]] == [ + {k: p[k] for k in keys} for p in fixture["panels"] + ] + assert {(c["ch"], c["dir"], c["encoding"]) for c in manifest["channels"]} == { + (c["ch"], c["dir"], c["encoding"]) for c in fixture["channels"] + } + + def strip_shares(node): + if isinstance(node, str): + return node + (key,) = [k for k in node if k in ("row", "col")] + return {key: [strip_shares(c) for c in node[key]]} + + assert strip_shares(manifest["layout"]) == strip_shares(fixture["layout"]) + + +def test_microduck_shared_streams_merge_once() -> None: + # Control + NavMap both bind nav_state and ui_command; Chat + Control + # both bind mode. Identical (dir, encoding, params) requests collapse to + # one channel each, at the max requested rate. + manifest = build_microduck(Row(Control(), NavMap(), Chat())) + names = [c["ch"] for c in manifest["channels"]] + assert len(names) == len(set(names)) + assert {"nav_state", "ui_command", "mode"} <= set(names) + assert [p["channels"].count("nav_state") for p in manifest["panels"]] == [1, 1, 0] + + +def test_microduck_duplicate_panels_merge_to_max_rate() -> None: + class SlowControl(Control): + def _channel_requests(self): + return tuple( + ChannelRequest( + r.stream, r.dir, r.encoding, r.max_hz / 2, r.params, delivery=r.delivery + ) + for r in super()._channel_requests() + ) + + manifest = build_microduck(Row(SlowControl(), Control())) + by_ch = {c["ch"]: c for c in manifest["channels"]} + assert by_ch["mode"]["maxHz"] == 10.0 + assert by_ch["ui_command"]["maxHz"] == 10.0 + assert [p["kind"] for p in manifest["panels"]] == ["control", "control"] + + +@pytest.mark.parametrize( + ("layout", "match"), + [ + # Same stream requested with two encodings inside one manifest. + (Control(nav_state="mode"), "conflicting requirements for stream 'mode'"), + (NavMap(goal="ui_command"), "conflicting requirements for stream 'ui_command'"), + (Row(Chat(mode="nav_state"), Control()), "conflicting requirements for stream 'nav_state'"), + # Stream exists but the bridge encodes it differently. + (Chat(chat="places"), "'places' encodes places.json.v1, not chat.json.v1"), + (NavMap(path="policy_state"), "'policy_state' encodes policy.json.v1, not path.json.v1"), + ( + Control(command="tele_cmd_vel"), + "'tele_cmd_vel' encodes twist.json.v1, not command.json.v1", + ), + (Chat(input="goal_request"), "'goal_request' encodes pose_goal.json.v1, not text.json.v1"), + # Unknown streams. + (Chat(chat="transcript"), "unknown stream 'transcript'"), + (NavMap(goal="goal"), "unknown tx stream 'goal'"), + ], + ids=[ + "control_nav_state_is_mode", + "navmap_goal_is_command", + "chat_mode_vs_control_nav_state", + "chat_chat_is_places", + "navmap_path_is_policy_state", + "control_command_is_twist", + "chat_input_is_goal", + "chat_unknown_rx", + "navmap_unknown_tx", + ], +) +def test_microduck_mismatched_encodings_raise(layout, match) -> None: + with pytest.raises(ValueError, match=match): + build_microduck(layout) + + +def test_microduck_tx_without_channel_table_entry_raises() -> None: + with pytest.raises(ValueError, match="tx stream 'ui_command' has no channel-table entry"): + build_microduck(Control(), tx_registry={"tele_cmd_vel": ("twist.json.v1", "latest")}) + + +def test_microduck_panels_are_keyword_only() -> None: + for panel_type in (Chat, NavMap, Control): + with pytest.raises(TypeError): + panel_type("agent") + + +@pytest.mark.parametrize( + "build", + [ + lambda: Chat(chat=""), + lambda: Chat(input=None), + lambda: NavMap(costmap=""), + lambda: NavMap(command=3), + lambda: Control(policies=""), + lambda: Control(nav_state=""), + ], + ids=[ + "chat_empty_chat", + "chat_none_input", + "navmap_empty_costmap", + "navmap_int_command", + "control_empty_policies", + "control_empty_nav_state", + ], +) +def test_microduck_panel_validation_errors(build) -> None: + with pytest.raises(ValueError): + build() + + +def test_microduck_manifest_pickles() -> None: + manifest = build_microduck(MICRODUCK_LAYOUT) + assert pickle.loads(pickle.dumps(manifest)) == MICRODUCK_MANIFEST + assert pickle.loads(pickle.dumps(MICRODUCK_LAYOUT)).children[0] == Control() + + +def test_tx_channel_defs_unpack_as_wire_triples() -> None: + # cockpit() and the bridge's manifest check both read TX_CHANNELS by + # unpacking each row as (ch, encoding, delivery). The rows carry the tx + # model and rate floor too, so that unpacking is what keeps the richer + # table compatible with both readers. + assert {ch: (encoding, delivery) for ch, encoding, delivery in TX_CHANNELS} == { + "tele_cmd_vel": ("twist.json.v1", "latest"), + "human_input": ("text.json.v1", "reliable"), + "goal_request": ("pose_goal.json.v1", "reliable"), + "ui_command": ("command.json.v1", "reliable"), + } + + +def test_read_only_chat_preserves_manifest_compatibility() -> None: + manifest = build_microduck(Chat(read_only=True)) + panel = manifest["panels"][0] + assert panel["params"]["readOnly"] is True + assert panel["channels"] == ["agent", "agent_idle", "mode", "human_input"] + parse_manifest(manifest) + + +def test_scene_fitting_is_an_additive_navmap_option() -> None: + manifest = build_microduck(NavMap(fit_places=True)) + assert manifest["panels"][0]["params"]["fitPlaces"] is True + parse_manifest(manifest) diff --git a/examples/microduck-world/.gitattributes b/examples/microduck-world/.gitattributes new file mode 100644 index 0000000000..de5780ca97 --- /dev/null +++ b/examples/microduck-world/.gitattributes @@ -0,0 +1,3 @@ +web/public/duck-preview.json filter=lfs diff=lfs merge=lfs -text +web/src/fonts/*.ttf filter=lfs diff=lfs merge=lfs -text +edge/worker-configuration.d.ts linguist-generated diff --git a/examples/microduck-world/.gitignore b/examples/microduck-world/.gitignore new file mode 100644 index 0000000000..be746f14b1 --- /dev/null +++ b/examples/microduck-world/.gitignore @@ -0,0 +1,34 @@ +/.venv +/vendor +/tools +cache/ +tmp/ +state/ +logs/ +config/* +!config/.gitkeep +/assets/microduck +MUJOCO_LOG.TXT +.env + +app/*.egg-info/ +**/__pycache__/ +.ruff_cache/ + +backups/ + +docs/stability-report.md + +/web/node_modules +web/dist/ + +edge/node_modules/ +edge/.wrangler/ +edge/.dev.vars* +edge/public/ +edge/dist/ + +/.venv-perception +._* + +relay/.public.json diff --git a/examples/microduck-world/.python-version b/examples/microduck-world/.python-version new file mode 100644 index 0000000000..f36fa5fd80 --- /dev/null +++ b/examples/microduck-world/.python-version @@ -0,0 +1 @@ +3.12.14 diff --git a/examples/microduck-world/README.md b/examples/microduck-world/README.md new file mode 100644 index 0000000000..da9da80148 --- /dev/null +++ b/examples/microduck-world/README.md @@ -0,0 +1,202 @@ +# Microduck World + +Source snapshot of the hosted 3v3 football demo, imported from application commit +`5ae1f01`. This folder is an independently installed application inside the DimOS +repository. Run its commands from this folder, not the repository root. + +It retains the production-tested framework revision in `dimos-revision.txt` +and the compatibility patch in `patches/`. Its nested `vendor/dimos` checkout +is a dependency, not a second copy committed here. This snapshot does not claim +runtime compatibility with every change on the enclosing PR branch. + +A persistent, headless Microduck world on Omarchy. Open +**https://sim.tule.world** and sign in with GitHub to play or watch. The private +operator gateway remains at **https://omarchy.tailca0707.ts.net:8443** on the tailnet. + +## Enter the world + +All six Microducks are visible together: Red 1 to Red 3 and Blue 1 to Blue 3. +Click an available duck to open its connection dialog. +Both teams spawn along opposite midfield sidelines, just outside the pitch and facing it. +Enter an optional player name in that dialog, then confirm Join. It appears above your duck for other +players and viewers. Use the Names checkbox to hide labels in your own view, +or choose a player from the follow menu. **Follow duck** locks a close camera +behind the duck and turns with its heading. Click and drag to unlock the camera; +click **Follow duck** again to lock it back on. **Watch the match** joins as a +spectator without taking a duck. Availability is live and every duck has one +exclusive controller. Hover or focus a duck card to preview its walking gait; +touch screens have a preview button. This animation runs in the browser. + +Every duck has the same full DimOS blueprint: agent conversation, MCP tools, +camera, map, navigation, exploration, policies and WASD controls. All six start +idle knowing the pitch and both locker-room locations. Their maps, observations, +named discoveries and conversations remain private. The four benchmark rooms +are preserved in a side wing reached through the narrow corridor behind the lockers. + +**Leave duck** releases the slot. A disconnected owner has 60 seconds to reconnect +to the same session. An available duck is absent from physics until claimed; a new +occupant receives fresh private knowledge. All six follow the same rules. The +legacy `?host` shortcut opens the Red 1 connection dialog. + +Public players and spectators authenticate with GitHub. Cloudflare serves the +website and relays its DimOS streams; the shared world and native football +detection camera run on Omarchy. See the +[implementation, validation and launch procedure](docs/public-launch.md). + +## Operate the world + +```bash +cd ~/projects/microduck-world +./service status +./service restart world +./service logs world +./world status +./world --mcp-port 9990 mcp list-tools +``` + +Choose the MCP port for the occupied duck: Red 1 to Red 3 use 9990 to 9992; +Blue 1 to Blue 3 use 9993 to 9995. For example, +`./world --mcp-port 9994 mcp list-tools` targets Blue 2. + +`./service` accepts install/start/stop/restart/status/logs and an optional +world/gateway target. Both services are enabled for boot and restart after failure. +User lingering is enabled, so an SSH login or monitor is not required. +`./world stop` and `./world restart` also use the supervisor when it owns the world. +Do not launch a second world stack. The single world process owns shared physics; +the supervisor launches each robot blueprint in a separate process tree. + +In each duck cockpit, the Control strip exposes the policies available for the selected robot variant. +Select walk for movement; stand deliberately holds position. In Teleop mode, +focus the Teleop panel, then hold WASD after the server acknowledges Teleop mode. Click the map or a room label to +navigate; its cancel button or Escape cancels navigation. Keyboard driving interrupts it. +The main world panel renders locally with Three.js. Drag to orbit, scroll to zoom, +and right-drag to pan. Overview frames the connected world; Follow duck tracks the robot. +Each browser has its own camera. Camera gestures do not command the duck. +The Duck camera also renders locally with Three.js. Both panels show FPS and offer +a Three.js / MuJoCo JPEG selector. The agent still receives its native MuJoCo camera. +The YOLO panel displays that native sensor image with YOLO ball boxes +computed on Omarchy by a shared DimOS perception module. Spectators can select +one occupied duck's public football feed. The image and boxes share a timestamp; +old frames and previous occupants' results are cleared. +The optional world JPEG view uses a shared follow camera capped at 12 FPS; its +server renderer runs only while someone selects it. +Panels can be moved, resized, minimized and maximized. Dark and light themes are available. +Click humancli to request Agent mode; the transcript stays visible in Teleop mode. +See [cockpit behavior and validation](docs/cockpit-ui.md). + +The pitch is 2.6 by 1.6 metres. Duck respawn returns to the midfield sideline. +Individual ball buttons drop each ball from two metres above midfield. +The wall scorer board records authenticated last-touch goals in SQLite across restarts; +own goals do not earn personal credit. + +The Agent is enabled with the authorized OpenAI credential. Select Agent in the +Control strip, then type in the Agent panel. Try "what can you see?" or +"go to the kitchen". Camera observation, room navigation, posture actions and +cancellation have been verified through browser chat. + +The private shell file `config/agent.env` is sourced by `./world`; keep it mode 600 +and restart the world after changing it. It is ignored by Git. Without +OPENAI_API_KEY the simulation still runs and the composer is read-only. + +Physics and live occupancy maps reset on world restart. Named-place memory uses +`state/robots/football-club-v3///`. The scene version prevents old +apartment coordinates from becoming the new prior. Reload retains a session; +leaving and rejoining starts fresh. +Boot registration and process recovery have been tested; a whole-PC reboot has not. + +## Project ownership + +- `app/microduck_world/`: blueprint, cockpit composition, prompt, scene validation, + private gateway, and focused tests. +- `web/`: project Three.js panel and build configuration; imports the pinned cockpit UI. +- `assets/scenes/apartment/`: scene XML, ScenePackage metadata, named places and viewer appearance. +- `assets/scenes/football/`: the connected pitch, goals, nets, scoreboard and turf texture. +- `assets/scenes/benchmark/`: preserved four-room source and translated side-wing scene. +- `assets/microduck/`: downloaded robot models and policies, with upstream licenses. +- `ops/systemd/`: service source; systemd installs registration symlinks in the account. +- `patches/`: the small shared DimOS fixes applied by setup. +- `vendor/dimos/`: independent source checkout pinned by `dimos-revision.txt`. +- `config/`: private runtime settings and TLS files. +- `state/`, `logs/`: persistent data and operational/test evidence. +- `backups/`: private local restore material for the pinned DimOS base. +- `cache/`, `tmp/`, `tools/`, `.venv/`: disposable caches and the local toolchain. + +## Sharing and reproducing this demo + +This is the application repository, built on the pinned DimOS framework. It is a +working hosted demo, not yet a turnkey installer for a fresh machine. The setup +script assumes the Python toolchain, vendor checkout, robot assets and policies +have been provisioned. See `dimos-revision.txt`, `env.sh`, and the setup notes below. + +Private credentials, TLS files, login databases, player state, logs, model downloads +and installed dependencies are excluded from Git. A new deployment needs its own +GitHub OAuth application, Cloudflare resources, runtime configuration and agent key. +The checked-in Cloudflare configuration identifies the existing demo deployment; +replace those resource IDs and domains before deploying your own copy. + +## Setup and checks + +The project toolchain and pinned vendor checkout are already provisioned on Omarchy. +Setup uses the vendor lockfile, installs the application requirements, and checks +dependency compatibility. The optional test extra avoids installing test tools in +a runtime-only setup. + +```bash +./service stop +./setup --test +./service start + +source env.sh +python -m pytest -c pyproject.toml --asyncio-mode=auto app/microduck_world -q +(cd web && deno task check && deno task test && deno task build) +``` + +Setup verifies the vendor revision and applies the project patch only when needed. +It fails on conflicting vendor edits instead of overwriting them. See +[implementation notes](docs/implementation-notes.md) for source restoration. + +The gateway's `/healthz` reports whether a robot is connected to the relay; it is +a readiness check, not proof that every sensor or AI provider works. A browser +opening the world during startup loads the project frontend and reconnects automatically. + +See [tailnet operations](docs/tailnet.md), [public launch](docs/public-launch.md), +the [checkpoint report](docs/overnight-report.md), and [client 3D rendering](docs/client-rendering.md). The connected football room is described in +[football](docs/football.md). See +[football club iteration](docs/football-club.md) for the current scene and +[multiplayer architecture and checks](docs/multiplayer.md) for the earlier checkpoint. + +## Overnight browser checks + +The bounded read-only check uses installed headless Chromium and writes +docs/stability-report.md, JSON checkpoints and hourly screenshots under logs/. +It does not drive the robot or call the agent. + +Run: source env.sh, then python ops/demo_browser_soak.py --hours 6 --interval 300. + +See docs/overnight-report.md for completed work and remaining limitations. + +## Simulation fidelity + +The robot uses the upstream 14-joint MuJoCo model and learned policies. Body mass, +inertia, actuator limits, gravity and contact forces participate in the dynamics; +these parameters have not been calibrated against a physical Microduck here. + +The RGB-D/agent camera and browser Duck camera use the exported `head_camera` mount +on the moving head. The project's `camera.py` converts that CAD site's +forward/left/up axes into MuJoCo camera axes. Camera intrinsics remain the model's +values and are not claimed to be a measured calibration of a physical unit. + +Automatic upright recovery is disabled. The amber **Respawn** button is an explicit +simulation reset: it cancels navigation, returns to Teleop, and places only the +selected duck upright at a clear midfield entrance slot assigned to its team. If all its team +bays are blocked, it waits for a clear bay. Its session, map frame, discoveries +and conversation are preserved. This is not a learned stand-up action or an agent +skill. The learned sit-to-stand behavior remains a separate policy action. + +Sensors remain idealized: exact odometry, rendered depth and additional virtual +range sensors. Landmarks are fixed geometry. Kicks no longer move the ball to the +foot: the duck must approach, align and make physical contact. The football room +adds three free balls, collision goals and direction-aware scoring. See +[football physics and checks](docs/football.md) for dimensions and approximations. +Jaw articulation is not implemented. These are explicit limits on +real-world transfer, not evidence of physical-robot validation. diff --git a/examples/microduck-world/app/microduck_world/ball_detection.py b/examples/microduck-world/app/microduck_world/ball_detection.py new file mode 100644 index 0000000000..16288d12fd --- /dev/null +++ b/examples/microduck-world/app/microduck_world/ball_detection.py @@ -0,0 +1,229 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""One shared DimOS detector for native MuJoCo head-camera observations.""" + +import base64 +import json +import logging +import threading +import time +from collections import OrderedDict +from dataclasses import dataclass +from typing import Any + +import cv2 +from dimos.core.core import rpc +from dimos.core.module import Module, ModuleConfig +from dimos.core.stream import In, Out +from dimos.web.codecs import web_encoder +from microduck_world.robot_io import ROBOT_IDS, RobotVision +from microduck_world.scene import PROJECT_ROOT +from reactivex.disposable import Disposable + +BALL_CAMERA_ENCODING = "ball-camera.json.v1" +BALL_CAMERA_CHANNELS = tuple(f"{robot}_ball_camera" for robot in ROBOT_IDS) +logger = logging.getLogger(__name__) + + +@web_encoder(BALL_CAMERA_ENCODING) +def encode_ball_camera(message: str) -> bytes: + return message.encode("utf-8") + + +@dataclass(frozen=True) +class BallBox: + xyxy: tuple[float, float, float, float] + confidence: float + + +@dataclass(frozen=True) +class FootballObservation: + """Pixels and detections share a timestamp and participant generation.""" + + robot: str + generation: str + timestamp: float + boxes: tuple[BallBox, ...] + + +class BallPerceptionConfig(ModuleConfig): + confidence: float = 0.25 + max_frame_age: float = 1.5 + model_name: str = "yolo11n.pt" + model_path: str = str(PROJECT_ROOT / "cache/models") + device: str | None = None + + +class BallPerception(Module): + config: BallPerceptionConfig + duck1_vision: In[RobotVision] + duck2_vision: In[RobotVision] + duck3_vision: In[RobotVision] + duck4_vision: In[RobotVision] + duck5_vision: In[RobotVision] + duck6_vision: In[RobotVision] + duck1_ball_camera: Out[str] + duck2_ball_camera: Out[str] + duck3_ball_camera: Out[str] + duck4_ball_camera: Out[str] + duck5_ball_camera: Out[str] + duck6_ball_camera: Out[str] + ball_detections: Out[FootballObservation] + + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + self._condition = threading.Condition() + self._pending: OrderedDict[str, RobotVision] = OrderedDict() + self._generations: dict[str, str] = {} + self._stopped = False + self._thread: threading.Thread | None = None + self._detector: Any = None + self._load_error = False + self._last_inference_error = 0.0 + + @rpc + def start(self) -> None: + super().start() + for robot in ROBOT_IDS: + self.register_disposable( + Disposable( + getattr(self, f"{robot}_vision").subscribe( + lambda vision, robot=robot: self.receive(robot, vision) + ) + ) + ) + self._thread = threading.Thread(target=self._run, daemon=True, name="football-perception") + self._thread.start() + + def receive(self, robot: str, vision: RobotVision) -> None: + if robot not in ROBOT_IDS or not vision.generation: + return + with self._condition: + if self._stopped: + return + # Replacement retains the duck's queue position: a faster camera + # cannot starve another duck, and only six RGB-D frames are retained. + self._generations[robot] = vision.generation + self._pending[robot] = vision + self._condition.notify() + + def _run(self) -> None: + try: + from dimos.perception.detection.detectors.yolo import Yolo2DDetector + + self._detector = Yolo2DDetector( + model_path=self.config.model_path, + model_name=self.config.model_name, + device=self.config.device, + ) + except Exception: + self._load_error = True + logger.exception("Football detector could not load") + try: + while True: + with self._condition: + self._condition.wait_for(lambda: self._stopped or bool(self._pending)) + if self._stopped: + break + robot, vision = self._pending.popitem(last=False) + if time.time() - vision.image.ts > self.config.max_frame_age: + continue + try: + payload, observation = self.process(robot, vision) + except Exception: + if time.monotonic() - self._last_inference_error >= 30: + logger.exception("Football detector inference failed") + self._last_inference_error = time.monotonic() + payload, observation = ( + self.camera_payload(robot, vision, (), "unavailable"), + None, + ) + with self._condition: + if self._stopped or self._generations.get(robot) != vision.generation: + continue + if time.time() - vision.image.ts > self.config.max_frame_age: + continue + getattr(self, f"{robot}_ball_camera").publish(json.dumps(payload)) + if observation is not None: + self.ball_detections.publish(observation) + finally: + if self._detector is not None: + self._detector.stop() + + def process( + self, robot: str, vision: RobotVision + ) -> tuple[dict[str, Any], FootballObservation | None]: + if self._load_error or self._detector is None: + return self.camera_payload(robot, vision, (), "unavailable"), None + # Stateless prediction is essential when a single model serves six + # cameras. The detector's persistent single-camera tracker is not used. + results = self._detector.model.predict( + source=vision.image.to_opencv(), + device=self._detector.device, + classes=[32], + conf=self.config.confidence, + iou=0.6, + imgsz=640, + verbose=False, + ) + boxes: list[BallBox] = [] + for result in results: + if result.boxes is None: + continue + for xyxy, confidence, cls in zip( + result.boxes.xyxy.cpu().tolist(), + result.boxes.conf.cpu().tolist(), + result.boxes.cls.cpu().tolist(), + strict=True, + ): + if int(cls) == 32: + boxes.append( + BallBox( + (float(xyxy[0]), float(xyxy[1]), float(xyxy[2]), float(xyxy[3])), + float(confidence), + ) + ) + observation = FootballObservation(robot, vision.generation, vision.image.ts, tuple(boxes)) + return self.camera_payload(robot, vision, observation.boxes, "ready"), observation + + def camera_payload( + self, robot: str, vision: RobotVision, boxes: tuple[BallBox, ...], status: str + ) -> dict[str, Any]: + pixels = vision.image.to_opencv() + ok, jpeg = cv2.imencode(".jpg", pixels, [cv2.IMWRITE_JPEG_QUALITY, 65]) + if not ok: + raise ValueError("Could not encode camera image") + return { + "robot": robot, + "generation": vision.generation, + "ts": vision.image.ts, + "width": pixels.shape[1], + "height": pixels.shape[0], + "status": status, + "image": "data:image/jpeg;base64," + base64.b64encode(jpeg).decode("ascii"), + "boxes": [{"xyxy": b.xyxy, "confidence": b.confidence} for b in boxes], + "model": self.config.model_name, + "class": "sports ball", + } + + @rpc + def stop(self) -> None: + with self._condition: + self._stopped = True + self._pending.clear() + self._condition.notify_all() + if self._thread is not None: + self._thread.join(timeout=15) + super().stop() diff --git a/examples/microduck-world/app/microduck_world/ball_physics.py b/examples/microduck-world/app/microduck_world/ball_physics.py new file mode 100644 index 0000000000..ae67766a4e --- /dev/null +++ b/examples/microduck-world/app/microduck_world/ball_physics.py @@ -0,0 +1,55 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""App-owned Pollen flat-floor physics, independent of shared DimOS constants.""" + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + import mujoco + +BALL_RADIUS = 0.05 +BALL_MASS = 0.03 +BALL_SPAWN_HEIGHT = BALL_RADIUS + 0.001 +BALL_FRICTION = (0.4, 0.01, 0.003) +BALL_SOLREF = (0.03, 0.4) +FLOOR_NAMES = ("floor", "football_floor", "club_floor", "tunnel_floor", "corridor_floor") +FLOOR_FRICTION = (1.0, 0.005, 0.0001) +FLOOR_SOLREF = (0.02, 1.0) +CONTACT_SOLIMP = (0.9, 0.95, 0.001, 0.5, 2.0) + + +def configure_contacts(spec: "mujoco.MjSpec", ball_names: tuple[str, ...]) -> None: + """Configure every physical ball and floor before mass/inertia are compiled. + + Pollen adds a sphere and plane outside the robot's named defaults. Keep + their equal default priority and solmix so MuJoCo combines both surfaces. + Decorative floors and ball paint remain outside this contact configuration. + """ + spec.option.timestep = 0.005 + spec.option.gravity = [0, 0, -9.81] + for name in (*FLOOR_NAMES, *(name + "_geom" for name in ball_names)): + geom = spec.geom(name) + ball = name not in FLOOR_NAMES + geom.friction = BALL_FRICTION if ball else FLOOR_FRICTION + geom.solref = BALL_SOLREF if ball else FLOOR_SOLREF + geom.condim = 6 if ball else 3 + geom.solimp = CONTACT_SOLIMP + geom.priority = 0 + geom.solmix = 1 + geom.margin = 0 + geom.gap = 0 + if ball: + geom.size = [BALL_RADIUS, 0, 0] + geom.mass = BALL_MASS diff --git a/examples/microduck-world/app/microduck_world/blueprints.py b/examples/microduck-world/app/microduck_world/blueprints.py new file mode 100644 index 0000000000..911e76fbd2 --- /dev/null +++ b/examples/microduck-world/app/microduck_world/blueprints.py @@ -0,0 +1,54 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Persistent shared physics and visitor lifecycle; robot stacks run independently.""" + +from dimos.core.coordination.blueprints import autoconnect +from microduck_world.ball_detection import BallPerception +from microduck_world.camera import HEAD_CAMERA +from microduck_world.cockpit import world_cockpit +from microduck_world.robot_io import ROBOT_IDS +from microduck_world.scene import load_world +from microduck_world.supervisor import RobotSupervisor +from microduck_world.world_sim import WorldSimModule + +_package, _scene = load_world() +cockpit_world = ( + autoconnect( + WorldSimModule.blueprint( + scene_xml=_package.mujoco_scene_path, + headless=True, + spawn_xy=_scene.spawn_xy, + camera_name=HEAD_CAMERA, + auto_stand=False, + enable_color=False, + enable_depth=False, + enable_pointcloud=False, + enable_mujoco_lidar=False, + chase_cam=False, + ), + world_cockpit("world"), + RobotSupervisor.blueprint(), + BallPerception.blueprint(), + ) + .remappings( + [ + (WorldSimModule, f"{robot}_{kind}", f"{robot}/hardware_{kind}") + for robot in ROBOT_IDS + for kind in ("command", "state", "vision") + ] + + [(BallPerception, f"{robot}_vision", f"{robot}/hardware_vision") for robot in ROBOT_IDS] + ) + .global_config(robot_model="microduck", viewer="none", n_workers=4) +) diff --git a/examples/microduck-world/app/microduck_world/camera.py b/examples/microduck-world/app/microduck_world/camera.py new file mode 100644 index 0000000000..216a904692 --- /dev/null +++ b/examples/microduck-world/app/microduck_world/camera.py @@ -0,0 +1,40 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Use the robot's exported head-camera mount and its forward/left/up frame.""" + +import mujoco +import numpy as np + +HEAD_CAMERA = "head_camera" + + +def configure_head_camera(spec: mujoco.MjSpec) -> None: + """Convert the CAD camera site frame to MuJoCo's right/up/back camera axes.""" + camera = spec.camera(HEAD_CAMERA) + mount = spec.site(HEAD_CAMERA) + if camera is None or mount is None: + raise ValueError("The robot must provide its head camera and mounting site") + # Site: X forward, Y left, Z up. Camera: X right, Y up, -Z forward. + optical = np.array([0.5, 0.5, -0.5, -0.5]) + rotation = np.empty(4) + mujoco.mju_mulQuat(rotation, np.asarray(mount.quat), optical) + camera.pos = list(mount.pos) + camera.quat = rotation.tolist() + + +def configure_clipping(model: mujoco.MjModel) -> None: + """Keep optical near/far distances metric as the world extent changes.""" + model.vis.map.znear = 0.005 / model.stat.extent + model.vis.map.zfar = 30.0 / model.stat.extent diff --git a/examples/microduck-world/app/microduck_world/cockpit.py b/examples/microduck-world/app/microduck_world/cockpit.py new file mode 100644 index 0000000000..1f7a48767c --- /dev/null +++ b/examples/microduck-world/app/microduck_world/cockpit.py @@ -0,0 +1,216 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Project cockpit composition; transport and panel implementations remain in DimOS.""" + +import os +from dataclasses import dataclass +from typing import Any, ClassVar + +from dimos.core.coordination.blueprints import Blueprint +from dimos.msgs.nav_msgs.Path import Path +from dimos.msgs.sensor_msgs.Image import Image +from dimos.robot.pollen.microduck import web_codecs # noqa: F401 -- register existing encoders +from dimos.web.cockpit import ( + Channel, + ChannelRequest, + Chat, + Col, + Control, + NavMap, + Panel, + Row, + Teleop, + cockpit, +) +from langchain_core.messages.base import BaseMessage +from microduck_world.ball_detection import BALL_CAMERA_CHANNELS, BALL_CAMERA_ENCODING +from microduck_world.comparison import COMPARE_CHANNEL, COMPARE_FPS +from microduck_world.relay import WorldBridge, relay_settings +from microduck_world.roster import ROSTER +from microduck_world.world_sim import WORLD_ENCODING, WORLD_FPS + +HEAD_SIZE = (640, 360) +HEAD_FPS = 6 +JPEG_QUALITY = 40 + +AGENT_ENABLED = bool(os.environ.get("OPENAI_API_KEY")) + + +@dataclass(frozen=True) +class World3D(Panel): + kind: ClassVar[str] = "world3d" + title: str = "Microduck World" + view: str = "world" + robot: str = "duck1" + + def _channel_requests(self) -> tuple[ChannelRequest, ...]: + return (ChannelRequest("world_state", "rx", WORLD_ENCODING, WORLD_FPS, delivery="latest"),) + + def _panel_params(self) -> dict[str, Any]: + jpeg = "color_image" if self.view == "pov" else COMPARE_CHANNEL + return {"view": self.view, "jpeg": jpeg, "robot": self.robot} + + +@dataclass(frozen=True) +class BallCamera(Panel): + kind: ClassVar[str] = "ball-camera" + title: str = "Football camera" + robot: str = "duck1" + + def _channel_requests(self) -> tuple[ChannelRequest, ...]: + return ( + ChannelRequest( + f"{self.robot}_ball_camera", "rx", BALL_CAMERA_ENCODING, 3.0, delivery="latest" + ), + ) + + def _panel_params(self) -> dict[str, Any]: + return {"robot": self.robot} + + +BALL_CHANNELS = tuple( + Channel(name, str, encoding=BALL_CAMERA_ENCODING, delivery="latest", max_hz=3.0) + for name in BALL_CAMERA_CHANNELS +) + +CHANNELS = ( + # Deliberately not bound in panel.channels: the SDK subscribes only while JPEG is selected. + Channel( + COMPARE_CHANNEL, + Image, + encoding="jpeg.v1", + delivery="latest", + max_hz=COMPARE_FPS, + rate_gate=False, + params={"quality": JPEG_QUALITY}, + ), + Channel( + "color_image", + Image, + encoding="jpeg.v1", + delivery="latest", + max_hz=HEAD_FPS, + rate_gate=False, + params={"quality": JPEG_QUALITY}, + ), + Channel( + "agent", + BaseMessage, + encoding="chat.json.v1", + max_hz=30.0, + resend_on_subscribe=True, + rate_gate=False, + replay_depth=200, + ), + Channel( + "agent_idle", + bool, + encoding="flag.json.v1", + max_hz=10.0, + resend_on_subscribe=True, + rate_gate=False, + ), + Channel( + "mode", + str, + encoding="mode.json.v1", + max_hz=10.0, + resend_on_subscribe=True, + rate_gate=False, + ), + Channel( + "policy_state", + str, + encoding="policy.json.v1", + max_hz=10.0, + resend_on_subscribe=True, + rate_gate=False, + ), + Channel( + "world_state", + str, + encoding=WORLD_ENCODING, + delivery="latest", + max_hz=WORLD_FPS, + resend_on_subscribe=True, + ), + Channel( + "path", + Path, + encoding="path.json.v1", + delivery="latest", + max_hz=5.0, + resend_on_subscribe=True, + ), + Channel( + "nav_state", + str, + encoding="navstate.json.v1", + max_hz=10.0, + resend_on_subscribe=True, + rate_gate=False, + ), + Channel( + "places", + str, + encoding="places.json.v1", + max_hz=2.0, + resend_on_subscribe=True, + rate_gate=False, + ), +) + + +def world_cockpit(robot: str = "duck1", generation: str = "") -> Blueprint: + """One full cockpit per robot; the world bridge only serves spectators.""" + observer = robot == "world" + layout = ( + World3D(robot="duck1") + if observer + else Col( + Control(), + Row( + World3D(robot=robot), + Col( + World3D(title="Duck camera", view="pov", robot=robot), + BallCamera(robot=robot), + NavMap(fit_places=True), + Teleop(title="Drive Microduck", mode="mode", max_linear=0.15, max_angular=0.6), + shares=[3, 3, 3, 2], + ), + Chat( + title="Agent" if AGENT_ENABLED else "Agent (API key required)", + read_only=not AGENT_ENABLED, + ), + shares=[6, 3, 3], + ), + ) + ) + channels = ( + tuple(c for c in CHANNELS if c.stream in ("world_state", COMPARE_CHANNEL)) + if observer + else CHANNELS + ) + tuple(c for c in BALL_CHANNELS if observer or c.stream == f"{robot}_ball_camera") + compiled = cockpit(layout=layout, channels=channels).blueprints[0] + result = WorldBridge.blueprint( + **compiled.kwargs, + robot_id="world" if observer else f"{robot}-{generation}", + robot_name="Football world" if observer else ROSTER[robot]["name"], + relay_url=None if observer else relay_settings().upstream_url, + local_port=int(relay_settings().upstream_url.rsplit(":", 1)[1]), + open_browser=False, + web_build=False, + ) + return result.namespace(robot, expose=("world_state", COMPARE_CHANNEL, *BALL_CAMERA_CHANNELS)) diff --git a/examples/microduck-world/app/microduck_world/comparison.py b/examples/microduck-world/app/microduck_world/comparison.py new file mode 100644 index 0000000000..4af5c9a4b4 --- /dev/null +++ b/examples/microduck-world/app/microduck_world/comparison.py @@ -0,0 +1,194 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""One bounded, demand-driven MuJoCo render shared by JPEG comparison viewers.""" + +import math +import threading +import time +from collections.abc import Callable +from copy import copy +from typing import Any + +import mujoco +import numpy as np +import requests +from dimos.msgs.sensor_msgs.Image import Image, ImageFormat +from microduck_world.football import Scoreboard +from numpy.typing import NDArray + +COMPARE_CHANNEL = "world_compare_image" +COMPARE_SIZE = (640, 360) +COMPARE_FPS = 12.0 +COMPARE_OFFSET = (0.9, -1.2, 0.8) +COMPARE_FOV = 42.0 + + +def comparison_requested(stats: Any) -> bool: + """The relay's aggregate subscriptions already handle multiple viewers/disconnects.""" + if not isinstance(stats, dict) or not isinstance(stats.get("perRobot"), dict): + return False + return any( + isinstance(robot, dict) + and isinstance(robot.get("subs"), list) + and COMPARE_CHANNEL in robot["subs"] + for robot in stats["perRobot"].values() + ) + + +def comparison_camera(data: mujoco.MjData, focus_body: int) -> mujoco.MjvCamera: + """Match the Three.js Follow duck view, shared across comparison viewers.""" + x, y, z = COMPARE_OFFSET + camera = mujoco.MjvCamera() + camera.type = mujoco.mjtCamera.mjCAMERA_FREE + camera.lookat[:] = data.xpos[focus_body] + camera.distance = math.sqrt(x * x + y * y + z * z) + camera.azimuth = math.degrees(math.atan2(-y, -x)) + camera.elevation = -math.degrees(math.atan2(z, math.hypot(x, y))) + return camera + + +class JpegComparison: + """Own render data and GL context on a worker; never step or lock the live simulation.""" + + def __init__( + self, model: mujoco.MjModel, stats_url: str, publish: Callable[[Image], None] + ) -> None: + # A private model also isolates visual settings from the agent camera. + self._model = model + self._stats_url = stats_url + self._publish = publish + self._stop = threading.Event() + self._lock = threading.Lock() + self._qpos: tuple[NDArray[np.float64], list[int]] | None = None + self._active = False + self._frames = 0 + self._render_ms = 0.0 + self._error: str | None = None + self._thread = threading.Thread(target=self._run, name="world-jpeg-comparison", daemon=True) + self._thread.start() + + def snapshot(self, qpos: NDArray[np.float64], lit: list[int] | None = None) -> None: + with self._lock: + if self._active: + self._qpos = (qpos.copy(), list(lit or [])) + + def status(self) -> dict[str, Any]: + with self._lock: + return { + "active": self._active, + "frames": self._frames, + "renderMs": round(self._render_ms, 2), + "error": self._error, + } + + def close(self) -> None: + self._stop.set() + self._thread.join(timeout=5.0) + if self._thread.is_alive(): + raise RuntimeError("JPEG comparison renderer did not stop") + + def _run(self) -> None: + renderer: mujoco.Renderer | None = None + model: mujoco.MjModel | None = None + data: mujoco.MjData | None = None + focus = 0 + scoreboard: Scoreboard | None = None + next_probe = 0.0 + next_frame = 0.0 + option = mujoco.MjvOption() + option.geomgroup[:] = [1, 1, 1, 0, 0, 0] + try: + with requests.Session() as http: + http.trust_env = False + while not self._stop.is_set(): + now = time.monotonic() + if now >= next_probe: + wanted = False + try: + response = http.get(self._stats_url, timeout=0.5) + response.raise_for_status() + wanted = comparison_requested(response.json()) + except (requests.RequestException, ValueError): + pass # Fail closed: no relay means no comparison rendering. + with self._lock: + self._active = wanted + if not wanted: + self._qpos = None + next_probe = time.monotonic() + 1.0 + with self._lock: + active = self._active + qpos = self._qpos + if not active: + if renderer is not None: + renderer.close() + renderer = None + data = None + model = None + self._stop.wait(max(0.01, next_probe - time.monotonic())) + continue + if qpos is None or now < next_frame: + self._stop.wait(0.01 if qpos is None else min(0.05, next_frame - now)) + continue + try: + started = time.monotonic() + if renderer is None: + model = copy(self._model) + scoreboard = Scoreboard(model) + model.vis.global_.fovy = COMPARE_FOV + model.vis.global_.offwidth = max( + model.vis.global_.offwidth, COMPARE_SIZE[0] + ) + model.vis.global_.offheight = max( + model.vis.global_.offheight, COMPARE_SIZE[1] + ) + data = mujoco.MjData(model) + focus = mujoco.mj_name2id(model, mujoco.mjtObj.mjOBJ_BODY, "trunk_base") + renderer = mujoco.Renderer( + model, height=COMPARE_SIZE[1], width=COMPARE_SIZE[0] + ) + assert model is not None and data is not None + data.qpos[:] = qpos[0] + assert scoreboard is not None + scoreboard.apply(model, qpos[1]) + mujoco.mj_forward(model, data) + renderer.update_scene( + data, camera=comparison_camera(data, focus), scene_option=option + ) + rgb = renderer.render() + self._publish( + Image( + data=rgb, + format=ImageFormat.RGB, + frame_id="comparison", + ts=time.time(), + ) + ) + with self._lock: + self._frames += 1 + self._render_ms = (time.monotonic() - started) * 1000 + self._error = None + except Exception as exc: + with self._lock: + self._error = str(exc)[:160] + if renderer is not None: + renderer.close() + renderer = None + self._stop.wait(1.0) + next_frame = max(started + 1.0 / COMPARE_FPS, time.monotonic()) + finally: + if renderer is not None: + renderer.close() + with self._lock: + self._active = False diff --git a/examples/microduck-world/app/microduck_world/conftest.py b/examples/microduck-world/app/microduck_world/conftest.py new file mode 100644 index 0000000000..725fa2f695 --- /dev/null +++ b/examples/microduck-world/app/microduck_world/conftest.py @@ -0,0 +1,38 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Hermetic module fixtures: replace only the external RPC boundary.""" + +from unittest.mock import Mock + +import pytest +from dimos.protocol.rpc.zenohrpc import ZenohRPC + + +@pytest.fixture +def module_factory(monkeypatch): + for method in ("__init__", "start", "serve_module_rpc", "stop"): + monkeypatch.setattr(ZenohRPC, method, Mock(return_value=None)) + modules = [] + + def create(cls, **kwargs): + module = cls(rpc_transport=ZenohRPC, **kwargs) + modules.append(module) + return module + + try: + yield create + finally: + for module in reversed(modules): + module.stop() diff --git a/examples/microduck-world/app/microduck_world/connection.py b/examples/microduck-world/app/microduck_world/connection.py new file mode 100644 index 0000000000..b086ae21d4 --- /dev/null +++ b/examples/microduck-world/app/microduck_world/connection.py @@ -0,0 +1,115 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Adapt one virtual robot to standard DimOS sensor and command streams.""" + +from dimos.core.core import rpc +from dimos.core.module import Module, ModuleConfig +from dimos.core.stream import In, Out +from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped +from dimos.msgs.geometry_msgs.Twist import Twist +from dimos.msgs.sensor_msgs.CameraInfo import CameraInfo +from dimos.msgs.sensor_msgs.Image import Image +from dimos.msgs.sensor_msgs.JointState import JointState +from dimos.msgs.sensor_msgs.PointCloud2 import PointCloud2 +from dimos.msgs.tf2_msgs.TFMessage import TFMessage +from microduck_world.robot_io import ( + Observation, + RobotCommand, + RobotState, + RobotVision, + allowed_generation, +) +from reactivex.disposable import Disposable + + +class ConnectionConfig(ModuleConfig): + generation: str + + +class SimRobotConnection(Module): + config: ConnectionConfig + hardware_state: In[RobotState] + hardware_vision: In[RobotVision] + hardware_command: Out[RobotCommand] + cmd_vel: In[Twist] + policy_request: In[str] + respawn_request: In[bool] + ball_drop_request: In[str] + odom: Out[PoseStamped] + joint_state: Out[JointState] + policy_state: Out[str] + color_image: Out[Image] + depth_image: Out[Image] + camera_info: Out[CameraInfo] + pointcloud: Out[PointCloud2] + observation: Out[Observation] + tf: Out[TFMessage] + + @rpc + def start(self) -> None: + super().start() + self.register_disposable(Disposable(self.hardware_state.subscribe(self.receive_state))) + self.register_disposable(Disposable(self.hardware_vision.subscribe(self.receive_vision))) + self.register_disposable(Disposable(self.cmd_vel.subscribe(self.drive))) + self.register_disposable(Disposable(self.policy_request.subscribe(self.request_policy))) + self.register_disposable(Disposable(self.respawn_request.subscribe(self.respawn))) + self.register_disposable(Disposable(self.ball_drop_request.subscribe(self.drop_ball))) + + def receive_state(self, state: RobotState) -> None: + if not allowed_generation(state.generation, self.config.generation): + return + self.odom.publish(state.odom) + self.joint_state.publish(state.joints) + self.policy_state.publish(state.policy) + + def receive_vision(self, vision: RobotVision) -> None: + if not allowed_generation(vision.generation, self.config.generation): + return + self.color_image.publish(vision.image) + self.depth_image.publish(vision.depth) + self.camera_info.publish(vision.camera_info) + self.tf.publish(vision.tf) + self.observation.publish( + Observation(vision.image, vision.depth, vision.camera_info, vision.camera_pose) + ) + self.pointcloud.publish( + PointCloud2.from_numpy(vision.points, frame_id="world", timestamp=vision.image.ts) + ) + + def drive(self, twist: Twist) -> None: + self.hardware_command.publish( + RobotCommand( + self.config.generation, + "twist", + (float(twist.linear.x), float(twist.linear.y), float(twist.angular.z)), + ) + ) + + def request_policy(self, request: str) -> None: + self.hardware_command.publish(RobotCommand(self.config.generation, "policy", request)) + + def drop_ball(self, ball: str) -> None: + self.hardware_command.publish(RobotCommand(self.config.generation, "drop_ball", ball)) + + def respawn(self, requested: bool) -> None: + if requested is True: + self.hardware_command.publish(RobotCommand(self.config.generation, "respawn", "")) + + @rpc + def stop(self) -> None: + try: + self.drive(Twist()) + finally: + super().stop() diff --git a/examples/microduck-world/app/microduck_world/control.py b/examples/microduck-world/app/microduck_world/control.py new file mode 100644 index 0000000000..1ed029df27 --- /dev/null +++ b/examples/microduck-world/app/microduck_world/control.py @@ -0,0 +1,51 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Human-only simulation controls alongside the existing Microduck cockpit router.""" + +import json + +from dimos.core.stream import Out +from dimos.robot.pollen.microduck.control_module import DuckControlModule + + +class WorldControl(DuckControlModule): + respawn_request: Out[bool] + ball_drop_request: Out[str] + + def _on_ui_command(self, raw: str) -> None: + try: + command = json.loads(raw) + except (ValueError, TypeError): + super()._on_ui_command(raw) + return + if isinstance(command, dict) and command.get("name") == "drop_ball": + args = command.get("args", {}) + from microduck_world.football import BALL_NAMES + + if isinstance(args, dict) and set(args) == {"ball"} and args["ball"] in BALL_NAMES: + self.ball_drop_request.publish(args["ball"]) + return + if not isinstance(command, dict) or command.get("name") != "respawn": + super()._on_ui_command(raw) + return + if command.get("args", {}) != {}: + return + # Switching mode also stops the project's frontier explorer. + self._set_mode("teleop") + with self._lock: + self._teleop_active = False + self._last_teleop_time = self._clock() + self._cancel_nav() + self.respawn_request.publish(True) diff --git a/examples/microduck-world/app/microduck_world/exploration.py b/examples/microduck-world/app/microduck_world/exploration.py new file mode 100644 index 0000000000..7d2b91248c --- /dev/null +++ b/examples/microduck-world/app/microduck_world/exploration.py @@ -0,0 +1,59 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Keep stock frontier exploration under this duck's control-mode ownership.""" + +import json +import threading +from typing import Any + +from dimos.agents.annotation import current_skill_context, skill +from dimos.agents.capabilities import CAP_MOVEMENT +from dimos.core.core import rpc +from dimos.core.stream import In +from dimos.navigation.frontier_exploration.wavefront_frontier_goal_selector import ( + WavefrontFrontierExplorer, +) +from reactivex.disposable import Disposable + + +class DuckExplorer(WavefrontFrontierExplorer): + mode: In[str] + + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + self._agent_mode = threading.Event() + + @rpc + def start(self) -> None: + super().start() + self.register_disposable(Disposable(self.mode.subscribe(self._on_mode))) + + def _on_mode(self, raw: str) -> None: + if json.loads(raw).get("mode") == "agent": + self._agent_mode.set() + else: + self._agent_mode.clear() + if self.is_exploration_active(): + self.stop_exploration() + + @skill(uses=[CAP_MOVEMENT], lifecycle="background") + def begin_exploration(self) -> str: + """Explore unknown frontiers in your own measured map. Requires Agent mode. + + Continues until end_exploration or exploration finishes. + """ + if not self._agent_mode.is_set(): + return "Select Agent mode before asking me to explore." + return super().begin_exploration(**{"_mcp_context": current_skill_context()}) diff --git a/examples/microduck-world/app/microduck_world/football.py b/examples/microduck-world/app/microduck_world/football.py new file mode 100644 index 0000000000..950cee110b --- /dev/null +++ b/examples/microduck-world/app/microduck_world/football.py @@ -0,0 +1,280 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Physical footballs, directional goal detection, and the room's scoreboard.""" + +import math +import time +from dataclasses import dataclass +from typing import Any + +import mujoco +import numpy as np +from dimos.robot.pollen.microduck.places import BALL_BODY +from microduck_world.ball_physics import BALL_RADIUS, configure_contacts +from numpy.typing import NDArray + +TEAMS = ("blue", "coral") +BALL_NAMES = (BALL_BODY, "football_ball_1", "football_ball_2", "football_ball_3") +DIGITS = ("abcdef", "bc", "abdeg", "abcdg", "bcfg", "acdfg", "acdefg", "abc", "abcdefg", "abcdfg") +OFF_COLOR = (0.018, 0.025, 0.024) +GROUND_TOLERANCE = 0.003 # MuJoCo soft-contact penetration at the floor. + + +def add_footballs(spec: mujoco.MjSpec) -> None: + """Attach free bodies after the robots so the host remains the first joint.""" + for index, name in enumerate(BALL_NAMES[1:], 1): + spawn = spec.site(f"football_spawn_{index}") + body = spec.worldbody.add_body(name=name, pos=list(spawn.pos)) + body.add_freejoint(name=name + "_freejoint") + body.add_geom( + name=name + "_geom", + type=mujoco.mjtGeom.mjGEOM_SPHERE, + size=[BALL_RADIUS, 0, 0], + rgba=[0.94, 0.95, 0.90, 1], + group=0, + ) + # Thin painted panels follow the same rigid ball; no extra collision or mass. + for patch, direction in enumerate( + ((0, 0, 1), (0, 0, -1), (1, 0, 0), (-1, 0, 0), (0, 1, 0), (0, -1, 0)) + ): + rotation = np.empty(4) + mujoco.mju_quatZ2Vec(rotation, np.asarray(direction, dtype=float)) + body.add_geom( + name=f"{name}_patch_{patch}", + type=mujoco.mjtGeom.mjGEOM_ELLIPSOID, + pos=(np.asarray(direction) * (BALL_RADIUS * (1 - 0.0008 / 0.035))).tolist(), + quat=rotation.tolist(), + size=(np.array([0.009, 0.009, 0.001]) * BALL_RADIUS / 0.035).tolist(), + mass=0, + contype=0, + conaffinity=0, + rgba=[0.035, 0.075, 0.085, 1], + group=1, + ) + # This includes the original benchmark ball created by the framework. + configure_contacts(spec, BALL_NAMES) + + +class Scoreboard: + """Lamp lookup shared by the physics publisher and private native renderers.""" + + def __init__(self, model: mujoco.MjModel) -> None: + self.ids = { + mujoco.mj_id2name(model, mujoco.mjtObj.mjOBJ_GEOM, i): i + for i in range(model.ngeom) + if (mujoco.mj_id2name(model, mujoco.mjtObj.mjOBJ_GEOM, i) or "").startswith("score_") + } + self.on_colors = {i: model.geom_rgba[i, :3].copy() for i in self.ids.values()} + + def lamps(self, scores: list[int]) -> list[int]: + lit: list[int] = [] + for team, score in zip(TEAMS, scores, strict=True): + for digit, character in enumerate(f"{min(score, 999):03}"): + lit.extend( + self.ids[f"score_{team}_{digit}_{segment}"] + for segment in DIGITS[int(character)] + ) + if score > 999: + lit.extend(self.ids[f"score_{team}_overflow_{axis}"] for axis in (0, 1)) + return lit + + def apply(self, model: mujoco.MjModel, lit: list[int]) -> None: + enabled = set(lit) + for i, color in self.on_colors.items(): + model.geom_rgba[i, :3] = color if i in enabled else OFF_COLOR + + +@dataclass +class Goal: + position: NDArray[np.float64] + rotation: NDArray[np.float64] + half_width: float + height: float + scoring_team: int + + def local(self, point: NDArray[np.float64]) -> NDArray[np.float64]: + return (point - self.position) @ self.rotation + + def clear_passage( + self, start: NDArray[np.float64], end: NDArray[np.float64], radius: float + ) -> bool: + """Check the swept sphere's intersection with the rectangular goal plane. + + Between physics samples the centre is interpolated linearly. Extrema of + y/z +/- sqrt(r*r - x*x) test the complete circular cross-section, including + fast diagonal shots and partial crossings, without requiring extra steps. + """ + delta = end - start + if abs(delta[0]) < 1e-12: + points = (start, end) + else: + lo = max(-radius, min(start[0], end[0])) + hi = min(radius, max(start[0], end[0])) + if lo > hi: + return True + xs = [lo, hi] + for slope in delta[1:] / delta[0]: + extreme = radius * slope / math.hypot(1, slope) + xs.extend(x for x in (extreme, -extreme) if lo < x < hi) + points = tuple(start + delta * ((x - start[0]) / delta[0]) for x in xs) + for point in points: + section = math.sqrt(max(0, radius * radius - point[0] * point[0])) + if ( + abs(point[1]) + section > self.half_width + or point[2] - section < -GROUND_TOLERANCE + or point[2] + section > self.height + ): + return False + return True + + +@dataclass +class Crossing: + previous: NDArray[np.float64] + armed: bool + entering: bool = False + + +class FootballMatch: + """Physics-thread-only counter. Scoring never moves a ball or applies a force.""" + + def __init__(self, model: mujoco.MjModel) -> None: + self.scoreboard = Scoreboard(model) + self.scores = [0, 0] + self.lit = self.scoreboard.lamps(self.scores) + self.goals = [] + for index, team in enumerate(TEAMS): + site = model.site("football_goal_" + team) + rotation = np.empty(9) + mujoco.mju_quat2Mat(rotation, site.quat) + self.goals.append( + Goal( + site.pos.copy(), + rotation.reshape(3, 3), + float(site.size[0]), + float(site.size[1]), + 1 - index, + ) + ) + self.balls = { + name: ( + int(model.joint(name + "_freejoint").qposadr[0]), + float(model.geom(name + "_geom").size[0]), + ) + for name in BALL_NAMES + } + self._crossings: dict[tuple[str, int], Crossing] = {} + self.last_goal: dict[str, Any] | None = None + self.ledger = None + self._last_touch = {} + self.drops = {name: 0 for name in self.balls} + + def drop_ball(self, data, ball): + if ball not in self.balls: + return False + center = np.array([0.0, 4.3, 2.0]) + radius = self.balls[ball][1] + for other, (adr, other_radius) in self.balls.items(): + if ( + other != ball + and np.linalg.norm(data.qpos[adr : adr + 3] - center) < radius + other_radius + 0.02 + ): + return False + adr = self.balls[ball][0] + dof = int(data.model.joint(ball + "_freejoint").dofadr[0]) + data.qpos[adr : adr + 7] = [*center, 1, 0, 0, 0] + data.qvel[dof : dof + 6] = 0 + data.qacc_warmstart[dof : dof + 6] = 0 + data.xfrc_applied[data.model.body(ball).id] = 0 + self._last_touch.pop(ball, None) + for key in list(self._crossings): + if key[0] == ball: + del self._crossings[key] + self.drops[ball] += 1 + return True + + def touches(self, data, robots, identities): + owners = {geom: r for r in robots.values() if r.active for geom in r.geoms} + ball_geoms = {data.model.geom(name + "_geom").id: name for name in self.balls} + touched = {} + for contact in data.contact[: data.ncon]: + if contact.dist > 0.003: + continue + a, b = map(int, contact.geom) + ball, robot = ( + (ball_geoms.get(a), owners.get(b)) + if a in ball_geoms + else (ball_geoms.get(b), owners.get(a)) + ) + if ball is None or robot is None: + continue + touched.setdefault(ball, {})[robot.id] = robot + from microduck_world.roster import ROSTER + + for ball, contacts in touched.items(): + self._last_touch[ball] = None + if len(contacts) != 1: + continue + robot = next(iter(contacts.values())) + identity = identities.get(robot.id) + if identity and identity["generation"] == robot.generation: + self._last_touch[ball] = {**identity, "team": ROSTER[robot.id]["team"]} + + def update(self, data: mujoco.MjData) -> None: + for ball, (adr, radius) in self.balls.items(): + for index, goal in enumerate(self.goals): + point = goal.local(data.qpos[adr : adr + 3]) + key = (ball, index) + state = self._crossings.get(key) + if state is None: + self._crossings[key] = Crossing(point.copy(), bool(point[0] <= -radius)) + continue + previous = state.previous + dx = point[0] - previous[0] + if state.armed and dx > 0 and previous[0] <= -radius < point[0]: + state.entering = True + if state.entering and not goal.clear_passage(previous, point, radius): + state.entering = False + if state.armed and dx > 0: + if previous[0] <= radius < point[0]: + if state.entering: + toucher = self._last_touch.pop(ball, None) + team = "blue" if goal.scoring_team == 0 else "red" + scorer = toucher if toucher and toucher["team"] == team else None + if scorer and self.ledger: + self.ledger.record(scorer, time.time()) + self.scores[goal.scoring_team] += 1 + self.lit = self.scoreboard.lamps(self.scores) + self.last_goal = { + "ball": ball, + "scorer": scorer["handle"] if scorer else None, + "team": TEAMS[goal.scoring_team], + "time": float(data.time), + } + state.armed = False + state.entering = False + if point[0] <= -radius: + state.armed = True + state.entering = False + state.previous = point.copy() + + def snapshot(self) -> dict[str, Any]: + return { + "drops": dict(self.drops), + "scores": list(self.scores), + "lit": list(self.lit), + "lastGoal": self.last_goal, + "scorers": list(self.ledger.rows) if self.ledger else [], + } diff --git a/examples/microduck-world/app/microduck_world/gateway.py b/examples/microduck-world/app/microduck_world/gateway.py new file mode 100644 index 0000000000..4b968eb46e --- /dev/null +++ b/examples/microduck-world/app/microduck_world/gateway.py @@ -0,0 +1,252 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Private HTTPS bootstrap and UDP gateway; the DimOS relay stays on loopback.""" + +import argparse +import re +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from ipaddress import IPv4Address, ip_network +from pathlib import Path +from typing import Any +from urllib.parse import urlsplit, urlunsplit + +import httpx +import uvicorn +from microduck_world.udp_forwarder import UdpForwarder +from pydantic import BaseModel, ConfigDict, Field, field_validator +from starlette.applications import Starlette +from starlette.requests import Request +from starlette.responses import FileResponse, HTMLResponse, JSONResponse, Response +from starlette.routing import Route +from starlette.staticfiles import StaticFiles + +STARTING_PAGE = """ + + + +Microduck World is starting +
+

Microduck World is starting

+

Reconnecting shortly. You can leave this page open.

+Try now +
""" + + +def unavailable(request: Request) -> Response: + headers = {"cache-control": "no-store", "retry-after": "3"} + if request.url.path == "/" and "text/html" in request.headers.get("accept", ""): + return HTMLResponse(STARTING_PAGE, status_code=503, headers=headers) + return JSONResponse({"status": "starting"}, status_code=503, headers=headers) + + +class GatewayConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + + bind_host: IPv4Address + port: int = Field(ge=1024, le=65535) + public_origin: str + upstream_url: str + cert_file: Path + key_file: Path + max_peers: int = Field(default=64, ge=1, le=256) + frontend_dir: Path | None = None + world_assets_dir: Path | None = None + + @field_validator("bind_host") + @classmethod + def private_interface(cls, host: IPv4Address) -> IPv4Address: + if not host.is_loopback and host not in ip_network("100.64.0.0/10"): + raise ValueError("Bind only to a Tailscale IPv4 address or loopback") + return host + + @field_validator("public_origin") + @classmethod + def https_origin(cls, value: str) -> str: + parsed = urlsplit(value) + if ( + parsed.scheme != "https" + or not parsed.hostname + or parsed.username + or parsed.path not in ("", "/") + or parsed.query + or parsed.fragment + ): + raise ValueError("public_origin must be an HTTPS origin without a path") + return value.rstrip("/") + + @field_validator("upstream_url") + @classmethod + def local_upstream(cls, value: str) -> str: + parsed = urlsplit(value) + if ( + parsed.scheme != "http" + or parsed.hostname != "127.0.0.1" + or parsed.username + or parsed.path not in ("", "/") + or parsed.query + or parsed.fragment + ): + raise ValueError("upstream_url must be a loopback HTTP origin") + return value.rstrip("/") + + +def rewrite_info( + info: dict[str, Any], config: GatewayConfig, forwarder: UdpForwarder +) -> dict[str, Any]: + """Keep the end-to-end certificate pin while publishing the private UDP endpoint.""" + upstream = urlsplit(info["wtUrl"]) + if upstream.scheme != "https" or upstream.hostname != "127.0.0.1" or not upstream.port: + raise ValueError("Relay advertised a non-loopback QUIC endpoint") + forwarder.set_target((upstream.hostname, upstream.port)) + public = urlsplit(config.public_origin) + return { + **info, + "wtUrl": urlunsplit(("https", public.netloc, upstream.path, upstream.query, "")), + } + + +def create_app(config: GatewayConfig) -> Starlette: + forwarder = UdpForwarder(max_peers=config.max_peers) + frontend = StaticFiles(directory=config.frontend_dir) if config.frontend_dir else None + + @asynccontextmanager + async def lifespan(app: Starlette) -> AsyncIterator[None]: + async with httpx.AsyncClient(timeout=10.0, trust_env=False) as client: + app.state.client = client + await forwarder.start(str(config.bind_host), config.port) + try: + yield + finally: + await forwarder.stop() + + async def serve(request: Request) -> Response: + # Tailnet membership is the access boundary; a foreign website must + # not read bootstrap data or use this as its own cross-origin relay. + if request.headers.get("host") != urlsplit(config.public_origin).netloc: + return Response("Unexpected host", status_code=421) + origin = request.headers.get("origin") + if origin is not None and origin != config.public_origin: + return Response("Origin not allowed", status_code=403) + if request.headers.get("sec-fetch-site") == "cross-site": + return Response("Cross-site access not allowed", status_code=403) + path = request.url.path + if frontend is not None and (path == "/" or path.startswith("/client/")): + file = "index.html" if path == "/" else path.removeprefix("/client/") + result = await frontend.get_response(file, request.scope) + result.headers["x-content-type-options"] = "nosniff" + result.headers["cache-control"] = "no-cache" + return result + if path.startswith("/world-assets/") and config.world_assets_dir is not None: + name = path.removeprefix("/world-assets/") + if re.fullmatch(r"scene-[a-f0-9]{20}\.json", name) is None: + return Response("Unknown world asset", status_code=404) + file_path = config.world_assets_dir / name + headers = { + "cache-control": "public, max-age=31536000, immutable", + "vary": "accept-encoding", + } + if "gzip" in request.headers.get("accept-encoding", ""): + file_path = file_path.with_suffix(".json.gz") + headers["content-encoding"] = "gzip" + if not file_path.is_file(): + return Response("World asset not ready", status_code=404) + headers["x-content-type-options"] = "nosniff" + return FileResponse(file_path, media_type="application/json", headers=headers) + session_info = re.fullmatch(r"/sessions/[a-f0-9-]{72}/api/info", path) is not None + if ( + path not in ("/", "/healthz", "/api/stats", "/api/info", "/api/lobby", "/sdk.js") + and not session_info + ): + return Response("Not found", status_code=404) + if request.method == "POST" and path != "/api/lobby": + return Response("Method not allowed", status_code=405) + body = bytearray() + if request.method == "POST": + async for chunk in request.stream(): + body.extend(chunk) + if len(body) > 1024: + return Response("Request too large", status_code=413) + health = path == "/healthz" + target = httpx.URL(config.upstream_url).copy_with( + path="/api/stats" if health else request.url.path, + query=request.url.query.encode(), + ) + try: + response = await request.app.state.client.request( + request.method, target, content=bytes(body) + ) + except httpx.RequestError: + return unavailable(request) + if health: + try: + stats = response.json() + robots = stats.get("robots") if isinstance(stats, dict) else None + ready = response.is_success and isinstance(robots, list) and len(robots) > 0 + except ValueError: + ready = False + return JSONResponse( + {"status": "ready" if ready else "starting"}, + status_code=200 if ready else 503, + headers={"cache-control": "no-store"}, + ) + if ( + (path == "/api/info" or session_info) + and request.method == "GET" + and response.is_success + ): + try: + info = rewrite_info(response.json(), config, forwarder) + except (ValueError, KeyError, TypeError): + return Response("Invalid relay discovery response", status_code=502) + return JSONResponse(info, headers={"cache-control": "no-store"}) + # Deliberately omit upstream wildcard CORS and content-encoding: + # httpx has already decoded the response body. + headers = { + key: response.headers[key] + for key in ("content-type", "cache-control") + if key in response.headers + } + headers["x-content-type-options"] = "nosniff" + return Response(response.content, status_code=response.status_code, headers=headers) + + return Starlette( + routes=[Route("/{path:path}", serve, methods=["GET", "HEAD", "POST"])], lifespan=lifespan + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", type=Path, required=True) + args = parser.parse_args() + config = GatewayConfig.model_validate_json(args.config.read_text()) + uvicorn.run( + create_app(config), + host=str(config.bind_host), + port=config.port, + ssl_certfile=str(config.cert_file), + ssl_keyfile=str(config.key_file), + access_log=False, + ) + + +if __name__ == "__main__": + main() diff --git a/examples/microduck-world/app/microduck_world/knowledge.py b/examples/microduck-world/app/microduck_world/knowledge.py new file mode 100644 index 0000000000..a044ca394d --- /dev/null +++ b/examples/microduck-world/app/microduck_world/knowledge.py @@ -0,0 +1,221 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Private, observation-backed place and object knowledge for one robot runtime.""" + +import json +import math +import threading +import time +from collections import OrderedDict +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import cv2 +import numpy as np +from dimos.agents.annotation import skill +from dimos.agents.skill_result import SkillResult +from dimos.core.core import rpc +from dimos.core.stream import In +from dimos.msgs.nav_msgs.OccupancyGrid import OccupancyGrid +from dimos.robot.pollen.microduck.skills import ( + MicroduckSkillContainer, + MicroduckSkillContainerConfig, +) +from microduck_world.robot_io import Observation +from reactivex.disposable import Disposable + + +@dataclass(frozen=True) +class CameraView: + id: str + observation: Observation + + def agent_encode(self) -> list[dict[str, Any]]: + image = self.observation.image + return [ + { + "type": "text", + "text": json.dumps( + { + "observation_id": self.id, + "width": self.observation.camera_info.width, + "height": self.observation.camera_info.height, + "coordinates": "Pixels: x from left, y from top.", + } + ), + }, + *image.agent_encode(), + ] + + +class KnowledgeConfig(MicroduckSkillContainerConfig): + knowledge_dir: str + + +def locate_pixel(observation: Observation, x: int, y: int) -> tuple[float, float, float]: + info = observation.camera_info + if not 0 <= x < info.width or not 0 <= y < info.height: + raise ValueError("Pixel is outside this camera image") + depth = float(observation.depth.data[y, x]) + if not math.isfinite(depth) or not 0.05 <= depth <= 6: + raise ValueError("This pixel has no usable measured depth") + point = np.array( + [(x - info.K[2]) * depth / info.K[0], (y - info.K[5]) * depth / info.K[4], depth] + ) + pose = observation.camera_pose + world = pose.orientation.to_rotation_matrix() @ point + pose.position.to_numpy() + return float(world[0]), float(world[1]), float(world[2]) + + +class DuckKnowledge(MicroduckSkillContainer): + config: KnowledgeConfig + observation: In[Observation] + global_costmap: In[OccupancyGrid] + + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + self._observations_lock = threading.Lock() + self._latest: Observation | None = None + self._seen: OrderedDict[str, Observation] = OrderedDict() + self._known_cells = 0 + self._known_area = 0.0 + self._journal: list[dict[str, Any]] = [] + + @rpc + def start(self) -> None: + super().start() + directory = Path(self.config.knowledge_dir) + directory.mkdir(parents=True, exist_ok=True) + path = directory / "observations.json" + if path.exists(): + self._journal = json.loads(path.read_text())[-100:] + self.register_disposable(Disposable(self.observation.subscribe(self._on_observation))) + self.register_disposable(Disposable(self.global_costmap.subscribe(self._on_map))) + + def _on_observation(self, observation: Observation) -> None: + with self._observations_lock: + self._latest = observation + + def _on_map(self, grid: OccupancyGrid) -> None: + with self._observations_lock: + self._known_cells = int(np.count_nonzero(grid.grid >= 0)) + self._known_area = self._known_cells * grid.resolution**2 + + @skill + def observe(self) -> CameraView | SkillResult: + """Look through your own camera. + + Returns an image and observation_id for recording objects or observations. + """ + with self._observations_lock: + observation = self._latest + if observation is None or time.time() - observation.image.ts > 5: + return SkillResult.fail("EXECUTION_TIMEOUT", "No recent camera observation") + id = f"{observation.image.ts:.6f}" + self._seen[id] = observation + while len(self._seen) > 8: + self._seen.popitem(last=False) + return CameraView(id, observation) + + def _seen_observation(self, id: str) -> Observation: + with self._observations_lock: + observation = self._seen.get(id) + if observation is None: + raise ValueError("Unknown observation. Call observe and use its observation_id.") + return observation + + def _record(self, observation: Observation, description: str) -> str: + description = description.strip() + if not description or len(description) > 1000: + raise ValueError("Description must contain 1–1000 characters") + directory = Path(self.config.knowledge_dir) + image_name = f"seen-{observation.image.ts:.6f}.jpg" + if not cv2.imwrite(str(directory / image_name), observation.image.data[:, :, ::-1]): + raise ValueError("Could not save camera evidence") + entry = { + "description": description, + "observed_at": observation.image.ts, + "image": image_name, + "camera_position": observation.camera_pose.position.to_numpy().tolist(), + } + with self._observations_lock: + self._journal = [*self._journal, entry][-100:] + (directory / "observations.json").write_text(json.dumps(self._journal, indent=2)) + return image_name + + @skill + def remember_object( + self, name: str, observation_id: str, pixel_x: int, pixel_y: int, description: str = "" + ) -> str: + """Annotate a visible object using your camera and measured depth. + + Args: + name: Your name for the visible object. + observation_id: ID returned by observe. + pixel_x: Horizontal pixel on the object, measured from the image left edge. + pixel_y: Vertical pixel on the object, measured from the image top edge. + description: What you observed; mention uncertainty when appropriate. + """ + try: + name = name.strip() + if not name or len(name) > 100: + raise ValueError("Object name must contain 1–100 characters") + observation = self._seen_observation(observation_id) + x, y, z = locate_pixel(observation, pixel_x, pixel_y) + image = self._record(observation, description or name) + self._places_memory().add( + name, + x, + y, + kind="object", + metadata={ + "source": "camera-depth", + "observation_id": observation_id, + "description": description, + "image": image, + "z": z, + "pixel": [pixel_x, pixel_y], + }, + ) + self._publish_places() + return f"Remembered {name} at ({x:.2f}, {y:.2f}) from my camera observation." + except ValueError as exc: + return f"Could not annotate object: {exc}" + + @skill + def record_observation(self, observation_id: str, description: str) -> str: + """Remember what you learned from one of your own camera observations. + + Args: + observation_id: ID returned by observe. + description: What you saw or inferred, explicitly noting uncertainty. + """ + try: + self._record(self._seen_observation(observation_id), description) + return "Saved this observation in my private memory." + except ValueError as exc: + return f"Could not remember observation: {exc}" + + @skill + def understanding(self) -> str: + """Summarize your measured map coverage, known places and recorded observations.""" + with self._observations_lock: + coverage = { + "observed_cells": self._known_cells, + "observed_area_m2": round(self._known_area, 2), + } + journal = list(self._journal[-20:]) + return json.dumps({"map": coverage, "places": self.list_places(), "observations": journal}) diff --git a/examples/microduck-world/app/microduck_world/physics_robots.py b/examples/microduck-world/app/microduck_world/physics_robots.py new file mode 100644 index 0000000000..1e79cfe3f2 --- /dev/null +++ b/examples/microduck-world/app/microduck_world/physics_robots.py @@ -0,0 +1,305 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Robot bodies and shared gait execution, owned by the physics thread.""" + +import json +import math +import threading +import time +from dataclasses import dataclass +from typing import Any + +import mujoco +import numpy as np +from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped +from dimos.msgs.sensor_msgs.JointState import JointState +from dimos.robot.pollen.microduck.gait import CONTROL_DT +from dimos.robot.pollen.microduck.policies import FALL_GRAVITY_Z, PolicyBank, PolicyScheduler +from dimos.robot.pollen.microduck.sim_module import ( + POLICY_DECIMATION, + MicroduckSimModuleConfig, + shape_twist, +) +from microduck_world.robot_io import ( + ROBOT_IDS, + VISITOR_IDS, + RobotCommand, + RobotState, +) +from microduck_world.roster import ROSTER +from numpy.typing import NDArray + + +@dataclass +class RobotBody: + id: str + bank: PolicyBank + scheduler: PolicyScheduler + actuators: list[int] + joint_qpos: list[int] + joint_qvel: list[int] + geoms: list[int] + collision: NDArray[np.int32] + origin: tuple[float, float] = (0.0, 0.0) + active: bool = False + generation: str = "" + fallen_since: float | None = None + respawns: int = 0 + + +class WorldRobots: + """Only step() mutates physics; subscribers update bounded mailboxes.""" + + def __init__( + self, + model: mujoco.MjModel, + bank: PolicyBank, + settings: dict[str, Any], + config: MicroduckSimModuleConfig, + ) -> None: + self.model, self.config = model, config + self.roster = settings.get("robots", ROSTER) + self.clearance = float(settings["clearance"]) + self.robots: dict[str, RobotBody] = {} + self._lock = threading.Lock() + self._wanted: dict[str, str] = {} + self._commands: dict[str, tuple[RobotCommand, float]] = {} + self._policies: dict[str, RobotCommand] = {} + self._respawns: dict[str, RobotCommand] = {} + self._lease_at = 0.0 + self._step = 0 + for id in ROBOT_IDS: + prefix = "" if id == "duck1" else id + "_" + scoped = bank.for_robot(model, prefix) + joints = [model.joint(prefix + name) for name in bank.joint_names] + actuators = [int(np.flatnonzero(model.actuator_trnid[:, 0] == j.id)[0]) for j in joints] + root = model.body(prefix + "trunk_base").id + geoms = [] + for i in range(model.ngeom): + ancestor = int(model.geom_bodyid[i]) + while ancestor and ancestor != root: + ancestor = int(model.body_parentid[ancestor]) + if ancestor == root: + geoms.append(i) + self.robots[id] = RobotBody( + id, + scoped, + PolicyScheduler(scoped.availability, scoped.variant), + actuators, + [int(j.qposadr[0]) for j in joints], + [int(j.dofadr[0]) for j in joints], + geoms, + np.array([model.geom_contype[geoms], model.geom_conaffinity[geoms]]), + ) + + def leases(self, occupants: dict[str, str]) -> None: + with self._lock: + self._wanted = {id: gen for id, gen in occupants.items() if id in VISITOR_IDS} + self._lease_at = time.monotonic() + + def command(self, id: str, command: RobotCommand) -> None: + if id not in ROBOT_IDS: + return + with self._lock: + if command.kind == "twist": + if not isinstance(command.value, tuple) or len(command.value) != 3: + return + if not all(math.isfinite(v) for v in command.value): + return + self._commands[id] = (command, time.monotonic()) + elif command.kind == "policy": + self._policies[id] = command + elif command.kind == "respawn": + self._respawns[id] = command + + def _place(self, robot: RobotBody, data: mujoco.MjData, xy: tuple[float, float]) -> None: + adr = robot.bank.root_qpos_adr + data.qpos[adr : adr + 2] = xy + robot.bank.initial_qpos(data) + robot.bank.reset() + data.ctrl[robot.actuators] = robot.bank.default_pose + + def _spawn(self, robot: RobotBody, data: mujoco.MjData, *, reset_origin: bool = True) -> bool: + player = self.roster[robot.id] + candidates = [ + player, + *(p for id, p in self.roster.items() if id != robot.id and p["team"] == player["team"]), + ] + for candidate in candidates: + x, y = candidate["spawn"] + if all( + other is robot + or not other.active + or math.hypot( + data.qpos[other.bank.root_qpos_adr] - x, + data.qpos[other.bank.root_qpos_adr + 1] - y, + ) + >= self.clearance + for other in self.robots.values() + ) and self._clear_spawn(data, x, y): + self._place(robot, data, (x, y)) + yaw = float(candidate["yaw"]) + adr = robot.bank.root_qpos_adr + data.qpos[adr + 3 : adr + 7] = (math.cos(yaw / 2), 0, 0, math.sin(yaw / 2)) + if reset_origin: + # All supplied places use this frame. Knowledge is still private. + robot.origin = (0.0, 0.0) + self.model.geom_contype[robot.geoms] = robot.collision[0] + self.model.geom_conaffinity[robot.geoms] = robot.collision[1] + robot.active = True + return True + return False + + def _clear_spawn(self, data: mujoco.MjData, x: float, y: float) -> bool: + """Reject fixed obstacles and free balls in the footprint above the floor.""" + radius = 0.18 + robot_geoms = {g for r in self.robots.values() for g in r.geoms} + for i in range(self.model.ngeom): + if i in robot_geoms or not ( + self.model.geom_contype[i] or self.model.geom_conaffinity[i] + ): + continue + position = data.geom_xpos[i] + if self.model.geom_type[i] == mujoco.mjtGeom.mjGEOM_BOX: + rotation = data.geom_xmat[i].reshape(3, 3) + extent = np.abs(rotation) @ self.model.geom_size[i] + if position[2] + extent[2] <= 0.035 or position[2] - extent[2] > 0.35: + continue + dx = max(abs(x - position[0]) - extent[0], 0) + dy = max(abs(y - position[1]) - extent[1], 0) + if math.hypot(dx, dy) < radius: + return False + elif self.model.geom_type[i] == mujoco.mjtGeom.mjGEOM_SPHERE: + size = float(self.model.geom_size[i, 0]) + if position[2] + size > 0.035 and position[2] - size < 0.35: + if math.hypot(x - position[0], y - position[1]) < radius + size: + return False + return True + + def step(self, data: mujoco.MjData) -> None: + now = time.monotonic() + with self._lock: + wanted = dict(self._wanted) if now - self._lease_at < 3 else {} + commands = dict(self._commands) + policies, self._policies = self._policies, {} + respawns, self._respawns = self._respawns, {} + for index, robot in enumerate(self.robots.values()): + generation = wanted.get(robot.id, "") + if generation != robot.generation or self._step == 0: + robot.active = False + robot.generation = generation + robot.fallen_since = None + robot.respawns = 0 + robot.scheduler = PolicyScheduler(robot.bank.availability, robot.bank.variant) + robot.bank.reset() + self.model.geom_contype[robot.geoms] = 0 + self.model.geom_conaffinity[robot.geoms] = 0 + if generation and not robot.active: + self._spawn(robot, data) + if not robot.active: + self._place(robot, data, (10 + index, 10)) + data.qpos[robot.bank.root_qpos_adr + 2] = -5 + continue + respawn = respawns.get(robot.id) + if respawn and respawn.generation == generation: + if self._spawn(robot, data, reset_origin=False): + robot.scheduler = PolicyScheduler(robot.bank.availability, robot.bank.variant) + robot.fallen_since = None + robot.respawns += 1 + # Old queued commands must not restart motion after a reset. + commands.pop(robot.id, None) + policies.pop(robot.id, None) + with self._lock: + self._commands.pop(robot.id, None) + self._policies.pop(robot.id, None) + else: + # Keep an explicit reset pending until its own team has space. + with self._lock: + self._respawns[robot.id] = respawn + self._commands.pop(robot.id, None) + commands.pop(robot.id, None) + request = policies.get(robot.id) + if request and request.generation == generation and isinstance(request.value, str): + try: + payload = json.loads(request.value) + if isinstance(payload, dict): + robot.scheduler.request(payload.get("policy"), payload.get("action", "")) + except (ValueError, TypeError): + pass + if self._step % POLICY_DECIMATION: + continue + twist = (0.0, 0.0, 0.0) + current = commands.get(robot.id) + if ( + current + and current[0].generation == generation + and now - current[1] < self.config.cmd_timeout + ): + value = current[0].value + if isinstance(value, tuple): + twist = value + robot.scheduler.set_twist(*shape_twist(self.config, *twist)) + if not robot.scheduler.suspend_fall_detector: + gravity = float(robot.bank.projected_gravity(data)[2]) + if gravity > FALL_GRAVITY_Z: + if robot.fallen_since is None: + robot.fallen_since = now + robot.scheduler.notify_fall( + now - robot.fallen_since > self.config.auto_stand_after + ) + else: + robot.fallen_since = None + robot.scheduler.notify_fall(False) + name, command = robot.scheduler.tick(CONTROL_DT) + data.ctrl[robot.actuators] = robot.bank.step(name, command, data) + self._step += 1 + + def states(self, data: mujoco.MjData) -> dict[str, RobotState]: + result = {} + now = time.time() + for robot in self.robots.values(): + if not robot.active: + continue + adr = robot.bank.root_qpos_adr + x, y, z, w, qx, qy, qz = data.qpos[adr : adr + 7] + result[robot.id] = RobotState( + robot.generation, + PoseStamped( + position=(x - robot.origin[0], y - robot.origin[1], z), + orientation=(qx, qy, qz, w), + frame_id="world", + ts=now, + ), + JointState( + name=list(robot.bank.joint_names), + position=data.qpos[robot.joint_qpos].tolist(), + velocity=data.qvel[robot.joint_qvel].tolist(), + ts=now, + ), + json.dumps( + {**robot.scheduler.snapshot(), "respawns": robot.respawns}, + separators=(",", ":"), + ), + ) + return result + + def snapshot(self) -> list[dict[str, Any]]: + return [ + {"id": r.id, "active": r.active, "generation": r.generation} + for r in self.robots.values() + ] + + def sensor_assignments(self) -> dict[str, tuple[str, tuple[float, float]]]: + return {r.id: (r.generation, r.origin) for r in self.robots.values() if r.active} diff --git a/examples/microduck-world/app/microduck_world/relay.py b/examples/microduck-world/app/microduck_world/relay.py new file mode 100644 index 0000000000..381d08749b --- /dev/null +++ b/examples/microduck-world/app/microduck_world/relay.py @@ -0,0 +1,107 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Project relay policy and discovery, using the existing bridge and transport.""" + +import asyncio +from dataclasses import replace +from pathlib import Path +from typing import Any, Literal + +import requests +from dimos.core.stream import In +from dimos.msgs.nav_msgs.Path import Path as NavPath +from dimos.msgs.sensor_msgs.Image import Image +from dimos.web.relay_bridge.protocol import Tx +from dimos.web.relay_bridge.relay_bridge_module import RelayBridgeModule, _Session +from dimos.web.relay_bridge.relay_process import RelayProcess +from langchain_core.messages.base import BaseMessage +from microduck_world.gateway import GatewayConfig +from microduck_world.scene import PROJECT_ROOT +from pydantic import BaseModel, ConfigDict, Field + + +def relay_settings() -> GatewayConfig: + return GatewayConfig.model_validate_json((PROJECT_ROOT / "config/tailnet.json").read_text()) + + +def private_relay_get(path: str) -> dict[str, Any]: + with requests.Session() as client: + client.trust_env = False + response = client.get(relay_settings().upstream_url + path, timeout=1) + response.raise_for_status() + value = response.json() + if not isinstance(value, dict): + raise ValueError("Invalid private relay response") + return value + + +class WorldCommand(BaseModel): + model_config = ConfigDict(strict=True, allow_inf_nan=False) + name: Literal["set_mode", "policy", "cancel_nav", "respawn", "drop_ball"] + args: dict[str, Any] = Field(default_factory=dict) + + +class WorldBridge(RelayBridgeModule): + world_state: In[str] + duck1_ball_camera: In[str] + duck2_ball_camera: In[str] + duck3_ball_camera: In[str] + duck4_ball_camera: In[str] + duck5_ball_camera: In[str] + duck6_ball_camera: In[str] + world_compare_image: In[Image] + agent: In[BaseMessage] + agent_idle: In[bool] + mode: In[str] + policy_state: In[str] + path: In[NavPath] + nav_state: In[str] + places: In[str] + + def _on_wire_tx(self, msg: Tx) -> None: + # Extend this project's command vocabulary while retaining the stock + # manifest, sequence, rate-limit, validation and publishing path. + handler = self._tx_defs.get(msg.ch) + if msg.ch == "ui_command" and handler is not None and handler.model is not WorldCommand: + self._tx_defs[msg.ch] = replace(handler, model=WorldCommand) + super()._on_wire_tx(msg) + + def _spawn_relay(self, open_browser: bool, serve_dir: Path | None) -> str: + self._relay = RelayProcess( + port=self.config.local_port, + web_dir=PROJECT_ROOT / "relay", + cockpit_dir=PROJECT_ROOT / "vendor/dimos/web/cockpit/dist", + sdk_dir=PROJECT_ROOT / "vendor/dimos/web/sdk/dist", + entrypoint=PROJECT_ROOT / "relay/main.ts", + ) + return self._relay.start().wt_url + + async def _connect_and_hello(self) -> _Session: + # Refresh discovery on every reconnect: the supervised relay's QUIC + # port and private registration credential rotate after a restart. + for attempt in range(60): + try: + info = await asyncio.to_thread(private_relay_get, "/internal/robot-info") + break + except requests.RequestException: + if attempt == 59: + raise + await asyncio.sleep(0.5) + previous = self._url + try: + self._url = info["wtUrl"] + return await super()._connect_and_hello() + finally: + self._url = previous diff --git a/examples/microduck-world/app/microduck_world/robot_blueprint.py b/examples/microduck-world/app/microduck_world/robot_blueprint.py new file mode 100644 index 0000000000..b822537861 --- /dev/null +++ b/examples/microduck-world/app/microduck_world/robot_blueprint.py @@ -0,0 +1,131 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The same independently runnable DimOS robot composition for all six ducks.""" + +import os +from uuid import UUID + +from dimos.agents.mcp.mcp_client import McpClient +from dimos.core.coordination.blueprints import Blueprint, autoconnect +from dimos.mapping.costmapper import CostMapper +from dimos.mapping.pointclouds.occupancy import HeightCostConfig +from dimos.mapping.voxels.module import VoxelGridMapper +from dimos.navigation.replanning_a_star.module import ReplanningAStarPlanner +from dimos.robot.pollen.microduck.config import MICRODUCK +from microduck_world.cockpit import world_cockpit +from microduck_world.connection import SimRobotConnection +from microduck_world.control import WorldControl +from microduck_world.exploration import DuckExplorer +from microduck_world.knowledge import DuckKnowledge +from microduck_world.robot_io import ROBOT_IDS +from microduck_world.robot_mcp import RobotMcpServer +from microduck_world.roster import ROSTER +from microduck_world.scene import PROJECT_ROOT, load_world + + +def robot_blueprint(robot: str, generation: str) -> Blueprint: + if robot not in ROBOT_IDS: + raise ValueError("Unknown robot") + UUID(generation) + player = ROSTER[robot] + port = int(player["mcp_port"]) + int(os.environ.get("MICRODUCK_MCP_PORT_OFFSET", "0")) + if not 1024 <= port <= 65535: + raise ValueError("MCP port offset is outside the allowed port range") + scene = load_world()[1] + directory = PROJECT_ROOT / "state/robots" / scene.id / robot / generation + prompt = f"""You operate {player["name"]}, a small biped on the {player["team"]} football team. +You have your own camera, range sensors, map, places and conversation. +Only use knowledge from your own tools, observations and the user's instructions. +The full-world browser view and other ducks' data are unavailable to you. +Start idle. Do not explore or move until instructed. Agent mode enables navigation. +Use observe for visual questions. Record useful findings with record_observation; +use remember_object with the observation ID and a pixel on the visible object. +Positions for object annotations come from your measured depth, not guesses. +Use remember_place to name your current location. Query understanding and list_places +to recall what you have learned. Rooms and object names appearing in tool examples +are examples, not evidence that those places exist here. +Use begin_exploration and end_exploration to discover your own map's frontiers. +Kicks only perform the physical motion: approach and align with a ball first. +A kick does not place or attract a ball. Never claim a goal without observing it. +Move slowly. Use movement tools one at a time; stop exploration before navigating +to a named place or performing a trick. Use list_policies for available actions. +Never pretend to see around an obstacle or infer exact unobserved room boundaries. +Distinguish observations from hypotheses. Keep replies brief. +""" + prompt += ( + "You know the pitch and both locker-room locations as supplied prior knowledge. " + "Your occupancy map and other discoveries come from your own sensors. " + "Both teams play free football, with no kickoff, timer or automatic resets. " + "Blue defends the west goal; red defends the east goal. " + "Teammates do not share their private observations or tools.\n" + ) + modules = [ + SimRobotConnection.blueprint(generation=generation), + VoxelGridMapper.blueprint(emit_every=1, voxel_size=0.03, device="CPU:0"), + CostMapper.blueprint( + config=HeightCostConfig( + resolution=0.03, can_pass_under=MICRODUCK.height_clearance + 0.05, can_climb=0.03 + ), + initial_safe_radius_meters=MICRODUCK.width_clearance + 0.15, + ), + ReplanningAStarPlanner.blueprint( + robot_width=MICRODUCK.width_clearance, + robot_rotation_diameter=MICRODUCK.rotation_diameter, + stuck_time_window=10.0, + stuck_threshold=0.15, + ), + WorldControl.blueprint(), + DuckExplorer.blueprint( + min_frontier_perimeter=0.15, + safe_distance=0.6, + lookahead_distance=2.0, + max_explored_distance=6.0, + goal_timeout=45.0, + ), + DuckKnowledge.blueprint( + rooms={name: scene.rooms[name] for name in ("football", "red_lockers", "blue_lockers")}, + objects={}, + scene=scene.id, + places_db=str(directory / "places.db"), + knowledge_dir=str(directory / "knowledge"), + ), + RobotMcpServer.blueprint(robot=robot, port=port), + ] + if os.environ.get("OPENAI_API_KEY"): + modules.append( + McpClient.blueprint( + system_prompt=prompt, + mcp_server_url=f"http://127.0.0.1:{port}/mcp", + trace_dir=directory / "agent-traces", + ) + ) + stack = ( + autoconnect(*modules) + .remappings( + [ + (VoxelGridMapper, "lidar", "pointcloud"), + ] + ) + .namespace(robot) + ) + return autoconnect(stack, world_cockpit(robot, generation)).global_config( + robot_model="microduck", + nerf_speed=0.5, + viewer="none", + n_workers=4, + mcp_port=port, + tool_stream_topic=f"/{robot}/{generation}/tool_streams", + listen_host="127.0.0.1", + ) diff --git a/examples/microduck-world/app/microduck_world/robot_io.py b/examples/microduck-world/app/microduck_world/robot_io.py new file mode 100644 index 0000000000..ff7871c7be --- /dev/null +++ b/examples/microduck-world/app/microduck_world/robot_io.py @@ -0,0 +1,69 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Typed virtual-hardware boundary; no scene or other-robot state crosses it.""" + +from dataclasses import dataclass +from typing import Literal + +import numpy as np +from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped +from dimos.msgs.sensor_msgs.CameraInfo import CameraInfo +from dimos.msgs.sensor_msgs.Image import Image +from dimos.msgs.sensor_msgs.JointState import JointState +from dimos.msgs.tf2_msgs.TFMessage import TFMessage +from microduck_world.roster import ROBOT_IDS +from numpy.typing import NDArray + +VISITOR_IDS = ROBOT_IDS + + +@dataclass(frozen=True) +class RobotCommand: + generation: str + kind: Literal["twist", "policy", "respawn", "drop_ball"] + value: tuple[float, float, float] | str + + +@dataclass(frozen=True) +class RobotState: + generation: str + odom: PoseStamped + joints: JointState + policy: str + + +@dataclass(frozen=True) +class RobotVision: + generation: str + image: Image + depth: Image + camera_info: CameraInfo + camera_pose: PoseStamped + tf: TFMessage + points: NDArray[np.float32] + + +@dataclass(frozen=True) +class Observation: + """A synchronized RGB-D observation in this robot's own map frame.""" + + image: Image + depth: Image + camera_info: CameraInfo + camera_pose: PoseStamped + + +def allowed_generation(actual: str, expected: str) -> bool: + return bool(expected) and actual == expected diff --git a/examples/microduck-world/app/microduck_world/robot_mcp.py b/examples/microduck-world/app/microduck_world/robot_mcp.py new file mode 100644 index 0000000000..a7d69d52bd --- /dev/null +++ b/examples/microduck-world/app/microduck_world/robot_mcp.py @@ -0,0 +1,43 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""One stock MCP server per robot process, with an explicit local tool boundary.""" + +from dimos.agents.mcp.mcp_server import McpServer +from dimos.core.core import rpc +from dimos.core.module import ModuleConfig +from dimos.core.rpc_client import RPCClient + + +class RobotMcpConfig(ModuleConfig): + robot: str + port: int + + +class RobotMcpServer(McpServer): + config: RobotMcpConfig + dedicated_worker = True + + def _start_server(self, port: int | None = None) -> None: + super()._start_server(self.config.port) + + @rpc + def on_system_modules(self, modules: list[RPCClient]) -> None: + prefix = self.config.robot + "/" + own = [module for module in modules if module.remote_name.startswith(prefix)] + # The framework's generic agent_send skill targets a global human_input + # topic. Each browser already has its own scoped input, so do not expose + # the server's operator/introspection skills to robot agents. + own = [module for module in own if not issubclass(module.actor_class, McpServer)] + super().on_system_modules(own) diff --git a/examples/microduck-world/app/microduck_world/roster.py b/examples/microduck-world/app/microduck_world/roster.py new file mode 100644 index 0000000000..69d59969b9 --- /dev/null +++ b/examples/microduck-world/app/microduck_world/roster.py @@ -0,0 +1,33 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shared player roster, also consumed by the lobby and browser.""" + +import json + +from microduck_world.scene import PROJECT_ROOT + +SETTINGS = json.loads((PROJECT_ROOT / "assets/scenes/apartment/multiplayer.json").read_text()) +ROSTER = SETTINGS["robots"] +ROBOT_IDS = tuple(ROSTER) + +if len(ROBOT_IDS) != 6 or any( + sum(player["team"] == team for player in ROSTER.values()) != 3 for team in ("red", "blue") +): + raise ValueError("The football roster requires three red and three blue players") + + +def prefix_for(robot: str) -> str: + """The engine's first body remains unprefixed; player permissions are equal.""" + return "" if robot == ROBOT_IDS[0] else robot + "_" diff --git a/examples/microduck-world/app/microduck_world/run_robot.py b/examples/microduck-world/app/microduck_world/run_robot.py new file mode 100644 index 0000000000..3e7f0e62fd --- /dev/null +++ b/examples/microduck-world/app/microduck_world/run_robot.py @@ -0,0 +1,52 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Run one robot blueprint against the persistent world.""" + +import argparse +import faulthandler +import signal +import threading + +from dimos.core.coordination.module_coordinator import ModuleCoordinator +from microduck_world.robot_blueprint import robot_blueprint +from microduck_world.robot_io import ROBOT_IDS + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--robot", choices=ROBOT_IDS, required=True) + parser.add_argument("--generation", required=True) + args = parser.parse_args() + stopped = threading.Event() + signal.signal(signal.SIGTERM, lambda *_: stopped.set()) + signal.signal(signal.SIGINT, lambda *_: stopped.set()) + # A missing RPC discovery reply can stall a startup. Save the blocked stack + # and exit so the supervisor can replace this process and its worker group. + # Hosted assets are already installed; a normal build takes a few seconds. + faulthandler.dump_traceback_later(60, exit=True) + try: + coordinator = ModuleCoordinator.build(robot_blueprint(args.robot, args.generation)) + finally: + faulthandler.cancel_dump_traceback_later() + try: + # The world owns the operator coordinator endpoint. Robot modules keep + # their own namespaced RPC endpoints and private MCP server. + stopped.wait() + finally: + coordinator.stop() + + +if __name__ == "__main__": + main() diff --git a/examples/microduck-world/app/microduck_world/scene.py b/examples/microduck-world/app/microduck_world/scene.py new file mode 100644 index 0000000000..4acdf915ed --- /dev/null +++ b/examples/microduck-world/app/microduck_world/scene.py @@ -0,0 +1,65 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Load project scene artifacts and named navigation locations.""" + +from pathlib import Path +from typing import Self + +from dimos.robot.pollen.microduck.places import RoomSpec +from dimos.simulation.scene_assets.spec import ScenePackage, load_scene_package +from pydantic import BaseModel, ConfigDict, Field, model_validator + +PROJECT_ROOT = Path(__file__).resolve().parents[2] + + +class WorldScene(BaseModel): + model_config = ConfigDict(extra="forbid", allow_inf_nan=False, str_strip_whitespace=True) + + id: str = Field(min_length=1) + spawn_xy: tuple[float, float] + rooms: dict[str, RoomSpec] + objects: dict[str, tuple[float, float]] + + @model_validator(mode="after") + def validate_places(self) -> Self: + names: set[str] = set() + for key, room in self.rooms.items(): + if key != room.name: + raise ValueError(f"Room key {key!r} must match its name {room.name!r}") + xmin, xmax, ymin, ymax = room.bounds + if xmin >= xmax or ymin >= ymax: + raise ValueError(f"Room {key!r} must have increasing x/y bounds") + if not room.contains(room.target[0], room.target[1]): + raise ValueError(f"Room {key!r} navigation target must be inside its bounds") + for name in (room.name, *room.aliases): + normalized = name.strip().casefold() + if not normalized or normalized in names: + raise ValueError(f"Place name or alias is empty or ambiguous: {name!r}") + names.add(normalized) + for name in self.objects: + normalized = name.strip().casefold() + if not normalized or normalized in names: + raise ValueError(f"Place name is empty or ambiguous: {name!r}") + names.add(normalized) + return self + + +def load_world() -> tuple[ScenePackage, WorldScene]: + package = load_scene_package(PROJECT_ROOT / "assets/scenes/apartment/scene.meta.json") + if package.mujoco_scene_path is None or package.objects_path is None: + raise ValueError("World scene requires MuJoCo geometry and place metadata") + if not package.mujoco_scene_path.is_file(): + raise FileNotFoundError(package.mujoco_scene_path) + return package, WorldScene.model_validate_json(package.objects_path.read_text()) diff --git a/examples/microduck-world/app/microduck_world/scorers.py b/examples/microduck-world/app/microduck_world/scorers.py new file mode 100644 index 0000000000..6875bc28f8 --- /dev/null +++ b/examples/microduck-world/app/microduck_world/scorers.py @@ -0,0 +1,63 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Persistent scorer totals. SQLite writes run on the lobby thread, not physics.""" + +import queue +import sqlite3 +from pathlib import Path +from uuid import uuid4 + + +class ScorerLedger: + def __init__(self, path: Path): + self.path = path + path.parent.mkdir(parents=True, exist_ok=True) + self.pending = queue.SimpleQueue() + with sqlite3.connect(path) as db: + db.execute( + "CREATE TABLE IF NOT EXISTS goals " + "(id TEXT PRIMARY KEY, user_id TEXT, handle TEXT, scored_at REAL)" + ) + self.rows = self._read(db) + + @staticmethod + def _read(db): + return [ + dict(handle=handle, goals=count) + for handle, count in db.execute( + "SELECT (SELECT handle FROM goals newer WHERE newer.user_id=g.user_id " + "ORDER BY scored_at DESC, rowid DESC LIMIT 1), COUNT(*) " + "FROM goals g GROUP BY user_id ORDER BY COUNT(*) DESC, g.user_id LIMIT 20" + ) + ] + + def record(self, identity, scored_at): + self.pending.put((str(uuid4()), identity["userId"], identity["handle"], scored_at)) + + def flush(self): + batch = [] + while not self.pending.empty(): + batch.append(self.pending.get_nowait()) + if not batch: + return + try: + with sqlite3.connect(self.path) as db: + db.executemany("INSERT OR IGNORE INTO goals VALUES (?,?,?,?)", batch) + rows = self._read(db) + self.rows = rows + except Exception: + for item in batch: + self.pending.put(item) + raise diff --git a/examples/microduck-world/app/microduck_world/sensors.py b/examples/microduck-world/app/microduck_world/sensors.py new file mode 100644 index 0000000000..673fd83fa0 --- /dev/null +++ b/examples/microduck-world/app/microduck_world/sensors.py @@ -0,0 +1,211 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Render synchronized, occluded measurements for each robot on a private scene copy.""" + +import math +import threading +import time +from collections.abc import Callable +from copy import copy + +import mujoco +import numpy as np +from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped +from dimos.msgs.geometry_msgs.Quaternion import Quaternion +from dimos.msgs.geometry_msgs.Transform import Transform +from dimos.msgs.geometry_msgs.Vector3 import Vector3 +from dimos.msgs.sensor_msgs.CameraInfo import CameraInfo +from dimos.msgs.sensor_msgs.Image import Image, ImageFormat +from dimos.msgs.tf2_msgs.TFMessage import TFMessage +from dimos.robot.pollen.microduck.sim_module import LIDAR_CAMERA_SPECS +from dimos.simulation.engines.mujoco_engine import camera_ray_directions +from dimos.utils.logging_config import setup_logger +from microduck_world.camera import HEAD_CAMERA +from microduck_world.football import Scoreboard +from microduck_world.robot_io import RobotVision +from numpy.typing import NDArray + +logger = setup_logger() +SENSOR_SIZE = (640, 360) +SENSOR_FPS = 3.0 +LIDAR_SIZE = (96, 48) + + +class RobotSensors: + def __init__(self, model: mujoco.MjModel, publish: Callable[[str, RobotVision], None]) -> None: + self._source = model + self._publish = publish + self._lock = threading.Lock() + self._stop = threading.Event() + self._snapshot: ( + tuple[NDArray[np.float64], dict[str, tuple[str, tuple[float, float]]], list[int]] | None + ) = None + self._thread = threading.Thread(target=self._run, name="robot-sensors", daemon=True) + self._error: str | None = None + + @property + def error(self) -> str | None: + with self._lock: + return self._error + + def start(self) -> None: + self._thread.start() + + def snapshot( + self, + qpos: NDArray[np.float64], + assignments: dict[str, tuple[str, tuple[float, float]]], + lit: list[int] | None = None, + ) -> None: + with self._lock: + self._snapshot = (qpos.copy(), dict(assignments), list(lit or [])) + + def close(self) -> None: + self._stop.set() + self._thread.join(timeout=10) + if self._thread.is_alive(): + raise RuntimeError("Robot sensor renderer did not stop") + + def _run(self) -> None: + renderer: mujoco.Renderer | None = None + try: + model = copy(self._source) + scoreboard = Scoreboard(model) + data = mujoco.MjData(model) + renderer = mujoco.Renderer(model, height=SENSOR_SIZE[1], width=SENSOR_SIZE[0]) + option = mujoco.MjvOption() + option.geomgroup[:] = [1, 1, 1, 0, 0, 0] + directions = camera_ray_directions(*LIDAR_SIZE, 140.0) + while not self._stop.is_set(): + started = time.monotonic() + with self._lock: + current = self._snapshot + if current is None: + self._stop.wait(0.05) + continue + qpos, assignments, lit = current + scoreboard.apply(model, lit) + data.qpos[:] = qpos + mujoco.mj_forward(model, data) + for id, (generation, origin) in assignments.items(): + if self._stop.is_set(): + break + prefix = "" if id == "duck1" else id + "_" + camera = model.camera(prefix + HEAD_CAMERA).id + ts = time.time() + offset = np.array([*origin, 0.0]) + position = data.cam_xpos[camera].copy() - offset + optical = data.cam_xmat[camera].reshape(3, 3) @ np.diag([1, -1, -1]) + orientation = Quaternion.from_rotation_matrix(optical) + pose = PoseStamped( + position=position.tolist(), orientation=orientation, frame_id="world", ts=ts + ) + width, height = SENSOR_SIZE + focal = height / (2 * math.tan(math.radians(float(model.cam_fovy[camera])) / 2)) + info = CameraInfo.from_intrinsics( + focal, + focal, + width / 2, + height / 2, + width, + height, + frame_id="camera_optical", + ) + info.ts = ts + renderer.update_scene(data, camera=camera, scene_option=option) + image = renderer.render().copy() + renderer.enable_depth_rendering() + depth = renderer.render().copy() + renderer.disable_depth_rendering() + points = self._range_points(model, data, prefix, directions) - offset + tf = TFMessage( + Transform( + translation=Vector3(*position), + rotation=orientation, + frame_id="world", + child_frame_id="camera_optical", + ts=ts, + ) + ) + self._publish( + id, + RobotVision( + generation, + Image( + data=image, format=ImageFormat.RGB, frame_id="camera_optical", ts=ts + ), + Image( + data=depth, + format=ImageFormat.DEPTH, + frame_id="camera_optical", + ts=ts, + ), + info, + pose, + tf, + points.astype(np.float32), + ), + ) + self._stop.wait(max(0.001, 1 / SENSOR_FPS - (time.monotonic() - started))) + except Exception as exc: + with self._lock: + self._error = str(exc) + logger.exception("Robot sensor pipeline failed") + finally: + if renderer is not None: + renderer.close() + + @staticmethod + def _range_points( + model: mujoco.MjModel, data: mujoco.MjData, prefix: str, directions: NDArray[np.float64] + ) -> NDArray[np.float64]: + root = model.body(prefix + "trunk_base").id + own = [] + for geom in range(model.ngeom): + ancestor = int(model.geom_bodyid[geom]) + while ancestor and ancestor != root: + ancestor = int(model.body_parentid[ancestor]) + if ancestor == root: + own.append(geom) + groups = model.geom_group[own].copy() + model.geom_group[own] = 5 + hits = [] + try: + for name, _ in LIDAR_CAMERA_SPECS: + camera = model.camera(prefix + name).id + origin = data.cam_xpos[camera].copy() + rays = directions @ data.cam_xmat[camera].reshape(3, 3).T + geom_ids = np.full(len(rays), -1, dtype=np.int32) + distance = np.full(len(rays), -1.0, dtype=np.float64) + mujoco.mj_multiRay( + model, + data, + origin, + rays.ravel(), + np.array([1, 0, 0, 1, 0, 0], dtype=np.uint8), + 1, + -1, + geom_ids, + distance, + None, + len(rays), + 6.0, + ) + valid = (distance >= 0.05) & (distance <= 6.0) + valid &= np.abs(directions[:, 1] * distance) <= 0.6 + hits.append(origin + rays[valid] * distance[valid, None]) + finally: + model.geom_group[own] = groups + return np.vstack(hits) diff --git a/examples/microduck-world/app/microduck_world/supervisor.py b/examples/microduck-world/app/microduck_world/supervisor.py new file mode 100644 index 0000000000..2ff9e1c8f0 --- /dev/null +++ b/examples/microduck-world/app/microduck_world/supervisor.py @@ -0,0 +1,147 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""One independently restartable DimOS runtime per robot visitor generation.""" + +import json +import os +import signal +import subprocess +import sys +import threading +import time +from contextlib import suppress +from dataclasses import dataclass +from typing import IO, Any + +import requests +from dimos.core.core import rpc +from dimos.core.module import Module +from dimos.utils.logging_config import setup_logger +from microduck_world.relay import relay_settings +from microduck_world.robot_io import VISITOR_IDS +from microduck_world.scene import PROJECT_ROOT + +logger = setup_logger() + + +@dataclass +class RobotProcess: + generation: str + process: subprocess.Popen[bytes] + log: IO[bytes] + + def close(self) -> None: + try: + self.process.send_signal(signal.SIGTERM) + except ProcessLookupError: + pass + try: + self.process.wait(timeout=15) + except subprocess.TimeoutExpired: + os.killpg(self.process.pid, signal.SIGKILL) + self.process.wait(timeout=5) + finally: + # A crashed/timed-out parent can leave live workers behind even when + # wait() returns immediately. Every runtime owns its process group. + with suppress(ProcessLookupError): + os.killpg(self.process.pid, signal.SIGKILL) + self.log.close() + + +class RobotSupervisor(Module): + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + self._stop = threading.Event() + self._thread: threading.Thread | None = None + self._children: dict[str, RobotProcess] = {} + + @rpc + def start(self) -> None: + super().start() + self._thread = threading.Thread(target=self._run, name="robot-supervisor", daemon=True) + self._thread.start() + + @rpc + def stop(self) -> None: + self._stop.set() + if self._thread is not None: + self._thread.join(timeout=65) + if self._thread.is_alive(): + raise RuntimeError("Robot runtimes did not stop") + super().stop() + + def _run(self) -> None: + settings = relay_settings() + wanted: dict[str, str] = {} + last_ok = 0.0 + retry: dict[str, float] = {} + try: + with requests.Session() as http: + http.trust_env = False + while not self._stop.is_set(): + try: + response = http.get( + settings.upstream_url + "/internal/assignments", timeout=1 + ) + response.raise_for_status() + value = response.json() + if not isinstance(value, dict) or not all( + k in VISITOR_IDS and isinstance(v, str) and v for k, v in value.items() + ): + raise ValueError("Invalid robot assignments") + wanted = value + last_ok = time.monotonic() + except (requests.RequestException, ValueError): + if time.monotonic() - last_ok > 3: + wanted = {} + for id, child in list(self._children.items()): + if child.generation != wanted.get(id) or child.process.poll() is not None: + child.close() + del self._children[id] + for id, generation in wanted.items(): + if id in self._children or time.monotonic() < retry.get(id, 0): + continue + log = (PROJECT_ROOT / f"logs/{id}.log").open("ab") + process = subprocess.Popen( + [ + sys.executable, + "-m", + "microduck_world.run_robot", + "--robot", + id, + "--generation", + generation, + ], + cwd=PROJECT_ROOT, + stdout=log, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + self._children[id] = RobotProcess(generation, process, log) + retry[id] = time.monotonic() + 10 + logger.info("Started robot runtime", robot=id, pid=process.pid) + (PROJECT_ROOT / "state/robot-runtimes.json").write_text( + json.dumps( + { + id: {"generation": c.generation, "pid": c.process.pid} + for id, c in self._children.items() + } + ) + ) + self._stop.wait(0.5) + finally: + for child in self._children.values(): + child.close() + self._children.clear() diff --git a/examples/microduck-world/app/microduck_world/test_ball_detection.py b/examples/microduck-world/app/microduck_world/test_ball_detection.py new file mode 100644 index 0000000000..fa716c55ce --- /dev/null +++ b/examples/microduck-world/app/microduck_world/test_ball_detection.py @@ -0,0 +1,79 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base64 +import time +from types import SimpleNamespace +from unittest.mock import Mock + +import cv2 +import numpy as np +from dimos.msgs.sensor_msgs.Image import Image, ImageFormat +from microduck_world.ball_detection import BallPerception + + +def vision(generation="first"): + pixels = np.zeros((36, 64, 3), dtype=np.uint8) + pixels[:, :, 0] = 220 + return SimpleNamespace( + generation=generation, image=Image(data=pixels, format=ImageFormat.RGB, ts=time.time()) + ) + + +def test_shared_queue_is_bounded_latest_per_duck_and_fair(module_factory): + module = module_factory(BallPerception) + for i in range(1, 7): + module.receive(f"duck{i}", vision()) + latest = vision("second") + module.receive("duck1", latest) + assert len(module._pending) == 6 + assert list(module._pending)[0] == "duck1" + assert module._pending["duck1"] is latest + assert module._generations["duck1"] == "second" + module.receive("unknown", vision()) + assert len(module._pending) == 6 + + +def test_frame_and_boxes_share_camera_identity_and_pixels(module_factory): + module = module_factory(BallPerception) + module._detector = Mock(device="cpu") + boxes = Mock() + boxes.xyxy.cpu().tolist.return_value = [[2, 3, 22, 23], [1, 1, 8, 8]] + boxes.conf.cpu().tolist.return_value = [0.9, 0.8] + boxes.cls.cpu().tolist.return_value = [32, 0] + module._detector.model.predict.return_value = [SimpleNamespace(boxes=boxes)] + camera = vision() + payload, observation = module.process("duck3", camera) + assert payload["robot"] == "duck3" + assert payload["generation"] == "first" + assert payload["ts"] == camera.image.ts + assert payload["boxes"] == [{"xyxy": (2.0, 3.0, 22.0, 23.0), "confidence": 0.9}] + assert observation.robot == "duck3" + assert observation.timestamp == camera.image.ts + module._detector.model.track.assert_not_called() + assert module._detector.model.predict.call_args.kwargs["classes"] == [32] + pixels = cv2.imdecode( + np.frombuffer(base64.b64decode(payload["image"].split(",")[1]), dtype=np.uint8), + cv2.IMREAD_COLOR, + ) + assert pixels.shape == (36, 64, 3) + assert pixels[0, 0, 2] > 200 and pixels[0, 0, 0] < 10 + + +def test_detector_failure_is_distinct_from_a_frame_with_no_ball(module_factory): + module = module_factory(BallPerception) + payload, observation = module.process("duck1", vision()) + assert payload["status"] == "unavailable" + assert payload["image"].startswith("data:image/jpeg;base64,") + assert observation is None diff --git a/examples/microduck-world/app/microduck_world/test_ball_physics.py b/examples/microduck-world/app/microduck_world/test_ball_physics.py new file mode 100644 index 0000000000..e11600fb6b --- /dev/null +++ b/examples/microduck-world/app/microduck_world/test_ball_physics.py @@ -0,0 +1,127 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Check compiled contacts in the six-robot scene, including the benchmark ball.""" + +import mujoco +import numpy as np +import pytest +from dimos.robot.pollen.microduck.assets_fetch import ensure_assets +from microduck_world.ball_physics import FLOOR_NAMES +from microduck_world.football import BALL_NAMES, FootballMatch +from microduck_world.scene import load_world +from microduck_world.visual_scene import export_scene +from microduck_world.world_sim import WorldSimModule + + +@pytest.fixture(scope="module") +def world(): + assets = ensure_assets() + module = WorldSimModule( + scene_xml=load_world()[0].mujoco_scene_path, + robot_mjcf=str(assets.robot_mjcf("default")), + headless=True, + auto_stand=False, + ) + spec = module._compose_spec() + return spec, spec.compile() + + +def test_threejs_receives_the_compiled_ball_radius(world): + _, model = world + scene = export_scene(model, {}) + geoms = {g["name"]: g for g in scene["geoms"]} + for name in BALL_NAMES: + assert geoms[name + "_geom"]["kind"] == "sphere" + assert geoms[name + "_geom"]["size"] == [0.05, 0, 0] + + +@pytest.mark.parametrize("name", BALL_NAMES) +def test_every_ball_has_upstream_mass_inertia_contacts_and_clear_reset_height(world, name): + spec, model = world + geom, body = model.geom(name + "_geom"), model.body(name) + np.testing.assert_allclose(geom.size, [0.05, 0, 0]) + np.testing.assert_allclose(body.mass, [0.03]) + np.testing.assert_allclose(body.inertia, [0.00003] * 3) + np.testing.assert_allclose(geom.friction, [0.4, 0.01, 0.003]) + np.testing.assert_allclose(geom.solref, [0.03, 0.4]) + np.testing.assert_allclose(geom.solimp, [0.9, 0.95, 0.001, 0.5, 2]) + assert geom.condim == 6 + adr = int(model.joint(name + "_freejoint").qposadr[0]) + assert model.qpos0[adr + 2] == pytest.approx(0.051) + data = mujoco.MjData(model) + mujoco.mj_forward(model, data) + assert all(geom.id not in c.geom or c.dist >= 0 for c in data.contact) + data.qpos[adr + 2] = 1 + mujoco.mj_resetData(model, data) + assert data.qpos[adr + 2] == pytest.approx(0.051) + for patch in spec.geoms: + if patch.name.startswith(name + "_patch_"): + assert patch.mass == 0 and patch.contype == 0 and patch.conaffinity == 0 + # Scaling the paint must leave it at the enlarged ball's surface. + assert 0.05 < np.linalg.norm(patch.pos) + patch.size[2] < 0.051 + + +@pytest.mark.parametrize("ball", BALL_NAMES) +@pytest.mark.parametrize("floor", FLOOR_NAMES) +def test_compiled_ball_floor_contact_mixing(world, ball, floor): + _, model = world + data = mujoco.MjData(model) + mujoco.mj_forward(model, data) + ground = model.geom(floor) + sphere = model.geom(ball + "_geom") + np.testing.assert_allclose(ground.friction, [1, 0.005, 0.0001]) + np.testing.assert_allclose(ground.solref, [0.02, 1]) + np.testing.assert_allclose(ground.solimp, [0.9, 0.95, 0.001, 0.5, 2]) + assert ground.condim == 3 + for geom in (ground, sphere): + assert geom.priority == 0 and geom.solmix == 1 + assert geom.margin == 0 and geom.gap == 0 + adr = int(model.joint(ball + "_freejoint").qposadr[0]) + # A slight overlap only in this fixture exposes the compiled contact pair. + data.qpos[adr : adr + 3] = data.geom_xpos[ground.id] + [0, 0, ground.size[2] + 0.0499] + mujoco.mj_forward(model, data) + contacts = [c for c in data.contact if set(c.geom) == {ground.id, sphere.id}] + assert len(contacts) == 1 + contact = contacts[0] + assert contact.dim == 6 + np.testing.assert_allclose(contact.friction, [1, 1, 0.01, 0.003, 0.003]) + np.testing.assert_allclose(contact.solref, [0.025, 0.7]) + np.testing.assert_allclose(contact.solimp, [0.9, 0.95, 0.001, 0.5, 2]) + assert model.opt.timestep == 0.005 + np.testing.assert_allclose(model.opt.gravity, [0, 0, -9.81]) + + +@pytest.mark.parametrize("ball", BALL_NAMES) +@pytest.mark.parametrize("sign, expected", [(1, [1, 0]), (-1, [0, 1])]) +def test_each_actual_scene_ball_physically_scores_without_reset_or_impulse( + world, ball, sign, expected +): + _, model = world + data = mujoco.MjData(model) + match = FootballMatch(model) + joint = model.joint(ball + "_freejoint") + adr, dof = int(joint.qposadr[0]), int(joint.dofadr[0]) + # Initial rolling shot is a test fixture; match.update cannot alter physics. + data.qpos[adr : adr + 3] = [sign * 0.85, 4.3, 0.05] + data.qvel[dof] = sign + data.qvel[dof + 4] = sign / 0.05 + for _ in range(400): + mujoco.mj_step(model, data) + before = data.qpos.copy(), data.qvel.copy() + match.update(data) + np.testing.assert_array_equal(data.qpos, before[0]) + np.testing.assert_array_equal(data.qvel, before[1]) + assert match.scores == expected + assert 1.356 < sign * data.qpos[adr] < 1.74 diff --git a/examples/microduck-world/app/microduck_world/test_camera.py b/examples/microduck-world/app/microduck_world/test_camera.py new file mode 100644 index 0000000000..47cd2e1a34 --- /dev/null +++ b/examples/microduck-world/app/microduck_world/test_camera.py @@ -0,0 +1,55 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mujoco +import numpy as np +import pytest +from microduck_world.camera import configure_clipping, configure_head_camera + + +def test_camera_uses_published_mount_and_follows_head_motion(): + spec = mujoco.MjSpec.from_string(""" + + + + + + + + + """) + configure_head_camera(spec) + model = spec.compile() + data = mujoco.MjData(model) + camera = model.camera("head_camera").id + site = model.site("head_camera").id + mujoco.mj_forward(model, data) + np.testing.assert_allclose(data.cam_xpos[camera], [0.05, 0, 0.2]) + np.testing.assert_allclose(-data.cam_xmat[camera].reshape(3, 3)[:, 2], [1, 0, 0]) + np.testing.assert_allclose(data.cam_xmat[camera].reshape(3, 3)[:, 1], [0, 0, 1]) + + data.qpos[model.joint("head_yaw").qposadr[0]] = np.pi / 2 + mujoco.mj_forward(model, data) + np.testing.assert_allclose(data.cam_xpos[camera], data.site_xpos[site], atol=1e-12) + np.testing.assert_allclose(data.cam_xpos[camera], [0, 0.05, 0.2], atol=1e-12) + np.testing.assert_allclose(-data.cam_xmat[camera].reshape(3, 3)[:, 2], [0, 1, 0], atol=1e-12) + + +@pytest.mark.parametrize("extent", [2.0, 15.0]) +def test_expanding_world_does_not_hide_nearby_objects_from_camera(extent): + model = mujoco.MjModel.from_xml_string("") + model.stat.extent = extent + configure_clipping(model) + assert model.vis.map.znear * model.stat.extent == pytest.approx(0.005) + assert model.vis.map.zfar * model.stat.extent == pytest.approx(30.0) diff --git a/examples/microduck-world/app/microduck_world/test_connection.py b/examples/microduck-world/app/microduck_world/test_connection.py new file mode 100644 index 0000000000..3810584450 --- /dev/null +++ b/examples/microduck-world/app/microduck_world/test_connection.py @@ -0,0 +1,62 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest.mock import Mock + +import numpy as np +from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped +from dimos.msgs.geometry_msgs.Twist import Twist +from dimos.msgs.sensor_msgs.CameraInfo import CameraInfo +from dimos.msgs.sensor_msgs.Image import Image, ImageFormat +from dimos.msgs.sensor_msgs.JointState import JointState +from dimos.msgs.tf2_msgs.TFMessage import TFMessage +from microduck_world.connection import SimRobotConnection +from microduck_world.robot_io import RobotCommand, RobotState, RobotVision + + +def test_previous_visitor_sensor_packets_cannot_enter_new_stack(module_factory, monkeypatch): + connection = module_factory(SimRobotConnection, generation="new") + published = {name: Mock() for name in connection.outputs} + for name, callback in published.items(): + monkeypatch.setattr(getattr(connection, name), "publish", callback) + image = Image(data=np.zeros((2, 2, 3), dtype=np.uint8), format=ImageFormat.RGB) + connection.receive_state(RobotState("old", PoseStamped(), JointState(), "{}")) + connection.receive_vision( + RobotVision( + "old", + image, + image, + CameraInfo(), + PoseStamped(), + TFMessage(), + np.empty((0, 3), dtype=np.float32), + ) + ) + assert not any(callback.called for callback in published.values()) + state = RobotState("new", PoseStamped(), JointState(), '{"active":"walk"}') + connection.receive_state(state) + published["odom"].assert_called_once_with(state.odom) + published["policy_state"].assert_called_once_with(state.policy) + connection.drive(Twist()) + assert published["hardware_command"].call_args.args[0].generation == "new" + + +def test_respawn_command_is_scoped_to_current_visitor(module_factory, monkeypatch): + connection = module_factory(SimRobotConnection, generation="current-visitor") + publish = Mock() + monkeypatch.setattr(connection.hardware_command, "publish", publish) + connection.respawn(False) + publish.assert_not_called() + connection.respawn(True) + publish.assert_called_once_with(RobotCommand("current-visitor", "respawn", "")) diff --git a/examples/microduck-world/app/microduck_world/test_control.py b/examples/microduck-world/app/microduck_world/test_control.py new file mode 100644 index 0000000000..179fc41b12 --- /dev/null +++ b/examples/microduck-world/app/microduck_world/test_control.py @@ -0,0 +1,68 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from unittest.mock import Mock + +from microduck_world.control import WorldControl + + +def test_manual_respawn_stops_navigation_and_returns_human_control(module_factory, monkeypatch): + control = module_factory(WorldControl, default_mode="agent") + navigation = Mock() + monkeypatch.setattr(control, "_navigation", navigation, raising=False) + mode = Mock() + reset = Mock() + velocity = Mock() + monkeypatch.setattr(control.mode, "publish", mode) + monkeypatch.setattr(control.respawn_request, "publish", reset) + monkeypatch.setattr(control.cmd_vel, "publish", velocity) + + control._on_ui_command('{"name":"respawn","args":{}}') + + navigation.cancel_goal.assert_called_once_with() + assert json.loads(mode.call_args.args[0])["mode"] == "teleop" + assert velocity.call_args.args[0].linear.x == 0 + reset.assert_called_once_with(True) + + +def test_regular_policy_commands_still_use_the_stock_router(module_factory, monkeypatch): + control = module_factory(WorldControl) + policy = Mock() + monkeypatch.setattr(control.policy_request, "publish", policy) + control._on_ui_command('{"name":"policy","args":{"policy":"walk","action":"start"}}') + request = json.loads(policy.call_args.args[0]) + assert (request["policy"], request["action"]) == ("walk", "start") + + +def test_ball_drop_passes_bridge_validation_without_changing_mode(module_factory, monkeypatch): + from microduck_world.relay import WorldCommand + + control = module_factory(WorldControl) + drop = Mock() + mode = Mock() + monkeypatch.setattr(control.ball_drop_request, "publish", drop) + monkeypatch.setattr(control.mode, "publish", mode) + raw = WorldCommand(name="drop_ball", args={"ball": "football_ball_1"}).model_dump_json() + control._on_ui_command(raw) + drop.assert_called_once_with("football_ball_1") + mode.assert_not_called() + + +def test_ball_drop_rejects_unknown_ball(module_factory, monkeypatch): + control = module_factory(WorldControl) + drop = Mock() + monkeypatch.setattr(control.ball_drop_request, "publish", drop) + control._on_ui_command('{"name":"drop_ball","args":{"ball":"unknown"}}') + drop.assert_not_called() diff --git a/examples/microduck-world/app/microduck_world/test_football.py b/examples/microduck-world/app/microduck_world/test_football.py new file mode 100644 index 0000000000..ef171beea1 --- /dev/null +++ b/examples/microduck-world/app/microduck_world/test_football.py @@ -0,0 +1,261 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from copy import copy + +import mujoco +import numpy as np +import pytest +from dimos.robot.pollen.microduck.places import add_ball_body +from microduck_world.ball_physics import BALL_RADIUS +from microduck_world.football import BALL_NAMES, FootballMatch, Scoreboard, add_footballs +from microduck_world.scene import load_world + + +@pytest.fixture +def pitch(): + spec = mujoco.MjSpec.from_file(str(load_world()[0].mujoco_scene_path)) + spec.option.timestep = 0.005 + add_ball_body(spec) + add_footballs(spec) + model = spec.compile() + return model, mujoco.MjData(model), FootballMatch(model) + + +def put(pitch, x, y=4.3, z=BALL_RADIUS, ball="football_ball_1"): + model, data, match = pitch + adr = int(model.joint(ball + "_freejoint").qposadr[0]) + data.qpos[adr : adr + 3] = (x, y, z) + match.update(data) + + +@pytest.mark.parametrize("sign, expected", [(1, [1, 0]), (-1, [0, 1])]) +def test_whole_ball_must_cross_and_only_once_until_it_returns_to_field(pitch, sign, expected): + _, data, match = pitch + put(pitch, sign * 1.2) + put(pitch, sign * 1.32) + assert match.scores == [0, 0] # Center over line, back of ball not yet over. + put(pitch, sign * (1.306 + BALL_RADIUS - 0.001)) + assert match.scores == [0, 0] # Would have scored with the old 3.5 cm radius. + put(pitch, sign * (1.306 + BALL_RADIUS + 0.001)) + assert match.scores == expected + before = data.qpos.copy() + for _ in range(20): + match.update(data) + np.testing.assert_array_equal(data.qpos, before) + put(pitch, sign * 1.335) + put(pitch, sign * 1.36) + assert match.scores == expected + put(pitch, sign * 1.2) + put(pitch, sign * 1.37) + assert match.scores == [n * 2 for n in expected] + + +@pytest.mark.parametrize("sign", [-1, 1]) +def test_ball_starting_in_net_or_crossing_backwards_does_not_score(pitch, sign): + put(pitch, sign * 1.6) + put(pitch, sign * 1.4) + put(pitch, sign * 1.1) + assert pitch[2].scores == [0, 0] + + +@pytest.mark.parametrize( + "y, z", [(4.9, BALL_RADIUS), (3.7, BALL_RADIUS), (4.3, 0.56), (4.79, BALL_RADIUS), (4.3, 0.47)] +) +def test_wide_shots_and_balls_overlapping_posts_or_crossbar_do_not_score(pitch, y, z): + put(pitch, 1.1, y, z) + put(pitch, 1.6, y, z) + assert pitch[2].scores == [0, 0] + + +def test_entering_from_side_behind_goal_line_does_not_score(pitch): + put(pitch, 1.1, 5.1) + put(pitch, 1.4, 5.1) + put(pitch, 1.4, 4.3) + assert pitch[2].scores == [0, 0] + + +def test_a_ball_landing_in_the_net_from_above_does_not_score(pitch): + put(pitch, 1.1, z=0.7) + put(pitch, 1.32, z=0.6) + put(pitch, 1.32, z=0.1) + put(pitch, 1.4, z=0.1) + assert pitch[2].scores == [0, 0] + + +def test_simultaneous_balls_score_independently_and_credit_opposite_goal(pitch): + model, data, match = pitch + for ball in BALL_NAMES: + adr = int(model.joint(ball + "_freejoint").qposadr[0]) + data.qpos[adr : adr + 3] = [1.1, 4.3, BALL_RADIUS] + match.update(data) + for ball in BALL_NAMES: + adr = int(model.joint(ball + "_freejoint").qposadr[0]) + data.qpos[adr] = 1.5 + match.update(data) + assert match.scores == [4, 0] + assert match.last_goal["team"] == "blue" + + +@pytest.mark.parametrize("sign, expected", [(1, [1, 0]), (-1, [0, 1])]) +def test_real_mujoco_ball_rolls_into_net_scores_and_is_retained(pitch, sign, expected): + model, data, match = pitch + ball = model.joint("football_ball_1_freejoint") + adr, vel = int(ball.qposadr[0]), int(ball.dofadr[0]) + put(pitch, sign * 0.85) + data.qvel[vel] = sign * 1.0 + data.qvel[vel + 4] = sign / BALL_RADIUS + for _ in range(400): + mujoco.mj_step(model, data) + match.update(data) + assert match.scores == expected + assert 1.34 < sign * data.qpos[adr] < 1.74 + assert abs(data.qvel[vel]) < 0.2 + np.testing.assert_allclose(model.body("football_ball_1").mass, [0.03]) + np.testing.assert_allclose(model.body("football_ball_1").inertia, [0.00003] * 3) + + +def test_real_post_contact_deflects_ball_without_a_phantom_goal(pitch): + model, data, match = pitch + ball = model.joint("football_ball_1_freejoint") + adr, vel = int(ball.qposadr[0]), int(ball.dofadr[0]) + put(pitch, 0.85, y=4.825) + data.qvel[vel] = 1 + data.qvel[vel + 4] = 1 / BALL_RADIUS + touched = False + post = model.geom("football_coral_post_1").id + for _ in range(300): + mujoco.mj_step(model, data) + match.update(data) + touched |= bool(np.any(data.contact.geom[: data.ncon] == post)) + assert touched + assert match.scores == [0, 0] + assert data.qpos[adr] < 1.3 + + +def test_player_tunnel_has_no_collision_and_side_is_still_a_wall(pitch): + model, data, _ = pitch + mujoco.mj_forward(model, data) + group = np.array([1, 0, 0, 0, 0, 0], dtype=np.uint8) + geom_id = np.zeros(1, dtype=np.int32) + through = mujoco.mj_ray( + model, data, np.array([0.0, 1.9, 0.2]), np.array([0.0, 1.0, 0.0]), group, 1, -1, geom_id + ) + assert through > 0.7 + blocked = mujoco.mj_ray( + model, data, np.array([1.8, 1.9, 0.2]), np.array([0.0, 1.0, 0.0]), group, 1, -1, geom_id + ) + assert blocked == pytest.approx(0.1) + + +def test_native_camera_copy_and_browser_receive_the_same_lamp_ids(pitch): + model, _, match = pitch + native = copy(model) + board = Scoreboard(native) + put(pitch, 1.1) + put(pitch, 1.5) + snapshot = match.snapshot() + board.apply(native, snapshot["lit"]) + assert snapshot["scores"] == [1, 0] + on = model.geom("score_blue_2_b").id + off = model.geom("score_blue_2_a").id + assert on in snapshot["lit"] and off not in snapshot["lit"] + np.testing.assert_array_equal(native.geom_rgba[on], model.geom_rgba[on]) + np.testing.assert_allclose(native.geom_rgba[off, :3], [0.018, 0.025, 0.024]) + np.testing.assert_allclose(model.geom_rgba[off, :3], [0.20, 0.53, 0.75]) + + +def test_large_scores_keep_full_count_and_light_overflow_indicator(pitch): + model, _, match = pitch + match.scores[:] = [999, 0] + put(pitch, 1.1) + put(pitch, 1.5) + assert match.snapshot()["scores"] == [1000, 0] + assert model.geom("score_blue_overflow_0").id in match.lit + + +def test_fast_diagonal_shot_checks_sphere_edge_between_samples(pitch): + # The centre fits at x=0, but the swept edge clips the right post. + put(pitch, 1.1, y=4.47) + put(pitch, 1.5, y=5.03) + assert pitch[2].scores == [0, 0] + + +def test_glancing_entry_that_fits_the_swept_opening_is_a_goal(pitch): + # Its leading tip enters near a post, then the ball travels inward. + # Requiring the full radius at the leading tip would reject this valid path. + put(pitch, 1.306 - BALL_RADIUS - 0.005, y=4.78) + put(pitch, 1.306 + BALL_RADIUS + 0.005, y=4.67) + assert pitch[2].scores == [1, 0] + + +@pytest.mark.parametrize( + "team, generation, expected", + [("duck4", "session", 1), ("duck1", "session", 0), ("duck4", "old-session", 0)], +) +def test_scorer_credit_uses_last_touch_owner_excludes_own_goals_and_stale_identity( + pitch, tmp_path, team, generation, expected +): + from types import SimpleNamespace + + from microduck_world.scorers import ScorerLedger + + model, _, match = pitch + ledger = ScorerLedger(tmp_path / "scores.sqlite3") + match.ledger = ledger + robot = SimpleNamespace(id=team, active=True, generation="session", geoms=[99999]) + contact = SimpleNamespace(geom=[model.geom("football_ball_1_geom").id, 99999], dist=0) + sensed = SimpleNamespace(model=model, contact=[contact], ncon=1) + match.touches( + sensed, + {team: robot}, + {team: {"generation": generation, "userId": "123", "handle": "player"}}, + ) + put(pitch, 1.0) + put(pitch, 1.5) + assert match.scores == [1, 0] + ledger.flush() + assert ledger.rows == ([{"handle": "player", "goals": 1}] if expected else []) + assert ScorerLedger(tmp_path / "scores.sqlite3").rows == ledger.rows + match.scores[:] = [0, 0] + assert ScorerLedger(tmp_path / "scores.sqlite3").rows == ledger.rows + + +@pytest.mark.parametrize("ball", BALL_NAMES) +def test_manual_drop_resets_one_ball_at_two_metres_without_score_or_old_credit(pitch, ball): + model, data, match = pitch + other_positions = data.qpos.copy() + adr = match.balls[ball][0] + match._last_touch[ball] = {"handle": "old"} + assert match.drop_ball(data, ball) + np.testing.assert_allclose(data.qpos[adr : adr + 7], [0, 4.3, 2, 1, 0, 0, 0]) + assert ball not in match._last_touch + for other, (other_adr, _) in match.balls.items(): + if other != ball: + np.testing.assert_array_equal( + data.qpos[other_adr : other_adr + 7], other_positions[other_adr : other_adr + 7] + ) + match.update(data) + assert match.scores == [0, 0] + assert match.drops[ball] == 1 + for _ in range(60): + mujoco.mj_step(model, data) + assert data.qpos[adr + 2] < 1.8 + + +def test_manual_drops_wait_for_clear_space(pitch): + _, data, match = pitch + assert match.drop_ball(data, BALL_NAMES[1]) + assert not match.drop_ball(data, BALL_NAMES[2]) + assert match.drops[BALL_NAMES[2]] == 0 diff --git a/examples/microduck-world/app/microduck_world/test_gateway.py b/examples/microduck-world/app/microduck_world/test_gateway.py new file mode 100644 index 0000000000..c0c2950f1d --- /dev/null +++ b/examples/microduck-world/app/microduck_world/test_gateway.py @@ -0,0 +1,201 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gzip +import socket +from collections.abc import Iterator + +import httpx +import pytest +from microduck_world.gateway import GatewayConfig, create_app, rewrite_info +from microduck_world.udp_forwarder import UdpForwarder +from starlette.testclient import TestClient + + +@pytest.fixture +def config() -> GatewayConfig: + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + return GatewayConfig( + bind_host="127.0.0.1", + port=port, + public_origin=f"https://world.test:{port}", + upstream_url="http://127.0.0.1:7780", + cert_file="unused.crt", + key_file="unused.key", + ) + + +@pytest.fixture +def client(config: GatewayConfig, monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]: + async def request(self, method, url, **kwargs): + return httpx.Response( + 200, + request=httpx.Request(method, url), + json={"wtUrl": "https://127.0.0.1:45678/viewer", "certHash": "relay-pin", "v": 5}, + headers={"access-control-allow-origin": "*"}, + ) + + monkeypatch.setattr(httpx.AsyncClient, "request", request) + with TestClient(create_app(config), base_url=config.public_origin) as client: + yield client + + +def test_discovery_preserves_certificate_pin_and_uses_gateway(client, config): + response = client.get("/api/info") + assert response.json() == { + "wtUrl": f"{config.public_origin}/viewer", + "certHash": "relay-pin", + "v": 5, + } + assert response.headers["cache-control"] == "no-store" + assert "access-control-allow-origin" not in response.headers + + +@pytest.mark.parametrize( + "headers", + [ + {"origin": "https://unrelated.test"}, + {"sec-fetch-site": "cross-site"}, + ], +) +def test_foreign_websites_cannot_read_bootstrap(client, headers): + assert client.get("/api/info", headers=headers).status_code == 403 + + +def test_wrong_host_is_rejected(client): + assert client.get("/api/info", headers={"host": "unrelated.test"}).status_code == 421 + + +def test_gateway_is_read_only_http_and_strips_upstream_cors(client): + assert client.post("/api/info").status_code == 405 + assert "access-control-allow-origin" not in client.get("/").headers + + +def test_discovery_cannot_redirect_udp_to_an_external_host(config): + with pytest.raises(ValueError, match="non-loopback"): + rewrite_info({"wtUrl": "https://203.0.113.1:443/viewer"}, config, UdpForwarder()) + + +@pytest.mark.parametrize("host", ["0.0.0.0", "192.168.1.10", "203.0.113.1"]) +def test_non_tailnet_bind_is_rejected(config, host): + with pytest.raises(ValueError, match="Tailscale"): + GatewayConfig.model_validate({**config.model_dump(), "bind_host": host}) + + +@pytest.fixture +def offline_client(client, monkeypatch): + async def offline(self, method, url, **kwargs): + raise httpx.ConnectError("offline") + + monkeypatch.setattr(httpx.AsyncClient, "request", offline) + return client + + +def test_browser_outage_page_retries_automatically(offline_client): + response = offline_client.get("/", headers={"accept": "text/html"}) + assert response.status_code == 503 + assert response.headers["cache-control"] == "no-store" + assert response.headers["retry-after"] == "3" + assert '' in response.text + assert "You can leave this page open" in response.text + + +def test_api_outage_returns_retryable_json(offline_client): + response = offline_client.get("/api/info") + assert response.status_code == 503 + assert response.headers["cache-control"] == "no-store" + assert response.headers["retry-after"] == "3" + assert response.json() == {"status": "starting"} + + +@pytest.mark.parametrize( + "stats,status", + [({"robots": [{"id": "duck"}]}, 200), ({"robots": []}, 503), ({}, 503), ([], 503)], +) +def test_readiness_requires_a_connected_robot(client, monkeypatch, stats, status): + async def reply(self, method, url, **kwargs): + assert url.path == "/api/stats" + return httpx.Response(200, json=stats) + + monkeypatch.setattr(httpx.AsyncClient, "request", reply) + response = client.get("/healthz") + assert response.status_code == status + assert response.json() == {"status": "ready" if status == 200 else "starting"} + assert response.headers["cache-control"] == "no-store" + + +@pytest.fixture +def visual_client(config, tmp_path): + frontend = tmp_path / "frontend" + assets = tmp_path / "world" + frontend.mkdir() + assets.mkdir() + (frontend / "index.html").write_text("World 3D") + (frontend / "app.js").write_text("console.log('world')") + (tmp_path / "private.txt").write_text("private") + (frontend / "escape.txt").symlink_to(tmp_path / "private.txt") + (assets / "scene-0123456789abcdef0123.json").write_text('{"version":1}') + (assets / "scene-0123456789abcdef0123.json.gz").write_bytes(gzip.compress(b'{"version":1}')) + settings = config.model_copy(update={"frontend_dir": frontend, "world_assets_dir": assets}) + with TestClient(create_app(settings), base_url=config.public_origin) as client: + yield client + + +def test_project_frontend_and_model_assets_keep_access_boundary(visual_client): + assert "World 3D" in visual_client.get("/").text + assert visual_client.get("/client/app.js").text == "console.log('world')" + response = visual_client.get( + "/world-assets/scene-0123456789abcdef0123.json", headers={"accept-encoding": "identity"} + ) + assert response.json() == {"version": 1} + assert "immutable" in response.headers["cache-control"] + assert visual_client.get("/client/escape.txt").status_code == 404 + assert visual_client.get("/world-assets/private.txt").status_code == 404 + assert ( + visual_client.get("/client/app.js", headers={"origin": "https://foreign.test"}).status_code + == 403 + ) + assert ( + visual_client.get( + "/world-assets/scene-0123456789abcdef0123.json", + headers={"sec-fetch-site": "cross-site"}, + ).status_code + == 403 + ) + + +def test_browser_receives_compressed_world_model(visual_client): + response = visual_client.get("/world-assets/scene-0123456789abcdef0123.json") + assert response.headers["content-encoding"] == "gzip" + assert response.json() == {"version": 1} + assert response.headers["vary"] == "accept-encoding" + + +def test_private_relay_routes_are_not_exposed(client): + for path in ("/internal/robot-info", "/internal/assignments", "/internal%2Frobot-info"): + assert client.get(path).status_code == 404 + + +def test_lobby_rejects_cross_origin_and_oversized_posts(client): + assert client.post("/api/lobby", headers={"origin": "https://evil.test"}).status_code == 403 + assert client.post("/api/lobby", content=b"x" * 1025).status_code == 413 + + +def test_discovery_keeps_session_ticket(config): + result = rewrite_info( + {"wtUrl": "https://127.0.0.1:45678/viewer?ticket=example"}, config, UdpForwarder() + ) + assert result["wtUrl"] == config.public_origin + "/viewer?ticket=example" diff --git a/examples/microduck-world/app/microduck_world/test_knowledge.py b/examples/microduck-world/app/microduck_world/test_knowledge.py new file mode 100644 index 0000000000..0cfe8fa209 --- /dev/null +++ b/examples/microduck-world/app/microduck_world/test_knowledge.py @@ -0,0 +1,111 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import time +from dataclasses import replace + +import numpy as np +import pytest +from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped +from dimos.msgs.sensor_msgs.CameraInfo import CameraInfo +from dimos.msgs.sensor_msgs.Image import Image, ImageFormat +from microduck_world.knowledge import CameraView, DuckKnowledge, locate_pixel +from microduck_world.robot_io import Observation + + +@pytest.fixture +def observation(): + return Observation( + Image(data=np.zeros((3, 3, 3), dtype=np.uint8), format=ImageFormat.RGB, ts=time.time()), + Image(data=np.full((3, 3), 2.0, dtype=np.float32), format=ImageFormat.DEPTH), + CameraInfo.from_intrinsics(2, 2, 1, 1, 3, 3), + PoseStamped(position=(3, 4, 5), orientation=(0, 0, 0, 1), frame_id="world"), + ) + + +def test_object_position_uses_pixel_depth_and_own_camera_pose(observation): + assert locate_pixel(observation, 2, 1) == (4, 4, 7) + + +@pytest.mark.parametrize("x,y", [(-1, 0), (0, 3), (3, 0)]) +def test_annotation_rejects_pixels_outside_own_image(observation, x, y): + with pytest.raises(ValueError, match="outside"): + locate_pixel(observation, x, y) + + +@pytest.mark.parametrize("depth", [float("nan"), float("inf"), 0, 7]) +def test_annotation_requires_measured_depth(observation, depth): + observation.depth.data[1, 1] = depth + with pytest.raises(ValueError, match="measured depth"): + locate_pixel(observation, 1, 1) + + +@pytest.fixture +def knowledge(module_factory, tmp_path): + def create(generation): + directory = tmp_path / generation + directory.mkdir(exist_ok=True) + return module_factory( + DuckKnowledge, + rooms={}, + objects={}, + scene=generation, + places_db=str(directory / "places.db"), + knowledge_dir=str(directory), + places_republish_s=0, + ) + + return create + + +def test_evidence_and_named_objects_stay_private_and_new_visitors_start_fresh( + knowledge, observation +): + first, other = knowledge("first"), knowledge("other") + first._on_observation(observation) + view = first.observe() + assert isinstance(view, CameraView) + assert "Unknown observation" in other.remember_object("box", view.id, 1, 1) + assert "Remembered box" in first.remember_object("box", view.id, 1, 1, "a visible box") + assert "box" in first.list_objects() + assert "box" not in other.list_objects() + first.stop() + resumed = knowledge("first") + assert "box" in resumed.list_objects() + fresh = knowledge("new-visitor") + assert "box" not in fresh.list_objects() + assert json.loads(fresh.understanding())["observations"] == [] + + +def test_only_images_returned_by_observe_can_be_annotated(knowledge, observation): + duck = knowledge("duck") + duck._on_observation(observation) + assert "Unknown observation" in duck.record_observation(str(observation.image.ts), "a box") + view = duck.observe() + assert isinstance(view, CameraView) + assert ( + duck.record_observation(view.id, "a box") == "Saved this observation in my private memory." + ) + assert json.loads(duck.understanding())["observations"][0]["description"] == "a box" + + +def test_stale_camera_is_not_presented_as_current(knowledge, observation): + duck = knowledge("duck") + stale = replace( + observation, + image=Image(data=observation.image.data, format=ImageFormat.RGB, ts=time.time() - 10), + ) + duck._on_observation(stale) + assert not duck.observe().success diff --git a/examples/microduck-world/app/microduck_world/test_physics_robots.py b/examples/microduck-world/app/microduck_world/test_physics_robots.py new file mode 100644 index 0000000000..09abc21729 --- /dev/null +++ b/examples/microduck-world/app/microduck_world/test_physics_robots.py @@ -0,0 +1,162 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from unittest.mock import Mock + +import mujoco +import numpy as np +import pytest +from dimos.robot.pollen.microduck.gait import MicroduckObserver +from dimos.robot.pollen.microduck.policies import PolicyBank +from dimos.robot.pollen.microduck.sim_module import MicroduckSimModuleConfig +from microduck_world.physics_robots import WorldRobots +from microduck_world.robot_io import RobotCommand +from microduck_world.roster import ROBOT_IDS + + +@pytest.fixture +def world(monkeypatch): + spec = mujoco.MjSpec() + for prefix in ("", *(id + "_" for id in ROBOT_IDS[1:])): + robot = mujoco.MjSpec.from_string(""" + + + + + + + + """) + spec.attach(robot, prefix=prefix, frame=spec.worldbody.add_frame()) + model = spec.compile() + data = mujoco.MjData(model) + banks = {} + for prefix in ("", *(id + "_" for id in ROBOT_IDS[1:])): + observer = MicroduckObserver(model, ["knee"], np.array([0.2]), prefix=prefix) + bank = Mock(spec=PolicyBank) + bank.joint_names = ["knee"] + bank.default_pose = observer.default_pose + bank.variant = "default" + bank.availability = {"walk": None} + bank.root_qpos_adr = observer.root_qpos_adr + bank.initial_qpos.side_effect = observer.initial_qpos + bank.projected_gravity.side_effect = observer.projected_gravity + bank.root_yaw.side_effect = observer.root_yaw + bank.step.return_value = observer.default_pose + banks[prefix] = bank + bank = banks[""] + bank.for_robot.side_effect = lambda model, prefix: banks[prefix] + clock = Mock(return_value=0.0) + monkeypatch.setattr("microduck_world.physics_robots.time.monotonic", clock) + robots = WorldRobots( + model, + bank, + { + "robots": { + id: {"team": "red" if n < 3 else "blue", "spawn": [n, 0], "yaw": 0} + for n, id in enumerate(ROBOT_IDS) + }, + "clearance": 0.4, + }, + MicroduckSimModuleConfig(auto_stand=False), + ) + robots.leases({id: "visitor" + str(n + 1) for n, id in enumerate(ROBOT_IDS)}) + for _ in range(4): + robots.step(data) + return robots, data, clock + + +def test_fallen_duck_stays_fallen_without_automatic_pose_reset(world): + robots, data, clock = world + duck = robots.robots["duck2"] + adr = duck.bank.root_qpos_adr + data.qpos[adr + 2 : adr + 7] = (0.03, 0, 1, 0, 0) + for _ in range(4): + robots.step(data) + clock.return_value = 2.5 + before = data.qpos.copy() + robots.step(data) + np.testing.assert_array_equal(data.qpos, before) + assert duck.scheduler.fallen + + +def test_respawn_only_resets_own_duck_and_preserves_its_map_frame(world): + robots, data, _ = world + duck = robots.robots["duck2"] + origin = duck.origin + adr = duck.bank.root_qpos_adr + data.qpos[adr : adr + 7] = (0.5, 0.6, 0.03, 0, 1, 0, 0) + data.qvel[:] = 0.4 + before_pos, before_vel = data.qpos.copy(), data.qvel.copy() + robots.command("duck2", RobotCommand("visitor2", "twist", (0.1, 0.0, 0.0))) + robots.command("duck2", RobotCommand("visitor2", "respawn", "")) + robots.step(data) + + np.testing.assert_allclose(data.qpos[adr : adr + 7], [1, 0, 0.125, 1, 0, 0, 0]) + for other_id in ("duck1", "duck3"): + other = robots.robots[other_id] + root = other.bank.root_qpos_adr + pos = list(range(root, root + 7)) + other.joint_qpos + vel_root = int( + robots.model.joint( + ("" if other_id == "duck1" else other_id + "_") + "trunk_base_freejoint" + ).dofadr[0] + ) + vel = list(range(vel_root, vel_root + 6)) + other.joint_qvel + np.testing.assert_array_equal(data.qpos[pos], before_pos[pos]) + np.testing.assert_array_equal(data.qvel[vel], before_vel[vel]) + assert duck.origin == origin + assert duck.generation == "visitor2" + assert json.loads(robots.states(data)["duck2"].policy)["respawns"] == 1 + assert "duck2" not in robots._commands + + +def test_previous_visitor_cannot_respawn_current_duck(world): + robots, data, _ = world + before = data.qpos.copy() + robots.command("duck2", RobotCommand("previous-visitor", "respawn", "")) + robots.step(data) + np.testing.assert_array_equal(data.qpos, before) + assert robots.robots["duck2"].respawns == 0 + + +def test_all_six_have_private_state_and_no_permanent_host(world): + robots, data, _ = world + assert set(robots.states(data)) == set(ROBOT_IDS) + robots.leases({}) + robots.step(data) + assert robots.states(data) == {} + + +def test_respawn_cannot_use_the_other_teams_available_spawns(world): + robots, data, _ = world + red = robots.robots["duck2"] + # Block all red bays using active robots, leaving blue bays irrelevant. + for id, x in [("duck4", 0), ("duck5", 1), ("duck6", 2)]: + data.qpos[ + robots.robots[id].bank.root_qpos_adr : robots.robots[id].bank.root_qpos_adr + 2 + ] = [x, 0] + root = red.bank.root_qpos_adr + data.qpos[root : root + 2] = [8, 0] + robots.command("duck2", RobotCommand("visitor2", "respawn", "")) + robots.step(data) + assert red.respawns == 0 + assert "duck2" in robots._respawns + assert data.qpos[root] == 8 + # Free its own bay and confirm that the pending request succeeds there. + data.qpos[robots.robots["duck5"].bank.root_qpos_adr] = 20 + robots.step(data) + assert red.respawns == 1 + assert data.qpos[root] == 1 diff --git a/examples/microduck-world/app/microduck_world/test_pitch_entrance.py b/examples/microduck-world/app/microduck_world/test_pitch_entrance.py new file mode 100644 index 0000000000..67042c2b01 --- /dev/null +++ b/examples/microduck-world/app/microduck_world/test_pitch_entrance.py @@ -0,0 +1,140 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Floor support and real-policy walking across the locker-to-pitch threshold.""" + +import math +from copy import copy, deepcopy + +import mujoco +import numpy as np +import pytest +from dimos.robot.pollen.microduck.policies import PolicyBank +from microduck_world.physics_robots import WorldRobots +from microduck_world.robot_io import RobotCommand +from microduck_world.roster import ROBOT_IDS, SETTINGS +from microduck_world.scene import PROJECT_ROOT, load_world +from microduck_world.world_sim import WorldSimModule + + +def test_pitch_entrance_has_continuous_level_floor_support(): + model = mujoco.MjModel.from_xml_path(str(load_world()[0].mujoco_scene_path)) + data = mujoco.MjData(model) + mujoco.mj_forward(model, data) + club, pitch = model.geom("club_floor"), model.geom("football_floor") + north = data.geom_xpos[club.id, 1] + club.size[1] + south = data.geom_xpos[pitch.id, 1] - pitch.size[1] + assert north == pytest.approx(south, abs=1e-10) + for x in np.linspace(-0.5, 0.5, 5): + for y in np.linspace(1.95, 2.2, 51): + geom = np.zeros(1, dtype=np.int32) + distance = mujoco.mj_ray( + model, + data, + np.array([x, y, 0.2]), + np.array([0.0, 0.0, -1.0]), + np.array([1, 0, 0, 0, 0, 0], dtype=np.uint8), + 1, + -1, + geom, + ) + assert distance == pytest.approx(0.2, abs=1e-10), (x, y, distance) + assert int(geom[0]) in (club.id, pitch.id) + + +@pytest.fixture(scope="module") +def entrance_world(): + module = WorldSimModule( + scene_xml=load_world()[0].mujoco_scene_path, + robot_mjcf=str(PROJECT_ROOT / "assets/microduck/robot/robot_allcollisions.xml"), + headless=True, + auto_stand=False, + ) + model = module._compose_spec().compile() + bank = PolicyBank(PROJECT_ROOT / "assets/microduck/policies", model) + return model, bank, module.config + + +@pytest.mark.parametrize("robot_id", ROBOT_IDS) +@pytest.mark.parametrize("direction", [1, -1]) +def test_real_walking_policy_crosses_pitch_threshold_without_falling( + entrance_world, robot_id, direction +): + source, bank, config = entrance_world + model = copy(source) + data = mujoco.MjData(model) + settings = deepcopy(SETTINGS) + index = ROBOT_IDS.index(robot_id) + x = (-0.3, 0, 0.3)[index % 3] + settings["robots"][robot_id].update( + spawn=[x, 1.4 if direction == 1 else 2.7], + yaw=direction * math.pi / 2, + ) + robots = WorldRobots(model, bank, settings, config) + robot = robots.robots[robot_id] + adr = robot.bank.root_qpos_adr + # Use the normal walk command, on both sides and the centre of the doorway. + # Smaller requests can select the scheduler's standing mode. + speed = 0.15 + crossed_at = None + for step in range(4400): + if step % 20 == 0: + robots.leases({robot_id: "entrance-test"}) + moving = step >= 400 and crossed_at is None + robots.command( + robot_id, + RobotCommand("entrance-test", "twist", (speed if moving else 0, 0, 0)), + ) + robots.step(data) + mujoco.mj_step(model, data) + if step >= 400: + assert robot.bank.projected_gravity(data)[2] < -0.9, ( + robot_id, + direction, + data.time, + data.qpos[adr : adr + 3], + ) + assert data.qpos[adr + 2] > 0.09 + y = data.qpos[adr + 1] + if crossed_at is None and (y > 2.45 if direction == 1 else y < 1.7): + crossed_at = step + if crossed_at is not None and step >= crossed_at + 200: + break # Remain upright for one second after stopping on the other side. + assert crossed_at is not None, (robot_id, direction, data.qpos[adr : adr + 3]) + + +def test_all_six_midfield_spawns_activate_and_remain_upright(entrance_world): + source, bank, config = entrance_world + model = copy(source) + data = mujoco.MjData(model) + robots = WorldRobots(model, bank, deepcopy(SETTINGS), config) + leases = {robot_id: f"spawn-{robot_id}" for robot_id in ROBOT_IDS} + for step in range(600): + if step % 20 == 0: + robots.leases(leases) + robots.step(data) + mujoco.mj_step(model, data) + for robot_id in ROBOT_IDS: + robot = robots.robots[robot_id] + assert robot.active, robot_id + adr = robot.bank.root_qpos_adr + x, y, z = data.qpos[adr : adr + 3] + expected_x, expected_y = SETTINGS["robots"][robot_id]["spawn"] + assert abs(x - expected_x) < 0.1 and abs(y - expected_y) < 0.1, (robot_id, x, y) + assert z > 0.09, (robot_id, z) + assert robot.bank.projected_gravity(data)[2] < -0.9, robot_id + # Use the same normal spawn path again for an explicit respawn. + robots.command("duck1", RobotCommand(leases["duck1"], "respawn", "")) + robots.step(data) + assert robots.robots["duck1"].respawns == 1 diff --git a/examples/microduck-world/app/microduck_world/test_relay.py b/examples/microduck-world/app/microduck_world/test_relay.py new file mode 100644 index 0000000000..abb9a42c6b --- /dev/null +++ b/examples/microduck-world/app/microduck_world/test_relay.py @@ -0,0 +1,41 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest.mock import Mock + +from dimos.web.relay_bridge.protocol import Tx +from dimos.web.relay_bridge.relay_bridge_module import TX_CHANNELS +from microduck_world.relay import WorldBridge + + +def test_respawn_uses_existing_command_guards_without_changing_global_bridge( + module_factory, monkeypatch +): + bridge = module_factory(WorldBridge) + handler = next(item for item in TX_CHANNELS if item.ch == "ui_command") + bridge._tx_defs["ui_command"] = handler + publish = Mock() + monkeypatch.setattr(bridge.ui_command, "publish", publish) + clock = Mock(return_value=10.0) + monkeypatch.setattr("dimos.web.relay_bridge.relay_bridge_module.time.monotonic", clock) + message = Tx(ch="ui_command", seq=1, data={"name": "respawn", "args": {}}) + + bridge._on_wire_tx(message) + bridge._on_wire_tx(message) + publish.assert_called_once_with('{"name":"respawn","args":{}}') + assert next(item for item in TX_CHANNELS if item.ch == "ui_command") is handler + + clock.return_value = 10.1 + bridge._on_wire_tx(Tx(ch="ui_command", seq=2, data={"name": "arbitrary-rpc"})) + publish.assert_called_once() diff --git a/examples/microduck-world/app/microduck_world/test_robot_mcp.py b/examples/microduck-world/app/microduck_world/test_robot_mcp.py new file mode 100644 index 0000000000..c5b28cec4f --- /dev/null +++ b/examples/microduck-world/app/microduck_world/test_robot_mcp.py @@ -0,0 +1,41 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest.mock import Mock + +from dimos.agents.mcp.mcp_server import McpServer, app +from microduck_world.knowledge import DuckKnowledge +from microduck_world.robot_mcp import RobotMcpServer + + +def test_agent_tools_exclude_foreign_robots_and_operator_endpoints(module_factory, monkeypatch): + server = module_factory(RobotMcpServer, robot="duck2", port=9991) + own = Mock(remote_name="duck2/duckknowledge", actor_class=DuckKnowledge) + foreign = Mock(remote_name="duck3/duckknowledge", actor_class=DuckKnowledge) + operator = Mock(remote_name="duck2/robotmcpserver", actor_class=RobotMcpServer) + register = Mock() + monkeypatch.setattr(McpServer, "on_system_modules", register) + server.on_system_modules([own, foreign, operator]) + register.assert_called_once_with([own]) + + +def test_mcp_calls_target_the_deployed_instance_not_the_shared_class(module_factory, monkeypatch): + for field, value in (("skills", []), ("skills_by_name", {}), ("rpc_calls", {})): + monkeypatch.setattr(app.state, field, value) + server = module_factory(RobotMcpServer, robot="duck2", port=9991) + info = Mock(func_name="observe", class_name="DuckKnowledge") + own = Mock(remote_name="duck2/duckknowledge", actor_class=DuckKnowledge) + own.get_skills.return_value = [info] + server.on_system_modules([own]) + assert app.state.rpc_calls["observe"].remote_name == "duck2/duckknowledge" diff --git a/examples/microduck-world/app/microduck_world/test_scene.py b/examples/microduck-world/app/microduck_world/test_scene.py new file mode 100644 index 0000000000..394c10999b --- /dev/null +++ b/examples/microduck-world/app/microduck_world/test_scene.py @@ -0,0 +1,123 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy + +import pytest +from microduck_world.scene import WorldScene, load_world +from pydantic import ValidationError + + +@pytest.fixture +def scene_data(): + return { + "id": "test-world", + "spawn_xy": [10.0, 20.0], + "rooms": { + "studio": { + "name": "studio", + "aliases": ["workspace"], + "bounds": [9.0, 12.0, 19.0, 22.0], + "target": [11.0, 21.0, 0.0], + } + }, + "objects": {"bench": [10.5, 21.5]}, + } + + +def test_world_metadata_supports_a_relocated_scene(scene_data): + scene = WorldScene.model_validate(scene_data) + assert scene.spawn_xy == (10.0, 20.0) + assert scene.rooms["studio"].target == (11.0, 21.0, 0.0) + assert scene.objects == {"bench": (10.5, 21.5)} + + +@pytest.mark.parametrize("name", ["studio", "WORKSPACE", ""]) +def test_landmarks_cannot_shadow_room_names_or_aliases(scene_data, name): + scene_data["objects"] = {name: [10.5, 21.5]} + with pytest.raises(ValidationError, match="empty or ambiguous"): + WorldScene.model_validate(scene_data) + + +def test_two_rooms_cannot_share_an_alias(scene_data): + second = copy.deepcopy(scene_data["rooms"]["studio"]) + second["name"] = "office" + scene_data["rooms"]["office"] = second + with pytest.raises(ValidationError, match="empty or ambiguous"): + WorldScene.model_validate(scene_data) + + +def test_room_key_must_match_the_displayed_name(scene_data): + scene_data["rooms"]["studio"]["name"] = "office" + with pytest.raises(ValidationError, match="must match"): + WorldScene.model_validate(scene_data) + + +def test_navigation_target_must_be_in_its_room(scene_data): + scene_data["rooms"]["studio"]["target"] = [0.0, 0.0, 0.0] + with pytest.raises(ValidationError, match="inside its bounds"): + WorldScene.model_validate(scene_data) + + +def test_room_bounds_cannot_be_reversed(scene_data): + scene_data["rooms"]["studio"]["bounds"] = [12.0, 9.0, 19.0, 22.0] + with pytest.raises(ValidationError, match="increasing"): + WorldScene.model_validate(scene_data) + + +@pytest.mark.parametrize("coordinate", [float("nan"), float("inf")]) +def test_non_finite_coordinates_are_rejected(scene_data, coordinate): + scene_data["objects"]["bench"] = [coordinate, 21.5] + with pytest.raises(ValidationError, match="finite"): + WorldScene.model_validate(scene_data) + + +def test_shipped_scene_loads_with_its_artifacts(): + package, scene = load_world() + assert package.mujoco_scene_path.is_file() + assert scene.id == "football-club-v3" + assert set(scene.rooms) == { + "kitchen", + "living", + "bedroom", + "office", + "football", + "red_lockers", + "blue_lockers", + "player_tunnel", + "benchmark_corridor", + } + + +def test_compact_lockers_keep_half_the_previous_area_and_midfield_spawns_are_clear(): + import json + + from microduck_world.roster import SETTINGS + from microduck_world.scene import PROJECT_ROOT + + places = json.loads((PROJECT_ROOT / "assets/scenes/apartment/places.json").read_text()) + for team in ("red", "blue"): + x0, x1, y0, y1 = places["rooms"][team + "_lockers"]["bounds"] + assert abs((x1 - x0) * (y1 - y0) - (2.55 * 3.25 / 2)) < 1e-6 + bays = [r["spawn"] for r in SETTINGS["robots"].values() if r["team"] == team] + assert len(bays) == 3 + expected_y = 3.25 if team == "red" else 5.35 + assert sorted(x for x, _ in bays) == [-0.6, 0.0, 0.6] + assert all(abs(y - expected_y) < 1e-6 for _, y in bays) + import math + from itertools import combinations + + assert all( + math.dist(a, b) >= SETTINGS["clearance"] for a, b in combinations(SETTINGS["spawns"], 2) + ) diff --git a/examples/microduck-world/app/microduck_world/test_sensors.py b/examples/microduck-world/app/microduck_world/test_sensors.py new file mode 100644 index 0000000000..f1820e19a6 --- /dev/null +++ b/examples/microduck-world/app/microduck_world/test_sensors.py @@ -0,0 +1,36 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mujoco +import numpy as np +from dimos.robot.pollen.microduck.sim_module import LIDAR_CAMERA_SPECS +from microduck_world.sensors import RobotSensors + + +def test_range_sensor_sees_front_wall_without_revealing_object_behind_it(): + spec = mujoco.MjSpec.from_string(""" + + + + + """) + for name, _ in LIDAR_CAMERA_SPECS: + spec.body("trunk_base").add_camera(name=name) + model = spec.compile() + data = mujoco.MjData(model) + mujoco.mj_forward(model, data) + original_groups = model.geom_group.copy() + points = RobotSensors._range_points(model, data, "", np.array([[0.0, 0.0, -1.0]])) + np.testing.assert_allclose(points, [[0, 0, -0.95]] * 3) + np.testing.assert_array_equal(model.geom_group, original_groups) diff --git a/examples/microduck-world/app/microduck_world/test_supervisor.py b/examples/microduck-world/app/microduck_world/test_supervisor.py new file mode 100644 index 0000000000..6687ef7bf1 --- /dev/null +++ b/examples/microduck-world/app/microduck_world/test_supervisor.py @@ -0,0 +1,63 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Robot process groups are cleaned even if their parent has already exited.""" + +import os +import select +import signal +import subprocess +import sys +from contextlib import suppress + +import pytest +from microduck_world.supervisor import RobotProcess + + +@pytest.fixture +def exited_parent_with_worker(tmp_path): + # The surviving worker holds the stdout pipe open after its parent exits. + script = """ +import subprocess +import sys +child = subprocess.Popen([sys.executable, '-c', 'import signal; signal.pause()']) +print(child.pid, flush=True) +""" + process = subprocess.Popen( + [sys.executable, "-c", script], stdout=subprocess.PIPE, start_new_session=True + ) + log = (tmp_path / "robot.log").open("ab") + try: + assert process.stdout is not None + assert int(process.stdout.readline()) > 0 + assert process.wait(timeout=5) == 0 + yield RobotProcess("test", process, log) + finally: + with suppress(ProcessLookupError): + os.killpg(process.pid, signal.SIGKILL) + process.wait(timeout=5) + if process.stdout is not None: + process.stdout.close() + log.close() + + +def test_close_reaps_workers_after_the_runtime_parent_exits(exited_parent_with_worker): + runtime = exited_parent_with_worker + assert not select.select([runtime.process.stdout], [], [], 0)[0] + + runtime.close() + + assert select.select([runtime.process.stdout], [], [], 5)[0] + assert runtime.process.stdout.read() == b"" + assert runtime.log.closed diff --git a/examples/microduck-world/app/microduck_world/test_tool_isolation.py b/examples/microduck-world/app/microduck_world/test_tool_isolation.py new file mode 100644 index 0000000000..a1730b54e9 --- /dev/null +++ b/examples/microduck-world/app/microduck_world/test_tool_isolation.py @@ -0,0 +1,54 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest.mock import Mock + +from dimos.agents.annotation import skill +from dimos.agents.mcp import tool_stream +from dimos.core.global_config import global_config + + +@skill +def open_stream() -> tool_stream.ToolStream: + """Start a test skill notification stream.""" + return tool_stream.ToolStream("explore") + + +def test_background_messages_and_stop_events_stay_on_the_own_runtime_topic(monkeypatch): + transport = Mock() + factory = Mock(return_value=transport) + monkeypatch.setattr(tool_stream, "make_transport", factory) + monkeypatch.setattr(global_config, "tool_stream_topic", "/duck2/session-a/tool_streams") + stream = open_stream() + try: + monkeypatch.setattr(global_config, "tool_stream_topic", "/duck3/session-b/tool_streams") + stream.send("Found a frontier") + factory.assert_called_once_with("/duck2/session-a/tool_streams") + finally: + stream.stop() + assert transport.publish.call_args.args[0]["method"] == tool_stream.TOOL_STREAM_STOPPED_METHOD + transport.stop.assert_called_once() + + +def test_agent_subscribes_only_to_its_configured_tool_updates(monkeypatch): + transport = Mock() + factory = Mock(return_value=transport) + monkeypatch.setattr(tool_stream, "make_transport", factory) + monkeypatch.setattr(global_config, "tool_stream_topic", "/duck3/session-b/tool_streams") + close = tool_stream.subscribe(Mock()) + try: + factory.assert_called_once_with("/duck3/session-b/tool_streams") + finally: + close() + transport.stop.assert_called_once() diff --git a/examples/microduck-world/app/microduck_world/test_udp_forwarder.py b/examples/microduck-world/app/microduck_world/test_udp_forwarder.py new file mode 100644 index 0000000000..97a456cf25 --- /dev/null +++ b/examples/microduck-world/app/microduck_world/test_udp_forwarder.py @@ -0,0 +1,97 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +from collections.abc import AsyncIterator +from typing import cast + +import pytest +import pytest_asyncio +from microduck_world.udp_forwarder import UdpForwarder + + +class Echo(asyncio.DatagramProtocol): + def __init__(self): + self.incoming = asyncio.Queue() + + def connection_made(self, transport): + self.transport = transport + + def datagram_received(self, data, addr): + self.incoming.put_nowait((data, addr)) + self.transport.sendto(data, addr) + + +class Inbox(asyncio.DatagramProtocol): + def __init__(self): + self.messages = asyncio.Queue() + + def datagram_received(self, data, addr): + self.messages.put_nowait(data) + + +@pytest_asyncio.fixture +async def rig() -> AsyncIterator[tuple]: + loop = asyncio.get_running_loop() + echo = Echo() + upstream, _ = await loop.create_datagram_endpoint(lambda: echo, local_addr=("127.0.0.1", 0)) + proxy = UdpForwarder(max_peers=2) + clients = [] + try: + await proxy.start("127.0.0.1", 0) + proxy.set_target(upstream.get_extra_info("sockname")) + target = proxy.transport.get_extra_info("sockname") + for _ in range(2): + inbox = Inbox() + transport, _ = await loop.create_datagram_endpoint(lambda: inbox, remote_addr=target) + clients.append((cast(asyncio.DatagramTransport, transport), inbox)) + yield proxy, echo, clients + finally: + for transport, _ in clients: + transport.close() + await proxy.stop() + upstream.close() + + +@pytest.mark.asyncio +async def test_bidirectional_packets_keep_clients_separate(rig): + _, echo, clients = rig + clients[0][0].sendto(b"first browser") + clients[1][0].sendto(b"second browser") + assert await asyncio.wait_for(clients[0][1].messages.get(), 2) == b"first browser" + assert await asyncio.wait_for(clients[1][1].messages.get(), 2) == b"second browser" + _, first_addr = await asyncio.wait_for(echo.incoming.get(), 2) + _, second_addr = await asyncio.wait_for(echo.incoming.get(), 2) + assert first_addr != second_addr + + +@pytest.mark.asyncio +async def test_peer_limit_and_relay_restart_cleanup(rig): + proxy, _, clients = rig + for transport, inbox in clients: + transport.sendto(b"connect") + assert await asyncio.wait_for(inbox.messages.get(), 2) == b"connect" + proxy.datagram_received(b"over capacity", ("127.0.0.1", 12345)) + assert ("127.0.0.1", 12345) not in proxy.peers + upstream_sockets = [peer.socket for peer in proxy.peers.values()] + new_port = 12346 if proxy.target[1] != 12346 else 12347 + proxy.set_target(("127.0.0.1", new_port)) + assert proxy.peers == {} + assert [sock.fileno() for sock in upstream_sockets] == [-1, -1] + + +def test_non_loopback_upstream_is_rejected(): + proxy = UdpForwarder() + with pytest.raises(ValueError, match="loopback"): + proxy.set_target(("203.0.113.1", 443)) diff --git a/examples/microduck-world/app/microduck_world/test_visitors.py b/examples/microduck-world/app/microduck_world/test_visitors.py new file mode 100644 index 0000000000..4ecae2078d --- /dev/null +++ b/examples/microduck-world/app/microduck_world/test_visitors.py @@ -0,0 +1,56 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mujoco +import numpy as np +import pytest +from dimos.robot.pollen.microduck.gait import MicroduckObserver + + +@pytest.fixture +def two_ducks(): + spec = mujoco.MjSpec() + for prefix in ("", "guest_"): + robot = mujoco.MjSpec.from_string(""" + + + + + """) + spec.attach(robot, prefix=prefix, frame=spec.worldbody.add_frame()) + model = spec.compile() + return model, mujoco.MjData(model) + + +def test_standing_one_duck_preserves_the_other_ducks_motion(two_ducks): + model, data = two_ducks + host = MicroduckObserver(model, ["knee"], np.array([0.2], dtype=np.float32)) + guest = MicroduckObserver(model, ["knee"], np.array([0.3], dtype=np.float32), prefix="guest_") + data.qvel[:] = 1.25 + guest.initial_qpos(data) + np.testing.assert_array_equal(data.qvel[: guest.root_qvel_adr], 1.25) + np.testing.assert_array_equal(data.qvel[guest.root_qvel_adr :], 0) + assert data.qpos[model.joint("guest_knee").qposadr[0]] == pytest.approx(0.3) + assert data.qpos[model.joint("knee").qposadr[0]] == 0 + assert host.root_qpos_adr != guest.root_qpos_adr + + +def test_observations_read_only_the_named_robot(two_ducks): + model, data = two_ducks + guest = MicroduckObserver(model, ["knee"], np.array([0.3], dtype=np.float32), prefix="guest_") + data.qpos[model.joint("guest_knee").qposadr[0]] = 0.7 + data.qpos[model.joint("knee").qposadr[0]] = 2 + mujoco.mj_forward(model, data) + obs = guest.build(data, np.zeros(1, dtype=np.float32), np.zeros(13, dtype=np.float32)) + assert obs[6] == pytest.approx(0.4) diff --git a/examples/microduck-world/app/microduck_world/test_visual_scene.py b/examples/microduck-world/app/microduck_world/test_visual_scene.py new file mode 100644 index 0000000000..0b0271e615 --- /dev/null +++ b/examples/microduck-world/app/microduck_world/test_visual_scene.py @@ -0,0 +1,159 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base64 +import gzip +import json +from io import BytesIO + +import mujoco +import numpy as np +import pytest +from microduck_world.comparison import COMPARE_OFFSET, comparison_camera, comparison_requested +from microduck_world.visual_scene import body_snapshot, export_scene, write_scene +from PIL import Image + + +@pytest.fixture +def model(): + return mujoco.MjModel.from_xml_string(""" + + + + + + + + + + + + + + """) + + +def test_visual_groups_and_compiled_mesh_buffers(model): + scene = export_scene(model, {}) + assert [geom["name"] for geom in scene["geoms"]] == ["floor", "visual", "limb_visual"] + mesh = scene["meshes"]["0"] + np.testing.assert_array_equal( + np.frombuffer(base64.b64decode(mesh["positions"]), dtype=" + + + + + + """) + scene = export_scene(model, {}) + assert len(scene["textures"]) == 1 + assert all(g["texture"] == {"id": "0", "repeat": [1.0, 2.0]} for g in scene["geoms"]) + with Image.open(BytesIO(base64.b64decode(scene["textures"]["0"]))) as image: + np.testing.assert_array_equal(np.asarray(image), model.tex_data.reshape(4, 8, 3)) diff --git a/examples/microduck-world/app/microduck_world/udp_forwarder.py b/examples/microduck-world/app/microduck_world/udp_forwarder.py new file mode 100644 index 0000000000..c06ba3076f --- /dev/null +++ b/examples/microduck-world/app/microduck_world/udp_forwarder.py @@ -0,0 +1,127 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Bounded UDP forwarding for a private gateway to a loopback QUIC relay.""" + +import asyncio +import logging +import socket +from contextlib import suppress +from dataclasses import dataclass +from ipaddress import ip_address +from typing import cast + +logger = logging.getLogger(__name__) +Address = tuple[str, int] + + +@dataclass +class Peer: + socket: socket.socket + last_seen: float + + +class UdpForwarder(asyncio.DatagramProtocol): + """Preserve a distinct upstream UDP connection for each browser endpoint.""" + + def __init__(self, max_peers: int = 64, idle_seconds: float = 60.0) -> None: + self.max_peers = max_peers + self.idle_seconds = idle_seconds + self.peers: dict[Address, Peer] = {} + self.target: Address | None = None + self.transport: asyncio.DatagramTransport | None = None + self._loop: asyncio.AbstractEventLoop | None = None + self._reaper: asyncio.Task[None] | None = None + + async def start(self, host: str, port: int) -> None: + self._loop = asyncio.get_running_loop() + await self._loop.create_datagram_endpoint(lambda: self, local_addr=(host, port)) + self._reaper = asyncio.create_task(self._expire_peers()) + + def connection_made(self, transport: asyncio.BaseTransport) -> None: + self.transport = cast(asyncio.DatagramTransport, transport) + + def set_target(self, target: Address) -> None: + if not ip_address(target[0]).is_loopback or not 1 <= target[1] <= 65535: + raise ValueError("UDP upstream must be a loopback address and valid port") + if target != self.target: + for addr in tuple(self.peers): + self._remove(addr) + self.target = target + + def datagram_received(self, data: bytes, addr: Address) -> None: + if self.target is None or self._loop is None: + return + peer = self.peers.get(addr) + if peer is None: + if len(self.peers) >= self.max_peers: + return + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + try: + sock.setblocking(False) + sock.connect(self.target) + self._loop.add_reader(sock.fileno(), self._reply, addr) + except OSError: + sock.close() + logger.exception("Could not open UDP upstream") + return + peer = Peer(sock, self._loop.time()) + self.peers[addr] = peer + peer.last_seen = self._loop.time() + try: + peer.socket.send(data) + except BlockingIOError: + pass # UDP is lossy; never queue stale control/video packets. + except OSError: + self._remove(addr) + + def _reply(self, addr: Address) -> None: + peer = self.peers.get(addr) + if peer is None or self.transport is None: + return + try: + data = peer.socket.recv(65535) + except BlockingIOError: + return + except OSError: + self._remove(addr) + return + self.transport.sendto(data, addr) + + def _remove(self, addr: Address) -> None: + peer = self.peers.pop(addr) + assert self._loop is not None + self._loop.remove_reader(peer.socket.fileno()) + peer.socket.close() + + async def _expire_peers(self) -> None: + assert self._loop is not None + while True: + await asyncio.sleep(min(5.0, self.idle_seconds)) + cutoff = self._loop.time() - self.idle_seconds + for addr, peer in tuple(self.peers.items()): + if peer.last_seen < cutoff: + self._remove(addr) + + async def stop(self) -> None: + if self._reaper is not None: + self._reaper.cancel() + with suppress(asyncio.CancelledError): + await self._reaper + self._reaper = None + for addr in tuple(self.peers): + self._remove(addr) + if self.transport is not None: + self.transport.close() + self.transport = None diff --git a/examples/microduck-world/app/microduck_world/visual_scene.py b/examples/microduck-world/app/microduck_world/visual_scene.py new file mode 100644 index 0000000000..91f09afb85 --- /dev/null +++ b/examples/microduck-world/app/microduck_world/visual_scene.py @@ -0,0 +1,151 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Export compiled MuJoCo visuals and body poses for the project web viewer.""" + +import base64 +import gzip +import hashlib +import json +from io import BytesIO +from pathlib import Path +from typing import Any, cast + +import mujoco +import numpy as np +from numpy.typing import NDArray +from PIL import Image as PixelImage + +# Scene geoms use group 0; robot visuals use group 2. Group 3 is collision-only. +VISIBLE_GROUPS = (0, 1, 2) +SUPPORTED_GEOMS = {"plane", "sphere", "capsule", "ellipsoid", "cylinder", "box", "mesh"} + + +def packed(values: NDArray[Any], dtype: str) -> str: + """Little-endian buffers retain mesh precision without JSON float expansion.""" + return base64.b64encode(np.asarray(values, dtype=dtype).tobytes()).decode("ascii") + + +def export_scene( + model: mujoco.MjModel, appearance: dict[str, Any], camera_name: str | None = None +) -> dict[str, Any]: + geoms: list[dict[str, Any]] = [] + meshes: dict[str, Any] = {} + textures: dict[str, str] = {} + bodies: set[int] = {0} + mesh_ids: dict[int, str] = {} + mesh_hashes: dict[str, str] = {} + for i in range(model.ngeom): + if int(model.geom_group[i]) not in VISIBLE_GROUPS: + continue + material = int(model.geom_matid[i]) + rgba = model.mat_rgba[material] if material >= 0 else model.geom_rgba[i] + if float(rgba[3]) == 0: + continue + kind = mujoco.mjtGeom(int(model.geom_type[i])).name.removeprefix("mjGEOM_").lower() + if kind not in SUPPORTED_GEOMS: + raise ValueError(f"3D viewer does not support visible geom type {kind!r}") + body = int(model.geom_bodyid[i]) + bodies.add(body) + geom: dict[str, Any] = { + "id": i, + "name": mujoco.mj_id2name(model, mujoco.mjtObj.mjOBJ_GEOM, i) or f"geom_{i}", + "body": body, + "kind": kind, + "size": model.geom_size[i].tolist(), + "position": model.geom_pos[i].tolist(), + "quaternion": model.geom_quat[i].tolist(), + "rgba": rgba.tolist(), + } + if material >= 0 and kind in ("box", "plane") and not model.mat_texuniform[material]: + texture = int(model.mat_texid[material, mujoco.mjtTextureRole.mjTEXROLE_RGB]) + if texture >= 0 and model.tex_type[texture] == mujoco.mjtTexture.mjTEXTURE_2D: + key = str(texture) + if key not in textures: + width, height = int(model.tex_width[texture]), int(model.tex_height[texture]) + channels, start = int(model.tex_nchannel[texture]), int(model.tex_adr[texture]) + pixels = model.tex_data[start : start + width * height * channels].reshape( + height, width, channels + ) + buffer = BytesIO() + PixelImage.fromarray(pixels).save(buffer, format="PNG") + textures[key] = base64.b64encode(buffer.getvalue()).decode("ascii") + geom["texture"] = {"id": key, "repeat": model.mat_texrepeat[material].tolist()} + if kind == "mesh": + mesh = int(model.geom_dataid[i]) + geom["part"] = model.mesh(mesh).name + if mesh not in mesh_ids: + va, vn = int(model.mesh_vertadr[mesh]), int(model.mesh_vertnum[mesh]) + fa, fn = int(model.mesh_faceadr[mesh]), int(model.mesh_facenum[mesh]) + buffers = { + "positions": packed(model.mesh_vert[va : va + vn], "= 0: + bodies.add(focus) + camera = None + if camera_name is not None: + camera_id = mujoco.mj_name2id(model, mujoco.mjtObj.mjOBJ_CAMERA, camera_name) + if camera_id < 0: + raise ValueError(f"Camera {camera_name!r} is missing from the world") + body = int(model.cam_bodyid[camera_id]) + bodies.add(body) + camera = { + "body": body, + "position": model.cam_pos[camera_id].tolist(), + "quaternion": model.cam_quat[camera_id].tolist(), + "fovy": float(model.cam_fovy[camera_id]), + "near": float(model.vis.map.znear * model.stat.extent), + "far": float(model.vis.map.zfar * model.stat.extent), + } + return { + "camera": camera, + "version": 1, + "up": "z", + "bodyIds": sorted(bodies), + "focusBody": max(0, focus), + "geoms": geoms, + "meshes": meshes, + "textures": textures, + "appearance": appearance, + } + + +def write_scene(scene: dict[str, Any], directory: Path) -> str: + """Publish immutable assets; a new model receives a different URL.""" + data = json.dumps(scene, separators=(",", ":"), allow_nan=False).encode() + digest = hashlib.sha256(data).hexdigest()[:20] + name = f"scene-{digest}.json" + directory.mkdir(parents=True, exist_ok=True) + for suffix, content in (("", data), (".gz", gzip.compress(data, compresslevel=6, mtime=0))): + path = directory / (name + suffix) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_bytes(content) + temporary.replace(path) + return name + + +def body_snapshot(data: mujoco.MjData, body_ids: list[int]) -> list[list[float]]: + """Read on the physics thread, after a step; quaternion order is MuJoCo wxyz.""" + poses = np.concatenate((data.xpos[body_ids], data.xquat[body_ids]), axis=1) + return cast(list[list[float]], np.round(poses, 6).tolist()) diff --git a/examples/microduck-world/app/microduck_world/world_sim.py b/examples/microduck-world/app/microduck_world/world_sim.py new file mode 100644 index 0000000000..80f2b7e5ca --- /dev/null +++ b/examples/microduck-world/app/microduck_world/world_sim.py @@ -0,0 +1,355 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Microduck simulation with a read-only visual state stream for the web world.""" + +import json +import threading +import time +from pathlib import Path +from typing import Any + +import mujoco +import requests +from dimos.core.core import rpc +from dimos.core.stream import In, Out +from dimos.msgs.sensor_msgs.Image import Image +from dimos.msgs.sensor_msgs.JointState import JointState +from dimos.robot.pollen.microduck.sim_module import ( + LIDAR_CAMERA_SPECS, + POV_CAMERA_NAME, + MicroduckSimModule, +) +from dimos.simulation.engines.mujoco_engine import MujocoEngine +from dimos.simulation.engines.robot_sim_binding import RobotSimSpec +from dimos.web.codecs import web_encoder +from microduck_world.ball_physics import BALL_SPAWN_HEIGHT +from microduck_world.camera import HEAD_CAMERA, configure_clipping, configure_head_camera +from microduck_world.comparison import JpegComparison +from microduck_world.football import BALL_BODY, BALL_NAMES, FootballMatch, add_footballs +from microduck_world.gateway import GatewayConfig +from microduck_world.physics_robots import WorldRobots +from microduck_world.robot_io import ROBOT_IDS, VISITOR_IDS, RobotCommand, RobotState, RobotVision +from microduck_world.scene import PROJECT_ROOT +from microduck_world.scorers import ScorerLedger +from microduck_world.sensors import RobotSensors +from microduck_world.visual_scene import body_snapshot, export_scene, write_scene + +WORLD_FPS = 30.0 +WORLD_ENCODING = "world.json.v1" + + +@web_encoder(WORLD_ENCODING) +def encode_world_state(message: str) -> bytes: + return message.encode("utf-8") + + +class WorldSimModule(MicroduckSimModule): + world_state: Out[str] + world_compare_image: Out[Image] + duck1_command: In[RobotCommand] + duck2_command: In[RobotCommand] + duck3_command: In[RobotCommand] + duck4_command: In[RobotCommand] + duck5_command: In[RobotCommand] + duck6_command: In[RobotCommand] + duck1_state: Out[RobotState] + duck2_state: Out[RobotState] + duck3_state: Out[RobotState] + duck4_state: Out[RobotState] + duck5_state: Out[RobotState] + duck6_state: Out[RobotState] + duck1_vision: Out[RobotVision] + duck2_vision: Out[RobotVision] + duck3_vision: Out[RobotVision] + duck4_vision: Out[RobotVision] + duck5_vision: Out[RobotVision] + duck6_vision: Out[RobotVision] + + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + self._football: FootballMatch | None = None + self._visual_body_ids: list[int] = [] + self._visual_asset = "" + self._last_world_publish = 0.0 + self._comparison: JpegComparison | None = None + self._robots: WorldRobots | None = None + self._scorer_identities = {} + self._drop_lock = threading.Lock() + self._ball_drops = {} + self._sensors: RobotSensors | None = None + self._lobby_stop = threading.Event() + self._lobby_thread: threading.Thread | None = None + self._visitor_settings = json.loads( + (PROJECT_ROOT / "assets/scenes/apartment/multiplayer.json").read_text() + ) + + @rpc + def start(self) -> None: + super().start() + settings = GatewayConfig.model_validate_json( + (PROJECT_ROOT / "config/tailnet.json").read_text() + ) + assert self._engine is not None + assert self._bank is not None + self._robots = WorldRobots( + self._engine.model, self._bank, self._visitor_settings, self.config + ) + for id in ROBOT_IDS: + + def drive(command: RobotCommand, robot: str = id) -> None: + assert self._robots is not None + if command.kind == "drop_ball": + if isinstance(command.value, str) and command.value in BALL_NAMES: + with self._drop_lock: + self._ball_drops[command.value] = ( + robot, + command.generation, + time.monotonic(), + ) + else: + self._robots.command(robot, command) + + self._subscribe(getattr(self, id + "_command"), drive) + self._sensors = RobotSensors(self._engine.model, self._publish_vision) + self._sensors.start() + self._lobby_stop.clear() + self._lobby_thread = threading.Thread( + target=self._poll_lobby, + args=(settings.upstream_url,), + daemon=True, + name="visitor-assignments", + ) + self._lobby_thread.start() + self._comparison = JpegComparison( + self._engine.model, + f"{settings.upstream_url}/api/stats", + self.world_compare_image.publish, + ) + + @rpc + def stop(self) -> None: + self._lobby_stop.set() + if self._lobby_thread is not None: + self._lobby_thread.join(timeout=3) + comparison, self._comparison = self._comparison, None + try: + if self._sensors is not None: + self._sensors.close() + if comparison is not None: + comparison.close() + finally: + super().stop() + if self._football and self._football.ledger: + self._football.ledger.flush() + + def _poll_lobby(self, origin: str) -> None: + with requests.Session() as client: + client.trust_env = False + while not self._lobby_stop.is_set(): + try: + response = client.get(origin + "/internal/assignments", timeout=1) + response.raise_for_status() + occupants = response.json() + if isinstance(occupants, dict) and all( + k in VISITOR_IDS and isinstance(v, str) and v for k, v in occupants.items() + ): + assert self._robots is not None + self._robots.leases(occupants) + identities = client.get(origin + "/internal/scorers", timeout=1) + identities.raise_for_status() + values = identities.json() + self._scorer_identities = { + robot: value + for robot, value in values.items() + if robot in ROBOT_IDS + and isinstance(value, dict) + and all( + isinstance(value.get(k), str) and value[k] + for k in ("generation", "userId", "handle") + ) + and value["generation"] == occupants.get(robot) + } + if self._football and self._football.ledger: + self._football.ledger.flush() + except (requests.RequestException, ValueError): + pass # The physics mailbox expires assignments if discovery stays unavailable. + self._lobby_stop.wait(0.5) + + def _gait_pre_step(self, engine: MujocoEngine) -> None: + if self._robots is not None: + self._robots.step(engine.data) + with self._drop_lock: + for ball, (robot_id, generation, queued) in list(self._ball_drops.items()): + robot = self._robots.robots[robot_id] + valid = ( + robot.active + and robot.generation == generation + and time.monotonic() - queued < 5 + ) + if not valid or ( + self._football and self._football.drop_ball(engine.data, ball) + ): + del self._ball_drops[ball] + host = self._robots.robots["duck1"] + engine.write_joint_command( + JointState( + position=engine.data.ctrl[host.actuators][self._engine_target_perm].tolist() + ) + ) + + def _publish_vision(self, robot: str, vision: RobotVision) -> None: + getattr(self, robot + "_vision").publish(vision) + + def _compose_spec(self) -> mujoco.MjSpec: + spec = super()._compose_spec() + assert self.config.robot_mjcf is not None + # Remove the inherited trunk-mounted POV workaround. Every view uses + # the actual head mount, including the private agent RGB-D renderer. + spec.delete(spec.camera(POV_CAMERA_NAME)) + configure_head_camera(spec) + # Bind the existing host engine to its own 14 actuators before guests are attached. + host_joints = tuple(a.target for a in spec.actuators) + self.config.robot_sim_spec = RobotSimSpec( + robot_id="duck1", + hardware_joints=host_joints, + model_joint_names=host_joints, + root_body_names=("trunk_base",), + root_joint_names=("trunk_base_freejoint",), + require_floating_base=True, + ) + for id in ROBOT_IDS[1:]: + guest = mujoco.MjSpec.from_file(str(self.config.robot_mjcf)) + configure_head_camera(guest) + for camera_name, _ in LIDAR_CAMERA_SPECS: + camera = spec.camera(camera_name) + guest.body("trunk_base").add_camera( + name=camera_name, + pos=list(camera.pos), + quat=list(camera.quat), + fovy=float(camera.fovy), + ) + spec.attach(guest, prefix=id + "_", frame=spec.worldbody.add_frame()) + spec.body(BALL_BODY).pos = [9.8, 0.075, BALL_SPAWN_HEIGHT] + add_footballs(spec) + return spec + + def _compose_model(self) -> mujoco.MjModel: + model = super()._compose_model() + configure_clipping(model) + # Initial data starts with unoccupied robots outside the arena. Keep the + # compiled scene extent unchanged so the host POV clipping stays correct. + for index, id in enumerate(ROBOT_IDS): + adr = int( + model.joint(("" if id == "duck1" else id + "_") + "trunk_base_freejoint").qposadr[0] + ) + model.qpos0[adr : adr + 3] = (10 + index, 10, -5) + # This hook runs during startup, before the engine starts its physics thread. + if self.config.scene_xml is None: + raise ValueError("WorldSimModule requires a scene XML") + appearance_path = Path(self.config.scene_xml).parent / "viewer.json" + appearance = json.loads(appearance_path.read_text()) if appearance_path.exists() else {} + scene = export_scene(model, appearance, self.config.camera_name) + scene["actors"] = [] + for id in ROBOT_IDS: + prefix = "" if id == "duck1" else id + "_" + root = model.body(prefix + "trunk_base").id + body_ids = [] + for body in scene["bodyIds"]: + ancestor = body + while ancestor and ancestor != root: + ancestor = int(model.body_parentid[ancestor]) + if ancestor == root: + body_ids.append(body) + color = self._visitor_settings["colors"][id] + rgb = [int(color[i : i + 2], 16) / 255 for i in (1, 3, 5)] + # Materials are scoped by each attached robot's prefix in MuJoCo. + tinted_materials: set[int] = set() + for geom in range(model.ngeom): + if int(model.geom_bodyid[geom]) not in body_ids: + continue + material = int(model.geom_matid[geom]) + rgba = model.mat_rgba[material] if material >= 0 else model.geom_rgba[geom] + if ( + model.geom_type[geom] == mujoco.mjtGeom.mjGEOM_MESH + and max(rgba[:3]) > 0.5 + and material not in tinted_materials + ): + rgba[:3] = [ + float(base) * 0.22 + tint * 0.78 + for base, tint in zip(rgba[:3], rgb, strict=True) + ] + if material >= 0: + tinted_materials.add(material) + scene["actors"].append( + { + "id": id, + "bodyIds": body_ids, + "focusBody": root, + "camera": { + **scene["camera"], + "body": int(model.camera(prefix + HEAD_CAMERA).bodyid[0]), + }, + "color": color, + } + ) + # Export the same colors the native cameras see. + for geom in scene["geoms"]: + index = geom["id"] + material = int(model.geom_matid[index]) + geom["rgba"] = ( + model.mat_rgba[material] if material >= 0 else model.geom_rgba[index] + ).tolist() + self._visual_body_ids = scene["bodyIds"] + name = write_scene(scene, PROJECT_ROOT / "state/viewer") + self._visual_asset = f"/world-assets/{name}" + self._football = FootballMatch(model) + self._football.ledger = ScorerLedger(PROJECT_ROOT / "state/football-scorers.sqlite3") + return model + + def _after_step(self, engine: MujocoEngine) -> None: + assert self._football is not None + if self._robots is not None: + self._football.touches(engine.data, self._robots.robots, self._scorer_identities) + self._football.update(engine.data) + now = time.monotonic() + if now - self._last_world_publish < 1.0 / WORLD_FPS: + return + self._last_world_publish = now + if self._robots is not None: + for robot, state in self._robots.states(engine.data).items(): + getattr(self, robot + "_state").publish(state) + if self._sensors is not None: + self._sensors.snapshot( + engine.data.qpos, self._robots.sensor_assignments(), self._football.lit + ) + comparison = self._comparison + if comparison is not None: + comparison.snapshot(engine.data.qpos, self._football.lit) + self.world_state.publish( + json.dumps( + { + "model": self._visual_asset, + "t": time.time(), + "simTime": float(engine.data.time), + "football": self._football.snapshot(), + "actors": self._robots.snapshot() if self._robots is not None else [], + "sensorError": self._sensors.error if self._sensors is not None else None, + "comparison": comparison.status() if comparison is not None else None, + "poses": body_snapshot(engine.data, self._visual_body_ids), + }, + separators=(",", ":"), + allow_nan=False, + ) + ) diff --git a/examples/microduck-world/assets/scenes/apartment/README.md b/examples/microduck-world/assets/scenes/apartment/README.md new file mode 100644 index 0000000000..5fcba2a9a3 --- /dev/null +++ b/examples/microduck-world/assets/scenes/apartment/README.md @@ -0,0 +1,18 @@ +# Initial apartment scene + +Copied from the DimOS Microduck four-room example at commit 536679f66d1f8c4a515edc9286c16ca3af4b89a0. +The original copyright and Apache-2.0 header are retained in scene.xml. + +This project now owns this scene copy. scene.meta.json follows the existing DimOS +ScenePackage contract; places.json supplies room targets, aliases, landmarks, +scene identity and the robot spawn. Keep landmark positions consistent with the +named XML geoms. Robot model assets are fetched separately into assets/microduck +and retain their upstream license terms. + +`viewer.json` owns browser-only appearance: background and ground colors, exposure, +per-geom color overrides keyed by XML geom name, and initial camera position/target. +Coordinates use MuJoCo meters and Z-up. Changes take effect after `./service restart world`; +the generated model URL changes automatically so browsers do not keep stale geometry. +The physics XML remains authoritative for collisions. To add a physical object, change +scene.xml; to change its browser color, use viewer.json. Textures and purely decorative +scene assets are future extensions of the project viewer, not demo modifications. diff --git a/examples/microduck-world/assets/scenes/apartment/multiplayer.json b/examples/microduck-world/assets/scenes/apartment/multiplayer.json new file mode 100644 index 0000000000..58b29b4d60 --- /dev/null +++ b/examples/microduck-world/assets/scenes/apartment/multiplayer.json @@ -0,0 +1,106 @@ +{ + "scene": "football-club-v3", + "clearance": 0.5, + "robots": { + "duck1": { + "team": "red", + "number": 1, + "name": "Red 1", + "spawn": [ + -0.6, + 3.25 + ], + "yaw": 1.5707963267948966, + "mcp_port": 9990 + }, + "duck2": { + "team": "red", + "number": 2, + "name": "Red 2", + "spawn": [ + 0.0, + 3.25 + ], + "yaw": 1.5707963267948966, + "mcp_port": 9991 + }, + "duck3": { + "team": "red", + "number": 3, + "name": "Red 3", + "spawn": [ + 0.6, + 3.25 + ], + "yaw": 1.5707963267948966, + "mcp_port": 9992 + }, + "duck4": { + "team": "blue", + "number": 1, + "name": "Blue 1", + "spawn": [ + -0.6, + 5.35 + ], + "yaw": -1.5707963267948966, + "mcp_port": 9993 + }, + "duck5": { + "team": "blue", + "number": 2, + "name": "Blue 2", + "spawn": [ + 0.0, + 5.35 + ], + "yaw": -1.5707963267948966, + "mcp_port": 9994 + }, + "duck6": { + "team": "blue", + "number": 3, + "name": "Blue 3", + "spawn": [ + 0.6, + 5.35 + ], + "yaw": -1.5707963267948966, + "mcp_port": 9995 + } + }, + "spawns": [ + [ + -0.6, + 3.25 + ], + [ + 0.0, + 3.25 + ], + [ + 0.6, + 3.25 + ], + [ + -0.6, + 5.35 + ], + [ + 0.0, + 5.35 + ], + [ + 0.6, + 5.35 + ] + ], + "colors": { + "duck1": "#e34d4e", + "duck2": "#e34d4e", + "duck3": "#e34d4e", + "duck4": "#418ee8", + "duck5": "#418ee8", + "duck6": "#418ee8" + } +} diff --git a/examples/microduck-world/assets/scenes/apartment/places.json b/examples/microduck-world/assets/scenes/apartment/places.json new file mode 100644 index 0000000000..31eab01c8c --- /dev/null +++ b/examples/microduck-world/assets/scenes/apartment/places.json @@ -0,0 +1,193 @@ +{ + "id": "football-club-v3", + "spawn_xy": [ + -0.6, + 3.25 + ], + "rooms": { + "kitchen": { + "name": "kitchen", + "aliases": [ + "space A" + ], + "bounds": [ + 8.3, + 10.3, + -0.825, + 1.175 + ], + "target": [ + 9.5, + 0.17500000000000004, + 0.0 + ] + }, + "living": { + "name": "living", + "aliases": [ + "space B", + "living room", + "lounge" + ], + "bounds": [ + 6.300000000000001, + 8.3, + -0.825, + 1.175 + ], + "target": [ + 7.1000000000000005, + 0.17500000000000004, + 3.14159 + ] + }, + "bedroom": { + "name": "bedroom", + "aliases": [ + "space C" + ], + "bounds": [ + 6.300000000000001, + 8.3, + -2.825, + -0.825 + ], + "target": [ + 7.1000000000000005, + -1.825, + 3.14159 + ] + }, + "office": { + "name": "office", + "aliases": [ + "space D", + "study" + ], + "bounds": [ + 8.3, + 10.3, + -2.825, + -0.825 + ], + "target": [ + 9.5, + -1.825, + 0.0 + ] + }, + "football": { + "name": "football", + "aliases": [ + "football field", + "pitch", + "soccer" + ], + "bounds": [ + -3.25, + 3.25, + 2.1, + 6.5 + ], + "target": [ + 0, + 3.05, + 1.5707963268 + ] + }, + "red_lockers": { + "name": "red_lockers", + "aliases": [ + "red locker room", + "red changing room" + ], + "bounds": [ + -3.25, + -0.7, + 0.375, + 2 + ], + "target": [ + -1.6, + 1.25, + 0 + ] + }, + "blue_lockers": { + "name": "blue_lockers", + "aliases": [ + "blue locker room", + "blue changing room" + ], + "bounds": [ + 0.7, + 3.25, + 0.375, + 2 + ], + "target": [ + 1.6, + 1.25, + 3.1415926536 + ] + }, + "player_tunnel": { + "name": "player_tunnel", + "aliases": [ + "player entrance", + "tunnel" + ], + "bounds": [ + -0.6, + 0.6, + -0.575, + 2 + ], + "target": [ + 0, + 1.25, + 1.5707963268 + ] + }, + "benchmark_corridor": { + "name": "benchmark_corridor", + "aliases": [ + "benchmark corridor", + "side corridor" + ], + "bounds": [ + 0.6, + 6.2, + -0.575, + 0.325 + ], + "target": [ + 4, + -0.175, + 0 + ] + } + }, + "objects": { + "red_box": [ + 9.8, + 0.675 + ], + "blue_box": [ + 6.800000000000001, + 0.675 + ], + "green_cylinder": [ + 6.800000000000001, + -2.325 + ], + "yellow_pillar": [ + 9.8, + -2.325 + ], + "orange_crate": [ + 8.9, + 0.875 + ] + } +} diff --git a/examples/microduck-world/assets/scenes/apartment/scene.meta.json b/examples/microduck-world/assets/scenes/apartment/scene.meta.json new file mode 100644 index 0000000000..5b8ee2c602 --- /dev/null +++ b/examples/microduck-world/assets/scenes/apartment/scene.meta.json @@ -0,0 +1,16 @@ +{ + "source_path": "scene.xml", + "package_dir": ".", + "alignment": { + "scale": 1.0, + "y_up": false + }, + "artifact_frames": { + "mujoco": "dimos_world" + }, + "artifacts": { + "mujoco_scene": "scene.xml", + "objects": "places.json" + }, + "entities": [] +} diff --git a/examples/microduck-world/assets/scenes/apartment/scene.xml b/examples/microduck-world/assets/scenes/apartment/scene.xml new file mode 100644 index 0000000000..d57fa2e33f --- /dev/null +++ b/examples/microduck-world/assets/scenes/apartment/scene.xml @@ -0,0 +1,107 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/examples/microduck-world/assets/scenes/apartment/viewer.json b/examples/microduck-world/assets/scenes/apartment/viewer.json new file mode 100644 index 0000000000..b38809e18d --- /dev/null +++ b/examples/microduck-world/assets/scenes/apartment/viewer.json @@ -0,0 +1,88 @@ +{ + "background": "#dfe8ed", + "ground": "#cbd8d9", + "exposure": 1.1, + "camera": { + "position": [ + 6, + -5, + 8 + ], + "target": [ + 0, + 3.2, + 0.12 + ] + }, + "colors": { + "floor_kitchen": "#e7cb84", + "floor_living": "#8fc4dc", + "floor_bedroom": "#b4a7dd", + "floor_office": "#9ebf9b", + "wall_south": "#d9ddd9", + "wall_east": "#d9ddd9", + "wall_west": "#d9ddd9", + "stub_east": "#b6c1bd", + "stub_west": "#b6c1bd", + "stub_north": "#b6c1bd", + "stub_south": "#b6c1bd", + "wall_north_west": "#d9ddd9", + "wall_north_east": "#d9ddd9", + "floor": "#cbd8d9", + "floor_red_lockers": "#bd6163", + "floor_blue_lockers": "#628ebd" + }, + "views": { + "football": { + "position": [ + 4.7, + 0.6, + 4.8 + ], + "target": [ + 0, + 4.3, + 0.12 + ], + "extent": [ + 6.7, + 4.5, + 1.4 + ] + }, + "lockers": { + "position": [ + 4, + -5, + 5 + ], + "target": [ + 0, + 1.15, + 0.1 + ], + "extent": [ + 6.7, + 1.8, + 1 + ] + }, + "benchmark": { + "position": [ + 13, + -8, + 6 + ], + "target": [ + 8.3, + -0.825, + 0.1 + ], + "extent": [ + 4.2, + 4.2, + 0.8 + ] + } + } +} diff --git a/examples/microduck-world/assets/scenes/benchmark/original.xml b/examples/microduck-world/assets/scenes/benchmark/original.xml new file mode 100644 index 0000000000..2275f23544 --- /dev/null +++ b/examples/microduck-world/assets/scenes/benchmark/original.xml @@ -0,0 +1,100 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/examples/microduck-world/assets/scenes/benchmark/places.json b/examples/microduck-world/assets/scenes/benchmark/places.json new file mode 100644 index 0000000000..4345c0769b --- /dev/null +++ b/examples/microduck-world/assets/scenes/benchmark/places.json @@ -0,0 +1,121 @@ +{ + "id": "apartment-v1", + "spawn_xy": [ + 0.0, + 0.0 + ], + "rooms": { + "kitchen": { + "name": "kitchen", + "aliases": [ + "space A" + ], + "bounds": [ + 0.0, + 2.0, + 0.0, + 2.0 + ], + "target": [ + 1.2, + 1.0, + 0.0 + ] + }, + "living": { + "name": "living", + "aliases": [ + "space B", + "living room", + "lounge" + ], + "bounds": [ + -2.0, + 0.0, + 0.0, + 2.0 + ], + "target": [ + -1.2, + 1.0, + 3.14159 + ] + }, + "bedroom": { + "name": "bedroom", + "aliases": [ + "space C" + ], + "bounds": [ + -2.0, + 0.0, + -2.0, + 0.0 + ], + "target": [ + -1.2, + -1.0, + 3.14159 + ] + }, + "office": { + "name": "office", + "aliases": [ + "space D", + "study" + ], + "bounds": [ + 0.0, + 2.0, + -2.0, + 0.0 + ], + "target": [ + 1.2, + -1.0, + 0.0 + ] + }, + "football": { + "name": "football", + "aliases": [ + "football field", + "pitch", + "soccer" + ], + "bounds": [ + -3.25, + 3.25, + 2.1, + 6.5 + ], + "target": [ + 0.9, + 3.05, + 0 + ] + } + }, + "objects": { + "red_box": [ + 1.5, + 1.5 + ], + "blue_box": [ + -1.5, + 1.5 + ], + "green_cylinder": [ + -1.5, + -1.5 + ], + "yellow_pillar": [ + 1.5, + -1.5 + ], + "orange_crate": [ + 0.6, + 1.7 + ] + } +} diff --git a/examples/microduck-world/assets/scenes/benchmark/scene.xml b/examples/microduck-world/assets/scenes/benchmark/scene.xml new file mode 100644 index 0000000000..fac6cecf0a --- /dev/null +++ b/examples/microduck-world/assets/scenes/benchmark/scene.xml @@ -0,0 +1,42 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/examples/microduck-world/assets/scenes/football/field.xml b/examples/microduck-world/assets/scenes/football/field.xml new file mode 100644 index 0000000000..617a704d6f --- /dev/null +++ b/examples/microduck-world/assets/scenes/football/field.xml @@ -0,0 +1,282 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/examples/microduck-world/assets/scenes/football/surface.png b/examples/microduck-world/assets/scenes/football/surface.png new file mode 100644 index 0000000000..6b158fa54e --- /dev/null +++ b/examples/microduck-world/assets/scenes/football/surface.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe0573bf7eedcb50de31d8087a9daf52c297aed6e6d45e54868abad73ba23fe4 +size 67654 diff --git a/examples/microduck-world/config/.gitkeep b/examples/microduck-world/config/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/examples/microduck-world/dimos-revision.txt b/examples/microduck-world/dimos-revision.txt new file mode 100644 index 0000000000..f05f749537 --- /dev/null +++ b/examples/microduck-world/dimos-revision.txt @@ -0,0 +1 @@ +536679f66d1f8c4a515edc9286c16ca3af4b89a0 diff --git a/examples/microduck-world/docs/ball-physics.md b/examples/microduck-world/docs/ball-physics.md new file mode 100644 index 0000000000..e4ed142cb6 --- /dev/null +++ b/examples/microduck-world/docs/ball-physics.md @@ -0,0 +1,130 @@ +# Pollen ball and flat-floor parity + +Verified 2026-09-08 against Hugging Face revision +`e81974b932c7ca1819843b7bb3dcd42e2993e98e`: + +- [game.js](https://huggingface.co/spaces/pollen-robotics/microduck-simulator/blob/e81974b932c7ca1819843b7bb3dcd42e2993e98e/app/src/game/game.js), lines 186-225: timestep, unqualified flat plane, and ball contact attributes. +- [constants.js](https://huggingface.co/spaces/pollen-robotics/microduck-simulator/blob/e81974b932c7ca1819843b7bb3dcd42e2993e98e/app/src/game/constants.js), lines 43-44 and 67-68: timestep, radius and parking height. +- [robot_allcollisions.xml](https://huggingface.co/spaces/pollen-robotics/microduck-simulator/blob/e81974b932c7ca1819843b7bb3dcd42e2993e98e/app/public/robot/mjlab/robot_allcollisions.xml): no root geom contact or option overrides. Robot-specific named defaults do not apply to the added ball or floor. The roller MJCF was also checked and has no root contact overrides. +- [package-lock.json](https://huggingface.co/spaces/pollen-robotics/microduck-simulator/blob/e81974b932c7ca1819843b7bb3dcd42e2993e98e/app/package-lock.json): upstream WASM is locked to MuJoCo 3.11.0. + +## Applied configuration + +| Parameter | All four balls | All five physical floor boxes | +| --- | --- | --- | +| Radius | 0.05 m | Existing shape and dimensions | +| Mass | 0.03 kg | Static | +| Sliding, torsional, rolling friction | 0.4, 0.01, 0.003 | 1, 0.005, 0.0001 | +| solref | 0.03, 0.4 | 0.02, 1 | +| condim | 6 | 3 | +| solimp | 0.9, 0.95, 0.001, 0.5, 2 | Same | +| priority / solmix | 0 / 1 | Same | +| margin / gap | 0 / 0 | Same | + +The compiled sphere inertia is 0.00003 kg m² on each axis. The benchmark ball +and three pitch balls share app-owned settings in `ball_physics.py`; no shared +framework constant or dependency changed. Floors already inherited these values; +the app now sets them explicitly during composition without altering geometry. +The floor names are `floor`, `football_floor`, `club_floor`, `tunnel_floor` and +`corridor_floor`. Decorative overlays remain noncolliding. + +All 20 ball-floor combinations were checked in the compiled six-robot scene. +Each contact has dimension 6, friction [1, 1, 0.01, 0.003, 0.003], solref +[0.025, 0.7] and the default solimp. These are the result of MuJoCo's maximum +friction and equal-weight contact mixing, not the ball coefficients alone. +See [MuJoCo contact mixing](https://mujoco.readthedocs.io/en/stable/modeling.html#contact-parameters). + +Spawn centres and reset qpos0 are z=0.051 m, providing 1 mm initial clearance. +The benchmark ball retains x/y [9.8, 0.075]; pitch balls retain their existing +x/y sites. Panel positions and sizes scale with radius; each panel has mass 0 +and both collision masks 0. Three.js receives the compiled 5 cm spheres. Scoring +already reads each compiled radius and now requires the full larger sphere to +pass the opening; the tests cover the extra 1.5 cm before a goal is awarded. + +## Validation + +The independent fixture in `ops/fixtures/pollen_flat_ball.xml` reproduces Pollen's +ball and flat plane without importing application constants. Its only placement +change is translating the horizontal parking position to the origin. Each +candidate is extracted from the compiled world ball and the existing pitch box, +with that box translated so its top is z=0. Ball inertia and simulation options +are checked before comparison. Both sides run on the same MuJoCo 3.10.0 process, +at a 0.005 s timestep with gravity [0, 0, -9.81]. No robot, policies, walls or +terrain affect these isolated comparisons. + +| Experiment | Reference and each of the four candidates | +| --- | --- | +| Rolling, initial 0.5 m/s and matching 10 rad/s spin | Below 0.01 m/s after 0.744627 m and 4.815 s | +| Maximum rolling state difference over 10 s | 6.22e-15 | +| Drop, centre initially at 0.5 m | First contact at 0.310 s | +| First rebound, centre / bottom height | 0.057960 m / 0.007960 m | +| Resting centre height | 0.049869 m | +| Maximum drop state difference over 10 s | 3.89e-16 | + +The drop produces up to 24.10 mm of transient soft-contact penetration on both +configurations, settling to 0.131 mm penetration. This is the verified upstream +soft-contact response, separate from the initial placement, which clears the +floor. The approximate 0.75 m / 5 s rolling sanity check is satisfied; it is not +used as an exact assertion across engine versions. + +Both existing ONNX kick policies made physical contact with the appropriate ankle, +moved the ball, scored once, and left the duck upright: + +| Policy | Net ball travel after 2 s | Peak speed | Score | +| --- | --- | --- | --- | +| Left kick | 0.810 m | 1.403 m/s | 1-0 | +| Right kick | 0.836 m | 1.260 m/s | 1-0 | + +All four actual scene balls also scored in both directions in isolated physics +tests. Scoring was verified not to mutate qpos or qvel. All six walking policies +ran for four simulation seconds from their locker spawns, and all five passages +between lockers, tunnel, pitch, corridor and benchmark wing remained clear. +The Python suite passed 132 tests, including whole-ball scoring, post/crossbar +misses, reset clearance, massless markings and the geometry exported to Three.js. +The 14 frontend tests, 15 lobby tests, TypeScript check, app physics type checks +and Ruff checks also passed. Regenerating the pitch preserved all 269 geom +attribute values; two XML number formats were normalized without changing values. + +Physics commit `a464222` was deployed to the existing world service. Its live +model is `scene-7b19186ca353a26ef125.json`: all four spheres report radius 0.05 m +and rest at z=0.049869 m at their original x/y positions. Six actors remain in +the streamed model. The live browser checkpoint passed all six walking previews, +connection-time naming, cancellation, mobile layout and the spectator path. +Temporary sessions started all six private robot runtimes; each exposed its 20 +human CLI/agent tools. The checkpoint sent no movement commands and released all +six slots. Live evidence is in `logs/ball-physics-checkpoint/live.json` and +`cli.json`, with browser results in `logs/picker-check/results.json`. + +Run the checks from the app repository after sourcing its environment: + +```bash +python -m pytest -c pyproject.toml --asyncio-mode=auto app/microduck_world -q +python ops/demo_ball_physics_checkpoint.py +MUJOCO_GL=egl python ops/demo_football_checkpoint.py +MUJOCO_GL=egl python ops/demo_club_checkpoint.py +``` + +Worktrees must set `PYTHONPATH` to their own `app` directory when reusing the +production virtual environment. Comparison reports and NumPy trajectories are +written to `logs/ball-physics-checkpoint/`; kick GIFs and measurements are in +`logs/football-checkpoint/`. + +## Runtime decision and remaining differences + +Keep the server's MuJoCo 3.10.0. Upstream uses WASM 3.11.0, but the required +contacts already compile correctly and both configurations match on the current +engine. The [3.11 release notes](https://mujoco.readthedocs.io/en/stable/changelog.html#version-3-11-0-july-27-2026) +include new contact features, an implicitfast integration change, a different +sleep tolerance and API/ABI changes. This scene uses Euler integration, default +Newton/pyramidal solving and no added adhesion or surface velocities. No upgrade +is needed for the requested ball-floor settings. This comparison does not claim +bit-identical native 3.10 and browser WASM 3.11 trajectories. + +Our finite box floors, football goals, rigid nets, walls and six-robot interactions +remain different from Pollen's single-robot arena. Tests establish parity for +isolated flat-floor contacts, not full-scene trajectory identity. Spawn centres +have 1 mm clearance instead of upstream's resting park height. The existing +server-authoritative physics, Three.js viewer, names, camera behavior, teams, +locker spawns, benchmark rooms, teleop, policies and human CLI remain in place. +There are no terrain bumps, artificial kick impulses, new kickoff/reset rules +or automatic ball respawns. diff --git a/examples/microduck-world/docs/client-rendering.md b/examples/microduck-world/docs/client-rendering.md new file mode 100644 index 0000000000..485f0fc701 --- /dev/null +++ b/examples/microduck-world/docs/client-rendering.md @@ -0,0 +1,150 @@ +# Client rendering checkpoint — 2026-09-05 + +Live on https://omarchy.tailca0707.ts.net:8443. Chrome/Chromium on the tailnet is +required by the existing WebTransport cockpit. This is a first visual foundation; +public access, visitor-owned ducks and the scene/UI redesign remain separate work. + +## Responsibilities + +MuJoCo on Omarchy remains authoritative for physics, policies, collisions, navigation +and the robot camera. The chase camera is disabled. The head camera remains 640×360 +at up to 6 fps, including the image used by the agent. It works without any visitor +providing a rendered frame. + +WorldSimModule extends the existing Microduck simulation hooks in the project package. +At startup it exports the compiled visible geoms and meshes, then publishes body poses +from the physics thread at a maximum of 30 Hz. No physics state is mutated by the viewer. +The existing DimOS stream, codec registry, relay and browser SDK carry world_state. +The initial model is identified by a content hash, so a restarted/changed model cannot +be paired accidentally with old geometry. Geometry is transferred once and cached. + +The project web entry registers world and POV variants of world3d, then mounts the +pinned stock cockpit with the SDK session exposed through a project context. +Controls, navigation, policies, chat and transport stay in the existing implementation. +Three.js renders the model on the visitor's GPU, interpolating poses and maintaining a +camera local to that browser. Orbit, zoom, pan, Overview and Follow duck affect only +that camera. A frozen pose stream displays a waiting indicator. Rendering stops while +the tab is hidden, and resources are disposed when the panel is removed. + +Source ownership: +- app/microduck_world/visual_scene.py: compiled model export and immutable assets. +- app/microduck_world/world_sim.py: existing sim hooks and the pose stream. +- app/microduck_world/cockpit.py: panel declaration and layout. +- web/src/: panel, renderer, protocol validation and styling. +- assets/scenes/apartment/viewer.json: scene-specific browser appearance. +- state/viewer/: ignored generated JSON/gzip model files; no secrets or policies. + +No new vendor or demo edits were required. The existing DimOS patch checksum is +unchanged: a4dd8f9074cb72ce7c9cc98ddec3f871b63c362308aeae2c043521ae7ee9155d. + +## Build and operate + +Source env.sh before commands. ./setup now installs frontend dependencies from +web/deno.lock, type-checks and builds the project UI. For a frontend-only change: + + (cd web && deno install --frozen && deno task check && deno task test && deno task build) + +Reload the browser after a frontend build. For scene/appearance or Python changes, +restart the world with ./service restart world. Gateway private configuration: + + "frontend_dir": "/home/tule/projects/microduck-world/web/dist", + "world_assets_dir": "/home/tule/projects/microduck-world/state/viewer" + +Only these generated/static directories are served, after gateway access checks. +The patch and all project code/assets remain within the project folder. + +## Validation + +- 34 project Python tests pass. New tests compare exported positions AND rotations + against real MuJoCo geom transforms after free-body movement and a joint rotation; + compare binary mesh buffers; check immutable/gzip model export; verify frontend and + model route access restrictions, including a symlink escape attempt. +- Strict mypy passes on the four new/changed runtime modules; Ruff check/format pass. +- Three frontend protocol tests, TypeScript check and production build pass. +- uv pip check: all 265 installed packages compatible. Existing vendor patch still + applies to the clean pinned checkout and matches the actual vendor diff exactly. +- Two independent Chromium sessions on the Mac received the same moving duck while + retaining separate cameras. Orbiting one changed its camera from (4.2,-5.2,4.5) + to approximately (-3.978,-3.141,6.322); the other stayed at its original camera. +- Mouse orbit, wheel zoom, right-drag pan, Follow duck, Overview, maximize and Escape + restore were exercised. Maximized canvas filled 1438×846 CSS pixels. +- A bounded 2-second W drive moved the rendered trunk about 24 cm; follow camera + moved with it and release returned all command velocities to zero. The test uses + DOM key events with code=KeyW because agent-browser 0.27.1 keydown omits code. +- Clicking the kitchen room reached approximately (1.082,0.891). Sit/stand changed + rendered trunk height from about 0.116 m to 0.059 m and back, without a fall. +- Agent chat called observe, received an image, and described the yellow/green floor, + gray column, white ball and yellow cylinder visible in its MuJoCo camera. +- Reload reused the model from cache (0 transferred bytes). Restarting the simulation + recovered live 3D and camera feeds in already-open browsers. +- Final state: Teleop, walk, no active goal, drive disarmed. No browser JS errors. + +Measured model: 89 geoms, 38 unique meshes, 17 body transforms. Initial asset is +10,383,726 bytes decoded / 4,279,166 bytes gzip. Live poses were observed around +26–28 Hz (30 Hz cap); head camera around 5.5–6 fps. The client draws frames locally +between pose updates. These are desktop smoke measurements, not a load benchmark. + +Evidence: logs/client-3d-overview.png, logs/client-3d-follow.png, +logs/client-3d-maximized.png. The earlier overnight soak predates this renderer. + +## Limits and next extensions + +The visitor pays the initial geometry download and local GPU cost. This exporter +covers the primitives/meshes and visible geom groups used by this scene, using colors +and computed mesh normals; it does not promise full MuJoCo texture/material parity, +skins, flex bodies or heightfields. Unsupported visible geom types fail explicitly. +Project lighting/colors already differ intentionally from the agent's camera. +Decorative additions must stay consistent with physical geometry where interaction +matters. Camera-wall occlusion is ordinary orbit-camera behavior; camera collision +avoidance is not implemented. Small-screen cockpit layout still needs the planned UI pass. + +Server logs still contain the previously observed shared-memory fallback and skipped +camera-render warnings; live streams and controls passed the checks above. No claim +of eliminating all server bottlenecks, reboot validation or overnight 3D soak is made. +The current access boundary is still the tailnet, not public visitor authorization. + + +## Camera comparison checkpoint — 2026-09-05 + +Both World and Duck camera now default to Three.js and display completed client +draws per second. This differs from simulation pose updates (maximum 30 Hz): poses +are interpolated between updates. The POV uses the actual MuJoCo camera's body, +local transform, field of view and clipping planes, with a fitted 16:9 viewport. + +Each panel has an independent Three.js / MuJoCo JPEG selector. The native duck feed +is still the agent's 640×360, maximum 6 FPS camera. The optional world JPEG uses a +fixed shared follow camera, 640×360, intentionally capped at 12 FPS to limit cost. +This cap is a configuration choice, not a claim about MuJoCo's maximum performance. + +Project comparison.py owns a separate render thread and private model/data. It +polls the existing relay's subscriptions once per second. With no world JPEG viewers +it releases its renderer and does not render or copy physics snapshots. With viewers +it renders one shared feed regardless of viewer count. The project panel explicitly +subscribes only while JPEG is selected and visible; Three.js drawing pauses for that +panel. No new vendor/demo changes or agent image calls were needed. + +Added ownership: comparison.py handles native comparison rendering; frameRate.ts +measures draws; viewerSession.ts exposes the existing SDK session for subscriptions. +The two browser panels share a single cached model download. + +Validation for this checkpoint: +- 42 Python tests, 6 frontend tests, strict TypeScript/build and runtime mypy pass. + Real MuJoCo tests verify camera world position/orientation, clipping planes and + the native follow camera position; subscription tests verify idle demand handling. +- Mac Chromium observed approximately 60 FPS for both Three.js panels, with poses + arriving around 26–28 Hz. Separate viewers retained independent backend choices. +- Native world production delivered 96 frames in 8.015 seconds; simulation time + advanced 8.015 seconds as well. Browser delivery was about 11.5 FPS world / 6 FPS + duck in the desktop sample. Indicative render time was about 6 ms per world frame. +- Switching all viewers back to Three.js made the native worker inactive; its frame + counter stayed unchanged and the additional GPU allocation was released. +- The bounded browser checkpoint captured both backends, verified hidden canvas + behavior and idle shutdown, then drove the duck about 24 cm while the new POV + followed it. Evidence is logs/camera-checkpoint.json, camera-three.png and + camera-jpeg.png. Run with: python ops/demo_camera_checkpoint.py. + +The bounded check runs headless Chromium with software WebGL on Omarchy. Its low +world FPS is not a client GPU benchmark. Mac automation later stalled on screenshot +capture, so final visual captures and the full flow used this independent browser. +The prior agent observation and overnight soak are earlier checkpoints, not reruns +of those checks after this change. diff --git a/examples/microduck-world/docs/cockpit-ui.md b/examples/microduck-world/docs/cockpit-ui.md new file mode 100644 index 0000000000..2adf261d95 --- /dev/null +++ b/examples/microduck-world/docs/cockpit-ui.md @@ -0,0 +1,52 @@ +# Cockpit presentation + +The application keeps its existing DimOS manifest layout. Each panel title bar supports dragging (or Alt + arrow keys), minimizing, and the existing maximize/restore action. Minimized panels reappear from the workspace toolbar; Reset panels restores the initial layout. Position changes are session-local. The dark/light preference persists in browser storage. + +Football camera uses the same frame as Duck camera. Its image and YOLO boxes still come from the paired server frame. The detection status remains below the feed. + +The app registers its humancli and Teleop presentations through the DimOS panel registry. `web/src/cockpit/` adapts the pinned Apache-2.0 DimOS UI components. A Vite plugin substitutes the local frame for shared panels and maps shared CSS colors to theme variables. The shared framework checkout is unchanged. + +Humancli retains DimOS's ChatLog, streamed rows, pending/retry handling, and send path. The transcript stays visible in Teleop mode. Input, send, and retry are disabled until Agent mode returns. Its logo and terminal colors follow humancli. + +Focusing Teleop requests `set_mode(teleop)` through the existing control command channel, then waits for the mode stream acknowledgment before arming the existing TeleopMachine. Focus loss, minimization, a hidden tab, disconnect, and unmount release the lease and stop motion. Keyboard messages remain scoped to Teleop. No global drive-key listeners were added. + +Validation on 2026-09-10: production build and TypeScript check passed; all 14 existing frontend tests passed. Live browser checks covered server-acknowledged mode switching, disabled/enabled chat input, camera dragging, panel reset, camera minimize/restore, Teleop disarm on minimize, and both themes. No browser errors were reported. Physics and services were not restarted for this frontend update. + + +## Follow-up layout and sign-out + +Teleop sits above two square camera feeds, with the map below. The default world camera starts at the pitch view. The Three.js POV uses the panel aspect ratio; the native football image and its SVG detections use the same centered crop. Detection still runs on the complete native frame. Maximized camera views remain square and fit the viewport. + +Clicking humancli requests Agent mode through the same control channel used by the mode buttons. The composer enables only after the mode stream acknowledges Agent mode. + +Sign-out clears both site cookies (session and OAuth state), deletes the corresponding server-side session and pending OAuth state, revokes access to the match, and clears local join/profile state. A site cannot clear cookies belonging to github.com, so this does not sign the user out of GitHub itself. + +Follow-up validation: 14 frontend tests and 12 edge tests passed, including session invalidation and deletion of both cookies. Type checks and production build passed. Live checks verified equal 180.125 px camera image dimensions, coincident image/overlay bounds, Agent mode acknowledgment, and a maximized 978 px square camera fitting inside the 1600 by 1100 viewport. The test duck was released. + + +## Rendering and account-choice correction + +Humancli now uses the full native sender-prefix width in wide panels and stacked metadata in narrow panels, avoiding text overlap. Its ASCII banner scales as vector text. Bold and inline code are rendered as safe React text elements; transport-only reasoning/function-call markers are omitted. Live checks verified tool calls, tool results, formatted coordinates, and no row overlap at normal and maximized sizes. + +GitHub authorization explicitly requests `prompt=select_account`, so an existing GitHub session presents account choice. This does not force re-entry of a GitHub password or clear GitHub-owned cookies. + +All six ducks now spawn facing the pitch at x=-0.3 (red) or x=0.3 (blue), with y=1.8, 1.2, and 0.6. These positions are in the central passage outside the midfield sideline. The 0.5 m inter-duck clearance is unchanged. A real-policy test activated all six, verified upright standing for three seconds, and checked manual respawn; all twelve pitch-entry walking cases passed. Validation: 26 scene/physics tests, 16 frontend tests, and 12 edge tests, plus type checks and build. + +Camera panel headers keep the title, renderer or duck selector, and window controls on one row. FPS appears over each feed. Drag the bottom-right handle to resize a panel, or focus the handle and use arrow keys. Reset panels restores the default layout. Native sensor images are 640×360; square defaults center-crop the image. YOLO processes the complete native frame. Duck Three.js FPS counts client draws; football FPS counts decoded detection images. + +The current layout places a 220px-wide YOLO window over the upper-left world view. Duck camera occupies the full controls-column width. Both feeds preserve native simulation 16:9 framing with letterboxing when resized, rather than cropping. Expanded camera framing uses the same camera mount and vertical field of view. Physical camera resolution and field of view remain provisional according to Pollen's press kit, so 640×360 is a simulation configuration, not a verified hardware specification. + +YOLO currently receives server-rendered MuJoCo RGB images, predicts sports-ball boxes on Omarchy, and sends JPEG images plus bounding-box JSON through the existing DimOS camera channels. The browser displays JPEG pixels and SVG boxes. Humancli currently receives raw images via observe; it does not subscribe to ball_detections. + +Proposed agent integration: route each FootballObservation to its owning robot generation, retain the synchronized RGB-D observation, and expose a read-only get_detections skill through that robot's MCP server. Return timestamp, class, confidence, box and depth-derived position, reject stale or previous-generation observations, and avoid giving one duck another duck's detections. Add per-duck tracking only when persistent object IDs are needed. This integration is planned, not implemented. + +Floating, expanded and minimized panels preserve their docked slot height so adjacent panels do not expand unexpectedly. Humancli requests Agent mode on pointer-down or keyboard focus and permits immediate local drafting; sending remains gated on the server mode update. Leaving the panel clears the local editing state. + +Public connection incident: repeated account-level replacements caused automatic reconnecting tabs to take ownership back from each other. Edge diagnostics confirmed the replacement reason with small pending queues. PublicTransport now notifies its owning session of an intentional replacement, and LobbyApp closes that session to stop retries. Workspace shows a replacement notice. Ordinary disconnects retain SDK reconnection. Reload existing tabs to load this behavior. Diagnostics log only reasons and queue counters. + +The marked pitch is now 2.6×1.6 m, centered at (0, 4.3). The surrounding floor, walls, rooms, goal dimensions and duck spawns remain unchanged. Goal centers move to x=±1.3 m; the ball spawn positions and pitch paint scale around midfield. + +A Three.js wall board beside the team scoreboard displays the top eight all-time GitHub scorers. DimOS attributes goals to the last authenticated duck contact; own goals, anonymous contacts, ambiguous simultaneous contacts, and stale participant generations receive no personal credit. Physical goals still count for the team. GitHub numeric IDs are the database keys, and authenticated handles are displayed. SQLite state is stored in state/football-scorers.sqlite3 and survives scoreboard resets and service restarts. Database writes occur on the lobby worker thread. The cosmetic board is absent from native MuJoCo RGB and YOLO images. + + +Midfield recovery update: duck spawn and manual respawn positions are x=-0.6, 0, 0.6 at y=3.25 for red and y=5.35 for blue, facing the pitch. The scorer board is 2.2 by 1.15 metres with larger handle and goal text. Teleop includes individual drop buttons for all three pitch balls and the benchmark ball. Each requests a server-authoritative reset to (0, 4.3, 2.0), clears velocity and previous goal attribution, and falls under normal MuJoCo gravity. A busy drop position queues briefly until clear; stale or revoked-player requests expire. The app extends its own DimOS bridge command validator with drop_ball; shared framework code is unchanged. diff --git a/examples/microduck-world/docs/dimtele-feasibility.md b/examples/microduck-world/docs/dimtele-feasibility.md new file mode 100644 index 0000000000..2c76614f37 --- /dev/null +++ b/examples/microduck-world/docs/dimtele-feasibility.md @@ -0,0 +1,100 @@ +# dimTELE feasibility for Microduck World + +Reviewed 2026-09-06. This is a source review and implementation proposal, not a deployed dimTELE integration or a public-network benchmark. + +## Decision + +**Feasible, but not a configuration-only migration.** dimTELE supplies reusable Internet teleoperation transports. Its current broker and browser application do not supply our anonymous, three-duck multiplayer experience. + +For the immediate goal of publishing `sim.tule.world`, I recommend an optional WebSocket browser transport in the existing DimOS web SDK/relay, carried through Cloudflare Tunnel. This retains the current cockpit, Three.js rendering, robot blueprints, admission rules and scene assets. It does not remove DimOS from the stack. dimTELE/WebRTC remains a useful subsequent option for camera video and physical-robot teleoperation. + +If proving dimTELE integration is the higher priority, first build a bounded one-duck prototype, including an independent read-only spectator and a forced connection failure. Expand to three ducks only after those paths work. Do not begin by replacing all three working blueprints. + +## What was inspected + +- DimOS checkpoint `e0676c6`: `dimos/core/transport.py`, `dimos/protocol/pubsub/impl/webrtc/providers/broker.py`, `video_track.py`, and `dimos/teleop/hosted/`. +- Broker repository `dimensionalOS/dimensional-teleop`, commit `895356b`: session routes, tenant authorization, Cloudflare orchestration, LiveKit token grants and browser connection code. This repository is private; it has not been copied into the public DimOS PR. +- Hosted project checkpoint `579c6e7`: project relay/lobby, gateway, cockpit declarations, world snapshots and the public-access plan. The live DimOS vendor remains on its existing tested pin plus patch. +- Current Cloudflare documentation for SFU DataChannels, TURN and Tunnel WebSockets. + +No broker credentials were created, no Cloudflare resources were provisioned, and no running duck was connected to dimTELE during this review. Permission to use the shared production broker for public visitor traffic, its deployed revision, account limits and billing remain unverified. + +## What dimTELE already provides + +DimOS has real `CloudflareTransport`, `CloudflareVideoTransport` and audio transport implementations. The robot makes outbound connections; browsers and robots exchange WebRTC media/data through Cloudflare Realtime SFU after HTTPS setup through the broker. Omarchy can continue hosting MuJoCo and every agent without exposing an inbound WebRTC port. + +The transport binds ordinary typed DimOS streams. Therefore the boundary can remain portable: each duck's transport adapter receives that duck's image/odometry and sends typed movement or skill requests to its existing controller. A future physical connection module can sit behind the same boundary. This does not make the MuJoCo ONNX locomotion executor itself hardware-ready. + +Three independently running duck processes fit three broker sessions. The inspected broker namespaces sessions by owner and a client-supplied robot ID, so explicit distinct robot IDs can coexist under an owner. The provider is a per-process singleton keyed by configuration; all broker-bound modules for one duck must share the appropriate worker/configuration. The shared world remains one simulation, not three simulations. + +WebRTC can carry game state as well as cameras. Cloudflare supports named DataChannels with multiple subscribers and delivery settings suitable for disposable state updates. Its current API also supports optional single-subscriber reply access; the inspected dimTELE broker uses separate forward and reverse channels instead. Platform support does not mean the broker has already implemented our multiplayer policy. [Cloudflare DataChannels](https://developers.cloudflare.com/realtime/sfu/datachannels/) + +TURN is needed for robust access from restrictive networks. Cloudflare provides UDP, TCP and TLS relay options; merely configuring STUN is insufficient for every visitor network. [Cloudflare TURN](https://developers.cloudflare.com/realtime/turn/) + +## Gaps that matter for this project + +| Requirement | Inspected dimTELE behavior | Required integration | +|---|---|---| +| Anyone can choose an available duck | Operators authenticate through Cognito; listing/joining checks robot owner or admin | A scoped visitor grant backed by our lobby; do not share an owner login or robot API key with visitors | +| Three exclusive duck slots | One operator claim per robot session; the same account may reclaim its own slot | Bind ownership to a unique participant and duck generation, including concurrent tabs, revocation and our reconnect grace | +| Read-only world viewer | Join schema accepts `viewer`, but Cloudflare channel bridging checks the bound operator and uses one stored operator session; shipped browser joins as operator | Complete spectator registration, independent subscriptions and cleanup; world-only permissions; spectator departure must not evict a controller | +| Three.js world, score and actors | Robot video/map/telemetry are built in; no world-state manifest contract | Feed the existing world snapshot to a public read-only channel, keep geometry/textures on HTTPS and retain client interpolation | +| Agent chat, tool activity, policies and annotated maps | Fixed command/state/map channels; no adapter for our web SDK's manifest, codecs and replay | Map those streams explicitly, or implement a general SDK transport adapter with bounded replay/backpressure | +| Microduck controls | Broker accepts only `go2` and `arm`; existing command handlers are robot-specific | Add a Microduck/generic robot capability path; call our controller rather than reusing Go2 action IDs | +| Persistent headless hosting | Provider closes after repeated terminal heartbeat responses; no automatic full redial in that path | Supervised reconnect, new SFU session/channel IDs, deadman stop and generation-safe restoration | +| Three independent robot minds | Transport sessions do not implement knowledge isolation | Preserve our per-duck blueprints and private stream wiring; never feed shared world geometry into agents | + +The optional LiveKit backend does not remove these gaps. In the inspected broker, viewer tokens still receive data-publishing permission. That is not proof that a viewer can move a robot, but it is insufficient evidence for a read-only boundary; robot-side sender/role checks and tighter grants would need validation. The current DimOS checkout also lacks the corresponding LiveKit provider, so this is not the shortest integration path. + +The DimOS provider warns above 32 KiB per data message but still sends it. That warning is not an enforced payload limit or fragmentation mechanism. Full maps, transcripts and JPEGs cannot be routed blindly through those channels. Use bounded frames/chunks, request a fresh state after reconnect, and use media tracks for video. No payload-limit or throughput benchmark has been performed here. + +## Architecture if we adopt dimTELE + +```text +Browser at sim.tule.world + ├─ HTTPS through Cloudflare Tunnel → Omarchy lobby and scene assets + ├─ scoped session authorization → dimTELE broker + └─ WebRTC through Cloudflare Realtime SFU + ├─ chosen duck: commands, private telemetry/chat/map, optional camera + └─ public world: read-only poses, actors and score + +Omarchy + ├─ one MuJoCo physics world + ├─ Duck 1 blueprint → its controller, sensors, map and agent → session 1 + ├─ Duck 2 blueprint → its controller, sensors, map and agent → session 2 + └─ Duck 3 blueprint → its controller, sensors, map and agent → session 3 +``` + +The public world needs its own logical read-only stream. It could initially use WebSocket through the existing relay, or become a separate SFU publisher. Spectators must not subscribe to every duck's private camera/map/chat merely to render the world. + +Our lobby should remain the authority for seat allocation and generation changes. Broker sessions should derive from that decision through short-lived scoped credentials. Maintaining two independent ownership systems would create release/reconnect races. A stale participant must lose both command access and private subscriptions. + +The existing production broker would require a supported extension and deployment by its maintainers. Alternatively, a separate broker could run on Omarchy behind Tunnel, but its current setup adds Cognito, a database and Cloudflare SFU/TURN credentials. Source access alone does not establish authorization to operate its shared service or redistribute private broker code. + +## Cost and performance implications + +MuJoCo physics and the agents' own server-rendered images continue unchanged. Browser Three.js rendering does not become smoother merely because the transport changes; the existing renderer already interpolates incoming state locally. + +Using WebRTC for camera video introduces encoding work on Omarchy and may reduce network use compared with individual JPEG frames. The inspected provider prefers H.264, but it does not establish an NVIDIA hardware-encoding configuration. Three camera tracks require a benchmark under physics and agent load. SFU fanout can reduce Omarchy's upload amplification as spectators grow, while introducing a managed service and its billing. + +For our current primary display, small world-state messages drive browser rendering and no continuous server video is required. This makes a WebSocket option attractive for the first public release. Cloudflare Tunnel explicitly supports WebSockets. [Cloudflare Tunnel FAQ](https://developers.cloudflare.com/cloudflare-one/faq/cloudflare-tunnels-faq/#does-cloudflare-tunnel-support-websockets) + +WebSocket's reliable TCP delivery can delay newer messages behind older traffic during loss or congestion. The implementation must cap queues, replace stale world/video updates, avoid accumulating old motion commands, prioritize control, preserve the deadman timeout, and reconnect without replaying movement. It is a smaller migration, not an automatic latency improvement. + +## File ownership and proposed checkpoints + +Generic SDK/relay transport code belongs in DimOS. A future generic broker capability or spectator correction belongs in dimTELE. Lobby grants, Microduck command adaptation, blueprints and deployment configuration belong in `microduck-world`. Football rules, scene geometry, colors and textures remain project assets. No scene-specific additions to DimOS demos are necessary. + +1. Choose the first public transport. For WebSocket, add it behind the existing SDK/relay interface with bounded control/state handling. For dimTELE, first prove one isolated duck plus one read-only spectator, including a forced redial. +2. Validate all three duck slots, duplicate tabs, fresh visitor generations, delayed/revoked commands, private-stream denial and world-only spectators. Do not drop existing chat, policies, maps or JPEG/Three.js comparison features. +3. Configure the Omarchy tunnel and domain, settle Duck 1 persistence/access, and enforce anonymous-agent and spectator budgets. Test from outside the tailnet, including a restrictive network and a slow client. +4. Measure physics timing, client rendering, command latency, reconnects and resource use with three controllers and representative spectators before switching the default public transport. + +One upgrade detail surfaced during the checkpoint: upstream `main` now recognizes acknowledged `pub` commands, while the hosted lobby currently gates its pinned protocol's command set (`tx`, twist, stop and teleop leases). Before upgrading that vendor pin, extend the admission policy and tests to every new write path, including `pub`. The working service was not upgraded during this checkpoint. + +## Source pointers + +- [DimOS hosted teleoperation](https://github.com/dimensionalOS/dimos/blob/e0676c6f3926e855c33dd397d4120ccf98073989/docs/capabilities/teleoperation/hosted.md) +- [DimOS broker provider](https://github.com/dimensionalOS/dimos/blob/e0676c6f3926e855c33dd397d4120ccf98073989/dimos/protocol/pubsub/impl/webrtc/providers/broker.py) +- Private broker revision `895356b`: `app/routers/sessions.py` (`create_session`, `_owns`, `_claim_operator_slot`, `join_session`, `bridge_datachannel`, `leave_session`); `app/services/auth.py`; `app/services/livekit.py`; `web/js/webrtc.js`. +- Project: `relay/lobby.ts`, `relay/main.ts`, `app/microduck_world/cockpit.py`, `app/microduck_world/world_sim.py`, and [public access](public-access.md). diff --git a/examples/microduck-world/docs/football-club.md b/examples/microduck-world/docs/football-club.md new file mode 100644 index 0000000000..9ac6c2aff5 --- /dev/null +++ b/examples/microduck-world/docs/football-club.md @@ -0,0 +1,206 @@ +# Football club iteration + +Six equivalent player slots share one MuJoCo world: Red 1 to Red 3 (`duck1` to +`duck3`) and Blue 1 to Blue 3 (`duck4` to `duck6`). The roster, team colors, +spawn bays, facing direction and private MCP ports are defined in +`assets/scenes/apartment/multiplayer.json` and consumed by Python and the lobby. + +## Scene and play + +The pitch retains its current dimensions, three field balls, goal geometry, nets, +physical kicking and direction-aware scoreboard. The score protocol retains its +existing `blue` and `coral` keys; the human UI calls the second team Red. There are +no kickoff, timer, automatic ball placement or new score reset rules. Existing +restart/reset behavior remains unchanged. + +Each team starts in its own locker room just south of the pitch. Each room is +2.55 by 1.625 metres, half its previous floor area. Its three spawn bays keep their +original lateral spacing and move forward to y=1.375. Both rooms open +into the central player tunnel. The narrow corridor behind the lockers leads east +to the original four-room benchmark scene, translated by (8.3, -0.825) metres. +The internal room layout and objects are retained. Its original ball moves with it. + +The roster holds three spawn bays per team. A spawn or explicit respawn uses the +requested bay if clear, then another clear bay in the same locker room. It never +uses the opposing team's room. A blocked request waits. Respawn cancels previous +motion while preserving that occupant's knowledge and conversation. + +## Knowledge and controls + +All six use an independent complete robot blueprint, including the existing agent, +MCP tools, teleop, learned policies, camera, mapper, planner and knowledge modules. +They share the world coordinate convention so the supplied pitch and locker-room +positions work for every duck. This does not share map contents or observations. +Only the pitch and two locker rooms are supplied semantic prior knowledge. + +Each occupant gets a UUID namespace and private database directory under +`state/robots/football-club-v3///`. Commands, sensor packets, +subscriptions and tool events are scoped to that generation. Duck 1 now has the +same lifecycle as every other slot. A new scene version avoids reusing coordinates +from older layouts. The human overview remains global and is not an agent sensor. + +## Lobby walking preview + +`ops/demo_record_walk.py` runs the existing Pollen ONNX walking policy in an +isolated MuJoCo instance and records a near-periodic 0.82-second gait. The resulting +`web/public/duck-preview.json` is about 90 KB before compression. At runtime the +lobby reuses geometry from its existing world preview and one Three.js renderer. +Static portraits are cached; only the active card animates, capped at 30 FPS. +Hover, keyboard focus and the touch preview button activate it. It fades to a +standing pose on exit and stops when hidden or offscreen. Deliberate hover, focus +and button requests still show the gait with reduced motion enabled; that +preference disables the card lift and pose transitions. +No live simulation or policy inference is started for a lobby animation. + +## Regenerate and verify + +Run from the app root with `source env.sh`: + +```bash +python ops/demo_build_football.py +python ops/demo_build_club.py +python ops/demo_record_walk.py +python ops/demo_club_checkpoint.py +python ops/demo_football_checkpoint.py +python -m pytest -c pyproject.toml --asyncio-mode=auto app/microduck_world -q +deno test -A --no-check --config vendor/dimos/web/relay/deno.json relay/lobby_test.ts +(cd web && deno task check && deno task test && deno task build) +``` + +The club checkpoint runs six unchanged gait policies, checks upright movement and +physical clearance through both locker doors, the pitch tunnel and side corridor, +and renders native scene screenshots. The football checkpoint checks foot contact +and ball travel without policy-triggered ball placement. Reports live under `logs/`. + +The application remains separate from the DimOS framework. This iteration does not +change the pinned vendor source, robot dynamics or learned policy weights. + +## Completed acceptance checks (2026-09-08) + +- Python: 98 passing tests. Lobby: 12 passing tests. Frontend: 9 passing tests, + TypeScript check and production build. Ruff passed for the application and new scripts. +- Six independent browser players connected with a seventh viewer. Occupied slots + returned HTTP 409. Reload retained the same duck; a viewer took a released slot. +- Under six-player load, simulation time advanced 10.005 seconds in 10.000 seconds + of wall time. This is a bounded load check, not a long-running performance guarantee. +- Driving Red 2 moved it about 27 cm; the other five changed position by less than + a millimetre during that check. +- Both native kick policies made foot contact and moved their test ball about 86 cm. +- The Blue 2 agent identified its team and all three prior locations, then navigated + from the blue locker room to the pitch and stopped upright. Its private camera + returned a current 640 by 360 observation, and human CLI calls reached its MCP server. +- Explicit respawn returned Blue 2 from the pitch to its own locker bay, restored + Teleop mode, and retained the conversation. +- The lobby animation advanced only for the active card and stopped after pointer + exit. Reduced motion retained static portraits. The 390-pixel mobile viewport + had no horizontal overflow. + +Repeat the browser acceptance check with all six player slots available using +`python ops/demo_football_lobby_checkpoint.py`. It releases its own test slots on exit. + +## Player identity and browser appearance + +All six duck cards are shown together. Clicking a duck opens a native modal +dialog for an optional player name; confirming it joins the selected slot. +Cancel and Escape close the prompt without claiming a duck. Spectators enter +directly without a name prompt. Blank names use the team +number. Names are normalized, limited to 24 characters, and unique among current +occupants (including reconnect reservations). Invalid or conflicting names leave +the current assignment untouched. Names persist across reload and respawn, clear +on departure or expiry, and belong to the occupant's UUID generation. + +The lobby shares display names with human browsers. Physics assignments remain +robot-to-generation pairs. Names are not added to sensors, agent prompts or MCP +tools. The browser checks the current generation before attaching a name to a duck. +It projects a small HTML label through the Three.js camera, using ordinary text +nodes. Labels add no WebGL draw calls. The Names checkbox is a local preference, +saved in the browser; the follow menu can select a player by display name. + +The lobby, Three.js world and Three.js duck camera share the same white-shell +palette, with team color on the face surround, thigh detail plates and ankle +details. Named mesh metadata selects the parts. These are browser material +overrides; native MuJoCo cameras and physical properties are unchanged. + +Automated checks for this iteration: 99 Python tests, 15 lobby tests and 11 +frontend tests, plus TypeScript checking and a production build. The native +checkpoint confirms all six gaits stay upright and the smaller rooms retain clear +passages to the pitch and benchmark wing. + +Browser acceptance for the compact rooms and player names also passed: + +- Red and blue lobby portraits show white shells with team accents. A 390-pixel + viewport has no horizontal overflow. +- A player named Ada appears above its duck in both the player and spectator + Three.js views. A case-insensitive duplicate is rejected before claiming a slot. +- Selecting Ada in the follow menu tracks its duck. The follow camera keeps the duck visible in its locker room; its current + close framing and wall handling are described below. +- Hiding names affects only that browser and persists through reload. The other + viewer keeps its labels. Respawn retains the name; departure hides it. +- Both test participants released their slots. All six slots were free afterward. + +Evidence is in `logs/name-check/` on the deployment host. The local macOS +headless browser stopped delivering animation frames even for a plain browser +callback, so visual acceptance used isolated Chromium contexts on the server. +Software-rendered headless frame rates are not a client performance measurement. + + +## Six-duck picker and hover follow-up + +The picker shows all six ducks together with separate spectator access. The name +prompt opens only after choosing a duck, focuses the input, and confirms the slot +on submit. Occupancy remains checked when connecting. The legacy host shortcut +uses the same name prompt for Red 1. + +The walking preview is attached inside the active card, so scrolling and card +transforms keep it aligned. React updates its active selection directly. Only one +Three.js canvas animates; all other portraits remain cached images. Reduced motion +no longer silently blocks a deliberately requested preview. + +Live Chromium checks passed for actual pointer movement over all six cards with +normal and reduced-motion settings, differing rendered walking poses, one active +canvas, and stopping after pointer exit. The connection prompt passed focus, +Escape/cancel without claiming a slot, a named join, and spectator entry without +a prompt. Desktop and 390-pixel mobile screenshots were inspected. Evidence is +in `logs/picker-check/` on the deployment host. Repeat with +`python ops/demo_picker_checkpoint.py` when at least one duck is free. The frontend's 11 tests, TypeScript +check and production build also passed. This update required no simulation restart. + + +## Close, fixed follow camera + +Follow duck and the player dropdown now select a fixed shoulder view about +0.8 metres from the aim point. The camera follows the duck's position and heading, +without copying its walking body's roll or pitch. A ray against scene geometry +shortens the offset when a wall or furniture would block the view. This is a +browser camera adjustment with no changes to physics or robot commands. + +A mouse press, click, tiny pointer movement or scroll keeps the camera locked. +Dragging at least four pixels switches to free orbit/pan within the same gesture. +The free camera then stays in place as the duck moves. Follow duck locks it again. +The toolbar shows the drag-to-unlock hint while following. + +Validation: 14 frontend tests passed, including translation/heading tracking, +roll/pitch isolation and wall clearance, plus TypeScript checking and the build. +The live Chromium check confirmed close framing, click and wheel staying locked, +a moving duck remaining centered, drag unlocking, the free camera remaining fixed +through another robot movement, and relocking. The test used two short navigation +goals for its own duck and released that duck afterward. Evidence is saved in +`logs/follow-check/` on the deployment host. No simulation restart was needed. + +## Pitch entrance floor repair, 2026-09-09 + +The club floor ended at y=2.05 while the pitch began at y=2.10. This left a +5 cm unsupported strip across the entrance. Real walking policies reproduced +falls at that boundary from left, centre and right approaches. Extending only +the club floor's north edge to y=2.10 removed the falls. Its south edge remains +y=0.275 and its top remains z=0; room dimensions, walls and contact settings +are unchanged. The club generator now produces the repaired floor. + +The earlier checkpoint checked horizontal doorway clearance at z=0.2 and only +walked briefly inside the lockers. It missed the missing floor at the threshold. +The new regression checks 255 downward rays across the entrance, then walks +each of the six ducks through it in both directions using the real policy and +checks they remain upright for a second after stopping. All 145 Python tests +passed, including ball contact and scoring regressions. The floor-support test +was also run before the fix and failed on the 5 cm gap. Before/after walking +measurements are in `logs/entrance-checkpoint/comparison.json` in the worktree. diff --git a/examples/microduck-world/docs/football.md b/examples/microduck-world/docs/football.md new file mode 100644 index 0000000000..1b25b9743d --- /dev/null +++ b/examples/microduck-world/docs/football.md @@ -0,0 +1,123 @@ +# Football room + +The six ducks spawn in two spaced columns at the central entrance just outside the midfield sideline. The +four benchmark rooms remain in the side wing, reached through a narrow corridor. +The playable pitch is 5.2 × 3.2 metres, inside a 6.7 × 4.4 metre enclosure. +Every floor meets at z=0 without a step. Two goals have a 1 metre clear +opening, 48 cm clearance below the crossbar, and 40 cm deep nets. + +## Try it + +Open https://omarchy.tailca0707.ts.net:8443/ and choose an available duck or watch. +Click **Football field** in the world panel to frame the pitch. That button +moves only your viewing camera. Drive straight onto the pitch from the midfield entrance, +or use the existing navigation tools. All six ducks remember the pitch and locker +locations, and retain their private sensors, policies, teleop and human CLI. + +Three balls start on the pitch. Approach and line up a ball in front of the +appropriate foot, stop, and select **kick left** or **kick right**. The original +apartment ball can also be brought onto the pitch. Selecting a kick no longer +teleports any ball to the foot. The learned policy and physical contact must move it. + +The scoreboard is mounted on the long wall, with matching blue/coral digits and a +compact browser score display. Blue scores into the coral goal; coral scores into +the blue goal. A goal requires the entire ball to cross the goal line from the +field, between the posts and below the crossbar. Crossing backward, entering from +the side, hitting a post, or bouncing in the net does not award another goal. +Each ball can score again after fully returning to the field. + +Goals do not reset or reposition balls. Scores and ball positions reset when the +world service restarts. Three digits and an overflow indicator display large +scores; the underlying count continues beyond 999. + +## Physical and visual model + +MuJoCo owns all motion, contact, ball trajectories and scoring. Physics runs at +200 Hz and the unchanged learned policies at 50 Hz. Three.js receives compiled +model geometry and body transforms; it never applies its own ball physics. + +All four balls, including the original benchmark ball, use Pollen's 5 cm radius, +30 g mass and solid-sphere inertia of 0.00003 kg m² on each axis. Ball friction is +[0.4, 0.01, 0.003], solref [0.03, 0.4], and condim 6. Existing flat floor boxes +keep friction [1, 0.005, 0.0001], solref [0.02, 1], and condim 3. Both surfaces +use default solimp [0.9, 0.95, 0.001, 0.5, 2], priority 0, solmix 1, margin 0 +and gap 0. MuJoCo therefore produces contact friction [1, 1, 0.01, 0.003, 0.003] +and solref [0.025, 0.7], with condim 6. + +Ball centres start and reset at z=0.051 m, leaving 1 mm of initial floor clearance. +Painted panels scale with the radius and remain massless and noncolliding. +Three.js receives the enlarged compiled geometry, and goal detection reads each +compiled ball radius to require the entire 10 cm ball to cross. See the +[physics comparison](ball-physics.md) for upstream sources and measured results. + +Posts, crossbars, benches and net cords have collision geometry. Nets are anchored +rigid capsule grids with compliant MuJoCo contacts, not flexible fabric. There is +no hidden wall behind a net. Flat turf uses the same generated PNG in both +MuJoCo and Three.js; the painted lines add no physical bumps or overlapping floor. + +The wall scoreboard is updated from the physics thread. Scoring examines the +swept ball volume at the goal plane, including angled and fast crossings, rather +than counting frames where a ball happens to be behind a goal. A 3 mm ground +tolerance accommodates MuJoCo's soft contact penetration. Score updates change +only lamp colors in the independent render models. Agents learn the score by +looking through their own cameras, not through an added global scoreboard tool. + +The full scene's size no longer inflates the head camera's near clipping plane: +it remains 5 mm, with a 30 metre far plane. The physical head mount and existing +explicit manual respawn remain. + +## Ownership and reproducibility + +- `assets/scenes/football/field.xml` and `surface.png` contain the generated room. +- `ops/demo_build_football.py` deterministically regenerates those assets. +- `assets/scenes/apartment/` owns the connection, named room and camera view. +- `app/microduck_world/football.py` owns ball composition, scoring and lamp states. +- `app/microduck_world/ball_physics.py` owns the app's ball and floor parameters. +- `web/src/` owns camera framing, texture loading, display and static mesh batching. + +No football scene or game logic was added to the shared DimOS demos or kernel. + +```bash +cd ~/projects/microduck-world +source env.sh +python ops/demo_build_football.py +python -m pytest -c pyproject.toml --asyncio-mode=auto app/microduck_world -q +(cd web && deno task check && deno task test && deno task build) +MUJOCO_GL=egl python ops/demo_football_checkpoint.py +python ops/demo_ball_physics_checkpoint.py +python ops/demo_football_browser.py +``` + +The physics checkpoint builds an isolated world and writes real-policy kick GIFs, +measurements and a native render under `logs/football-checkpoint/`. Its initial +ball placement is test setup, never an action in the running world. The browser +checkpoint uses an available Duck 1 slot, drives briefly, tests manual respawn, +checks camera modes and spectator permissions, captures desktop/mobile screens, +and releases its slot. It does not send agent messages. + +## Verification, 2026-09-06 + +The final hosted suite passed 96 Python tests, 9 frontend tests and 12 lobby +admission tests. TypeScript checking, the production build, focused Python type +checks and linting of the new football modules and scripts also passed. + +The checkpoint was rerun with the live default `robot_allcollisions.xml` model +(the earlier script selected `robot_walk.xml`). Both real ONNX kick policies +made foot contact and scored in isolated MuJoCo; the measured results were unchanged: +left kick moved the ball 0.862 m with a 0.941 m/s peak speed; right kick moved it +0.858 m with a 0.803 m/s peak speed. The duck stayed upright in both tests. +Tests also cover both goal directions, full-ball clearance, post/crossbar misses, +reverse and side entry, repeat prevention, net retention, floor continuity, +ball mass/inertia, native scoreboard lamps and camera transforms. + +The browser checkpoint verified the Three.js room and POV, scoreboard display, +walking, manual respawn, the 640 × 360 MuJoCo camera, observer restrictions and +a 390 px mobile lobby without horizontal overflow. Static batching reduced the +captured field view from 370 to 160 draw calls. + +The server's headless Chromium test uses SwiftShader, not its NVIDIA GPU. With two +WebGL panels it produced about 2 FPS, and nominal 50 ms timers stalled up to +2.2 seconds. Walking was therefore checked with the JPEG renderers selected; +the existing safety timeouts were preserved. This is a software-renderer limit, +not a measurement of a hardware-accelerated client browser. No physical Microduck +or measured turf/ball calibration was available for hardware validation. diff --git a/examples/microduck-world/docs/implementation-notes.md b/examples/microduck-world/docs/implementation-notes.md new file mode 100644 index 0000000000..72e327287c --- /dev/null +++ b/examples/microduck-world/docs/implementation-notes.md @@ -0,0 +1,83 @@ +# Implementation boundaries + +Historical implementation notes. For the current six-duck football application, +start with [the README](../README.md) and [cockpit behavior](cockpit-ui.md). +This package is now included at `examples/microduck-world` for review; it remains +independently installed and pins its own tested DimOS dependency. + +## Application versus DimOS + +The hosted world is an external Python package with a dimos.blueprints entry point. +Its composition, agent prompt, room/landmark metadata, scene XML, gateway, and +operational files belong to this project. The built-in blueprint registry is untouched. + +The cumulative shared patch changes reusable behavior, including: +- Camera canvases fill their available area; paired feeds adapt to panel shape. +- Control strips use content height. +- NavMap optionally fits scene metadata, sharing one transform for imagery, overlays and clicks. +- Chat has an additive read-only display option, preserving the existing manifest format. +- The upstream integration includes a simulated upright-reset helper. This project disables automatic recovery and exposes a human-only Respawn button; see README simulation fidelity. +- Responses-based MCP agents receive camera images in the current tool result, + fixing delayed observations and unnecessary retries. +- Reliable channel subscriptions request cached state for late viewers. Replay + requests survive subscription coalescing; chat retains stable message numbers + so existing viewers do not duplicate the history. + +This checkpoint also binds MCP calls to deployed instance names, adds a +configurable per-runtime background tool topic, and exposes existing command +shaping and camera-ray helpers for simulation adapters. + +Corresponding shared tests are included. No new hosted-world application logic +was added under DimOS demos. The Chat read-only option is a UI affordance; +it does not authorize or secure transport commands. + +Scene metadata now rejects ambiguous names/aliases, reversed bounds, room targets +outside their room, and non-finite coordinates. Existing ScenePackage and RoomSpec +types are reused. The ball can already be disabled through MicroduckSimModule.ball_body; +there is no need for another project-specific simulation implementation. + +## Reproducibility + +`dimos-revision.txt` pins the base; `patches/dimos-hosted-world.patch` records the +shared changes. Setup checks the pin and accepts an already-applied patch. +Project dependencies are actually installed, and `uv pip check` verifies compatibility. +Test dependencies are a separate optional extra. + +The pinned commit is available in `dimensionalOS/dimos`. To provision its source +from this application directory: + +```bash +mkdir -p vendor +git clone https://github.com/dimensionalOS/dimos.git vendor/dimos +git -C vendor/dimos checkout "$(cat dimos-revision.txt)" +``` + +Install Python 3.12, uv, Deno and the Linux native dependencies described by the +pinned framework before running `./setup --test`. Fetch robot assets through the +pinned Microduck asset downloader. The HTTPS gateway additionally needs a local +`config/tailnet.json` matching `GatewayConfig`, certificates and private keys; +public hosting needs the resources described in [public launch](public-launch.md). +The live server's private backups and configuration are intentionally excluded. + +## Remaining boundaries + +One shared physics world hosts three independently built robot blueprints. +Duck 1 retains the host annotations; Duck 2 and Duck 3 learn from their own +sensors and keep private maps, contexts, tool topics and knowledge. See +[multiplayer](multiplayer.md) for lifecycle and isolation details. + +The tailnet gateway preserves certificate pinning and adds trusted HTTPS, but the +underlying relay still trusts reachable peers. Public authorization must cover +robot registration and every command channel, not only keyboard teleop. +See public-access.md before exposing an internet-facing endpoint. + +All service source and logs live in this directory. User systemd only maintains +registration symlinks outside it. Physics state is transient; named places persist. +TLS certificate renewal remains manual. + +## Client world panel + +The project now owns a Three.js panel under web/ and exports the compiled MuJoCo +model through WorldSimModule. It reuses the cockpit panel registry and existing SDK +without extending the shared DimOS patch. See [client rendering](client-rendering.md) +for the protocol, assets, setup and browser checkpoint. diff --git a/examples/microduck-world/docs/lobby-checkpoint.md b/examples/microduck-world/docs/lobby-checkpoint.md new file mode 100644 index 0000000000..cf028ab088 --- /dev/null +++ b/examples/microduck-world/docs/lobby-checkpoint.md @@ -0,0 +1,66 @@ +# Character selection checkpoint — 2026-09-05 + +The tailnet home page now offers Duck 1, Duck 2, Duck 3 and spectator entry. +Availability is live. Duck 1 is an exclusive slot, including the existing `?host` +shortcut. A conflicting claim returns HTTP 409 without replacing the owner. + +The real robot meshes supply the portraits; no duplicate model downloads or three +extra live camera feeds were added. Portraits are rendered once and their temporary +GPU resources are disposed. Gold, teal and violet shells come from project scene +metadata and appear in native MuJoCo images and the exported Three.js model. +Duck 1 advertises four supplied room annotations and five known objects. Visitors +start fresh. Spectators see room labels and active duck positions on an overview. + +All code and assets remain under `/home/tule/projects/microduck-world`. This +checkpoint changes no DimOS vendor files, patches, built-in registry or demos. +The existing cockpit's agent, mapping, policy and control panels remain in use. +Chakra Petch is hosted locally with its OFL license in the distributed client assets. + +## Validation + +- 65 project Python tests, including a real-process regression test proving that + worker descendants are removed when their parent has already exited. +- A deliberately blocked test startup exited in 62.7 seconds with its stack + recorded, proving the 60-second startup watchdog terminates a stalled build. +- 12 lobby relay tests, including three simultaneous identities, exact duck + selection, host conflicts, disconnect reservations and cross-robot denial. +- Six frontend tests, TypeScript checking, production build, project Ruff and + strict mypy for all 19 production Python files. +- Desktop 1440×1000 and mobile 390×844 screenshots visually reviewed, with all + portrait images loaded and no horizontal document overflow. +- Four independent Chromium contexts reached Duck 2, Duck 3, Duck 1 and spectator + views simultaneously. Both Three.js panels rendered for every duck. A fourth + player could not take any occupied slot, and the host alias returned 409. +- Duck 2 moved about 29 cm while Ducks 1 and 3 remained stationary. Reload kept + Duck 2. Leaving Duck 3 returned to the lobby, and the spectator took that released + slot as a new visitor. Test sessions were released; only the resident runtime + remained afterward. +- The exact configured OpenAI key is absent from reachable project history, + project source and built browser assets. `config/agent.env` remains ignored, + untracked and mode 0600. Detailed results are in the private audit log. + +Evidence lives under ignored `logs/`: `multiplayer-checkpoint.json`, +`multiplayer-lobby.png`, `multiplayer-lobby-mobile.png`, `multiplayer-world.png`, +`key-exposure-audit.json` and `startup-timeout-check.json`. + +## Issues found and practical limits + +The Mac browser CLI stalled during screenshot capture; server Chromium supplied +the reviewed screenshots. Four simultaneous software WebGL views also overloaded +an early test run. The final test verified each Three.js view, then switched prior +contexts to native feeds before opening the next, keeping control timers responsive. +The spectator screenshot's low FPS reflects headless software rendering, not a +measurement of normal client hardware. This was a functional check, not a public +load test or an AI-provider billing test. + +One repeated visitor startup stalled during module wiring. The project runtime now +bounds startup to 60 seconds, writes the blocked stack to its private log and exits +for supervisor replacement. Cleanup always removes that runtime's entire process +group, including workers left by an exited parent. The final simultaneous-login +run completed normally. The underlying intermittent RPC discovery stall is not +claimed resolved; bounded recovery and long-duration launch testing remain relevant +before public release. + +`sim.tule.world` is not live. DNS/ingress, public transport, Duck 1 access rules and +anonymous agent limits are still pending; see [public-access.md](public-access.md). +Rotate the key that appeared in chat before public launch. diff --git a/examples/microduck-world/docs/lobby-design.md b/examples/microduck-world/docs/lobby-design.md new file mode 100644 index 0000000000..a09d4e10e6 --- /dev/null +++ b/examples/microduck-world/docs/lobby-design.md @@ -0,0 +1,26 @@ +# Character-select lobby + +The entry screen behaves like choosing a playable character. The actual Microduck +model is the hero, with three distinct shell colors and a fourth spectator option. +The spectator preview uses the scene's room coordinates and live body poses. + +Palette: deep blue #142443, panel blue #213758, cloud white #eff4ff, +Duck 1 gold #ffcf62, Duck 2 teal #62d8cb, Duck 3 violet #b49aff. +Typography: locally hosted Chakra Petch for titles and actions; system sans for +short instructions. Labels use sentence case. Duck numbers are identities, not +ornamental numbering. Availability and map knowledge have explicit text labels. + +Desktop: title and occupancy above one row of three robot choices and a quieter +spectator choice. Narrow layouts become two columns, then one column. Each choice +is one keyboard-accessible action. The actual cockpit retains its working layout. + + Microduck World 1 / 3 ducks occupied + Choose your Microduck + [ Gold robot ] [ Teal robot ] [ Violet robot ] [ Labeled room map ] + [ Duck 1 ] [ Duck 2 ] [ Duck 3 ] [ Spectator ] + [ Map loaded ] [ Start fresh] [ Start fresh ] [ Watch the world ] + +Review against the brief: avoid a marketing hero and generic gradient cards. Spend +visual emphasis on the recognizable robot portraits, with color indicating a real +in-world identity. Render portraits once with a temporary WebGL context and reuse +them as images; do not create three additional continuously rendering world feeds. diff --git a/examples/microduck-world/docs/multiplayer.md b/examples/microduck-world/docs/multiplayer.md new file mode 100644 index 0000000000..3dbb902952 --- /dev/null +++ b/examples/microduck-world/docs/multiplayer.md @@ -0,0 +1,173 @@ +# Independent ducks — 2026-09-05 + +This is the previous checkpoint. See [the football club iteration](football-club.md) +for the current six-player roster, common prior knowledge and scene layout. + +There are three robot identities: Duck 1 is the host; Duck 2 and Duck 3 are visitor +slots. Each runs the same independently built DimOS robot blueprint in its own +process tree. They share one MuJoCo physics world. Observers do not occupy a slot. + +Every new visitor starts idle with no supplied room names, object names or previous +visitor knowledge. Sensors immediately build a local map of visible surfaces; +movement and exploration wait for user instructions. Select Agent mode and ask +“explore the space”, “stop exploring”, “what can you see?” or “remember this place”. +Duck 1 keeps the apartment's supplied room and landmark annotations. + +The home page offers three explicit character choices and a spectator option. +Duck 1 is white, Duck 2 teal and Duck 3 violet; these shell colors are applied in +MuJoCo and exported to the human Three.js view. Existing robot meshes are reused. +Duck 1 shows “Map loaded”; Ducks 2 and 3 show “No previous map”. The spectator card and +world overview display room labels and live duck positions for humans only. +All three choices have exclusive control slots. “Leave duck” returns to the lobby; +“Back to lobby” lets a spectator choose a duck. `?host` remains a Duck 1 shortcut, +and cannot displace another controller. + +## Composition + +```mermaid +flowchart TB + world[Shared MuJoCo physics and sensor renderer] + lobby[Lobby and robot supervisor] + lobby --> d1 + lobby --> d2 + lobby --> d3 + world <-->|Own commands and measurements| d1[Duck 1 blueprint] + world <-->|Own commands and measurements| d2[Duck 2 blueprint] + world <-->|Own commands and measurements| d3[Duck 3 blueprint] + subgraph robot[Inside each duck blueprint] + connection[Robot connection] --> mapping[Voxel map and cost map] + mapping --> planner[A-star navigation and frontier exploration] + planner --> control[Control mode and movement] + control --> connection + connection --> knowledge[Own camera observations and places] + knowledge <--> agent[Own MCP server and agent conversation] + agent --> planner + browser[Own cockpit] --> control + browser <--> agent + end + world --> spectator[Global Three.js view for humans] +``` + +`robot_blueprint.py` composes standard DimOS mapping, planning, control, frontier +exploration and MCP modules. `run_robot.py` builds it through the pinned version's +`ModuleCoordinator.build`. The root `blueprints.py` contains the physics world, +observer bridge and lifecycle supervisor. It does not contain a shared robot agent. + +The connection publishes ordinary DimOS `color_image`, `depth_image`, `camera_info`, +`tf`, `pointcloud`, `odom`, `joint_state` and policy streams. A synchronized +`observation` bundle binds a camera image to its measured depth and optical pose. +Visitors use a map frame translated to their spawn; their initial position is near +(0, 0). The browser world continues to use the common world coordinate frame. + +## Isolation and lifecycle + +- Each runtime has its own namespaced streams, module RPC addresses, MCP endpoint, + conversation, background tool topic and map. The MCP server only registers that + robot's skills; generic operator tools such as global `agent_send` are excluded. +- The simulator emits only the addressed robot's measurements. Sensors are + occluded by geometry. A duck may see another duck through its camera or range + sensor, just as a physical robot could. It receives no other robot's odometry, + semantic map or memory. +- Global poses and scene assets serve the human Three.js panels. They are not + connected to agent skills or perception inputs. +- A visitor session owns a UUID generation. Commands and measurements carry it; + packets from an earlier occupant are rejected. The visitor's relay identity also + includes this UUID, preventing old cockpit caches from reaching a new visitor. +- Reload or a brief disconnect retains the same runtime and knowledge. Leaving + releases the slot immediately. A disconnect without reconnection expires after + 60 seconds. The supervisor stops expired runtimes and creates fresh processes + for replacement visitors. Each robot has an independent command watchdog. + +This is isolation of agent data and tool access, not an operating-system sandbox +for hostile code on Omarchy. Tailnet access still supplies the deployment's trust +boundary. Public hosting needs a Duck 1 access decision, admission limits, agent budgets +and a tested public transport route before exposure to the internet. See +[public-access.md](public-access.md). + +## Knowledge and future perception + +`DuckKnowledge` extends the existing Microduck skill container and PlacesMemory. +Agents can label their current location, save image-backed observations and use +`remember_object` to annotate a pixel in an image they actually observed. Object +positions are computed from that image's measured depth, intrinsics and camera +pose. Unknown observation IDs, out-of-bounds pixels and invalid depth are rejected. +Room names can be remembered as places; automatic room segmentation is not present. +The text description is still an LLM interpretation, not a verified detector label. + +Duck 1's place database remains `state/places.db`. Visitor databases, evidence and +agent traces live under `state/robots///`. Old session files are +retained for inspection but never loaded by replacement visitors. Occupancy maps +and conversations live in each running process; a world restart resets these. + +YOLO is not installed in this checkpoint. Add a detector to the common robot +blueprint, consuming that duck's `color_image`; fuse detections with its depth, +calibration and TF for map annotations. No global scene labels are needed. +A physical Microduck can reuse the stack by replacing the simulator connection +with hardware drivers producing equivalent streams. Ground-truth simulation +odometry and range sensing still need real localization and sensor equivalents; +this has not been validated on hardware. + +## Code ownership and operation + +All application code lives in this project. `physics_robots.py` owns robot bodies +and gait mailboxes; `sensors.py` renders robot measurements; `connection.py` adapts +them to DimOS; `knowledge.py` and `exploration.py` extend reusable robot capabilities. +The cockpit remains composition in `cockpit.py` plus the project Three.js panels. +Scene XML, known host places and spawn settings remain under `assets/scenes/`. +The superseded manual-only visitor controller and shared prompt were removed. +No files under DimOS demos were changed. + +Reusable DimOS additions for this step are: MCP calls bind deployed module-instance +names, background tool topics are configurable per runtime, and existing gait +command shaping and camera ray directions have public reuse points. The cumulative +vendor patch remains `patches/dimos-hosted-world.patch` against `dimos-revision.txt`. + +`config/robots.json` configures the three loopback MCP ports; `setup` copies +`ops/robots.example.json` when no config exists. `logs/duck1.log`, `duck2.log` and +`duck3.log` hold runtime logs; `state/robot-runtimes.json` holds current PIDs and +session generations. All child runtimes belong to `microduck-world.service`. + +## Verification + +The browser checkpoint used independent host, two visitor and observer sessions on +the Mac. It verified visitor capacity, private empty initial knowledge, native and +Three.js cameras, user-requested exploration, stopping, object/place annotation, +independent posture control, reload and fresh visitor handoff. Duck 2 expanded its +measured cost map from about 2.9 to 16.6 square metres, observed a red cube, located +it using measured depth and named its current place. Duck 3 remained idle with an +empty conversation and no copied annotations. Coverage is a cost-map metric, not a +percentage of the apartment's floor area. + +Detailed recorded results are in `logs/independence-live-checkpoint.json`. The +existing `ops/demo_multiplayer_checkpoint.py` has been updated for two visitors +plus an observer and can rerun the bounded movement/reconnect browser check. + +Python tests cover private evidence, fresh databases, stale sensor generations, +MCP filtering and dispatch, scoped background events and a real MuJoCo occlusion +case. Lobby tests include a stale-runtime subscription rejection. Frontend type +checking, protocol/FPS tests and the production build are also part of validation. +Whole-PC reboot, hardware deployment and public internet load testing remain untested. + +### Automated check results + +- Project Python suite: 64 passed. +- Shared gait, policies and simulator: 96 passed, one optional test skipped. +- MCP notification/server unit tests: 34 passed. Two live-server tests are excluded + while the hosted world owns their fixed port; the real agents were tested through + browser chat instead. +- Lobby: 8 passed. Shared relay registry: 45 passed. +- Frontend: type check, 6 tests and production build passed. +- Project Ruff and strict mypy: passed (19 production files). +- Dependency compatibility: all 266 installed packages compatible. +- Clean pinned-source patch application: passed; details and SHA-256 are recorded + in `logs/independence-patch-check.json`. + +The pinned vendor's full Ruff rules still report seven existing warnings in the +MCP server and MuJoCo engine (broad exception handling and one import style rule). +Those pre-existing lines were not expanded as part of this change. + +The updated repeatable browser acceptance run passed with both visitor slots free: +Duck 2 and Duck 3 connected, a third visitor observed, movement remained isolated, +reload retained the duck, and the observer took a released slot. Both test visitors +were explicitly released at teardown. See `logs/independent-browser-check.log` and +`logs/multiplayer-checkpoint.json`. diff --git a/examples/microduck-world/docs/overnight-report.md b/examples/microduck-world/docs/overnight-report.md new file mode 100644 index 0000000000..82921f4bed --- /dev/null +++ b/examples/microduck-world/docs/overnight-report.md @@ -0,0 +1,168 @@ +# Microduck World — checkpoint report, 5 September 2026 + +Project: /home/tule/projects/microduck-world on Omarchy. +Browser URL: https://omarchy.tailca0707.ts.net:8443 + +## Completed checkpoints + +### 1. Cockpit functionality + +Restored Control, mode and policy streams, Agent transcript, ObserveSkill, and +McpServer. The external blueprint conditionally includes McpClient when an OpenAI +key is configured. The scene-specific agent prompt uses project room metadata. + +Browser checks exercised stand, sit/stand in both directions, left/right kicks, +roulade, and ground pick. All supported actions completed without a reported +policy error or fallen state. Roller policies correctly explain that this robot +variant does not support them. + +The running MCP server exposes 18 tools on loopback. Direct list_places returned +the four rooms and five landmarks from project assets. Direct observe returned +a valid 640 × 360 JPEG from the robot camera. These checks used no LLM. + +The user authorized the local OpenAI key on 5 September. It was verified against +OpenAI, transferred over SSH stdin, stored in ignored config/agent.env with mode +600, and loaded by the supervised world. The live blueprint now includes McpClient. + +Browser chat tests passed for scene discovery, camera observation, kitchen +navigation, sit/stand, and operator cancellation. Kitchen navigation reached +(1.06, 0.88), confirmed independently from odometry. Cancelling a later living-room +trip cleared the goal and the agent correctly reported that it stopped early. + +The first live camera test exposed a shared MCP bug: the image was queued for a +later user turn, so the agent initially reported no image and retried. The client +now returns multimodal content directly in the Responses tool result. The retest +produced one observe call, an image tool result, and one accurate answer in about +three seconds. Fourteen MCP client tests and strict mypy passed. Regression tests +also verify the actual request serialization without contacting a provider. + +A second-viewer check exposed missing chat history while another browser stayed +connected. The shared relay now requests cached reliable-channel state for late +viewers; the robot preserves those requests when subscription bursts coalesce. +Requests are bounded by the current channel set, and stable chat message numbers +prevent duplicates. After the final deploy, a real list_places conversation +replayed completely into a reloaded second Mac browser while the first retained +exactly one copy of each message. Both 640 × 360 feeds remained live. A Teleop/Agent mode round trip retained the history; Teleop correctly hides +the composer behind the mode notice. The duck was left standing in Teleop mode. + +### 2. Panels and navigation + +Camera canvases fill their available surface with aspect ratio preserved. +Tall panels stack the chase and head feeds at full width; wide panels use an +inset. The head feed now renders at 640 × 360. The Control strip no longer +reserves unused vertical space. + +NavMap optionally fits room/object metadata, using the same transform for the +costmap, labels, robot and click coordinates. There are no apartment coordinates +hardcoded into this option. + +Browser checks covered 1440 × 900 and 1280 × 720 layouts, maximize/restore, +live feeds, map goals, cancellation, and keyboard release. A kitchen click sent +(1.2, 1.0), entered following_path, and cancellation cleared the goal. Keyboard +drive produced vx=0.15 and measurable movement; release returned vx to zero. +An initial key press sent immediately with the asynchronous arm request was +ignored; the check passed after waiting for the armed state. + +### 3. Headless operation and recovery + +Project-owned systemd user services supervise the world and HTTPS/UDP gateway. +Both are enabled, and tule has Linger=yes, allowing operation without a desktop +login. The relay's existing --open-browser false option disables desktop browser +launch on this headless host. Wrapper commands route restarts to systemd and prevent duplicate worlds. + +Deliberately killing each service's main process verified automatic recovery. +The browser resumed after both world and gateway failures without manual reload. +A real server reboot has NOT been tested. + +Added /healthz and a browser startup page that retries while the simulation +loads. The startup page recovered automatically in the browser. Readiness means +a robot is registered, not that every sensor or the LLM is healthy. + +### 4. Project organization and setup + +Scene XML, room/landmark metadata, cockpit composition, prompt, gateway, +deployment scripts and operational documentation live in the external project. +No hosted application logic was added to DimOS demos or the built-in registry. + +The shared patch contains reusable camera layout, optional read-only Chat, +optional scene-fitting NavMap, spawn-relative fall recovery, immediate MCP image +results, and late-viewer history replay, with tests. +Scene validation rejects ambiguous aliases, invalid bounds/targets and non-finite +coordinates. + +Setup pins the existing DimOS revision, applies the patch idempotently, installs +project dependencies and checks dependency compatibility. A full setup --test +run passed with 264 compatible packages. The vendor checkout lacks older Git +objects, so a full-history bundle was impossible. A project-local Git-directory +backup can restore the pinned source tree; restoring and applying the final +patch was verified. It is not a full-history or off-machine backup. + +### 5. Public access preparation + +docs/public-access.md records the relay audit and required authorization design. +The current relay trusts reachable peers; generic goal, policy and agent +commands are not all protected by the keyboard controller lease. An HTTP-only +tunnel does not cover the UDP transport. + +The application remains tailnet-only. Public authorization, internet exposure, +the later sim.tule.world domain, visitor-owned ducks and scene redesign have +NOT been implemented. + +## Validation + +- 147 cockpit frontend tests passed; TypeScript check passed. +- 94 Python cockpit tests passed. +- 29 external-project tests passed. +- 5 targeted simulation fall-recovery tests passed. +- 14 MCP client tests passed, including immediate camera delivery and wire-format checks. +- 296 Python relay protocol, bridge and transport-session tests passed with asyncio enabled. +- 76 TypeScript relay registry and protocol tests passed. +- Strict mypy passed for gateway, UDP forwarder, scene, agent, MCP client and + the changed relay protocol, bridge and transport session. +- Project Ruff checks and dependency compatibility checks passed. +- Browser regressions ran from the Mac over tailnet and real headless Chromium + on Omarchy through a loopback-only SSH CDP tunnel. Final two-viewer replay + verification and the final screenshot used separate Mac browser sessions. + +The six-hour browser soak finished with 73/73 checkpoints passing, from +04:39:24 to 10:39:24 UTC on 5 September. It checked trusted HTTPS readiness and +actual browser chase/head camera, odometry and policy sequence advancement every +five minutes, with hourly screenshots. Both services recorded zero restarts during +the run. This run preceded agent enablement and the later shared fixes; it is +not a six-hour soak of the final version. The live agent and two-viewer replay +checks above cover those changes. The final source deploy was at 15:58 UTC. + +Evidence is in docs/stability-report.md and +logs/browser-soak-20260905T043853Z/. Shared test logs, live runtime logs and +additional browser captures are under logs/. The final desktop screenshot is +logs/final-cockpit-20260905.png. After adding test-only HTTP type +stubs, all 265 installed packages passed the dependency compatibility check. + +## Remaining work and limits + +1. Implement and test public transport authorization before exposing the domain. + The current public-access deliverable is the audit and permission design. +2. Test an intentional full reboot when convenient. +3. Renew the Tailscale certificate before 4 December 2026; renewal is manual. +4. The existing Zenoh memlock warning remains; local transport falls back, while + browser chase video measured approximately 18–19 fps. +5. Domain publication, visitor-owned ducks and scene redesign remain deferred. + +The earlier credential-transfer block was resolved by explicit user authorization. +Live agent validation used the authorized OpenAI account. No API key is included +in the source repository or this report. + +## Movement handoff correction — 5 September, 16:13 UTC + +The final handoff had left the duck on stand. The user then reported that neither +WASD nor room clicks moved it. Live state confirmed Teleop mode and the stand +policy, with no fallen/locked state. Selecting walk restored movement: a two-second +W hold changed odometry from about (0.008, 0.004) to (0.182, -0.021), and clicking +the kitchen label reached (1.045, 0.902). The duck is now left on walk with no +movement command held. The Control strip explains that stand holds position and +walk is needed to drive or navigate. + +The browser automation CLI's letter-key command emitted an empty physical code; +the movement retest used a DOM keyboard event with code KeyW and a guaranteed +keyup. The real room-label test used mouse input. Both were verified against +robot odometry, not only UI command values. diff --git a/examples/microduck-world/docs/public-access.md b/examples/microduck-world/docs/public-access.md new file mode 100644 index 0000000000..716f50d82a --- /dev/null +++ b/examples/microduck-world/docs/public-access.md @@ -0,0 +1,78 @@ +# Public access: sim.tule.world + +Historical preparation notes. The current implementation and remaining launch +checks are in [public-launch.md](public-launch.md). That implementation uses six +equal player slots, GitHub identity, fresh private context for every duck, and a +Cloudflare WSS relay. The three-slot and anonymous-access discussion below +describes an earlier deployment. + +The running deployment remains private to the tailnet at +`https://omarchy.tailca0707.ts.net:8443/`. The new character-select lobby is not a +public launch. DNS, public ingress and domain TLS have not been configured. + +## Intended experience + +Visitors choose an available Duck 1, Duck 2 or Duck 3, or watch without occupying a +slot. Each duck has one controller and its own agent tools, camera and knowledge. +Duck 1 retains the supplied apartment annotations; visitor ducks start fresh. +The host is a robot identity, not an administrator role. + +The current tailnet deployment allows any tailnet user to claim any available duck. +`?host` remains an alias for Duck 1 and respects the same exclusive ownership check. +Whether public Duck 1 should require host authentication is an open product decision. +If it is public, its persistent conversation and knowledge must be treated as shared +public data; private host conversation must be cleared or separated before launch. + +## Existing boundaries + +- Opaque participant tickets bind a viewer to one runtime. The relay rejects + spectator commands, private spectator subscriptions and cross-robot watch/control. +- All three duck slots are exclusive. Reload retains ownership; leaving releases + immediately; a disconnected owner has 60 seconds to reconnect. +- Visitor generations isolate commands, sensor streams, conversations and maps + from previous occupants. The shared world stream is for human visualization. +- Robot registration requires a rotating server-only credential. Its discovery + endpoint, assignments and all MCP servers are loopback-only. +- The HTTPS gateway validates Host and Origin and exposes an allowlist of page, + asset and session routes. Configuration, logs, state databases, source files and + `/internal/` are not served. UDP peer storage and participant storage are bounded. + +These are application boundaries, not a sandbox for running untrusted code on the +server. No visitor code uploads or execution are offered. + +## API key + +The OpenAI key is loaded only by server processes from ignored `config/agent.env` +(mode 0600). It is not a frontend environment variable. The key audit scans every +reachable project revision and built browser file for the exact configured value +without printing it. Results are recorded in ignored `logs/key-exposure-audit.json`. + +The key previously appeared in the operator's chat, so replace it before a public +launch. Keeping it out of JavaScript does not prevent API charges: an anonymous +visitor can still ask their server-side agent to perform work. + +## Remaining release work + +1. Confirm DNS management and the public network route for `sim.tule.world`. +2. Choose public Duck 1 ownership and conversation persistence rules, and implement + authentication if that identity is private. +3. Bound anonymous agent work: admission and command rates, concurrent turns, + per-session limits and a global usage ceiling with an operator shutoff. Bound + spectator load and long-lived idle sessions as part of the same admission policy. +4. Configure domain TLS and the chosen ingress; retain the private gateway's bind + guard until a separate public configuration and its tests are ready. +5. Test from outside the tailnet: all four entry paths, ownership conflicts, + revoked/forged sessions, denied cross-duck actions, reload, disconnect, server + restart, slow viewers and enforcement of agent limits. + +## Transport requirement + +The browser currently uses HTTPS over TCP plus WebTransport/QUIC over UDP. An +HTTP-only proxy or tunnel may serve the lobby while leaving the feeds disconnected. +A public route must carry the required UDP traffic with trusted HTTPS discovery, or +we must implement and test an alternate browser transport before choosing an +HTTP-only ingress. A DNS record alone is not enough. + +Project admission policy belongs in this repository. Generic transport support +belongs in DimOS. Scene metadata and colors remain in `assets/scenes/apartment/`; +public hosting does not require adding scene-specific logic to DimOS demos. diff --git a/examples/microduck-world/docs/public-launch-evidence.json b/examples/microduck-world/docs/public-launch-evidence.json new file mode 100644 index 0000000000..d5a7285a92 --- /dev/null +++ b/examples/microduck-world/docs/public-launch-evidence.json @@ -0,0 +1,71 @@ +{ + "checked_date": "2026-09-09", + "scope": "Read-only inventory and existing statistics. No load test or deployment.", + "omarchy": { + "cpu": "Intel Core i9-10900", + "physical_cores": 10, + "hardware_threads": 20, + "installed_ram_gb": 64, + "os_reported_ram_gib": 62, + "gpu": "NVIDIA GeForce RTX 3070 Ti", + "vram_mib": 8192, + "driver": "610.57.04", + "occupied_players": 0, + "connected_players": 0, + "viewers": 0, + "world_service_memory_bytes": 2992111616, + "gateway_service_memory_bytes": 73293824, + "gpu_utilization_percent": 0, + "gpu_memory_used_mib": 863 + }, + "world_stream_accumulated": { + "frames_in": 183117, + "bytes_in": 1127451093, + "mean_frame_bytes": 6156.998492766919, + "configured_max_hz": 30, + "estimated_mbps_per_viewer_at_30hz": 1.4776796382640607, + "note": "Historical counters, not an active full-load bandwidth measurement. Excludes transport overhead, cameras, maps, chat and asset downloads." + }, + "aws_price_query": { + "service": "AmazonEC2", + "region": "us-east-1", + "location": "US East (N. Virginia)", + "operating_system": "Linux", + "tenancy": "Shared", + "purchase": "OnDemand", + "preinstalled_software": "NA", + "capacity_status": "Used", + "records": [ + { + "instance": "g6.2xlarge", + "vcpu": "8", + "memory": "32 GiB", + "usd_per_hour": "0.9776000000", + "usd_per_730_hours": "713.6480000000", + "sku": "XCA3J9EEM739PU6M", + "rate_code": "XCA3J9EEM739PU6M.JRTCKXETXF.6YS6EN2CT7", + "effective_date": "2026-09-01T00:00:00Z" + }, + { + "instance": "g6.4xlarge", + "vcpu": "16", + "memory": "64 GiB", + "usd_per_hour": "1.3232000000", + "usd_per_730_hours": "965.9360000000", + "sku": "KXDXMVEY4HRJRGG4", + "rate_code": "KXDXMVEY4HRJRGG4.JRTCKXETXF.6YS6EN2CT7", + "effective_date": "2026-09-01T00:00:00Z" + }, + { + "instance": "g6.8xlarge", + "vcpu": "32", + "memory": "128 GiB", + "usd_per_hour": "2.0144000000", + "usd_per_730_hours": "1470.5120000000", + "sku": "WDZZWUA642MPM79G", + "rate_code": "WDZZWUA642MPM79G.JRTCKXETXF.6YS6EN2CT7", + "effective_date": "2026-09-01T00:00:00Z" + } + ] + } +} diff --git a/examples/microduck-world/docs/public-launch-plan.md b/examples/microduck-world/docs/public-launch-plan.md new file mode 100644 index 0000000000..9392703329 --- /dev/null +++ b/examples/microduck-world/docs/public-launch-plan.md @@ -0,0 +1,158 @@ +# Microduck public launch plan + +This is the planning record. See [public-launch.md](public-launch.md) for the +implemented architecture, measured load and current deployment status. + +Prepared 2026-09-09. Planning only. Hardware, current occupancy, existing relay counters and AWS prices were read without modifying the running services. No public infrastructure was provisioned and no new load test was run. + +## Recommended first release + +One shared match, with six controllable ducks and additional read-only spectators. The user confirmed this scope and selected GitHub sign-in. Keep the simulation on Omarchy initially, publish the existing website on a public HTTPS domain, and add a public WebSocket transport. dimTELE is optional and is not a dependency for this release. + +The user requires MuJoCo to remain responsible for all physics and DimOS to run the robot stack. Simulation, sensor capture, perception, policies, teleop, mapping and the human CLI therefore remain composed through DimOS on Omarchy. Cloudflare handles website infrastructure and transport. Three.js displays server state and cosmetic UI. Neither Cloudflare nor the browser becomes a second physics authority. + +Preserve the six ducks, teams, locker-room spawns, smaller benchmark rooms, names, picker animation, follow camera, teleop, policies, human CLI, scoreboard behavior and current ball physics. Three.js continues drawing the world in each browser. MuJoCo remains authoritative on the server. Do not add kickoff rules, automated resets or kicks as part of hosting or perception. + +Proposed first capacity target: six players plus 20 spectators. This is a load-test target and proposed admission limit, not a measured maximum. Enable that many connections only after the combined workload passes the launch checks. + +Pending product decisions: + +- Public hostname. `sim.tule.world` was proposed in the earlier hosting document, but no public route has been configured. +- Spectator authentication. GitHub sign-in is confirmed for players. Requiring it for spectators too is now the proposed default for reducing anonymous connection abuse; this scope is awaiting the user's answer. The landing page and previews can remain public. +- Camera visibility. Controller access is included. Spectator access to duck cameras is awaiting the user's answer, and should never expose private chat, maps or commands. + +## Product definitions for implementation + +| Definition | Decision or proposed first-release behavior | +| :--- | :--- | +| Identity | GitHub sign-in, confirmed. Use GitHub's immutable user ID internally; keep the duck's custom display name separate from account identity. | +| Entry flow | Public landing page and all six animated duck previews. Pick a duck, sign in with GitHub if needed, enter the duck name at connection, then atomically claim the slot. Recheck availability after sign-in and offer another duck or spectator mode if it was taken. | +| Roles | Player, spectator and operator. Players control only their claimed duck. Spectators receive permitted read-only streams. Operators can release seats, revoke sessions and block abusive accounts. | +| Seats | One active duck per GitHub account and one controller per duck. Keep the existing 60-second reconnect grace and generation isolation. Additional visitors can watch when the six seats are occupied. | +| Screens and states | Landing/picker, sign-in callback/error, connection/name prompt, cockpit, spectator view, and a small operator view. Explicit connecting, occupied, reconnecting, server-offline and capacity-reached states. | +| Persistence | Store account identity, preferred display name, sessions, account restrictions and usage totals. Preserve existing per-session robot knowledge isolation; returning with the same GitHub account does not automatically load a previous player's memory. | +| Anti-spam | Enforce account and connection limits, name validation, request limits and agent spending budgets on the server. GitHub authentication provides an identity to enforce against; it does not prove that a visitor is human or prevent multiple accounts. | +| Perception | YOLO runs on Omarchy as a DimOS perception component, using native MuJoCo head-camera RGB. Start with 2D ball detection and the current 3 Hz source rate. | +| Hosting | Proposed Cloudflare Workers, one match-scoped Durable Object relay, and D1 for account/session records. Omarchy initiates authenticated outbound WSS connections. | +| Operations | Separate staging and production configuration, versioned scene/model assets, monitoring, rollback and an admission limit established by the combined load test. | + +GitHub sign-in needs an application registration owned by the chosen GitHub account or organization, a public hostname and exact callback URL. Request only identity access; repository permissions are unnecessary for this product. Exchange the authorization code on the server, use state and PKCE, revalidate the GitHub identity, and issue the site's own secure HttpOnly session cookie. Keep GitHub tokens out of browser storage and out of robot command messages. A framework auth integration can implement this flow without adding another identity provider. [GitHub authorization flow](https://docs.github.com/en/apps/oauth-apps/building-oauth-apps/authorizing-oauth-apps) + +The minimal data contracts are `User`, `Session`, `DuckLease`, `WorldSnapshot`, `CameraFrame` and `BallDetections`. A lease binds the GitHub user ID, world ID, duck ID and runtime generation. A camera/detection pair binds that same duck and generation to a frame ID and capture timestamp. Define command acknowledgments, expiry and reconnection behavior before implementing the public relay. + +## Hosting architecture + +```text +Browser + Three.js world, GitHub sign-in, cockpit and detector panel + | +Cloudflare Worker + Static assets, authentication endpoints and session validation + D1: account/session records and account restrictions + | +Cloudflare Durable Object, one per match + Authenticated WSS relay, read-only world distribution, bounded queues + | +Authenticated outbound WSS connection initiated by Omarchy + DimOS transport adapter and authoritative duck ownership + DimOS MuJoCo world and six locomotion policies + Occupied ducks' sensors, maps, agents and human CLI + Shared DimOS perception worker using native head-camera RGB +``` + +The proposed Cloudflare Worker serves the application and auth endpoints; a match-scoped Durable Object accepts the browser connections and Omarchy's authenticated publisher connection. Cloudflare documents this WebSocket server capability. D1 provides managed SQL storage for account/session records. Visitors need only their browser, and Omarchy needs outbound connectivity. Public router port forwarding and a Cloudflare Tunnel are not required for this proposed relay path. [Workers Static Assets](https://developers.cloudflare.com/workers/static-assets/), [Durable Object WebSockets](https://developers.cloudflare.com/durable-objects/best-practices/websockets/), [D1](https://developers.cloudflare.com/d1/) + +Keep transport connections scoped to the match and separate private duck streams from the shared world feed. Authentication can issue a bounded identity grant, but Omarchy remains the final authority for duck claims and accepted commands. The relay must obtain an acknowledged claim and generation from Omarchy, never allocate seats from an independent cached roster. If Omarchy disconnects, show the world as offline/stale and accept no movement until fresh ownership and state are available. Do not save every pose or camera frame to the account database. + +A self-managed CPU server running the same WSS relay remains a fallback if Cloudflare's latency or limits fail the prototype. The earlier EC2 relay proposal is therefore an alternative, not an additional required server. Measure relay placement, private-link latency and restricted-network behavior before selecting the final public path. + +The existing gateway is deliberately restricted to loopback or Tailscale addresses and forwards WebTransport over UDP. Putting its web page behind an HTTPS proxy alone will not make the live simulation work publicly. Implement an optional WSS adapter that preserves the current protocol, lobby authorization and reconnect behavior. Keep the private gateway mode intact. Generic SDK/relay changes belong in the appropriate framework repository; Microduck deployment, admission and UI changes belong in this application. + +Separate control traffic from camera and bulk map transfers, discard superseded state/video before transmission, and bound queued bytes. On reconnect, request fresh state and revalidate ownership; do not replay old movement commands. Keep the existing movement deadman behavior. + +The public relay should distribute one upstream world feed to many viewers. This is additional implementation work: a plain reverse proxy forwarding one connection per visitor would still multiply Omarchy's upload use. Keep duck commands and private data scoped to the controller's current session and generation. Internal endpoints and MCP ports remain private; preserve the human CLI through the existing DimOS tool route with authenticated access. + +Cloudflare Tunnel is an alternative for direct HTTPS/WSS ingress to an origin. It does not itself provide the shared relay or a transparent browser UDP path. For the initial 3 Hz camera panel, prototype bounded binary JPEG messages with matching detection metadata on a separate WSS media connection. Use an appropriate paid Developer Platform configuration and confirm its media use, traffic limits and account pricing before deployment; do not assume a free CDN/tunnel is an unrestricted video service. If smoother video becomes a requirement or TCP delivery fails the latency target, use Cloudflare Realtime SFU/WebRTC for camera tracks while retaining DimOS and the same ownership rules. That is a separate media adapter, not a change to physics. [Cloudflare routing](https://developers.cloudflare.com/tunnel/routing/), [service terms](https://www.cloudflare.com/service-specific-terms-application-services/), [Realtime](https://developers.cloudflare.com/realtime/) + +Budget the Worker, Durable Object messages/active duration and storage separately. Platform connection ceilings do not establish our tested capacity; continuous world traffic also should not be costed as an idle, hibernating relay. No Cloudflare resources, subscriptions or identity applications have been created in this planning work. [Durable Object pricing](https://developers.cloudflare.com/durable-objects/platform/pricing/) + +## Football detection feed + +Use the duck's actual native RGB sensor image as the detector input. The current source is 640 by 360 pixels at a configured maximum of 3 frames per second per active duck. A cockpit subscription cap of 6 Hz does not increase that source rate. Six occupied ducks therefore offer up to 18 source images per second for detection. + +Start with the existing DimOS detector integration and measure accuracy before choosing different weights. A GitHub API review of DimOS checkpoint `e0676c6` verified `Yolo2DDetector`, which imports Ultralytics, defaults to `yolo11n.pt` and selects CUDA when available. `Detection2DModule` consumes `color_image` and publishes typed `Detection2DArray` messages. The Microduck blueprint does not yet wire this detector. Verify compatibility with the application's exact deployed dependency pin during implementation. [DimOS YOLO adapter](https://github.com/dimensionalOS/dimos/blob/e0676c6/dimos/perception/detection/detectors/yolo.py), [DimOS detection module](https://github.com/dimensionalOS/dimos/blob/e0676c6/dimos/perception/detection/module2D.py) + +Ultralytics provides model weights, image inference, boxes/class scores, training/evaluation tools and export to runtimes such as ONNX and TensorRT. In this architecture it is a library used by DimOS's perception component. Local inference does not require sending camera frames to an Ultralytics cloud service. It supplies the object-recognition implementation, while DimOS manages the robot pipeline, MuJoCo supplies physics and sensor rendering, and Three.js displays the website. [Prediction](https://docs.ultralytics.com/modes/predict/), [Training](https://docs.ultralytics.com/modes/train/), [Export](https://docs.ultralytics.com/modes/export/) + +COCO provides the broader `sports ball` class, not a football-specific classifier. Test the pitch balls and original benchmark ball explicitly, including small, distant, occluded and moving appearances. Fine-tune on representative rendered images if the pretrained model misses them. [Ultralytics COCO documentation](https://docs.ultralytics.com/datasets/detect/coco/) + +Load one model in a shared DimOS-managed worker. Feed it bounded per-duck queues that retain the newest frame, with fair scheduling between ducks. Detection must run outside the physics stepping loop and must not block locomotion, sensors or teleop. Stop processing inactive duck sessions and clear their results when a new player claims the slot. A spectator never creates another copy of inference. + +The inspected YOLO adapter calls `track(..., persist=True)`. Do not interleave six independent camera streams through that persistent tracker: it would share temporal identity state across ducks. For the initial bounding-box feature, use stateless prediction behind the DimOS detector interface with serialized/batched inference. If tracking is added, maintain separate tracking state per duck and runtime generation while sharing model weights. Preserve DimOS message types and explicitly publish matched full-frame images; the inspected module's `detected_image_*` outputs are object crops, not the requested full-camera overlay. Optional 3D annotations must use measured depth and calibration, not assumed depth. + +YOLO operates on image pixels, not inside a renderer. MuJoCo-rendered RGB is recommended because it already comes from the authoritative head camera and continues working with no browser connected. Browser-rendered Three.js RGB could also be analyzed, but it would make perception depend on the viewer's camera, rendering settings and connection, and would require uploading those frames to Omarchy. A separate server-side Three.js camera renderer is possible but would duplicate an existing sensor path. Keep Three.js for the world display and camera-panel overlay. + +Publish the source frame together with its duck ID, generation, frame ID, timestamp, dimensions and detections. Draw boxes and confidence in the browser over that exact image, correcting for resizing and letterboxing. A box from the native camera must not be drawn on an independently rendered Three.js image or the following camera. Show `Ball detected`, `No ball detected`, or a stale/unavailable state as appropriate. `No ball detected` is not proof that no ball exists. + +Keep the smooth Three.js world view alongside this sensor panel. Begin with the existing sensor rate. If a smoother detection view is wanted, separate RGB capture scheduling from expensive depth and lidar work and benchmark 5 to 10 Hz before raising the source rate. Detector speed alone does not establish camera throughput. + +For validation, render segmentation can supply test labels, but deployed detection must consume image pixels rather than simulator ball coordinates. Hold out camera positions and include empty views, balls behind walls, white duck feet, field lines and multiple balls. Record precision, recall by visible ball size, false positives and end-to-end frame age. Do not promise detection accuracy before this evaluation. + +Choose code and weight licenses before integration. Ultralytics offers AGPL-3.0 and an enterprise option; resolve their fit with the application's intended distribution instead of assuming that every YOLO implementation and checkpoint has the same license. [Ultralytics licensing options](https://www.ultralytics.com/license) + +## Omarchy capacity evidence + +Read-only inventory: + +| Component | Omarchy | +| :--- | :--- | +| CPU | Intel Core i9-10900, 10 physical cores, 20 hardware threads | +| RAM | 64 GB installed, about 62 GiB reported by the OS | +| GPU | NVIDIA GeForce RTX 3070 Ti, 8,192 MiB VRAM | +| NVIDIA driver | 610.57.04 | +| Current occupancy | Zero players and zero viewers at inspection | +| Current world service memory | About 2.8 GiB, while unoccupied | + +The earlier six-player validation advanced simulation time by 10.005 seconds during 10.000 seconds of wall time. This establishes real-time operation during that bounded test, not sustained public capacity with YOLO enabled. The low GPU utilization and memory snapshot above are idle observations and cannot establish remaining capacity under six active players. [Existing six-duck validation](football-club.md) + +Each occupied duck launches its own agent/runtime pipeline, configured with four workers, plus mapping and sensor work. Spectators receive the shared world state and render locally, so player and spectator counts must be measured separately. Hosted language-model API use also needs its own request and spend budget. + +Existing relay counters recorded 1,127,451,093 bytes over 183,117 world frames: an accumulated average of about 6,157 bytes per frame. At the configured 30 Hz this projects to 1.48 Mbps per receiving browser, before network overhead, assets, cameras, maps and chat. These are historical counters, not a fresh active-load measurement. + +| World-only spectators | Projected world-state payload traffic | +| :--- | :--- | +| 20 | About 30 Mbps | +| 50 | About 74 Mbps | +| 100 | About 148 Mbps | + +With edge fanout, this traffic leaves the public relay, while Omarchy sends approximately one shared copy upstream. A transparent proxy alone provides no such saving. Measure the actual uplink and private-link latency before selecting the final architecture and limits. + +The current 128-participant retention cap and default 64 UDP-peer cap are separate implementation limits, not capacity benchmarks. A future WSS path needs explicit limits too. Test limits and reconnection headroom deliberately in staging rather than increasing production caps to claim scale. + +## AWS equivalent and cost + +There is no exact instance equivalent to this desktop CPU and GPU combination. `g6.4xlarge` is the first comparison candidate: similar RAM, fewer CPU threads, and a different GPU with more VRAM. Its L4 is intended for inference and graphics workloads, but this does not establish equivalent MuJoCo, rendering or YOLO performance. AWS lists G6 GPU memory as 24 GB in the product table and approximately 22 GiB in the instance specification. [AWS G6 specifications](https://aws.amazon.com/ec2/instance-types/g6/), [EC2 CPU/core specifications](https://docs.aws.amazon.com/ec2/latest/instancetypes/ac.html) + +| Option | vCPUs | RAM | GPU | USD/hour | USD/month at 730 hours | +| :--- | :--- | :--- | :--- | :--- | :--- | +| g6.2xlarge | 8 | 32 GiB | One NVIDIA L4 | 0.9776 | 713.65 | +| g6.4xlarge | 16 | 64 GiB | One NVIDIA L4 | 1.3232 | 965.94 | +| g6.8xlarge | 32 | 128 GiB | One NVIDIA L4 | 2.0144 | 1,470.51 | + +Prices were queried directly from the AWS Price List API on 2026-09-09: US East (N. Virginia), Linux, shared tenancy, on-demand, no preinstalled paid software. Returned rates were effective 2026-09-01. Totals exclude storage, public IPv4, data transfer, DNS, monitoring, model API use, taxes and any separate relay. These are reference-region prices; availability, quota, latency and price in the eventual deployment region need confirmation. [AWS Price List Query API](https://docs.aws.amazon.com/awsaccountbilling/latest/aboutv2/using-price-list-query-api.html), [EC2 pricing and additional charges](https://aws.amazon.com/ec2/pricing/on-demand/) + +Start an eventual cloud benchmark with g6.4xlarge. Consider g6.2xlarge only if profiling shows that the occupied app fits its CPU and RAM budget. Consider g6.8xlarge only if CPU measurements justify it; its extra CPU/RAM do not add another GPU. Validate native EGL rendering, NVIDIA drivers, the same MuJoCo version, all six policies, camera throughput and detector performance before migrating. + +For the first release, reusing Omarchy avoids the full GPU-instance bill. It introduces dependence on local power, internet upload and server uptime. The proposed Cloudflare relay has separate platform costs; the CPU-server fallback has separate compute and traffic costs. Quote the chosen path using measured traffic and the account's existing usage. + +## Implementation order and launch checks + +1. Establish the scene baseline. The pitch entrance repair is committed as `efc2c9b`, with 145 Python tests passing, including six ducks crossing in both directions. Production remains on `be43a3e`. Deploying that repair and restarting the shared world still needs the previously requested approval; automatic review rejected the restart because it would interrupt the live session. The planning request does not authorize that restart. +2. Add the detector on staging. Prove exact image/box synchronization, six-stream scheduling, accuracy on actual balls and graceful inference failure while physics continues. Do not change the physics engine or contact settings for this feature. +3. Add the public WSS path and world relay. Preserve all existing cockpit streams and permission boundaries. Test simultaneous claims, six active slots, rejected seventh claims, read-only viewers, multiple tabs, stale generations, reconnects and slow connections. The existing per-duck runtime startup and shutdown should remain authoritative. +4. Configure public access after choosing the domain, GitHub application owner and spectator permissions. Implement GitHub sign-in, the account/session store, one-duck-per-account admission, TLS, request limits, agent spend limits, useful busy/offline states and monitoring. Cache immutable scene assets. Verify that a visitor outside the tailnet can load and connect without installing anything. +5. Measure staging capacity with all six players walking and exercising normal policies, sensors, maps and the human CLI, with YOLO on. Add 10, 20, 25 and 50 world-only spectators in stages; stop when latency or resources degrade. If spectator cameras are enabled, test that as a separate profile. Include cold page loads, reconnect bursts and slow clients. +6. Use a 30-minute target-load soak plus external browser checks. Proposed acceptance targets: simulation time stays within 1 percent of wall time without accumulating lag; no stale movement is replayed; p95 input-to-visible-motion is below 200 ms on a suitable nearby test connection; p95 detection-frame age is below 750 ms at the initial 3 Hz; queues remain bounded; and measured CPU, GPU, RAM and upload retain roughly 25 percent headroom. Measure client frame time and field entrance walking as well. These are proposed acceptance targets, not results already obtained. +7. Publish with the largest tested admission limit below the first degraded workload. Keep a known-good deployment for rollback and alert on stopped simulation progress, sensor/inference failures and connection errors. A process merely being alive is not sufficient health evidence. + +Raw inventory, counters, AWS SKUs, rate codes and calculated monthly costs are saved in [public-launch-evidence.json](public-launch-evidence.json). diff --git a/examples/microduck-world/docs/public-launch-validation.json b/examples/microduck-world/docs/public-launch-validation.json new file mode 100644 index 0000000000..7580c002a1 --- /dev/null +++ b/examples/microduck-world/docs/public-launch-validation.json @@ -0,0 +1,77 @@ +{ + "date": "2026-09-09", + "publicDeployment": "live at https://sim.tule.world", + "tests": { + "python": 148, + "relay": 20, + "frontend": 14, + "cloudflare": 11 + }, + "nativeCameraSanityCheck": { + "mujoco": "3.10.0", + "device": "cuda", + "model": "yolo11n.pt", + "threshold": 0.25, + "frames": 102, + "visible_ball_frames": 96, + "detected_visible_frames": 78, + "empty_false_positive_frames": 0, + "inference_ms_p50": 7.84287101123482, + "inference_ms_p95": 8.024680020753294, + "matched_visible_frames_iou50": 78 + }, + "isolatedCapacity": { + "players": 6, + "spectators": 20, + "durationSeconds": 45.04383532999782, + "transport": "local DimOS WebTransport", + "cameraSubscriptions": 26, + "failures": [], + "realTimeFactor": { + "min": 0.9987795404958633, + "median": 0.9997402019935304, + "max": 1.0003852221605751 + }, + "worldHz": { + "min": 25.08667371953237, + "median": 25.619488028619784, + "max": 25.619488028619784 + }, + "cameraHz": { + "min": 1.820448889381995, + "median": 1.8870506780179217, + "max": 1.9314518704418728 + }, + "cameraAgeMax": { + "min": 0.07103109359741211, + "median": 0.08788347244262695, + "max": 0.1393885612487793 + }, + "aggregatePayloadMiBPerSecond": 4.855771277775277, + "unavailableCameraFrames": 0 + }, + "limitations": [ + "Capacity was measured in a short local protocol load test; public browser functionality was verified separately.", + "All 18 YOLO misses were textured pitch balls at 0.3 m across all six head cameras.", + "Existing production world remained running alongside the isolated test." + ], + "cloudflareVersion": "40187f0e-0eec-4051-8c3a-be8e7c599018", + "publicChecks": { + "githubOAuth": "passed, public profile only", + "duckJoin": "passed, Red 1 named Tule", + "reload": "same participant generation retained", + "worldUpdateHzObserved": [ + 27, + 29 + ], + "policyCommands": "stand and walk reached MuJoCo", + "teleop": "control lease acquired", + "agentChat": "ready response received", + "humanCliTools": 20, + "anonymousControlStatus": 401, + "privateRobotInfoStatus": 404, + "logout": "cookie cleared and duck released", + "spectatorSignIn": "passed, live 28 Hz world with no player slot occupied", + "cloudflareStandardEnabled": true + } +} diff --git a/examples/microduck-world/docs/public-launch.md b/examples/microduck-world/docs/public-launch.md new file mode 100644 index 0000000000..56bf1c4aa4 --- /dev/null +++ b/examples/microduck-world/docs/public-launch.md @@ -0,0 +1,210 @@ +# Public Microduck demo + +Deployed at **https://sim.tule.world** on 2026-09-10 UTC. GitHub authentication, +the outbound Cloudflare connection and native DimOS football perception are live. +The existing production checkout was fast-forwarded after its six slots were +confirmed empty. Its service names, private configuration and human CLI paths +were preserved. A private configuration and frontend backup is available under +`backups/public-launch-2026-09-10/` on Omarchy. + +## What runs where + +```mermaid +flowchart LR + Browser[Browser: Three.js and DimOS Session] <-->|HTTPS / WSS| Edge[Cloudflare Worker and MatchRelay] + Edge --- Identity[GitHub OAuth and D1 sessions] + Origin[Omarchy: DimOS relay] <-->|Outbound WSS| Edge + Origin <--> World[DimOS: shared MuJoCo physics] + World --> Camera[Native MuJoCo head cameras] + Camera --> YOLO[DimOS BallPerception: one CUDA YOLO model] + YOLO --> Origin + Origin <--> Ducks[Six isolated DimOS robot blueprints] +``` + +The Worker serves the built frontend and exported scene. Its single Durable Object +routes unchanged DimOS protocol frames. The application uses the stock DimOS +browser Session, codecs, subscriptions, command sequencing and teleop state +machine through a WebSocket adapter. The existing Registry on Omarchy owns duck +leases, generations, control authorization and private subscriptions. + +Omarchy establishes the connection outward. No public inbound port, Cloudflare +Tunnel or dimTELE account is required. The private HTTPS/WebTransport gateway and +the loopback MCP endpoints remain usable. The `world` launcher continues calling +the DimOS CLI, including its status, daemon and human MCP commands. + +MuJoCo remains version 3.10.0 with the existing timestep, gravity, floor, ball +contacts and goals. Hosting adds no physics authority, reset rule or kick impulse. +The six ducks, team spawns, smaller benchmark rooms, names, picker animation, +follow camera, policies, navigation and agent context remain in the application. + +## Admission and identity + +- The homepage and previews are public. Playing and watching require GitHub. +- GitHub identity uses immutable user IDs. OAuth requests no repository or email + scopes. PKCE and single-use, five-minute state protect the callback. +- The seven-day session is a Secure, HttpOnly, SameSite=Lax cookie. D1 stores its + hash. GitHub access tokens are discarded after fetching the account identity. +- One account owns one participant and one live browser connection. Reload + replaces its old connection; existing 60-second duck reconnect grace applies. +- Six player slots and twenty connected spectator places are the initial limits. +- The user enters their duck name immediately before claiming an available slot. +- Spectators can select an active duck's public football camera. Private chat, + maps, discoveries, controls and MCP services are not included in that feed. +- Requests are limited per account, or per IP before sign-in. Control traffic and + queued bytes are bounded separately. Bad client frames disconnect that client. +- Logout and operator bans revoke the participant. Operators are explicitly + allowlisted by GitHub ID; the first operator is `6902572`. + +## Football perception + +`BallPerception` is a DimOS module with six native `RobotVision` inputs. It uses +DimOS's `Yolo2DDetector` to load one `yolo11n.pt` model, then runs stateless +prediction on the CUDA GPU. The six cameras do not share persistent tracker state. +Only the COCO `sports ball` class is selected. This is a general ball detector, +not a trained football-specific classifier. + +The module keeps one latest frame per duck, schedules ducks fairly, and rejects +old frames or previous participant generations. Detection runs outside the physics +callback. It publishes a typed `FootballObservation` for DimOS consumers and a +camera message containing the exact JPEG, timestamp, generation and boxes. The +browser decodes the image before displaying its overlay and clears stale frames. +`Detector unavailable` is distinct from `No ball detected`. + +The smooth world and normal duck view still render with Three.js. Football boxes +appear on the native sensor image they were measured from. No browser image upload +or Ultralytics cloud service is used. + +Runtime pins: Torch `2.7.1+cu128`, torchvision `0.22.1+cu128`, Ultralytics `8.3.203`, +Polars `1.33.1`, NumPy `2.3.5` and Pillow `12.2.0`. The separate `.venv-perception` +adds the existing base environment through a `.pth` file, preserving the pinned +DimOS installation and its `opencv-contrib-python` `4.13.0.92` provider of `cv2`. +Ultralytics's plain `opencv-python` dependency is intentionally satisfied by that +existing contrib build. `uv pip check` does not include the base `.pth` environment; +the setup script checks the combined runtime's package metadata instead. + +Weights: official Ultralytics assets release `v8.3.0/yolo11n.pt`, SHA-256 +`0ebbc80d4a7680d14987a577cd21342b65ecfd94632bd9a8da63ae6417644ee1`. +The Ultralytics code and weights retain their upstream licensing terms. + +## Validation + +| Check | Result | +| :--- | :--- | +| Python application regressions | 148 passed, including ball contacts and six-duck pitch entrances | +| DimOS application relay | 20 passed, including identity, cross-duck denial, reconnect and corrupt-client isolation | +| Frontend regressions | 14 passed; typecheck and production build passed | +| Cloudflare runtime | 11 passed, including OAuth, bans, bounded requests, slow viewers, origin loss and the browser transport adapter | +| Browser smoke test | All six choices, connection name prompt, active duck cockpit, native YOLO camera and existing controls verified; no browser console errors | +| Public agent and CLI | Agent replied `ready` through the public cockpit; the original human CLI listed all 20 tools for the signed-in duck | + +The bounded capacity test used the actual isolated DimOS world and 26 protocol +clients: six player blueprints plus twenty spectators, each subscribed to the world +and one football camera. After startup and a 15-second warmup, a 45-second sample +measured: + +| Metric | Result | +| :--- | :--- | +| Physics real-time factor | Median 0.9997, range 0.9988 to 1.0004 | +| World updates | Median 25.62 Hz, minimum 25.09 Hz | +| Camera updates | Median 1.89 Hz, range 1.82 to 1.93 Hz | +| Maximum observed camera frame age | 139 ms on localhost | +| Camera unavailable frames / failed clients | 0 / 0 | +| Aggregate delivered application payload | 4.86 MiB/s across all 26 subscribers | + +This is a short local WebTransport load sample. It excludes browser rendering, +public network latency, OAuth traffic, agent inference, cold asset downloads and +Cloudflare egress measurements. It establishes an initial admission target, not +the maximum possible audience. The existing production world was also running on +Omarchy. Startup briefly skipped camera frames while six robot stacks initialized; +the measured steady-state workload retained real-time physics. The public origin +batches shared frames before upload; that uplink saving still needs measurement +through the deployed Cloudflare route. + +The isolated native camera test used all six head cameras and all four actual scene +balls at 0.3, 0.5, 0.8 and 1.2 m, plus six empty views. Of 96 visible-ball images, +78 had a prediction overlapping the segmentation-derived ball box by IoU at least +0.5. Empty views had zero detections. All 18 misses were the three textured pitch +balls at 0.3 m, across all ducks. The benchmark ball was detected at every tested +distance. Median inference time was 7.84 ms, p95 8.02 ms after the first frame. +These staged samples are a sanity check, not an accuracy benchmark for moving, +occluded or distant footballs. Close-range reliability remains an improvement area. + +Raw test evidence lives under ignored `state/validation/`; a compact nonsecret +summary is committed in `docs/public-launch-validation.json`. + +## Public connection checks + +The real GitHub callback completed using public-profile-only authorization. The +player returned to the Red 1 name prompt, joined as Tule, and received live DimOS +state and the matched native YOLO camera. Public world updates settled around +27 to 29 Hz. A policy switch to stand reached the simulator, switching back to +walk succeeded, teleop acquired its control lease, and the public agent replied. +Reload retained the same duck, name and participant generation. Logout cleared +the session and released its duck. Signing back in through Watch the match opened +a spectator connection at 28 Hz with zero player slots occupied. Its football +camera correctly waits for a selected duck to be occupied. Unauthenticated +control and session-discovery requests return 401; the private robot-registration +route returns 404. The public lobby and scene preview expose all six duck slots. + +The first Internet test revealed that a 16-frame acknowledgment window was too +small for the cockpit's combined streams. The deployed relay now allows 64 +outstanding downstream frames and 32 upstream messages, with the existing 2 MiB +byte bounds and timeouts still enforced. A regression test reproduces two healthy +40-frame bursts before acknowledgments return, and the slow-client disconnection +test still passes. The browser recovered automatically after the edge deployment. + +## Deployment procedure + +GitHub OAuth app `Microduck Football · DimOS` is registered under `aromeoes`. +Homepage: `https://sim.tule.world`. Callback: +`https://sim.tule.world/auth/callback`. Client ID: `Ov23lifchwSuJsZK26hd`. + +The Cloudflare personal account and active `tule.world` zone have been verified. +D1 `microduck-identity` has been created and migration `0001_identity.sql` applied. +The Worker, custom-domain route and public bridge are deployed. Cloudflare first +required registering the account subdomain `tule-world.workers.dev`; public +`workers.dev` routing remains disabled. The active Worker version is +`40187f0e-0eec-4051-8c3a-be8e7c599018`. Secrets are installed as Worker secrets and +in the mode-0600 origin bridge configuration. The server now runs the CUDA overlay +from its original production directory, with the test worktree linking to it. + +1. With the target world stopped, run `bash ops/setup-perception` if that runtime + is not already provisioned. Preserve the project's downloaded robot assets, + pinned vendor patch, private `config/agent.env`, and existing operator setup. + That file is never copied to browser assets. The earlier credential-exposure + note in `docs/public-access.md` records a prior rotation recommendation. Verify + whether that credential was already rotated when enabling public agent usage. +2. Build the frontend with `cd web && deno task build`. Start the world once to + export its native scene, then run `python ops/prepare-public-assets.py`. +3. In `edge`, run `npm ci`, `npm run check`, `npm test`, and `npm run build`. + D1 migrations use `npx wrangler d1 migrations apply microduck-identity --remote`. +4. Read the GitHub client secret from the operator's mode-0600 local handoff file. + Generate a fresh 32-byte hex host secret. Write both to a temporary mode-0600 + JSON secrets file, without printing them or putting values in command arguments. +5. Verify that `sim.tule.world` has no conflicting DNS record or Worker route. + Deploy using `npx wrangler deploy --domain sim.tule.world --secrets-file PATH`. + The account reports Workers Standard enabled; no pricing plan was changed. + Billing subscription details are not available to the current OAuth token. + Continuous traffic must be costed as an active Durable Object, including + incoming acknowledgments. +6. Install `relay/.public.json` mode 0600 on Omarchy: + `{"url":"wss://sim.tule.world/bridge","secret":"HOST_SECRET_VALUE"}`. + This ignored file is in the Deno relay's existing permitted read directory. +7. Cut over one systemd-managed world to this branch, preserving its private + configuration and DimOS CLI registration. Check occupancy first, avoid two + public world authorities, and stop the temporary test services. Keep the old + application commit/configuration available for rollback. +8. Verify the real GitHub callback, logout, account switching, occupied-slot + conflicts, player and spectator WSS feeds, teleop, policies, the human CLI and + agent availability. Test one remote/slow viewer and an origin restart. Verify + anonymous clients cannot connect or fetch private session information. + +Wrangler's development dependency audit currently reports a transitive Sharp / +libheif advisory via Miniflare. The affected image-processing library is used by +local development tooling and is not bundled into this Worker. This application +does not use an image transformation binding or process HEIF inputs. Do not apply +the audit's suggested downgrade to Wrangler 4.15.2 without compatibility testing. + +Rollback removes the public bridge configuration and restores the prior application +commit/service configuration. Disconnecting the origin closes public clients and +discards pending control traffic. OAuth/D1 resources can remain for a later retry. diff --git a/examples/microduck-world/docs/tailnet.md b/examples/microduck-world/docs/tailnet.md new file mode 100644 index 0000000000..de8de2e87a --- /dev/null +++ b/examples/microduck-world/docs/tailnet.md @@ -0,0 +1,83 @@ +# Private browser access + +Target URL: https://omarchy.tailca0707.ts.net:8443 + +The gateway is running with a trusted Tailscale HTTPS certificate and was +verified from the Mac over the tailnet. It binds only the configured tailnet IPv4 address. The +public internet and sim.tule.world are not enabled by this configuration. + +## First certificate + +From a terminal with access to sudo on Omarchy: + +```bash +cd ~/projects/microduck-world +sudo tailscale cert --cert-file=config/tls/omarchy.crt --key-file=config/tls/omarchy.key omarchy.tailca0707.ts.net +sudo chown tule:tule config/tls/omarchy.crt config/tls/omarchy.key +chmod 600 config/tls/omarchy.key +./gateway-service start +``` + +Tailscale HTTPS must be enabled for the tailnet. If the cert command reports that +it is disabled, enable HTTPS in the tailnet's DNS settings and retry. Certificates +expire; repeat issuance before expiry, then stop/start the gateway to load the new +files. Automatic renewal is not configured; service startup at boot is enabled. + +## Operations + +```bash +./gateway-service status +./gateway-service logs +./gateway-service stop +./gateway-service start +``` + +The gateway and world are supervised user services, enabled for boot. +Linger is enabled for tule, so they do not require an interactive login. +Service source, configuration, keys, and logs live here; systemd maintains account +registration symlinks. Use ./service to manage both, or ./gateway-service for +gateway-only compatibility. Process crash recovery has been tested; the whole PC +has not been rebooted for validation. + +GET /healthz returns ready only while a robot is registered with the relay. +The project frontend remains available during relay restarts and retries its connection. + +## Connection design + +Browser HTTPS -> project gateway TCP 8443 -> project frontend/assets or loopback HTTP relay. +The private config/tailnet.json sets frontend_dir to this project's web/dist and +world_assets_dir to state/viewer. The gateway serves / and /client/* from the built +frontend and /world-assets/scene-.json from generated visual assets. Existing +Host/Origin/cross-site checks apply to all these routes. Model responses are immutable +and gzip-compressed; configuration, TLS keys and robot policy files are not served. +Browser QUIC -> project gateway UDP 8443 -> existing loopback QUIC relay. + +/api/info retains the relay's ephemeral certificate hash and rewrites only its +advertised endpoint. The browser receives that pin over trusted HTTPS and verifies +the original relay certificate end-to-end. No certificate checks are disabled, +and the DimOS relay and SDK source remain unchanged. A relay restart changes the +upstream port; the next bootstrap refresh updates forwarding and retires old +connections. UDP peers are capped at 64 and idle mappings expire after 60 seconds. + +Tailnet membership/ACLs are the current access boundary. Wrong Host, foreign Origin +and cross-site browser requests are rejected; wildcard CORS is stripped. This is +not the future public visitor/session authorization layer. The UDP endpoint can +carry the relay's protocol, so only trusted tailnet members should have access. + +## Validation so far + +- 12 focused tests cover discovery/pin preservation, wrong origins and hosts, + private interface validation, read-only HTTP, separate UDP clients, connection + limits and upstream restart cleanup. +- Strict mypy passed on the two gateway source files; Ruff passed. +- The actual running relay delivered a 10,882-byte JPEG through the UDP forwarder. +- Trusted HTTPS verified from the Mac without certificate exceptions. +- Chromium on the Mac received live video at roughly 18 fps, reconnected after + reload, and moved the simulated duck with keyboard control. Key release + returned commanded velocity to zero. +- Certificate expires 2026-12-04; renewal remains a manual operation. + +## Reference + +User lingering starts the user service manager at boot and keeps it after logout: +[systemd loginctl documentation](https://www.freedesktop.org/software/systemd/man/252/loginctl.html). diff --git a/examples/microduck-world/docs/validation.md b/examples/microduck-world/docs/validation.md new file mode 100644 index 0000000000..3aac0875ac --- /dev/null +++ b/examples/microduck-world/docs/validation.md @@ -0,0 +1,25 @@ +# Baseline validation — 2026-09-04 + +Executed on Omarchy using Python 3.12.14, Deno 2.6.10 and the RTX 3070 Ti. + +- Headless MuJoCo EGL render produced a nonuniform 160x120 RGB frame. +- ScenePackage and typed place metadata loaded successfully. +- Every project landmark's coordinates matched its XML geom. +- Project blueprint imported and ran through external entry-point discovery. +- Seven modules started and DimOS reported a successful health check. +- Persistent places database was created at state/places.db. +- Ruff check and format check passed on project Python sources. +- setup completed a second time; uv pip check found all 260 packages compatible. +- Chromium connected to the running cockpit and displayed both camera feeds, + costmap, named rooms and landmarks. Chase video observed at roughly 18.5–19 fps; + head camera roughly 6 fps. Relay reported no frame drops in the sampled interval. +- A 2.5-second simulated W-key hold moved x from about 0.001 m to 0.304 m; + release returned the panel's commanded velocities to zero. +- A nearby map goal entered following_path and reached its destination. +- A separate room goal was cancelled through the UI; state became cancelled. +- Page reload reacquired video and state with teleop disarmed. +- Server restart passed its health check and the browser reconnected. + +Evidence: logs/baseline.png. This is a baseline smoke check, not a public-load, +multi-client authorization, real-robot, or boot-recovery test. Occasional camera +renders were skipped by the engine's clock-protection mechanism under load. diff --git a/examples/microduck-world/edge/migrations/0001_identity.sql b/examples/microduck-world/edge/migrations/0001_identity.sql new file mode 100644 index 0000000000..796f6ee24d --- /dev/null +++ b/examples/microduck-world/edge/migrations/0001_identity.sql @@ -0,0 +1,18 @@ +CREATE TABLE users ( + id TEXT PRIMARY KEY, + login TEXT NOT NULL, + preferred_name TEXT NOT NULL DEFAULT '', + banned INTEGER NOT NULL DEFAULT 0, + created_at INTEGER NOT NULL +); +CREATE TABLE sessions ( + hash TEXT PRIMARY KEY, + user_id TEXT NOT NULL REFERENCES users(id), + expires_at INTEGER NOT NULL +); +CREATE INDEX sessions_user ON sessions(user_id); +CREATE TABLE oauth_states ( + hash TEXT PRIMARY KEY, + verifier TEXT NOT NULL, + expires_at INTEGER NOT NULL +); diff --git a/examples/microduck-world/edge/package-lock.json b/examples/microduck-world/edge/package-lock.json new file mode 100644 index 0000000000..fbfc95fe01 --- /dev/null +++ b/examples/microduck-world/edge/package-lock.json @@ -0,0 +1,2982 @@ +{ + "name": "microduck-public-edge", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "microduck-public-edge", + "devDependencies": { + "@cloudflare/vitest-plugin": "1.1.6", + "@types/node": "22.20.2", + "typescript": "5.9.2", + "vitest": "4.1.11", + "wrangler": "4.126.0" + } + }, + "node_modules/@cloudflare/kv-asset-handler": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/@cloudflare/kv-asset-handler/-/kv-asset-handler-0.5.0.tgz", + "integrity": "sha512-jxQYkj8dSIzc0cD6cMMNdOc1UVjqSqu8BZdor5s8cGjW2I8BjODt/kWPVdY+u9zj3ms75Q5qaZgnxUad83+eAg==", + "dev": true, + "license": "MIT OR Apache-2.0", + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@cloudflare/unenv-preset": { + "version": "2.16.1", + "resolved": "https://registry.npmjs.org/@cloudflare/unenv-preset/-/unenv-preset-2.16.1.tgz", + "integrity": "sha512-ECxObrMfyTl5bhQf/lZCXwo5G6xX9IAUo+nDMKK4SZ8m4Jvvxp52vilxyySSWh2YTZz8+HQ07qGH/2rEom1vDw==", + "dev": true, + "license": "MIT OR Apache-2.0", + "peerDependencies": { + "unenv": "2.0.0-rc.24", + "workerd": ">1.20260305.0 <2.0.0-0" + }, + "peerDependenciesMeta": { + "workerd": { + "optional": true + } + } + }, + "node_modules/@cloudflare/vitest-plugin": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/@cloudflare/vitest-plugin/-/vitest-plugin-1.1.6.tgz", + "integrity": "sha512-cT+Z7klp7Nx5YIWx63eVxK+SDeh386QbMtxNRVE/WiHLTmfAHapMX1hcysd7ky3CfVMdhafSSVWTBKXgtJFh5g==", + "dev": true, + "license": "MIT", + "dependencies": { + "cjs-module-lexer": "1.2.3", + "esbuild": "0.28.1", + "miniflare": "5.20260908.0-alpha", + "wrangler": "4.130.0", + "zod": "4.4.3" + }, + "peerDependencies": { + "@vitest/runner": "^4.1.0", + "@vitest/snapshot": "^4.1.0", + "vitest": "^4.1.0" + } + }, + "node_modules/@cloudflare/vitest-plugin/node_modules/@cloudflare/workerd-darwin-64": { + "version": "1.20260908.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20260908.1.tgz", + "integrity": "sha512-t3juyCXFn12OklBL0S7UC98py3nLEmuioBOUOazeyeVsLUu8fv+pfJrR+XzwSH2Uh6rCXZNogRh+LtV0YpzMcQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@cloudflare/vitest-plugin/node_modules/@cloudflare/workerd-darwin-arm64": { + "version": "1.20260908.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20260908.1.tgz", + "integrity": "sha512-I1nwA4qm/fUNSKhPSO76YA6JAhcA0KU63yFcYHN6AiDowGbDyfjDPH9KCVopT5rI1LY4GfbdAi5b9gODqZsAkQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@cloudflare/vitest-plugin/node_modules/@cloudflare/workerd-linux-64": { + "version": "1.20260908.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20260908.1.tgz", + "integrity": "sha512-s/h5uSW1UC6dGeVKSqrPLpTu+vo0fKJZCNEokW1Ol1qcCB2oN6WCRHhoyzo4Y69oONAXRgLfLBYaHWe7z51Vnw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@cloudflare/vitest-plugin/node_modules/@cloudflare/workerd-linux-arm64": { + "version": "1.20260908.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20260908.1.tgz", + "integrity": "sha512-PP/nUKl0R6colwfocggL8dctO/dFMqMft12unzFUkoAJr0aXQeT+ia0JuNmOUWXe6EfvyCSmAbijEsTKQ5t/ew==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@cloudflare/vitest-plugin/node_modules/@cloudflare/workerd-windows-64": { + "version": "1.20260908.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20260908.1.tgz", + "integrity": "sha512-jkaS5EKKTvzAdIlbMfOYrHoTucWVRwYBwIig+xRsjXQv5BXTLA2Llg+iM8djlKtk0CjkztZhXmuxuqoqWKZmFw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@cloudflare/vitest-plugin/node_modules/miniflare": { + "version": "5.20260908.0-alpha", + "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-5.20260908.0-alpha.tgz", + "integrity": "sha512-BHIknb0u6vLIvb+R8PsUAzgoWWk3OlW85DrV2SG0EydfSpIba28YT0rH6xZThjnSwDxzNuW3FDRNSWvbiI/DVw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cspotcode/source-map-support": "0.8.1", + "sharp": "0.35.2", + "undici": "7.29.0", + "workerd": "1.20260908.1", + "ws": "8.21.0", + "youch": "4.1.0-beta.10" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@cloudflare/vitest-plugin/node_modules/workerd": { + "version": "1.20260908.1", + "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20260908.1.tgz", + "integrity": "sha512-rYhpW6NWHD++p34ej+VXWzi5Pdnmd7ncdbB/ux4s+i3mf7IeOpW3O4roqClQ+Z8ernvyMCknBLCb76z1rA+a7Q==", + "dev": true, + "hasInstallScript": true, + "license": "Apache-2.0", + "bin": { + "workerd": "bin/workerd" + }, + "engines": { + "node": ">=16" + }, + "optionalDependencies": { + "@cloudflare/workerd-darwin-64": "1.20260908.1", + "@cloudflare/workerd-darwin-arm64": "1.20260908.1", + "@cloudflare/workerd-linux-64": "1.20260908.1", + "@cloudflare/workerd-linux-arm64": "1.20260908.1", + "@cloudflare/workerd-windows-64": "1.20260908.1" + } + }, + "node_modules/@cloudflare/vitest-plugin/node_modules/wrangler": { + "version": "4.130.0", + "resolved": "https://registry.npmjs.org/wrangler/-/wrangler-4.130.0.tgz", + "integrity": "sha512-fzNjnTzyZl31PGJOFXbiLeZqEIToQ9KOzkkvGdQz4wlB+BoHnl3BRwCR5xg8AqYyzVunvvMHlMzvlbThQ7rrAA==", + "dev": true, + "license": "MIT OR Apache-2.0", + "dependencies": { + "@cloudflare/kv-asset-handler": "0.5.0", + "@cloudflare/unenv-preset": "2.16.1", + "blake3-wasm": "2.1.5", + "esbuild": "0.28.1", + "miniflare": "5.20260908.0-alpha", + "path-to-regexp": "6.3.0", + "unenv": "2.0.0-rc.24", + "workerd": "1.20260908.1" + }, + "bin": { + "cf-wrangler": "bin/cf-wrangler.js", + "wrangler": "bin/wrangler.js", + "wrangler2": "bin/wrangler.js" + }, + "engines": { + "node": ">=22.0.0" + }, + "optionalDependencies": { + "fsevents": "2.3.3" + }, + "peerDependencies": { + "@cloudflare/workers-types": "^5.20260908.1" + }, + "peerDependenciesMeta": { + "@cloudflare/workers-types": { + "optional": true + } + } + }, + "node_modules/@cspotcode/source-map-support": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", + "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "0.3.9" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@emnapi/runtime": { + "version": "1.11.3", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.3.tgz", + "integrity": "sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.1.tgz", + "integrity": "sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.1.tgz", + "integrity": "sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.1.tgz", + "integrity": "sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.1.tgz", + "integrity": "sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.1.tgz", + "integrity": "sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.1.tgz", + "integrity": "sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.1.tgz", + "integrity": "sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.1.tgz", + "integrity": "sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.1.tgz", + "integrity": "sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.1.tgz", + "integrity": "sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.1.tgz", + "integrity": "sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.1.tgz", + "integrity": "sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.1.tgz", + "integrity": "sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.1.tgz", + "integrity": "sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.1.tgz", + "integrity": "sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.1.tgz", + "integrity": "sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.1.tgz", + "integrity": "sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.1.tgz", + "integrity": "sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.1.tgz", + "integrity": "sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.1.tgz", + "integrity": "sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.1.tgz", + "integrity": "sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.1.tgz", + "integrity": "sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.1.tgz", + "integrity": "sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.1.tgz", + "integrity": "sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.1.tgz", + "integrity": "sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.1.tgz", + "integrity": "sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@img/colour": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", + "integrity": "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@img/sharp-darwin-arm64": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.35.2.tgz", + "integrity": "sha512-eEieHsMksAW4IiO5NzauESRl2D2qz3J/kwUxUrSfV06A93eEaRfMpHXyUb1mAqrR7i8U9A0GRqE9pjn6u1Jjpg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-arm64": "1.3.1" + } + }, + "node_modules/@img/sharp-darwin-x64": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.35.2.tgz", + "integrity": "sha512-BaktuGPCeHJMARpodR8jK4uKiZrPAy9WrfQW0sdI37clracq8Bp01AYS3SZgi5FS/y5twa9t4+LIuuxQjqRrWw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-x64": "1.3.1" + } + }, + "node_modules/@img/sharp-freebsd-wasm32": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-freebsd-wasm32/-/sharp-freebsd-wasm32-0.35.2.tgz", + "integrity": "sha512-YoAxdnd8hPUkvLHd3bWY+YA8nw3xM/RyRopYucNsWHVSan8NLVM3X2volsfoRDcXdUJPg6tXahSd7HXPK7lRnw==", + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "dependencies": { + "@img/sharp-wasm32": "0.35.2" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-arm64": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.3.1.tgz", + "integrity": "sha512-4V/M3roRMTYjiwZY9IOVQOE8OyeCxFAkYmyZDrZl51uOKjibm3oeEJ4WAmLxutAfzFbC9jqUiPs2gbnGflH+7g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.3.1.tgz", + "integrity": "sha512-c0/DxItpJv2+dGhgycJBBgotdqruGYDvA79drdh0MD1dFpy7JzJ/PlXwi1H4rFf0eTy8tgbI91aHDnZIceY3jQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.3.1.tgz", + "integrity": "sha512-aGGy9aWzXgHBG7HNyQPWorZthlp7+x6fDRoPAQbGO3ThcttuTyKIx3NuSHb6zb4gBNq6/yNn9f1cy9nFKS/Vmg==", + "cpu": [ + "arm" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.3.1.tgz", + "integrity": "sha512-JznefmcK9j1JKPz8AkQDh89kjojubyfOasWBPKfzMIhPwsgDy9evpE/naJTXXXmghS1iFwR8u/kTwh/I2/+GCw==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-ppc64": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.3.1.tgz", + "integrity": "sha512-1EkwGNCZk6iWNCMWqrvdJ+r1j0PT1zIz60CNPhYnJlK/zyeWqlsPZIe+ocBVqPF8k/Ssee/NCk+tE9Ryrko6ng==", + "cpu": [ + "ppc64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-riscv64": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.3.1.tgz", + "integrity": "sha512-Ilays+w2bXdnxzxtQdmXR62u8o8GYa3eL4+Gr+1KiE4xperMZUslRaVPJwwPkzlHEjGfXAfRVAa/7CYCtSqsBw==", + "cpu": [ + "riscv64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-s390x": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.3.1.tgz", + "integrity": "sha512-VfBwVHQTbRoj4XlpA/KLZ7ltgMpz+4WSejFzQ+GnoImjo1PtEJ59QB2qR1xQEeRPYIkNrPIm2L4cICMvz4C2ew==", + "cpu": [ + "s390x" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.3.1.tgz", + "integrity": "sha512-+c8ukgwU62DS54nCAjw7keOfHUkmr0B5QHEdcOqRnodF/MNXJbVI8Eopoj4B/0H8Asr65I+A4Amrn7a85/md6A==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.3.1.tgz", + "integrity": "sha512-qlKb/pwbkAi1WMsJrYHk7CuDrd12s27U2QnRhFYUoJNrRCmkosMTttuRFat/DDB3IlDm5qE1TJgZ4JDnHX8Ldw==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-x64": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.3.1.tgz", + "integrity": "sha512-yO21HwoUVLN8Qa+/SBjQLMYwBWAVJjeGPNe+hc0OUeMeifEtJqu5a1c4HayE1nNpDih9y3/KkoltfkDodmKAlg==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-linux-arm": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.35.2.tgz", + "integrity": "sha512-SE4kzF2mepn6z+6E7L6lsV8FzuLL6IPQdyX8ZiwROAG/G8td+hP/m7FsFPwidtrF19gvajuC9l6TxAVcsA4S7A==", + "cpu": [ + "arm" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.3.1" + } + }, + "node_modules/@img/sharp-linux-arm64": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.35.2.tgz", + "integrity": "sha512-af12Pnd0ZGu2HfP8NayB0kk6eC/lrfbQE6HlR4jD+34wdJ1Vw9TF6TMn6ZvffT+WgqVsl0hRbmNvz2u/23VmwA==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.3.1" + } + }, + "node_modules/@img/sharp-linux-ppc64": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.35.2.tgz", + "integrity": "sha512-hYSBm7zcNtDCozCxQHYZJiu63b/bXsgRZuOxCIBZsStMM9Vap47iFHdbX4kCvQsblPB/k+clhELpdQJHQLSHvg==", + "cpu": [ + "ppc64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-ppc64": "1.3.1" + } + }, + "node_modules/@img/sharp-linux-riscv64": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.35.2.tgz", + "integrity": "sha512-qQt0Kc13+Hoan/Awq/qMSQw3L+RI1NCRPgD5cUJ/1WSSmIoysLOc72jlRM3E0OHN9Yr313jgeQ2T+zW+F03QFA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-riscv64": "1.3.1" + } + }, + "node_modules/@img/sharp-linux-s390x": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.35.2.tgz", + "integrity": "sha512-E4fLLfRPzDLlEeDaTzI98OFLcv++WL5ChLLMwPoVd0CIoZQqupBSNbOisPL5am9XsbQ9T84+iiMpUvbFtkunbA==", + "cpu": [ + "s390x" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-s390x": "1.3.1" + } + }, + "node_modules/@img/sharp-linux-x64": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.35.2.tgz", + "integrity": "sha512-gi0zFJJRLswfCZmHtJdikXPOc5u7qamSOS3NHedLqLd4W8Q0NqjdBr6TTRIgsfFjqfTsHFgdfvJ9LwqSgcHiAA==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.3.1" + } + }, + "node_modules/@img/sharp-linuxmusl-arm64": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.35.2.tgz", + "integrity": "sha512-siWbOW1u6HFnFLrp0waKyW7VEf7jYvcDWdrXEFa8AkdAQgEvuu5Fz8/Y70w9EeqAdwDtfU012BhEHHaDqvQNzg==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-arm64": "1.3.1" + } + }, + "node_modules/@img/sharp-linuxmusl-x64": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.35.2.tgz", + "integrity": "sha512-YBqMMcjDi4QGYiSn4vNOYBhmlC4z5AXqkOUUqI2e0AFA4urNv4ESgOgwNl3K+4etQhha0twXlzeF20bbULm9Yg==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-x64": "1.3.1" + } + }, + "node_modules/@img/sharp-wasm32": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.35.2.tgz", + "integrity": "sha512-Mrv4JQNYVQ94xH+jzZ9r+gowleN8mv2FTgKT+PI6bx5C0G8TdNYndu161pg2i7uoBwxy2ImPMHrJOM2LZef7Bw==", + "dev": true, + "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.11.1" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-webcontainers-wasm32": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-webcontainers-wasm32/-/sharp-webcontainers-wasm32-0.35.2.tgz", + "integrity": "sha512-QNV27pxs9wpApEiCfvHM1RDoP1w1+2KrUWWDPEhEwg+latvOrfuhWrHWZKwdSFwU6jh3myjw/yOCRsUIuOft3g==", + "cpu": [ + "wasm32" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "@img/sharp-wasm32": "0.35.2" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-arm64": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.35.2.tgz", + "integrity": "sha512-BiVRYc/t6/Vl3e1hBx0hugG4oN9Pydf4fgMSpxTQJmwGUg/YoXTWHiFeRymHfCZzifxu4F4rpk/I67D0LQ20wQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-ia32": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.35.2.tgz", + "integrity": "sha512-YYEhx9PImCC7T0tI8JDMi4DB9LwLCXCU5OWNYEXAxh5Q1ShKkyC6byxzoBJ3gEFDnH2lQckWuDe70G7mB2XJog==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-x64": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.35.2.tgz", + "integrity": "sha512-imoOyBcoM/iiUr4J6VPpCNjPnjvP/Gks95898yB8YqoGGYmHYbOyCuNv9FMhFgtaiHFGbHW8bxKqRV6VjtXThQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz", + "integrity": "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", + "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.0.3", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + }, + "node_modules/@oxc-project/types": { + "version": "0.149.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.149.0.tgz", + "integrity": "sha512-Efcc+iF0j3Bf67YjEqIqWXbX5XddXoK/Mw4K1/JuXwRCZ8N16VR7iT23nlCc9XrveFVh/E5Rqs2StT0V8v9LdA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/oxc-project" + } + }, + "node_modules/@poppinss/colors": { + "version": "4.1.6", + "resolved": "https://registry.npmjs.org/@poppinss/colors/-/colors-4.1.6.tgz", + "integrity": "sha512-H9xkIdFswbS8n1d6vmRd8+c10t2Qe+rZITbbDHHkQixH5+2x1FDGmi/0K+WgWiqQFKPSlIYB7jlH6Kpfn6Fleg==", + "dev": true, + "license": "MIT", + "dependencies": { + "kleur": "^4.1.5" + } + }, + "node_modules/@poppinss/dumper": { + "version": "0.6.5", + "resolved": "https://registry.npmjs.org/@poppinss/dumper/-/dumper-0.6.5.tgz", + "integrity": "sha512-NBdYIb90J7LfOI32dOewKI1r7wnkiH6m920puQ3qHUeZkxNkQiFnXVWoE6YtFSv6QOiPPf7ys6i+HWWecDz7sw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@poppinss/colors": "^4.1.5", + "@sindresorhus/is": "^7.0.2", + "supports-color": "^10.0.0" + } + }, + "node_modules/@poppinss/exception": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@poppinss/exception/-/exception-1.2.3.tgz", + "integrity": "sha512-dCED+QRChTVatE9ibtoaxc+WkdzOSjYTKi/+uacHWIsfodVfpsueo3+DKpgU5Px8qXjgmXkSvhXvSCz3fnP9lw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@rolldown/binding-android-arm-eabi": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm-eabi/-/binding-android-arm-eabi-1.2.8.tgz", + "integrity": "sha512-tN5aztYkKCte4i5SIrrz5yK/HMjEuCqCSCJa418jOV8tZ1cBY3YF2otxB1ktPxzsLA1BeTqwapK0bfjxNvHJVw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-android-arm64": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.2.8.tgz", + "integrity": "sha512-dIYTWl9XprMUiQFoc55KUyk/oS8SKYH3zFl0LTR7RT0Xj4hgSVyuJcroH8JUu8RcpF8fTB6E0aOwCkZoYPcDSQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-arm64": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.2.8.tgz", + "integrity": "sha512-PCSDQGXD2IyTEFrcgPyBM8jJuGmrbCMuoIOXdbEGVemruKACXoLQJrb+A45Z0L5t1RQkdfJprAYPkikbh7dzdA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-x64": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.2.8.tgz", + "integrity": "sha512-Uk7lRsGhPFHVX/sAUC6D5H9Ol30dFHd6iquokll2th3LpdJ3F5CzQB+7DHn0Ri2mG+U7k2zXiPHDrwZenXhwSA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-freebsd-x64": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.2.8.tgz", + "integrity": "sha512-DjszaTEVogPqA5bYzsEeqDCQxbcp2fexQwKcRspYji2yzR68fCf+e4fx6kBSRDwX5/brZaHw/hWS9+A/+/w9sQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm-gnueabihf": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.2.8.tgz", + "integrity": "sha512-zmwa7FTmdzB6aaEEuuls18H6Ap5JmJPSoPTuXixeJZV6tG40SyLkApQtz1g8ptZtiEKqj9OM0oNLPh1AgvE31Q==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-gnu": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.2.8.tgz", + "integrity": "sha512-KdYQDPHwJVnbFwdTGMgxsI9SqblBlz6STGM+w1We/d5B8OWWidYH0MwkU/uA1wM5fIpO2MkOVxXrNzzuZhw9ew==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-musl": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.2.8.tgz", + "integrity": "sha512-jFJTifHnNPY+yzOoNZQfSIysrVyXzEQPhPnOUjmD1bcQGHH6s7c8cViKWar8YplQImE5N9JRqMCLrM2CdxOrZA==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-ppc64-gnu": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.2.8.tgz", + "integrity": "sha512-FhiOziBDWPBjbcmRzfLyIJnaP7AVMFXT7YCXPjXxj7wKU3vx24RjrCNN/zjvVa+N2vVoHJwCoUBvsrN/DG3zIA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-s390x-gnu": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.2.8.tgz", + "integrity": "sha512-WnHfADMzOV2Y55wlx1hzzQnar/wDt/VdvWSD99r18Mz9ylNieIGOkRx3UV21h7m/eJvjySYJkO26VvGNFkwsIQ==", + "cpu": [ + "s390x" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-gnu": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.2.8.tgz", + "integrity": "sha512-H9tRr5ibfXFVLxbPOseVewewFpl28zcEdjRDt2FTUZU7odxP0gEv1ki4/kGmcGOh78oRwZuuQllGLZ9zTJp84g==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-musl": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.2.8.tgz", + "integrity": "sha512-UefiqfM3D6IVNlZ8tSGs9+Ejjud2T+oxO0IHADU45Y+lyEjD2dVFyZHbkfX0LUb5Zugo/oIv1eCO/KVYhgYJYA==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-openharmony-arm64": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.2.8.tgz", + "integrity": "sha512-637Ke4kWSy6rp9cxQ9gMOXlxPgIw/c1beASV4M//3+9I4uwBVOOl74G+e3zyU3u19U7RkRl/HuewixZ/Z6+Rjg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-arm64-msvc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.2.8.tgz", + "integrity": "sha512-xWBkPOF1Q9k/Gv1nQXnVdLxKu74jXppuOM4Z3mnypVUJJJwLsMl7hNJGRAUJoG8A5MgOI1ACKM+wBFxSJzKy4A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-x64-msvc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.2.8.tgz", + "integrity": "sha512-uz2ZvfgXbxqNwijjjbxrnvALwpyODDcgc1T1N8N3rf/DXKQmaFwmB4LX4yyjggpwN2obdQLb2rgirX5ffCWYng==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.1.tgz", + "integrity": "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@sindresorhus/is": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-7.2.0.tgz", + "integrity": "sha512-P1Cz1dWaFfR4IR+U13mqqiGsLFf1KbayybWwdd2vfctdV6hDpUkgCY0nKOLLTMSoRd/jJNjtbqzf13K8DCCXQw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sindresorhus/is?sponsor=1" + } + }, + "node_modules/@speed-highlight/core": { + "version": "1.2.24", + "resolved": "https://registry.npmjs.org/@speed-highlight/core/-/core-1.2.24.tgz", + "integrity": "sha512-qeW2e1l78afw8VhRPfPQ1Gjj+KU5XFQ/OFV5ti6eTa9bruO7mJyZtA4vw0ofqmA3tKCkROE9xLk3VZoeRc98nw==", + "dev": true, + "license": "CC0-1.0" + }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/chai": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", + "integrity": "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/deep-eql": "*", + "assertion-error": "^2.0.1" + } + }, + "node_modules/@types/deep-eql": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", + "integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/estree": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", + "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "22.20.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.2.tgz", + "integrity": "sha512-xlvWf4Vs9n1PEVYwP1n4vvG07M6y8WgvJ2t0vbrWTmijsIHp1cS+uJ2kMIRdY3nHZK0nCYKrPeD171+SzF4/zw==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@vitest/expect": { + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.11.tgz", + "integrity": "sha512-VX2x5vNJXET47KAFzwERI+KRMtTTCSWTfSMKsW7JsUsXV4psq++e3DvZpuTDOpHcxytiDs6p2nhVb2tVDiiUYw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@standard-schema/spec": "^1.1.0", + "@types/chai": "^5.2.2", + "@vitest/spy": "4.1.11", + "@vitest/utils": "4.1.11", + "chai": "^6.2.2", + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/mocker": { + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.11.tgz", + "integrity": "sha512-2XJVD55d1o5AZous5CCGKS74g/riOj9odEt2bQpCVZeblHyHdnMeFl4jl0XjU21stf4mbjUkew2eXQZt65g5CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/spy": "4.1.11", + "estree-walker": "^3.0.3", + "magic-string": "^0.30.21" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "msw": "^2.4.9", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "msw": { + "optional": true + }, + "vite": { + "optional": true + } + } + }, + "node_modules/@vitest/pretty-format": { + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.11.tgz", + "integrity": "sha512-yiZzPbGTS9Sr/JpFl8zHrcIkAofNbFV6k21vIgQN/cY/oxZeXhJv5sc/MBJ5jFKWmWs+oJHw0UXLZjmf931+Vw==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/runner": { + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.11.tgz", + "integrity": "sha512-LztvUgdwMNJMIkj3hQnnxiC2Xy1zNxq928W/xhjCLaNCzqTZOudjwbQf6v9IntZGPw132i2Lq2rgTRZHD3JHNw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/utils": "4.1.11", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/snapshot": { + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.11.tgz", + "integrity": "sha512-pN7ikn1ON7h8ee4gIAp4AzyK+zBtJPzVbqOgu5LCEh4VaJVbPQcgYQYJIMGQPXVeJJq1fnfazis7a5pFNPahog==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "4.1.11", + "@vitest/utils": "4.1.11", + "magic-string": "^0.30.21", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/spy": { + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.11.tgz", + "integrity": "sha512-apNa/prQy2qCeywhnixOHPRCgGNhvg7T4Dapfl1GahLp/R+uhBm5cPyFoNVyqsNd2h1nJxL6BqqdIjiABL60YA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/utils": { + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.11.tgz", + "integrity": "sha512-zTCVGpyFsGWBhllOyKlTw/vnr6D9qxsfSDyfbyZmTyjHw5N/VuvzHpHoQjm2ZJzn4RJgx5w4r7V0er69CmLgPQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "4.1.11", + "convert-source-map": "^2.0.0", + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/assertion-error": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", + "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/blake3-wasm": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/blake3-wasm/-/blake3-wasm-2.1.5.tgz", + "integrity": "sha512-F1+K8EbfOZE49dtoPtmxUQrpXaBIl3ICvasLh+nJta0xkz+9kF/7uet9fLnwKqhDrmj6g+6K3Tw9yQPUg2ka5g==", + "dev": true, + "license": "MIT" + }, + "node_modules/chai": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/chai/-/chai-6.2.2.tgz", + "integrity": "sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/cjs-module-lexer": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-1.2.3.tgz", + "integrity": "sha512-0TNiGstbQmCFwt4akjjBg5pLRTSyj/PkWQ1ZoO2zntmg9yLqSRxwEa4iCfQLGjqhiqBfOJa7W/E8wfGrTDmlZQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "dev": true, + "license": "MIT" + }, + "node_modules/cookie": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-1.1.1.tgz", + "integrity": "sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/error-stack-parser-es": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/error-stack-parser-es/-/error-stack-parser-es-1.0.5.tgz", + "integrity": "sha512-5qucVt2XcuGMcEGgWI7i+yZpmpByQ8J1lHhcL7PwqCwu9FPP3VUXzT4ltHe5i2z9dePwEHcDVOAfSnHsOlCXRA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/es-module-lexer": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.3.2.tgz", + "integrity": "sha512-poHGpORABojJJucnV9KbOavETW8lBVnphkW77ER5/BQ5Fz7oXSoCNek7IH3vR5nRjdsEz926ibFYX8KtLQmdyw==", + "dev": true, + "license": "MIT" + }, + "node_modules/esbuild": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.1.tgz", + "integrity": "sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.28.1", + "@esbuild/android-arm": "0.28.1", + "@esbuild/android-arm64": "0.28.1", + "@esbuild/android-x64": "0.28.1", + "@esbuild/darwin-arm64": "0.28.1", + "@esbuild/darwin-x64": "0.28.1", + "@esbuild/freebsd-arm64": "0.28.1", + "@esbuild/freebsd-x64": "0.28.1", + "@esbuild/linux-arm": "0.28.1", + "@esbuild/linux-arm64": "0.28.1", + "@esbuild/linux-ia32": "0.28.1", + "@esbuild/linux-loong64": "0.28.1", + "@esbuild/linux-mips64el": "0.28.1", + "@esbuild/linux-ppc64": "0.28.1", + "@esbuild/linux-riscv64": "0.28.1", + "@esbuild/linux-s390x": "0.28.1", + "@esbuild/linux-x64": "0.28.1", + "@esbuild/netbsd-arm64": "0.28.1", + "@esbuild/netbsd-x64": "0.28.1", + "@esbuild/openbsd-arm64": "0.28.1", + "@esbuild/openbsd-x64": "0.28.1", + "@esbuild/openharmony-arm64": "0.28.1", + "@esbuild/sunos-x64": "0.28.1", + "@esbuild/win32-arm64": "0.28.1", + "@esbuild/win32-ia32": "0.28.1", + "@esbuild/win32-x64": "0.28.1" + } + }, + "node_modules/estree-walker": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0" + } + }, + "node_modules/expect-type": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.4.0.tgz", + "integrity": "sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/kleur": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/kleur/-/kleur-4.1.5.tgz", + "integrity": "sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/lightningcss": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.33.0.tgz", + "integrity": "sha512-WkUDrojuJs0xkgGf2udWxa3yGBRxPtxUkB79i6aCZLRgc7PM8fZe9TosfPDcvEpQZbuFASnHYmRLBLUbmLOIIA==", + "dev": true, + "license": "MPL-2.0", + "dependencies": { + "detect-libc": "^2.0.3" + }, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + }, + "optionalDependencies": { + "lightningcss-android-arm64": "1.33.0", + "lightningcss-darwin-arm64": "1.33.0", + "lightningcss-darwin-x64": "1.33.0", + "lightningcss-freebsd-x64": "1.33.0", + "lightningcss-linux-arm-gnueabihf": "1.33.0", + "lightningcss-linux-arm64-gnu": "1.33.0", + "lightningcss-linux-arm64-musl": "1.33.0", + "lightningcss-linux-x64-gnu": "1.33.0", + "lightningcss-linux-x64-musl": "1.33.0", + "lightningcss-win32-arm64-msvc": "1.33.0", + "lightningcss-win32-x64-msvc": "1.33.0" + } + }, + "node_modules/lightningcss-android-arm64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-android-arm64/-/lightningcss-android-arm64-1.33.0.tgz", + "integrity": "sha512-gEpRTalKdosp4Bb8qWtc2iOgE5SeIHlpS1up9bFq2wAyYhl1UdTObYiHe98zEM9SQvSoqQZ1IQD0JNpg3Ml5pg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-arm64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.33.0.tgz", + "integrity": "sha512-Sciaz8eenNTKn9b3t7+xr0ipTp9YxKQY4npwQ3mrRuL0BAVHBLyZxofhaKBAVtzmtRZ/zTyo0/to4B1uWG/Djg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-x64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.33.0.tgz", + "integrity": "sha512-Z5UPAxzrjlWNNyGy6i65cJzzvgJ5D3T6wMvs+gWpY9d7qRhANrxqAp6LhxIgZhWEw18RfJTGcRxjuLIBr+m8XQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-freebsd-x64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.33.0.tgz", + "integrity": "sha512-QQM/Ti/hQajJwCY+RiWuCZ9sdtI/XQk7nDK5vC8kkdwixezOlDgvDx7+RT+QjK6FcFT4MpsuoBnHIo/O3StRRg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm-gnueabihf": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.33.0.tgz", + "integrity": "sha512-N7FVBe6iS24MlM6R/4RBTxGhQheZGs7tiQ9U32UtF75NzP5Q7xWPRqLBCKxlRQRk3rY1jCIPLzx7WzOhuUIRLQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-gnu": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.33.0.tgz", + "integrity": "sha512-j2v/itmy4HlNxlc6voKXYgBqNi0Ng2LShg4z7GufpEgs05P+2suBVyi9I6YHq5uoVFx9ETin3eCEhLVyXGQnKg==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-musl": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.33.0.tgz", + "integrity": "sha512-yiO5ROMuYQgXbC60yjZU5CYSFZGKXL0HFATXt9mHJn1+zW55oCtMI9NfcVhYLMFDL7gV7oBPon/EmMMGg2OvtQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-gnu": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.33.0.tgz", + "integrity": "sha512-ar+Ju7LmcN0Jo4FpL4hpFybwNG9/3A/Br5KW2n2jyODg3MEZXaDYADdemoNS+BDNfMgKvylJLj4S5tyRActuAg==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-musl": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.33.0.tgz", + "integrity": "sha512-RYiYbkokw0trfKqqzfF55lginwEPrD3OJDfTuJzFs1MK6iFnDenaz1fqLLtX4ITG3OktJQXOeTaw1awrBAlZPw==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-arm64-msvc": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.33.0.tgz", + "integrity": "sha512-1K+MPfLSFVpphzpdbfkhlWk6wBrTObBzS2T6db10PNOZgR9GoVsAWzwNyuhUYYbTp23j+4RrncfujZ4uAzXvwA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-x64-msvc": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.33.0.tgz", + "integrity": "sha512-OlEICDx/Xl0FqSp4bry8zFnCvGpig3Gl4gCquvYwHuqJKEC1+n9NgDniFvqHGmMv1ZkqDJrDqKKSykTDX+ehuA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/magic-string": { + "version": "0.30.21", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", + "integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.5" + } + }, + "node_modules/nanoid": { + "version": "3.3.18", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.18.tgz", + "integrity": "sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/obug": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/obug/-/obug-2.2.1.tgz", + "integrity": "sha512-XrsrhT5sybtKI6wakr2SPOlGZWWYbUXZ7a0jT8/QOeAPau+1X/bSegNe5YR75oJmEZQbKningirmGOEJCIk61Q==", + "dev": true, + "funding": [ + "https://github.com/sponsors/sxzz", + "https://opencollective.com/debug" + ], + "license": "MIT", + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/path-to-regexp": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.3.0.tgz", + "integrity": "sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/pathe": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", + "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", + "dev": true, + "license": "MIT" + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", + "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/postcss": { + "version": "8.5.28", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.28.tgz", + "integrity": "sha512-RRuzqDtt5Y9h3quz5hWhK+TPnsmVs6WwSU6LkJMeY4HstUEDuYTG8UJSdawMRzmzAtV+KEoG8N3Qg2qLy5vM/A==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.18", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/rolldown": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.2.8.tgz", + "integrity": "sha512-Z67nTmhZe7anqnM/EjI392w5i/ANUinjip7QYsOyN37oayduxt3ksdX0hf5OOamkAd53BiIHfbfSzfUmzKFQqQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@oxc-project/types": "=0.149.0", + "@rolldown/pluginutils": "^1.0.0" + }, + "bin": { + "rolldown": "bin/cli.mjs" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "optionalDependencies": { + "@rolldown/binding-android-arm-eabi": "1.2.8", + "@rolldown/binding-android-arm64": "1.2.8", + "@rolldown/binding-darwin-arm64": "1.2.8", + "@rolldown/binding-darwin-x64": "1.2.8", + "@rolldown/binding-freebsd-x64": "1.2.8", + "@rolldown/binding-linux-arm-gnueabihf": "1.2.8", + "@rolldown/binding-linux-arm64-gnu": "1.2.8", + "@rolldown/binding-linux-arm64-musl": "1.2.8", + "@rolldown/binding-linux-ppc64-gnu": "1.2.8", + "@rolldown/binding-linux-s390x-gnu": "1.2.8", + "@rolldown/binding-linux-x64-gnu": "1.2.8", + "@rolldown/binding-linux-x64-musl": "1.2.8", + "@rolldown/binding-openharmony-arm64": "1.2.8", + "@rolldown/binding-win32-arm64-msvc": "1.2.8", + "@rolldown/binding-win32-x64-msvc": "1.2.8" + } + }, + "node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/sharp": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.35.2.tgz", + "integrity": "sha512-FVtFjtBCMiJS6yb5CX7Sop45WFMpeGw6oRKuJnXYgf/f1ms/D7LE/ZUSNxnW7rZ/dbslQWYkoqFHGPaDBtaK4w==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@img/colour": "^1.1.0", + "detect-libc": "^2.1.2", + "semver": "^7.8.4" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-darwin-arm64": "0.35.2", + "@img/sharp-darwin-x64": "0.35.2", + "@img/sharp-freebsd-wasm32": "0.35.2", + "@img/sharp-libvips-darwin-arm64": "1.3.1", + "@img/sharp-libvips-darwin-x64": "1.3.1", + "@img/sharp-libvips-linux-arm": "1.3.1", + "@img/sharp-libvips-linux-arm64": "1.3.1", + "@img/sharp-libvips-linux-ppc64": "1.3.1", + "@img/sharp-libvips-linux-riscv64": "1.3.1", + "@img/sharp-libvips-linux-s390x": "1.3.1", + "@img/sharp-libvips-linux-x64": "1.3.1", + "@img/sharp-libvips-linuxmusl-arm64": "1.3.1", + "@img/sharp-libvips-linuxmusl-x64": "1.3.1", + "@img/sharp-linux-arm": "0.35.2", + "@img/sharp-linux-arm64": "0.35.2", + "@img/sharp-linux-ppc64": "0.35.2", + "@img/sharp-linux-riscv64": "0.35.2", + "@img/sharp-linux-s390x": "0.35.2", + "@img/sharp-linux-x64": "0.35.2", + "@img/sharp-linuxmusl-arm64": "0.35.2", + "@img/sharp-linuxmusl-x64": "0.35.2", + "@img/sharp-webcontainers-wasm32": "0.35.2", + "@img/sharp-win32-arm64": "0.35.2", + "@img/sharp-win32-ia32": "0.35.2", + "@img/sharp-win32-x64": "0.35.2" + } + }, + "node_modules/siginfo": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", + "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==", + "dev": true, + "license": "ISC" + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/stackback": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", + "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==", + "dev": true, + "license": "MIT" + }, + "node_modules/std-env": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/std-env/-/std-env-4.2.0.tgz", + "integrity": "sha512-oCUKSupKTHX53EyjDtuZQ64pjLJ6yYCtpmEw0goYxtjG9KpbRe8KAsl2tBUGU9DyMcJ0RwJ8GqJAFzMXcXW1Rw==", + "dev": true, + "license": "MIT" + }, + "node_modules/supports-color": { + "version": "10.2.2", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-10.2.2.tgz", + "integrity": "sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/chalk/supports-color?sponsor=1" + } + }, + "node_modules/tinybench": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", + "integrity": "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/tinyexec": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.3.1.tgz", + "integrity": "sha512-GCvB3aoys96IuDFBMcTB46JOR6mdMtAToqwiW8JlWhsoh1mhHi/xn9ss/Dg7N555GiJyEt2qzoG/NHCwM6h1EA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/tinyglobby": { + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/tinyrainbow": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.1.1.tgz", + "integrity": "sha512-yau8yJdTt989Mm0Bd/236QnzEiPf2xLLTqUZRUJOo/3CB078LSwzei343DgtJVmfJKJE3TMINY1u42SQsP6mXw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "dev": true, + "license": "0BSD", + "optional": true + }, + "node_modules/typescript": { + "version": "5.9.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.2.tgz", + "integrity": "sha512-CWBzXQrc/qOkhidw1OzBTQuYRbfyxDXJMVJ1XNwUHGROVmuaeiEm3OslpZ1RV96d7SKKjZKrSJu3+t/xlw3R9A==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici": { + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.29.0.tgz", + "integrity": "sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20.18.1" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/unenv": { + "version": "2.0.0-rc.24", + "resolved": "https://registry.npmjs.org/unenv/-/unenv-2.0.0-rc.24.tgz", + "integrity": "sha512-i7qRCmY42zmCwnYlh9H2SvLEypEFGye5iRmEMKjcGi7zk9UquigRjFtTLz0TYqr0ZGLZhaMHl/foy1bZR+Cwlw==", + "dev": true, + "license": "MIT", + "dependencies": { + "pathe": "^2.0.3" + } + }, + "node_modules/vite": { + "version": "8.2.2", + "resolved": "https://registry.npmjs.org/vite/-/vite-8.2.2.tgz", + "integrity": "sha512-cFKLV/PRgAUlIRm5WjMjJ86jrftzpqcgH+Us+DS8mI3CDNiH30Whrz8uHL3+MOLPAgqbMBAqWdAHAphOAM+z/Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "lightningcss": "^1.33.0", + "picomatch": "^4.0.5", + "postcss": "^8.5.26", + "rolldown": "~1.2.4", + "tinyglobby": "^0.2.17" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^20.19.0 || >=22.12.0", + "@vitejs/devtools": "^0.4.0 || ^0.5.0", + "esbuild": "^0.27.0 || ^0.28.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "@vitejs/devtools": { + "optional": true + }, + "esbuild": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/vitest": { + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.11.tgz", + "integrity": "sha512-fhACrNXUidIbGSBr5FlbuBkO7VWC1ZyLl0DO4CU2DrQoAPxX84Ysxs+HeGQpii5lZWV1Q4gBZTTu49mF+A6Edw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/expect": "4.1.11", + "@vitest/mocker": "4.1.11", + "@vitest/pretty-format": "4.1.11", + "@vitest/runner": "4.1.11", + "@vitest/snapshot": "4.1.11", + "@vitest/spy": "4.1.11", + "@vitest/utils": "4.1.11", + "es-module-lexer": "^2.0.0", + "expect-type": "^1.3.0", + "magic-string": "^0.30.21", + "obug": "^2.1.1", + "pathe": "^2.0.3", + "picomatch": "^4.0.3", + "std-env": "^4.0.0-rc.1", + "tinybench": "^2.9.0", + "tinyexec": "^1.0.2", + "tinyglobby": "^0.2.15", + "tinyrainbow": "^3.1.0", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0", + "why-is-node-running": "^2.3.0" + }, + "bin": { + "vitest": "vitest.mjs" + }, + "engines": { + "node": "^20.0.0 || ^22.0.0 || >=24.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@edge-runtime/vm": "*", + "@opentelemetry/api": "^1.9.0", + "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", + "@vitest/browser-playwright": "4.1.11", + "@vitest/browser-preview": "4.1.11", + "@vitest/browser-webdriverio": "4.1.11", + "@vitest/coverage-istanbul": "4.1.11", + "@vitest/coverage-v8": "4.1.11", + "@vitest/ui": "4.1.11", + "happy-dom": "*", + "jsdom": "*", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "@edge-runtime/vm": { + "optional": true + }, + "@opentelemetry/api": { + "optional": true + }, + "@types/node": { + "optional": true + }, + "@vitest/browser-playwright": { + "optional": true + }, + "@vitest/browser-preview": { + "optional": true + }, + "@vitest/browser-webdriverio": { + "optional": true + }, + "@vitest/coverage-istanbul": { + "optional": true + }, + "@vitest/coverage-v8": { + "optional": true + }, + "@vitest/ui": { + "optional": true + }, + "happy-dom": { + "optional": true + }, + "jsdom": { + "optional": true + }, + "vite": { + "optional": false + } + } + }, + "node_modules/why-is-node-running": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", + "integrity": "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==", + "dev": true, + "license": "MIT", + "dependencies": { + "siginfo": "^2.0.0", + "stackback": "0.0.2" + }, + "bin": { + "why-is-node-running": "cli.js" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrangler": { + "version": "4.126.0", + "resolved": "https://registry.npmjs.org/wrangler/-/wrangler-4.126.0.tgz", + "integrity": "sha512-glDq/nxaeQwue0XMIeupfwlu2jVxnFs+wjDFT71DQuURN5LsVdCwu0Af/1ep10U5xr1rKPGhHDEDkKG9nxE/dg==", + "dev": true, + "license": "MIT OR Apache-2.0", + "dependencies": { + "@cloudflare/kv-asset-handler": "0.5.0", + "@cloudflare/unenv-preset": "2.16.1", + "blake3-wasm": "2.1.5", + "esbuild": "0.28.1", + "miniflare": "5.20260825.0-alpha", + "path-to-regexp": "6.3.0", + "unenv": "2.0.0-rc.24", + "workerd": "1.20260825.1" + }, + "bin": { + "cf-wrangler": "bin/cf-wrangler.js", + "wrangler": "bin/wrangler.js", + "wrangler2": "bin/wrangler.js" + }, + "engines": { + "node": ">=22.0.0" + }, + "optionalDependencies": { + "fsevents": "2.3.3" + }, + "peerDependencies": { + "@cloudflare/workers-types": "^5.20260825.1" + }, + "peerDependenciesMeta": { + "@cloudflare/workers-types": { + "optional": true + } + } + }, + "node_modules/wrangler/node_modules/@cloudflare/workerd-darwin-64": { + "version": "1.20260825.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20260825.1.tgz", + "integrity": "sha512-oHu38dwaUuzAilyTb0QkQ1YxU/kzqzIQybCvQKAhiK1CGtQS9h0MmjIZYogv3g8cFGGY2k+Wxs0wV9hHK8z78g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/wrangler/node_modules/@cloudflare/workerd-darwin-arm64": { + "version": "1.20260825.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20260825.1.tgz", + "integrity": "sha512-ak5zh8YGEjxQQ78bVo7gzU+tcg2fSFxMIjOPZtWk56a/rIYLbGu6ECcliqnYfMUlragg68H0JuVpfdr3BR5Alw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/wrangler/node_modules/@cloudflare/workerd-linux-64": { + "version": "1.20260825.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20260825.1.tgz", + "integrity": "sha512-bNQvzz6NemWAwixDRz1fQa5T+E5lS4xpB7A/H/72ULxrjVpHmq8CGFPSbdmRp3dvgBjZTgp7wHdGLISLSVd9Gg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/wrangler/node_modules/@cloudflare/workerd-linux-arm64": { + "version": "1.20260825.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20260825.1.tgz", + "integrity": "sha512-a5E61YsnNCHHQMnmYsbVXInzeYqFqAMwm/wo16dWW4klXDr6T1bm7u1h5G7ZkxVM7+rtc69oe0yVHjDEqzpYVg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/wrangler/node_modules/@cloudflare/workerd-windows-64": { + "version": "1.20260825.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20260825.1.tgz", + "integrity": "sha512-EokzVY2suzRSzeURi2HpHnySR5mo6aF5V1klFIqFOZp2YJyXXTI8AvQgYzhlmGTZY3LNL41jjkUQunOM2OErgQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/wrangler/node_modules/miniflare": { + "version": "5.20260825.0-alpha", + "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-5.20260825.0-alpha.tgz", + "integrity": "sha512-ZwlF6LuX43ilx9EwMRDKHenoGXiNdcKSyGx5aPaJhuozujVZasb2lRR7t3ojJZSnVzwDPsBBYCu8lLnc+/KIgQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cspotcode/source-map-support": "0.8.1", + "sharp": "0.35.2", + "undici": "7.29.0", + "workerd": "1.20260825.1", + "ws": "8.21.0", + "youch": "4.1.0-beta.10" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/wrangler/node_modules/workerd": { + "version": "1.20260825.1", + "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20260825.1.tgz", + "integrity": "sha512-ccS6TEaaRxgONKawiYGnFYUVGfr2pmN1b7mrNtw0ADVhFaYsEIVoCHnQ4UjhM9EJDzuaNgAWFY82nzVrjWpuOA==", + "dev": true, + "hasInstallScript": true, + "license": "Apache-2.0", + "bin": { + "workerd": "bin/workerd" + }, + "engines": { + "node": ">=16" + }, + "optionalDependencies": { + "@cloudflare/workerd-darwin-64": "1.20260825.1", + "@cloudflare/workerd-darwin-arm64": "1.20260825.1", + "@cloudflare/workerd-linux-64": "1.20260825.1", + "@cloudflare/workerd-linux-arm64": "1.20260825.1", + "@cloudflare/workerd-windows-64": "1.20260825.1" + } + }, + "node_modules/ws": { + "version": "8.21.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz", + "integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/youch": { + "version": "4.1.0-beta.10", + "resolved": "https://registry.npmjs.org/youch/-/youch-4.1.0-beta.10.tgz", + "integrity": "sha512-rLfVLB4FgQneDr0dv1oddCVZmKjcJ6yX6mS4pU82Mq/Dt9a3cLZQ62pDBL4AUO+uVrCvtWz3ZFUL2HFAFJ/BXQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@poppinss/colors": "^4.1.5", + "@poppinss/dumper": "^0.6.4", + "@speed-highlight/core": "^1.2.7", + "cookie": "^1.0.2", + "youch-core": "^0.3.3" + } + }, + "node_modules/youch-core": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/youch-core/-/youch-core-0.3.3.tgz", + "integrity": "sha512-ho7XuGjLaJ2hWHoK8yFnsUGy2Y5uDpqSTq1FkHLK4/oqKtyUU1AFbOOxY4IpC9f0fTLjwYbslUz0Po5BpD1wrA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@poppinss/exception": "^1.2.2", + "error-stack-parser-es": "^1.0.5" + } + }, + "node_modules/zod": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.4.3.tgz", + "integrity": "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} diff --git a/examples/microduck-world/edge/package.json b/examples/microduck-world/edge/package.json new file mode 100644 index 0000000000..ca57926123 --- /dev/null +++ b/examples/microduck-world/edge/package.json @@ -0,0 +1,19 @@ +{ + "name": "microduck-public-edge", + "private": true, + "type": "module", + "scripts": { + "dev": "wrangler dev --port 8787", + "types": "wrangler types", + "check": "tsc --noEmit", + "test": "vitest run", + "build": "wrangler deploy --dry-run --outdir dist" + }, + "devDependencies": { + "@cloudflare/vitest-plugin": "1.1.6", + "@types/node": "22.20.2", + "typescript": "5.9.2", + "vitest": "4.1.11", + "wrangler": "4.126.0" + } +} diff --git a/examples/microduck-world/edge/src/auth.ts b/examples/microduck-world/edge/src/auth.ts new file mode 100644 index 0000000000..dc04bde2b0 --- /dev/null +++ b/examples/microduck-world/edge/src/auth.ts @@ -0,0 +1,227 @@ +import { timingSafeEqual } from "node:crypto"; +import type { Identity } from "../../shared/publicWire.ts"; + +export type User = Identity & { + preferred_name: string; + banned: number; + expires_at: number; +}; +const SESSION_COOKIE = "__Host-microduck"; +const STATE_COOKIE = "__Host-microduck-oauth"; +export const SESSION_SECONDS = 7 * 86400; +const encoder = new TextEncoder(); + +export function randomToken(): string { + return Array.from( + crypto.getRandomValues(new Uint8Array(32)), + (b) => b.toString(16).padStart(2, "0"), + ).join(""); +} +export async function digest(value: string): Promise { + return Array.from( + new Uint8Array( + await crypto.subtle.digest("SHA-256", encoder.encode(value)), + ), + (b) => b.toString(16).padStart(2, "0"), + ).join(""); +} +export async function equalSecret(a: string, b: string): Promise { + const [x, y] = await Promise.all([digest(a), digest(b)]); + return timingSafeEqual(encoder.encode(x), encoder.encode(y)); +} +export function cookie(req: Request, name: string): string { + return req.headers.get("cookie")?.split(";").map((s) => s.trim()).find((s) => + s.startsWith(`${name}=`) + )?.slice(name.length + 1) ?? ""; +} +function setCookie(name: string, value: string, age: number): string { + return `${name}=${value}; Path=/; HttpOnly; Secure; SameSite=Lax; Max-Age=${age}`; +} +export function json(value: unknown, status = 200): Response { + return Response.json(value, { + status, + headers: { "cache-control": "no-store" }, + }); +} +export function sameOrigin(req: Request, env: Env): boolean { + return req.headers.get("origin") === env.PUBLIC_ORIGIN; +} +export async function userFor(req: Request, env: Env): Promise { + const token = cookie(req, SESSION_COOKIE); + if (!/^[a-f0-9]{64}$/.test(token)) return null; + return env.DB.prepare( + "SELECT u.id, u.login, u.preferred_name, u.banned, s.expires_at FROM sessions s JOIN users u ON u.id=s.user_id WHERE s.hash=? AND s.expires_at>? AND u.banned=0", + ).bind(await digest(token), Date.now()).first(); +} +export function admin(user: Identity, env: Env): boolean { + return env.ADMIN_GITHUB_IDS.split(",").map((s) => s.trim()).includes(user.id); +} + +export async function authRoute( + req: Request, + env: Env, +): Promise { + const url = new URL(req.url); + if ( + ["/api/auth", "/auth/login", "/auth/callback"].includes(url.pathname) && + req.method !== "GET" + ) return json({ error: "Method not allowed" }, 405); + if (url.pathname === "/api/auth") { + const user = await userFor(req, env); + return json({ + required: true, + provider: "github", + user: user + ? { + id: user.id, + login: user.login, + preferredName: user.preferred_name, + admin: admin(user, env), + } + : null, + configured: !!env.GITHUB_CLIENT_ID && !!env.GITHUB_CLIENT_SECRET, + }); + } + if (url.pathname === "/auth/login") { + if (!env.GITHUB_CLIENT_ID || !env.GITHUB_CLIENT_SECRET) { + return json({ error: "GitHub sign-in is being configured." }, 503); + } + const state = randomToken(); + const verifier = randomToken(); + const raw = new Uint8Array( + await crypto.subtle.digest("SHA-256", encoder.encode(verifier)), + ); + const challenge = btoa(String.fromCharCode(...raw)).replaceAll("+", "-") + .replaceAll("/", "_").replaceAll("=", ""); + await env.DB.batch([ + env.DB.prepare("DELETE FROM oauth_states WHERE expires_at 512 || + !(await equalSecret(state, cookie(req, STATE_COOKIE))) + ) { + return json({ + error: "Sign-in expired. Return to the homepage and try again.", + }, 400); + } + const saved = await env.DB.prepare( + "DELETE FROM oauth_states WHERE hash=? AND expires_at>? RETURNING verifier", + ).bind(await digest(state), Date.now()).first<{ verifier: string }>(); + if (!saved) { + return json({ error: "Sign-in expired. Please try again." }, 400); + } + const tokenResponse = await fetch( + "https://github.com/login/oauth/access_token", + { + method: "POST", + headers: { + accept: "application/json", + "content-type": "application/json", + }, + body: JSON.stringify({ + client_id: env.GITHUB_CLIENT_ID, + client_secret: env.GITHUB_CLIENT_SECRET, + code, + code_verifier: saved.verifier, + redirect_uri: `${env.PUBLIC_ORIGIN}/auth/callback`, + }), + signal: AbortSignal.timeout(10_000), + }, + ); + const token = await tokenResponse.json<{ access_token?: string }>(); + if (!tokenResponse.ok || !token.access_token) { + return json({ error: "GitHub could not complete sign-in." }, 502); + } + const profileResponse = await fetch("https://api.github.com/user", { + headers: { + authorization: `Bearer ${token.access_token}`, + accept: "application/vnd.github+json", + "user-agent": "Microduck-DimOS", + "x-github-api-version": "2022-11-28", + }, + signal: AbortSignal.timeout(10_000), + }); + const profile = await profileResponse.json< + { id?: number; login?: string } + >(); + if ( + !profileResponse.ok || !Number.isSafeInteger(profile.id) || + !profile.login || !/^[a-zA-Z0-9-]{1,39}$/.test(profile.login) + ) { + return json({ error: "Could not verify your GitHub identity." }, 502); + } + const id = String(profile.id); + await env.DB.prepare( + "INSERT INTO users(id,login,created_at) VALUES(?,?,?) ON CONFLICT(id) DO UPDATE SET login=excluded.login", + ).bind(id, profile.login, Date.now()).run(); + if ( + (await env.DB.prepare("SELECT banned FROM users WHERE id=?").bind(id) + .first<{ banned: number }>())?.banned + ) return json({ error: "This account cannot join the demo." }, 403); + const session = randomToken(); + await env.DB.prepare( + "INSERT INTO sessions(hash,user_id,expires_at) VALUES(?,?,?)", + ).bind(await digest(session), id, Date.now() + SESSION_SECONDS * 1000) + .run(); + const headers = new Headers({ + location: env.PUBLIC_ORIGIN, + "cache-control": "no-store", + }); + headers.append( + "set-cookie", + setCookie(SESSION_COOKIE, session, SESSION_SECONDS), + ); + headers.append("set-cookie", setCookie(STATE_COOKIE, "", 0)); + return new Response(null, { status: 302, headers }); + } + if (url.pathname === "/auth/logout") { + if (req.method !== "POST" || !sameOrigin(req, env)) { + return json({ error: "Forbidden" }, 403); + } + const user = await userFor(req, env); + if (user) { + await env.DB.prepare("DELETE FROM sessions WHERE hash=?").bind( + await digest(cookie(req, SESSION_COOKIE)), + ).run(); + await env.MATCH.getByName(env.MATCH_ID).revoke(user.id); + } + const pendingState = cookie(req, STATE_COOKIE); + if (pendingState) await env.DB.prepare("DELETE FROM oauth_states WHERE hash=?") + .bind(await digest(pendingState)).run(); + const headers = new Headers({ "cache-control": "no-store" }); + headers.append("set-cookie", setCookie(SESSION_COOKIE, "", 0)); + headers.append("set-cookie", setCookie(STATE_COOKIE, "", 0)); + return new Response(null, { status: 204, headers }); + } + return null; +} diff --git a/examples/microduck-world/edge/src/index.ts b/examples/microduck-world/edge/src/index.ts new file mode 100644 index 0000000000..01257a6a18 --- /dev/null +++ b/examples/microduck-world/edge/src/index.ts @@ -0,0 +1,227 @@ +import { + admin, + authRoute, + equalSecret, + json, + sameOrigin, + userFor, +} from "./auth.ts"; +export { MatchRelay } from "./room.ts"; + +class RequestError extends Error { + constructor(message: string, readonly status: number) { + super(message); + } +} + +async function boundedJson(request: Request, limit = 4096): Promise { + if (!request.body) return null; + const reader = request.body.getReader(); + const chunks: Uint8Array[] = []; + let length = 0; + for (;;) { + const { value, done } = await reader.read(); + if (done) break; + length += value.length; + if (length > limit) { + await reader.cancel(); + throw new RequestError("Request too large", 413); + } + chunks.push(value); + } + const bytes = new Uint8Array(length); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.length; + } + try { + return JSON.parse(new TextDecoder().decode(bytes)); + } catch { + throw new RequestError("Invalid JSON", 400); + } +} + +async function handle(request: Request, env: Env): Promise { + const url = new URL(request.url); + const room = env.MATCH.getByName(env.MATCH_ID); + let authenticated: Awaited> = null; + try { + if (url.pathname === "/bridge") { + const protocols = + request.headers.get("sec-websocket-protocol")?.split(",").map((s) => + s.trim() + ) ?? []; + if ( + !env.HOST_SECRET || protocols[0] !== "microduck-origin" || + !(await equalSecret(protocols[1] ?? "", env.HOST_SECRET)) + ) return json({ error: "Forbidden" }, 403); + return room.fetch(new Request("https://match/host", request)); + } + if ( + url.pathname.startsWith("/auth/") || url.pathname.startsWith("/api/") || + url.pathname === "/connect" || url.pathname.startsWith("/sessions/") + ) { + const ip = request.headers.get("cf-connecting-ip") ?? "local"; + authenticated = await userFor(request, env); + const rate = await env.REQUEST_LIMITER.limit({ + key: authenticated ? `user:${authenticated.id}` : `ip:${ip}`, + }); + if (!rate.success) { + return json({ error: "Too many requests. Please wait a minute." }, 429); + } + } + const auth = await authRoute(request, env); + if (auth) return auth; + if ( + request.method === "GET" && + (url.pathname === "/api/lobby" || url.pathname === "/api/preview") + ) { + const result = await room.http(null, url.pathname); + return json(JSON.parse(result.body), result.status); + } + if (url.pathname === "/healthz") { + const result = await room.http(null, "/healthz"); + return json(JSON.parse(result.body), result.status); + } + if ( + url.pathname.startsWith("/api/") || + url.pathname.startsWith("/sessions/") || url.pathname === "/connect" + ) { + const user = authenticated; + if (!user) { + return json({ + error: "Sign in with GitHub to join.", + login: "/auth/login", + }, 401); + } + if ( + (request.method !== "GET" || url.pathname === "/connect") && + !sameOrigin(request, env) + ) return json({ error: "Forbidden origin" }, 403); + if (url.pathname === "/connect") { + const headers = new Headers(request.headers); + headers.set("x-user-id", user.id); + headers.set("x-user-login", user.login); + headers.set("x-session-expires", String(user.expires_at)); + return room.fetch( + new Request( + `https://match/viewer?ticket=${ + encodeURIComponent(url.searchParams.get("ticket") ?? "") + }`, + { headers }, + ), + ); + } + if (/^\/sessions\/[a-zA-Z0-9-]{1,100}\/api\/info$/.test(url.pathname)) { + const ticket = url.pathname.split("/")[2]; + const result = await room.http( + user, + "/api/session", + JSON.stringify({ token: ticket }), + ); + if (result.status !== 200) { + return json(JSON.parse(result.body), result.status); + } + const body = JSON.parse(result.body) as { v: number }; + return json({ + wtUrl: `${env.PUBLIC_ORIGIN.replace(/^http/, "ws")}/connect?ticket=${ + encodeURIComponent(ticket) + }`, + certHash: "", + v: body.v, + }); + } + if (url.pathname === "/api/lobby" && request.method === "POST") { + const body = await boundedJson(request); + const result = await room.http( + user, + "/api/lobby", + JSON.stringify(body), + ); + if (result.status === 200) { + const name = + (JSON.parse(result.body) as { displayName?: string }).displayName; + if ( + name && body && typeof body === "object" && "action" in body && + body.action === "join" + ) { + await env.DB.prepare("UPDATE users SET preferred_name=? WHERE id=?") + .bind(name, user.id).run(); + } + } + return json(JSON.parse(result.body), result.status); + } + if ( + url.pathname === "/api/admin/ban" && request.method === "POST" && + admin(user, env) + ) { + const body = await boundedJson(request) as { + userId?: string; + banned?: boolean; + }; + if ( + !body || typeof body.userId !== "string" || + !/^\d+$/.test(body.userId) || typeof body.banned !== "boolean" || + body.userId === user.id + ) return json({ error: "Invalid account" }, 400); + await env.DB.prepare("UPDATE users SET banned=? WHERE id=?").bind( + body.banned ? 1 : 0, + body.userId, + ).run(); + if (body.banned) { + await env.DB.prepare("DELETE FROM sessions WHERE user_id=?").bind( + body.userId, + ).run(); + await room.revoke(body.userId); + } + return json({ ok: true }); + } + return json({ error: "Not found" }, 404); + } + if (request.method !== "GET" && request.method !== "HEAD") { + return new Response("Method not allowed", { status: 405 }); + } + return env.ASSETS.fetch(request); + } catch (error) { + if (error instanceof RequestError) { + return json({ error: error.message }, error.status); + } + // Do not log callback codes, session cookies, bridge protocols or secrets. + console.error( + JSON.stringify({ + event: "request_failed", + path: url.pathname.startsWith("/sessions/") + ? "/sessions/…" + : url.pathname, + kind: error instanceof Error ? error.name : "Error", + }), + ); + return json({ + error: "The request could not be completed. Please try again.", + }, 503); + } +} + +export default { + async fetch(request, env): Promise { + const response = await handle(request, env); + if (response.status === 101) return response; + const headers = new Headers(response.headers); + headers.set("x-content-type-options", "nosniff"); + headers.set("referrer-policy", "no-referrer"); + headers.set( + "permissions-policy", + "camera=(), microphone=(), geolocation=()", + ); + headers.set( + "content-security-policy", + "default-src 'self'; script-src 'self' 'wasm-unsafe-eval'; style-src 'self' 'unsafe-inline'; img-src 'self' data: blob:; connect-src 'self' wss://sim.tule.world; worker-src 'self' blob:; frame-ancestors 'none'; base-uri 'none'; form-action 'self'", + ); + return new Response(response.body, { + status: response.status, + statusText: response.statusText, + headers, + }); + }, +} satisfies ExportedHandler; diff --git a/examples/microduck-world/edge/src/room.ts b/examples/microduck-world/edge/src/room.ts new file mode 100644 index 0000000000..0333bdbc32 --- /dev/null +++ b/examples/microduck-world/edge/src/room.ts @@ -0,0 +1,342 @@ +import { DurableObject } from "cloudflare:workers"; +import { + type Identity, + MAX_CLIENTS, + MAX_CONTROL_BYTES, + MAX_FRAME_BYTES, + MAX_PENDING_DOWNLOADS, + MAX_PENDING_UPLOADS, + pack, + serverFrame, + unpack, +} from "../../shared/publicWire.ts"; + +type Client = { + socket: WebSocket; + user: Identity; + pending: Map; + bytes: number; + seq: number; + count: number; + window: number; + expires: number; + clockOffset: number | null; + uploads: { bytes: number; at: number }[]; +}; +type Reply = { status: number; body: string }; + +/** Transport only: the stock DimOS registry on Omarchy authorizes every command. */ +export class MatchRelay extends DurableObject { + private host: WebSocket | null = null; + private hostSeen = 0; + private uploadBytes = 0; + private clients = new Map(); + private pending = new Map< + string, + { resolve: (r: Reply) => void; timer: ReturnType } + >(); + + constructor(ctx: DurableObjectState, env: Env) { + super(ctx, env); + // Continuous simulation traffic already keeps this object active. Close peers + // on host/credit timeout; no stale command is retained for a later reconnect. + setInterval(() => this.sweep(), 5000); + } + + async fetch(request: Request): Promise { + const url = new URL(request.url); + if (request.headers.get("upgrade")?.toLowerCase() !== "websocket") { + return new Response("WebSocket required", { status: 426 }); + } + if (url.pathname === "/host") { + if (this.host) { + return new Response("Origin already connected", { status: 409 }); + } + const pair = new WebSocketPair(); + this.host = pair[1]; + this.hostSeen = Date.now(); + pair[1].binaryType = "arraybuffer"; + pair[1].accept(); + pair[1].addEventListener( + "message", + (e) => this.fromHost(pair[1], e.data), + ); + pair[1].addEventListener("close", () => this.hostClosed(pair[1])); + pair[1].addEventListener("error", () => this.hostClosed(pair[1])); + return new Response(null, { + status: 101, + webSocket: pair[0], + headers: { "sec-websocket-protocol": "microduck-origin" }, + }); + } + if (url.pathname !== "/viewer") { + return new Response("Not found", { status: 404 }); + } + if (!this.host || Date.now() - this.hostSeen > 15_000) { + return new Response("World offline", { status: 503 }); + } + const connectingUser = request.headers.get("x-user-id"); + for (const [oldId, old] of this.clients) { + if (old.user.id === connectingUser) { + this.closeClient(oldId, "Replaced by your new connection"); + } + } + if (this.clients.size >= MAX_CLIENTS) { + return new Response("The world is full", { status: 503 }); + } + const user = { + id: request.headers.get("x-user-id") ?? "", + login: request.headers.get("x-user-login") ?? "", + }; + const expires = Number(request.headers.get("x-session-expires")); + if ( + !/^\d+$/.test(user.id) || !Number.isFinite(expires) || + expires <= Date.now() + ) return new Response("Sign in again", { status: 401 }); + const id = crypto.randomUUID(); + const pair = new WebSocketPair(); + pair[1].binaryType = "arraybuffer"; + pair[1].accept(); + const client: Client = { + socket: pair[1], + user, + pending: new Map(), + bytes: 0, + seq: 0, + count: 0, + window: Date.now(), + expires, + clockOffset: null, + uploads: [], + }; + this.clients.set(id, client); + pair[1].addEventListener("message", (e) => this.fromClient(id, e.data)); + pair[1].addEventListener("close", () => this.closeClient(id)); + pair[1].addEventListener("error", () => this.closeClient(id)); + this.host.send( + JSON.stringify({ + t: "open", + id, + user, + ticket: url.searchParams.get("ticket") ?? "", + }), + ); + return new Response(null, { status: 101, webSocket: pair[0] }); + } + + async http( + user: Identity | null, + path: string, + bodyJson = "null", + ): Promise { + if (!this.host || Date.now() - this.hostSeen > 15_000) { + return { + status: 503, + body: JSON.stringify({ + error: "The world is offline. Please try again shortly.", + }), + }; + } + if (this.pending.size >= 64) { + return { + status: 503, + body: JSON.stringify({ error: "The world is busy." }), + }; + } + const id = crypto.randomUUID(); + const message = JSON.stringify({ + t: "http", + id, + user, + path, + body: JSON.parse(bodyJson), + }); + if (message.length > MAX_CONTROL_BYTES) { + return { + status: 413, + body: JSON.stringify({ error: "Request too large" }), + }; + } + return new Promise((resolve) => { + const timer = setTimeout(() => { + this.pending.delete(id); + resolve({ + status: 504, + body: JSON.stringify({ error: "The world did not respond." }), + }); + }, 7000); + this.pending.set(id, { resolve, timer }); + try { + this.host!.send(message); + } catch { + this.hostClosed(this.host!); + } + }); + } + + async revoke(userId: string): Promise { + for (const [id, client] of this.clients) { + if (client.user.id === userId) this.closeClient(id, "Session revoked"); + } + this.host?.send(JSON.stringify({ t: "revoke", userId })); + } + + private fromHost(socket: WebSocket, data: string | ArrayBuffer): void { + if (socket !== this.host) return; + this.hostSeen = Date.now(); + try { + if (typeof data === "string") { + if (data.length > MAX_FRAME_BYTES) { + throw new Error("Origin message too large"); + } + const msg = JSON.parse(data); + if (msg.t === "ping") socket.send(JSON.stringify({ t: "pong" })); + else if (msg.t === "received") { + const item = this.clients.get(msg.id)?.uploads.shift(); + if (item) this.uploadBytes -= item.bytes; + } else if (msg.t === "reply") { + const p = this.pending.get(msg.id); + if ( + p && Number.isInteger(msg.status) && msg.status >= 200 && + msg.status <= 599 + ) { + clearTimeout(p.timer); + this.pending.delete(msg.id); + p.resolve({ status: msg.status, body: JSON.stringify(msg.body) }); + } + } else if (msg.t === "close") { + this.closeClient(msg.id, "Session changed. Reconnecting."); + } + return; + } + const { header, payload } = unpack(new Uint8Array(data)); + if (header.kind !== 0 && header.kind !== 1) { + throw new Error("Invalid origin frame kind"); + } + for (const id of header.to) { + const client = this.clients.get(id); + if (!client) continue; + // Credit is released only when the browser receives a message. Large or + // suspended viewers cannot create an unbounded Cloudflare send queue. + if ( + client.pending.size >= MAX_PENDING_DOWNLOADS || + client.bytes + payload.length > MAX_FRAME_BYTES + ) { + this.closeClient(id, "Connection too slow. Reconnect to resume."); + continue; + } + const seq = ++client.seq; + client.pending.set(seq, { bytes: payload.length, at: Date.now() }); + client.bytes += payload.length; + client.socket.send(serverFrame(header.kind, seq, payload)); + } + } catch { + this.hostClosed(socket); + } + } + + private fromClient(id: string, data: string | ArrayBuffer): void { + const c = this.clients.get(id); + if (!c) return; + try { + if (Date.now() - c.window > 1000) { + c.window = Date.now(); + c.count = 0; + } + if (++c.count > 300) throw new Error("Message rate exceeded"); + if (typeof data === "string") { + if (data.length > 100) throw new Error("Invalid acknowledgment"); + const ack = JSON.parse(data); + if (ack.t !== "ack" || !Number.isInteger(ack.n)) { + throw new Error("Invalid acknowledgment"); + } + const pending = c.pending.get(ack.n); + if (pending) { + c.bytes -= pending.bytes; + c.pending.delete(ack.n); + } + return; + } + if (!this.host || c.expires <= Date.now()) { + throw new Error("Session expired"); + } + const bytes = new Uint8Array(data); + if ( + bytes.length < 9 || bytes.length > MAX_CONTROL_BYTES + 9 || + ![0, 2].includes(bytes[0]) + ) throw new Error("Invalid control"); + const sentAt = new DataView( + bytes.buffer, + bytes.byteOffset, + bytes.byteLength, + ).getFloat64(1); + if (!Number.isFinite(sentAt)) throw new Error("Invalid client clock"); + c.clockOffset ??= Date.now() - sentAt; + if (Math.abs(Date.now() - sentAt - c.clockOffset) > 1500) { + throw new Error("Stale control"); + } + if ( + c.uploads.length >= MAX_PENDING_UPLOADS || + this.uploadBytes + bytes.length > MAX_FRAME_BYTES + ) throw new Error("Origin backpressure"); + c.uploads.push({ bytes: bytes.length, at: Date.now() }); + this.uploadBytes += bytes.length; + this.host.send( + pack( + { to: [id], kind: bytes[0] as 0 | 2, sentAt: Date.now() }, + bytes.subarray(9), + ), + ); + } catch (error) { + console.log("viewer_rejected", { reason: error instanceof Error ? error.message : "Invalid client message" }); + this.closeClient( + id, + "Session expired or connection delayed. Please reconnect.", + ); + } + } + + private closeClient(id: string, reason = "Disconnected"): void { + const c = this.clients.get(id); + if (!c) return; + console.log("viewer_closed", { reason, pendingFrames: c.pending.size, pendingBytes: c.bytes, uploads: c.uploads.length, messagesThisSecond: c.count }); + this.clients.delete(id); + this.uploadBytes -= c.uploads.reduce((sum, item) => sum + item.bytes, 0); + try { + c.socket.close(1000, reason); + } catch { /* already closed */ } + try { + this.host?.send(JSON.stringify({ t: "close", id })); + } catch { /* origin closed */ } + } + private hostClosed(host: WebSocket): void { + if (host !== this.host) return; + this.host = null; + try { + host.close(1011, "Origin disconnected"); + } catch { /* already closed */ } + for (const id of this.clients.keys()) { + this.closeClient(id, "World reconnecting"); + } + for (const p of this.pending.values()) { + clearTimeout(p.timer); + p.resolve({ + status: 503, + body: JSON.stringify({ error: "World reconnecting" }), + }); + } + this.pending.clear(); + } + private sweep(): void { + if (this.host && Date.now() - this.hostSeen > 15_000) { + this.hostClosed(this.host); + } + for (const [id, c] of this.clients) { + if ( + c.expires <= Date.now() || c.uploads.some((p) => + Date.now() - p.at > 1500 + ) || [...c.pending.values()].some((p) => Date.now() - p.at > 5000) + ) this.closeClient(id, "Session expired or viewer stalled"); + } + } +} diff --git a/examples/microduck-world/edge/test/auth.test.ts b/examples/microduck-world/edge/test/auth.test.ts new file mode 100644 index 0000000000..9036022aff --- /dev/null +++ b/examples/microduck-world/edge/test/auth.test.ts @@ -0,0 +1,205 @@ +import { env, exports } from "cloudflare:workers"; +import { + afterEach, + beforeAll, + beforeEach, + describe, + expect, + it, + vi, +} from "vitest"; +import { digest } from "../src/auth.ts"; +import migration from "../migrations/0001_identity.sql?raw"; + +const origin = "https://sim.tule.world"; +const request = (path: string, init?: RequestInit) => + exports.default.fetch(origin + path, { redirect: "manual", ...init }); +const token = "b".repeat(64); +async function session(id = "100", expires = Date.now() + 60_000) { + await env.DB.prepare( + "INSERT OR REPLACE INTO users(id,login,created_at) VALUES(?,?,?)", + ).bind(id, "player", Date.now()).run(); + await env.DB.prepare( + "INSERT OR REPLACE INTO sessions(hash,user_id,expires_at) VALUES(?,?,?)", + ).bind(await digest(token), id, expires).run(); + return `__Host-microduck=${token}`; +} +beforeAll(async () => { + for ( + const statement of migration.split(";").filter((s: string) => s.trim()) + ) await env.DB.prepare(statement).run(); +}); +beforeEach(async () => { + await env.DB.batch([ + env.DB.prepare("DELETE FROM sessions"), + env.DB.prepare("DELETE FROM oauth_states"), + env.DB.prepare("DELETE FROM users"), + ]); +}); +afterEach(() => vi.restoreAllMocks()); + +describe("GitHub identity boundary", () => { + it("keeps anonymous clients out of control and session discovery", async () => { + for (const path of ["/connect", "/sessions/fake/api/info"]) { + expect((await request(path)).status).toBe(401); + } + expect( + (await request("/api/lobby", { + method: "POST", + body: '{"action":"create"}', + })).status, + ).toBe(401); + const response = await request("/api/auth"); + expect(await response.json()).toMatchObject({ + required: true, + provider: "github", + user: null, + }); + }); + it("does not trust forged identity headers or cross-origin requests", async () => { + expect( + (await request("/api/lobby", { + method: "POST", + headers: { "x-user-id": "100", origin }, + body: "{}", + })).status, + ).toBe(401); + const cookie = await session(); + expect( + (await request("/api/lobby", { + method: "POST", + headers: { cookie, origin: "https://attacker.example" }, + body: "{}", + })).status, + ).toBe(403); + expect( + (await request("/connect", { + headers: { cookie, origin: "https://attacker.example" }, + })).status, + ).toBe(403); + }); + it("rejects expired and banned sessions", async () => { + const cookie = await session("100", Date.now() - 1); + expect((await request("/connect", { headers: { cookie, origin } })).status) + .toBe(401); + await session(); + await env.DB.prepare("UPDATE users SET banned=1 WHERE id='100'").run(); + expect((await request("/connect", { headers: { cookie, origin } })).status) + .toBe(401); + }); + it("requires the configured server credential for the outbound origin", async () => { + expect( + (await request("/bridge", { + headers: { + upgrade: "websocket", + "sec-websocket-protocol": "microduck-origin,wrong", + }, + })).status, + ).toBe(403); + }); + it("rejects malformed and oversized JSON before forwarding it to Omarchy", async () => { + const cookie = await session(); + const headers = { cookie, origin }; + expect( + (await request("/api/lobby", { method: "POST", headers, body: "{" })) + .status, + ).toBe(400); + expect( + (await request("/api/lobby", { + method: "POST", + headers, + body: JSON.stringify({ name: "x".repeat(5000) }), + })).status, + ).toBe(413); + }); + it("uses single-use OAuth state, PKCE and a secure HttpOnly site session", async () => { + const login = await request("/auth/login"); + expect(login.status).toBe(302); + const authorize = new URL(login.headers.get("location")!); + expect(authorize.origin).toBe("https://github.com"); + expect(authorize.searchParams.get("scope")).toBe(""); + expect(authorize.searchParams.get("prompt")).toBe("select_account"); + expect(authorize.searchParams.get("code_challenge_method")).toBe("S256"); + const state = authorize.searchParams.get("state")!; + const stateCookie = login.headers.get("set-cookie")!.split(";")[0]; + const callback = `/auth/callback?code=test-code&state=${state}`; + expect((await request(callback)).status).toBe(400); + const mocked = vi.spyOn(globalThis, "fetch").mockImplementation( + async (input, init) => { + if (String(input) === "https://github.com/login/oauth/access_token") { + const body = JSON.parse(String(init?.body)); + expect(body.code_verifier).toHaveLength(64); + expect(body.redirect_uri).toBe(`${origin}/auth/callback`); + return Response.json({ access_token: "github-test-access" }); + } + if (String(input) === "https://api.github.com/user") { + return Response.json({ id: 100, login: "player" }); + } + throw new Error("Unexpected outbound request"); + }, + ); + const response = await request(callback, { + headers: { cookie: stateCookie }, + }); + expect(response.status).toBe(302); + const cookies = response.headers.get("set-cookie")!; + expect(cookies).toContain("HttpOnly"); + expect(cookies).toContain("Secure"); + expect(cookies).toContain("SameSite=Lax"); + expect(cookies).not.toContain("github-test-access"); + expect( + await env.DB.prepare("SELECT COUNT(*) AS count FROM sessions").first(), + ).toEqual({ count: 1 }); + expect( + (await request(callback, { headers: { cookie: stateCookie } })).status, + ).toBe(400); + expect(mocked).toHaveBeenCalledTimes(2); + }); + it("sign-out invalidates the session and pending OAuth state and clears both cookies", async () => { + const sessionCookie = await session(); + const login = await request("/auth/login"); + const oauthCookie = login.headers.get("set-cookie")!.split(";")[0]; + const cookies = `${sessionCookie}; ${oauthCookie}`; + const response = await request("/auth/logout", { method: "POST", headers: { origin, cookie: cookies } }); + expect(response.status).toBe(204); + const cleared = response.headers.getSetCookie(); + expect(cleared).toHaveLength(2); + expect(cleared.some(c => c.startsWith("__Host-microduck="))).toBe(true); + expect(cleared.some(c => c.startsWith("__Host-microduck-oauth="))).toBe(true); + for (const c of cleared) expect(c).toContain("Max-Age=0"); + expect(await env.DB.prepare("SELECT COUNT(*) AS count FROM sessions").first()).toEqual({ count: 0 }); + expect(await env.DB.prepare("SELECT COUNT(*) AS count FROM oauth_states").first()).toEqual({ count: 0 }); + expect(await (await request("/api/auth", { headers: { cookie: cookies } })).json()).toMatchObject({ user: null }); + }); + it("checks admin identity and revokes a banned account's sessions", async () => { + let cookie = await session(); + expect( + (await request("/api/admin/ban", { + method: "POST", + headers: { cookie, origin }, + body: JSON.stringify({ userId: "200", banned: true }), + })).status, + ).toBe(404); + await env.DB.prepare( + "INSERT INTO users(id,login,created_at) VALUES('200','target',0)", + ).run(); + await env.DB.prepare( + "INSERT INTO sessions(hash,user_id,expires_at) VALUES('target','200',?)", + ).bind(Date.now() + 1000).run(); + cookie = await session("6902572"); + expect( + (await request("/api/admin/ban", { + method: "POST", + headers: { cookie, origin }, + body: JSON.stringify({ userId: "200", banned: true }), + })).status, + ).toBe(200); + expect( + await env.DB.prepare("SELECT banned FROM users WHERE id='200'").first(), + ).toEqual({ banned: 1 }); + expect( + await env.DB.prepare("SELECT hash FROM sessions WHERE user_id='200'") + .first(), + ).toBeNull(); + }); +}); diff --git a/examples/microduck-world/edge/test/publicTransport.test.ts b/examples/microduck-world/edge/test/publicTransport.test.ts new file mode 100644 index 0000000000..93f10a056c --- /dev/null +++ b/examples/microduck-world/edge/test/publicTransport.test.ts @@ -0,0 +1,140 @@ +import { env } from "cloudflare:workers"; +import { afterEach, expect, it, vi } from "vitest"; +import { PublicTransport } from "../../web/src/publicTransport.ts"; +import { pack, unpack } from "../../shared/publicWire.ts"; + +afterEach(() => vi.unstubAllGlobals()); + +it("carries SDK streams through the real edge relay and acknowledges received frames", async () => { + const match = env.MATCH.getByName(crypto.randomUUID()); + const hostResponse = await match.fetch("https://test/host", { + headers: { upgrade: "websocket" }, + }); + const host = hostResponse.webSocket!; + host.binaryType = "arraybuffer"; + host.accept(); + const received: (string | ArrayBuffer)[] = []; + let clientId = ""; + host.addEventListener("message", (event) => { + received.push(event.data); + if (typeof event.data === "string") { + const message = JSON.parse(event.data); + if (message.t === "open") clientId = message.id; + } else { + const { header } = unpack(new Uint8Array(event.data)); + host.send(JSON.stringify({ t: "received", id: header.to[0] })); + } + }); + + // Workers returns an accepted socket instead of a browser constructor. This + // shim changes only that API surface; both websocket endpoints and the DO + // credit/account routing below run in the actual Workers test runtime. + class BrowserSocket extends EventTarget { + static OPEN = 1; + readyState = 0; + bufferedAmount = 0; + binaryType = "arraybuffer"; + socket?: WebSocket; + constructor() { + super(); + void this.open(); + } + async open() { + const response = await match.fetch("https://test/viewer?ticket=example", { + headers: { + upgrade: "websocket", + "x-user-id": "100", + "x-user-login": "player", + "x-session-expires": String(Date.now() + 60000), + }, + }); + this.socket = response.webSocket!; + this.socket.binaryType = "arraybuffer"; + this.socket.accept(); + this.socket.addEventListener( + "message", + (e) => + this.dispatchEvent(new MessageEvent("message", { data: e.data })), + ); + this.socket.addEventListener("close", () => { + this.readyState = 3; + this.dispatchEvent(new CloseEvent("close")); + }); + this.readyState = 1; + this.dispatchEvent(new Event("open")); + } + send(data: string | Uint8Array) { + this.socket!.send(data); + } + close(code?: number, reason?: string) { + this.socket?.close(code, reason); + } + } + vi.stubGlobal("WebSocket", BrowserSocket); + const transport = new PublicTransport({ + wtUrl: "wss://test/connect", + certHash: "", + v: 5, + }); + try { + const control = await transport.createBidirectionalStream(); + const writer = control.writable.getWriter(); + await writer.write(new Uint8Array([1, 2, 3])); + await vi.waitFor(() => + expect(received.some((m) => typeof m !== "string")).toBe(true) + ); + const inbound = unpack( + new Uint8Array( + received.find((m): m is ArrayBuffer => typeof m !== "string")!, + ), + ); + expect(inbound.header.kind).toBe(0); + expect([...inbound.payload]).toEqual([1, 2, 3]); + + const reader = control.readable.getReader(); + for (let n = 0; n < 100; n++) { + host.send(pack({ to: [clientId], kind: 0 }, new Uint8Array([n]))); + expect([...(await reader.read()).value!]).toEqual([n]); + } + const streams = transport.incomingUnidirectionalStreams.getReader(); + host.send(pack({ to: [clientId], kind: 1 }, new Uint8Array([7, 8]))); + const dataReader = (await streams.read()).value!.getReader(); + expect([...(await dataReader.read()).value!]).toEqual([7, 8]); + expect((await dataReader.read()).done).toBe(true); + + const datagrams = transport.datagrams.writable.getWriter(); + await datagrams.write(new Uint8Array([9])); + await vi.waitFor(() => + expect(received.filter((m) => typeof m !== "string")).toHaveLength(2) + ); + const last = received.filter((m): m is ArrayBuffer => typeof m !== "string") + .at(-1)!; + expect(unpack(new Uint8Array(last)).header.kind).toBe(2); + transport.close(); + expect(await transport.closed).toEqual({ reason: "Session ended" }); + } finally { + transport.close(); + host.close(); + } +}); + +it("reports replacement once without treating ordinary disconnects as ownership changes", async () => { + let socket: EventTarget; + class Socket extends EventTarget { + binaryType = "arraybuffer"; + constructor() { super(); socket = this; queueMicrotask(() => this.dispatchEvent(new Event("open"))); } + close() {} + } + vi.stubGlobal("WebSocket", Socket); + const replaced = vi.fn(); + const transport = new PublicTransport({ wtUrl: "wss://test", certHash: "", v: 5 }, replaced); + await transport.ready; + socket!.dispatchEvent(new CloseEvent("close", { reason: "Replaced by your new connection" })); + expect(replaced).toHaveBeenCalledTimes(1); + expect((await transport.closed).reason).toBe("Replaced by your new connection"); + const ordinary = new PublicTransport({ wtUrl: "wss://test", certHash: "", v: 5 }, replaced); + await ordinary.ready; + socket!.dispatchEvent(new CloseEvent("close", { reason: "World reconnecting" })); + expect(replaced).toHaveBeenCalledTimes(1); + expect((await ordinary.closed).reason).toBe("World reconnecting"); +}); diff --git a/examples/microduck-world/edge/test/room.test.ts b/examples/microduck-world/edge/test/room.test.ts new file mode 100644 index 0000000000..c316e3c52b --- /dev/null +++ b/examples/microduck-world/edge/test/room.test.ts @@ -0,0 +1,114 @@ +import { env } from "cloudflare:workers"; +import { expect, it } from "vitest"; +import { + clientFrame, + MAX_PENDING_DOWNLOADS, + pack, +} from "../../shared/publicWire.ts"; + +function message(socket: WebSocket): Promise { + return new Promise((resolve) => + socket.addEventListener("message", (e) => resolve(e.data), { once: true }) + ); +} +async function room() { + const room = env.MATCH.getByName(crypto.randomUUID()); + const response = await room.fetch("https://test/host", { + headers: { upgrade: "websocket" }, + }); + const host = response.webSocket!; + host.binaryType = "arraybuffer"; + host.accept(); + return { room, host }; +} +async function viewer(room: ReturnType) { + const response = await room.fetch("https://test/viewer?ticket=example", { + headers: { + upgrade: "websocket", + "x-user-id": "100", + "x-user-login": "player", + "x-session-expires": String(Date.now() + 60000), + }, + }); + expect(response.status).toBe(101); + const socket = response.webSocket!; + socket.binaryType = "arraybuffer"; + socket.accept(); + return socket; +} +it("routes opaque DimOS frames, returns receive credit and disconnects slow viewers", async () => { + const { room: match, host } = await room(); + const opened = message(host); + const client = await viewer(match); + const info = JSON.parse(await opened as string); + expect(info.user).toEqual({ id: "100", login: "player" }); + const inbound = message(host); + client.send(clientFrame(0, new Uint8Array([1, 2, 3]))); + expect(typeof await inbound).not.toBe("string"); + host.send(JSON.stringify({ t: "received", id: info.id })); + const frame = message(client); + host.send(pack({ to: [info.id], kind: 1 }, new Uint8Array([4, 5, 6]))); + expect([...new Uint8Array(await frame as ArrayBuffer).slice(5)]).toEqual([ + 4, + 5, + 6, + ]); + client.send(JSON.stringify({ t: "ack", n: 1 })); + const closed = new Promise((resolve) => + client.addEventListener("close", resolve, { once: true }) + ); + for (let i = 0; i < MAX_PENDING_DOWNLOADS + 2; i++) { + host.send(pack({ to: [info.id], kind: 1 }, new Uint8Array([7]))); + } + await closed; + host.close(); +}); +it("allows a healthy cockpit burst while Internet acknowledgments are in flight", async () => { + const { room: match, host } = await room(); + const opened = message(host); + const client = await viewer(match); + const info = JSON.parse(await opened as string); + try { + for (let batch = 0; batch < 2; batch++) { + const sequences: number[] = []; + const burst = new Promise((resolve, reject) => { + const onClose = () => + reject(new Error("Healthy client was disconnected")); + const onMessage = (event: MessageEvent) => { + sequences.push(new DataView(event.data as ArrayBuffer).getUint32(1)); + if (sequences.length === 40) { + client.removeEventListener("message", onMessage); + client.removeEventListener("close", onClose); + resolve(); + } + }; + client.addEventListener("message", onMessage); + client.addEventListener("close", onClose, { once: true }); + }); + // About half a second of the multi-channel cockpit stream, before ACKs + // make their return trip. The old 16-frame cap rejected this workload. + for (let i = 0; i < 40; i++) { + host.send(pack({ to: [info.id], kind: 1 }, new Uint8Array(1024))); + } + await burst; + for (const n of sequences) client.send(JSON.stringify({ t: "ack", n })); + } + } finally { + client.close(); + host.close(); + } +}); + +it("fails closed when the origin disconnects and does not retain old controls", async () => { + const { room: match, host } = await room(); + const opened = message(host); + const client = await viewer(match); + await opened; + const closed = new Promise((resolve) => + client.addEventListener("close", resolve, { once: true }) + ); + host.close(); + await closed; + const response = await match.http(null, "/api/lobby"); + expect(response.status).toBe(503); +}); diff --git a/examples/microduck-world/edge/tsconfig.json b/examples/microduck-world/edge/tsconfig.json new file mode 100644 index 0000000000..243286ccc4 --- /dev/null +++ b/examples/microduck-world/edge/tsconfig.json @@ -0,0 +1,19 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "Bundler", + "strict": true, + "noEmit": true, + "skipLibCheck": true, + "allowImportingTsExtensions": true, + "types": [ + "node" + ] + }, + "include": [ + "src/**/*.ts", + "worker-configuration.d.ts", + "../shared/**/*.ts" + ] +} diff --git a/examples/microduck-world/edge/vitest.config.ts b/examples/microduck-world/edge/vitest.config.ts new file mode 100644 index 0000000000..2a6ca83a0e --- /dev/null +++ b/examples/microduck-world/edge/vitest.config.ts @@ -0,0 +1,6 @@ +import { cloudflareTest } from "@cloudflare/vitest-plugin"; +import { defineConfig } from "vitest/config"; +export default defineConfig({ + plugins: [cloudflareTest({ wrangler: { configPath: "./wrangler.jsonc" }, miniflare: { bindings: { GITHUB_CLIENT_SECRET: "test-only-github-secret", HOST_SECRET: "a".repeat(64) } } })], + test: { include: ["test/**/*.test.ts"] }, +}); diff --git a/examples/microduck-world/edge/worker-configuration.d.ts b/examples/microduck-world/edge/worker-configuration.d.ts new file mode 100644 index 0000000000..5346c512fb --- /dev/null +++ b/examples/microduck-world/edge/worker-configuration.d.ts @@ -0,0 +1,15111 @@ +/* eslint-disable */ +// Generated by Wrangler by running `wrangler types` (hash: 30a6c7e1c022078d071cf89e478c5756) +// Runtime types generated with workerd@1.20260825.1 2026-09-09 nodejs_compat +interface __BaseEnv_Env { + DB: D1Database; + REQUEST_LIMITER: RateLimit; + ASSETS: Fetcher; + PUBLIC_ORIGIN: "https://sim.tule.world"; + GITHUB_CLIENT_ID: "Ov23lifchwSuJsZK26hd"; + ADMIN_GITHUB_IDS: "6902572"; + MATCH_ID: "football-v1"; + GITHUB_CLIENT_SECRET: string; + HOST_SECRET: string; + MATCH: DurableObjectNamespace; +} +declare namespace Cloudflare { + interface GlobalProps { + mainModule: typeof import("./src/index"); + durableNamespaces: "MatchRelay"; + } + interface Env extends __BaseEnv_Env {} +} +interface Env extends __BaseEnv_Env {} +type StringifyValues> = { + [Binding in keyof EnvType]: EnvType[Binding] extends string ? EnvType[Binding] : string; +}; +declare namespace NodeJS { + interface ProcessEnv extends StringifyValues> {} +} + +// Begin runtime types +/*! ***************************************************************************** +Copyright (c) Cloudflare. All rights reserved. +Copyright (c) Microsoft Corporation. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. You may obtain a copy of the +License at http://www.apache.org/licenses/LICENSE-2.0 +THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +MERCHANTABLITY OR NON-INFRINGEMENT. +See the Apache Version 2.0 License for specific language governing permissions +and limitations under the License. +***************************************************************************** */ +/* eslint-disable */ +// noinspection JSUnusedGlobalSymbols +declare var onmessage: never; +/** + * The **`DOMException`** interface represents an abnormal event (called an exception) that occurs as a result of calling a method or accessing a property of a web API. This is how error conditions are described in web APIs. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/DOMException) + */ +declare class DOMException extends Error { + constructor(message?: string, name?: string); + /** + * The **`message`** read-only property of the DOMException interface returns a string representing a message or description associated with the given error name. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/DOMException/message) + */ + readonly message: string; + /** + * The **`name`** read-only property of the DOMException interface returns a string that contains one of the strings associated with an error name. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/DOMException/name) + */ + readonly name: string; + /** + * The **`code`** read-only property of the DOMException interface returns one of the legacy error code constants, or 0 if none match. + * @deprecated + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/DOMException/code) + */ + readonly code: number; + static readonly INDEX_SIZE_ERR: number; + static readonly DOMSTRING_SIZE_ERR: number; + static readonly HIERARCHY_REQUEST_ERR: number; + static readonly WRONG_DOCUMENT_ERR: number; + static readonly INVALID_CHARACTER_ERR: number; + static readonly NO_DATA_ALLOWED_ERR: number; + static readonly NO_MODIFICATION_ALLOWED_ERR: number; + static readonly NOT_FOUND_ERR: number; + static readonly NOT_SUPPORTED_ERR: number; + static readonly INUSE_ATTRIBUTE_ERR: number; + static readonly INVALID_STATE_ERR: number; + static readonly SYNTAX_ERR: number; + static readonly INVALID_MODIFICATION_ERR: number; + static readonly NAMESPACE_ERR: number; + static readonly INVALID_ACCESS_ERR: number; + static readonly VALIDATION_ERR: number; + static readonly TYPE_MISMATCH_ERR: number; + static readonly SECURITY_ERR: number; + static readonly NETWORK_ERR: number; + static readonly ABORT_ERR: number; + static readonly URL_MISMATCH_ERR: number; + static readonly QUOTA_EXCEEDED_ERR: number; + static readonly TIMEOUT_ERR: number; + static readonly INVALID_NODE_TYPE_ERR: number; + static readonly DATA_CLONE_ERR: number; + get stack(): any; + set stack(value: any); +} +type WorkerGlobalScopeEventMap = { + fetch: FetchEvent; + scheduled: ScheduledEvent; + queue: QueueEvent; + unhandledrejection: PromiseRejectionEvent; + rejectionhandled: PromiseRejectionEvent; +}; +declare abstract class WorkerGlobalScope extends EventTarget { + EventTarget: typeof EventTarget; +} +/* The **`console`** object provides access to the debugging console (e.g., the Web console in Firefox). * + * The **`console`** object provides access to the debugging console (e.g., the Web console in Firefox). + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console) + */ +interface Console { + "assert"(condition?: boolean, ...data: any[]): void; + /** + * The **`console.clear()`** static method clears the console if possible. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/clear_static) + */ + clear(): void; + /** + * The **`console.count()`** static method logs the number of times that this particular call to count() has been called. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/count_static) + */ + count(label?: string): void; + /** + * The **`console.countReset()`** static method resets counter used with console.count(). + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/countReset_static) + */ + countReset(label?: string): void; + /** + * The **`console.debug()`** static method outputs a message to the console at the "debug" log level. The message is only displayed to the user if the console is configured to display debug output. In most cases, the log level is configured within the console UI. This log level might correspond to the Debug or Verbose log level. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/debug_static) + */ + debug(...data: any[]): void; + /** + * The **`console.dir()`** static method displays a list of the properties of the specified JavaScript object. In browser consoles, the output is presented as a hierarchical listing with disclosure triangles that let you see the contents of child objects. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/dir_static) + */ + dir(item?: any, options?: any): void; + /** + * The **`console.dirxml()`** static method displays an interactive tree of the descendant elements of the specified XML/HTML element. If it is not possible to display as an element the JavaScript Object view is shown instead. The output is presented as a hierarchical listing of expandable nodes that let you see the contents of child nodes. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/dirxml_static) + */ + dirxml(...data: any[]): void; + /** + * The **`console.error()`** static method outputs a message to the console at the "error" log level. The message is only displayed to the user if the console is configured to display error output. In most cases, the log level is configured within the console UI. The message may be formatted as an error, with red colors and call stack information. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/error_static) + */ + error(...data: any[]): void; + /** + * The **`console.group()`** static method creates a new inline group in the Web console log, causing any subsequent console messages to be indented by an additional level, until console.groupEnd() is called. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/group_static) + */ + group(...data: any[]): void; + /** + * The **`console.groupCollapsed()`** static method creates a new inline group in the console. Unlike console.group(), however, the new group is created collapsed. The user will need to use the disclosure button next to it to expand it, revealing the entries created in the group. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/groupCollapsed_static) + */ + groupCollapsed(...data: any[]): void; + /** + * The **`console.groupEnd()`** static method exits the current inline group in the console. See Using groups in the console in the console documentation for details and examples. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/groupEnd_static) + */ + groupEnd(): void; + /** + * The **`console.info()`** static method outputs a message to the console at the "info" log level. The message is only displayed to the user if the console is configured to display info output. In most cases, the log level is configured within the console UI. The message may receive special formatting, such as a small "i" icon next to it. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/info_static) + */ + info(...data: any[]): void; + /** + * The **`console.log()`** static method outputs a message to the console. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/log_static) + */ + log(...data: any[]): void; + /** + * The **`console.table()`** static method displays tabular data as a table. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/table_static) + */ + table(tabularData?: any, properties?: string[]): void; + /** + * The **`console.time()`** static method starts a timer you can use to track how long an operation takes. You give each timer a unique name, and may have up to 10,000 timers running on a given page. When you call console.timeEnd() with the same name, the browser will output the time, in milliseconds, that elapsed since the timer was started. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/time_static) + */ + time(label?: string): void; + /** + * The **`console.timeEnd()`** static method stops a timer that was previously started by calling console.time(). + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/timeEnd_static) + */ + timeEnd(label?: string): void; + /** + * The **`console.timeLog()`** static method logs the current value of a timer that was previously started by calling console.time(). + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/timeLog_static) + */ + timeLog(label?: string, ...data: any[]): void; + /* The **`console.timeStamp()`** static method adds a single marker to the browser's Performance tool (Firefox bug 1387528, Chrome). This lets you correlate a point in your code with the other events recorded in the timeline, such as layout and paint events. */ + timeStamp(label?: string): void; + /** + * The **`console.trace()`** static method outputs a stack trace to the console. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/trace_static) + */ + trace(...data: any[]): void; + /** + * The **`console.warn()`** static method outputs a warning message to the console at the "warning" log level. The message is only displayed to the user if the console is configured to display warning output. In most cases, the log level is configured within the console UI. The message may receive special formatting, such as yellow colors and a warning icon. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/warn_static) + */ + warn(...data: any[]): void; +} +declare const console: Console; +type BufferSource = ArrayBufferView | ArrayBuffer; +type TypedArray = Int8Array | Uint8Array | Uint8ClampedArray | Int16Array | Uint16Array | Int32Array | Uint32Array | Float32Array | Float64Array | BigInt64Array | BigUint64Array; +declare namespace WebAssembly { + class CompileError extends Error { + constructor(message?: string); + } + class RuntimeError extends Error { + constructor(message?: string); + } + type ValueType = "anyfunc" | "externref" | "f32" | "f64" | "i32" | "i64" | "v128"; + interface GlobalDescriptor { + value: ValueType; + mutable?: boolean; + } + class Global { + constructor(descriptor: GlobalDescriptor, value?: any); + value: any; + valueOf(): any; + } + type ImportValue = ExportValue | number; + type ModuleImports = Record; + type Imports = Record; + type ExportValue = Function | Global | Memory | Table; + type Exports = Record; + class Instance { + constructor(module: Module, imports?: Imports); + readonly exports: Exports; + } + interface MemoryDescriptor { + initial: number; + maximum?: number; + shared?: boolean; + } + class Memory { + constructor(descriptor: MemoryDescriptor); + readonly buffer: ArrayBuffer; + grow(delta: number): number; + } + type ImportExportKind = "function" | "global" | "memory" | "table"; + interface ModuleExportDescriptor { + kind: ImportExportKind; + name: string; + } + interface ModuleImportDescriptor { + kind: ImportExportKind; + module: string; + name: string; + } + abstract class Module { + static customSections(module: Module, sectionName: string): ArrayBuffer[]; + static exports(module: Module): ModuleExportDescriptor[]; + static imports(module: Module): ModuleImportDescriptor[]; + } + type TableKind = "anyfunc" | "externref"; + interface TableDescriptor { + element: TableKind; + initial: number; + maximum?: number; + } + class Table { + constructor(descriptor: TableDescriptor, value?: any); + readonly length: number; + get(index: number): any; + grow(delta: number, value?: any): number; + set(index: number, value?: any): void; + } + function instantiate(module: Module, imports?: Imports): Promise; + function validate(bytes: BufferSource): boolean; +} +/** + * The **`ServiceWorkerGlobalScope`** interface of the Service Worker API represents the global execution context of a service worker. + * Available only in secure contexts. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ServiceWorkerGlobalScope) + */ +interface ServiceWorkerGlobalScope extends WorkerGlobalScope { + DOMException: typeof DOMException; + WorkerGlobalScope: typeof WorkerGlobalScope; + btoa(data: string): string; + atob(data: string): string; + setTimeout(callback: (...args: any[]) => void, msDelay?: number): number; + setTimeout(callback: (...args: Args) => void, msDelay?: number, ...args: Args): number; + clearTimeout(timeoutId: number | null): void; + setInterval(callback: (...args: any[]) => void, msDelay?: number): number; + setInterval(callback: (...args: Args) => void, msDelay?: number, ...args: Args): number; + clearInterval(timeoutId: number | null): void; + queueMicrotask(task: Function): void; + structuredClone(value: T, options?: StructuredSerializeOptions): T; + reportError(error: any): void; + fetch(input: RequestInfo | URL, init?: RequestInit): Promise; + self: ServiceWorkerGlobalScope; + crypto: Crypto; + caches: CacheStorage; + scheduler: Scheduler; + performance: Performance; + Cloudflare: Cloudflare; + readonly origin: string; + Event: typeof Event; + ExtendableEvent: typeof ExtendableEvent; + CustomEvent: typeof CustomEvent; + PromiseRejectionEvent: typeof PromiseRejectionEvent; + FetchEvent: typeof FetchEvent; + TailEvent: typeof TailEvent; + TraceEvent: typeof TailEvent; + ScheduledEvent: typeof ScheduledEvent; + MessageEvent: typeof MessageEvent; + CloseEvent: typeof CloseEvent; + ReadableStreamDefaultReader: typeof ReadableStreamDefaultReader; + ReadableStreamBYOBReader: typeof ReadableStreamBYOBReader; + ReadableStream: typeof ReadableStream; + WritableStream: typeof WritableStream; + WritableStreamDefaultWriter: typeof WritableStreamDefaultWriter; + TransformStream: typeof TransformStream; + ByteLengthQueuingStrategy: typeof ByteLengthQueuingStrategy; + CountQueuingStrategy: typeof CountQueuingStrategy; + ErrorEvent: typeof ErrorEvent; + MessageChannel: typeof MessageChannel; + MessagePort: typeof MessagePort; + EventSource: typeof EventSource; + ReadableStreamBYOBRequest: typeof ReadableStreamBYOBRequest; + ReadableStreamDefaultController: typeof ReadableStreamDefaultController; + ReadableByteStreamController: typeof ReadableByteStreamController; + WritableStreamDefaultController: typeof WritableStreamDefaultController; + TransformStreamDefaultController: typeof TransformStreamDefaultController; + CompressionStream: typeof CompressionStream; + DecompressionStream: typeof DecompressionStream; + TextEncoderStream: typeof TextEncoderStream; + TextDecoderStream: typeof TextDecoderStream; + Headers: typeof Headers; + Body: typeof Body; + Request: typeof Request; + Response: typeof Response; + WebSocket: typeof WebSocket; + WebSocketPair: typeof WebSocketPair; + WebSocketRequestResponsePair: typeof WebSocketRequestResponsePair; + AbortController: typeof AbortController; + AbortSignal: typeof AbortSignal; + TextDecoder: typeof TextDecoder; + TextEncoder: typeof TextEncoder; + navigator: Navigator; + Navigator: typeof Navigator; + URL: typeof URL; + URLSearchParams: typeof URLSearchParams; + URLPattern: typeof URLPattern; + Blob: typeof Blob; + File: typeof File; + FormData: typeof FormData; + Crypto: typeof Crypto; + SubtleCrypto: typeof SubtleCrypto; + CryptoKey: typeof CryptoKey; + CacheStorage: typeof CacheStorage; + Cache: typeof Cache; + FixedLengthStream: typeof FixedLengthStream; + IdentityTransformStream: typeof IdentityTransformStream; + HTMLRewriter: typeof HTMLRewriter; +} +declare function addEventListener(type: Type, handler: EventListenerOrEventListenerObject, options?: EventTargetAddEventListenerOptions | boolean): void; +declare function removeEventListener(type: Type, handler: EventListenerOrEventListenerObject, options?: EventTargetEventListenerOptions | boolean): void; +/** + * The **`dispatchEvent()`** method of the EventTarget sends an Event to the object, (synchronously) invoking the affected event listeners in the appropriate order. The normal event processing rules (including the capturing and optional bubbling phase) also apply to events dispatched manually with dispatchEvent(). + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventTarget/dispatchEvent) + */ +declare function dispatchEvent(event: WorkerGlobalScopeEventMap[keyof WorkerGlobalScopeEventMap]): boolean; +/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/btoa) */ +declare function btoa(data: string): string; +/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/atob) */ +declare function atob(data: string): string; +/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/setTimeout) */ +declare function setTimeout(callback: (...args: any[]) => void, msDelay?: number): number; +/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/setTimeout) */ +declare function setTimeout(callback: (...args: Args) => void, msDelay?: number, ...args: Args): number; +/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/clearTimeout) */ +declare function clearTimeout(timeoutId: number | null): void; +/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/setInterval) */ +declare function setInterval(callback: (...args: any[]) => void, msDelay?: number): number; +/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/setInterval) */ +declare function setInterval(callback: (...args: Args) => void, msDelay?: number, ...args: Args): number; +/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/clearInterval) */ +declare function clearInterval(timeoutId: number | null): void; +/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/queueMicrotask) */ +declare function queueMicrotask(task: Function): void; +/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/structuredClone) */ +declare function structuredClone(value: T, options?: StructuredSerializeOptions): T; +/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/reportError) */ +declare function reportError(error: any): void; +/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/fetch) */ +declare function fetch(input: RequestInfo | URL, init?: RequestInit): Promise; +declare const self: ServiceWorkerGlobalScope; +/** +* The Web Crypto API provides a set of low-level functions for common cryptographic tasks. +* The Workers runtime implements the full surface of this API, but with some differences in +* the [supported algorithms](https://developers.cloudflare.com/workers/runtime-apis/web-crypto/#supported-algorithms) +* compared to those implemented in most browsers. +* +* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/web-crypto/) +*/ +declare const crypto: Crypto; +/** +* The Cache API allows fine grained control of reading and writing from the Cloudflare global network cache. +* +* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/cache/) +*/ +declare const caches: CacheStorage; +/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/scheduler) */ +declare const scheduler: Scheduler; +/** +* The Workers runtime supports a subset of the Performance API, used to measure timing and performance, +* as well as timing of subrequests and other operations. +* +* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/performance/) +*/ +declare const performance: Performance; +declare const Cloudflare: Cloudflare; +declare const origin: string; +declare const navigator: Navigator; +interface TestController { +} +interface ExecutionContext { + waitUntil(promise: Promise): void; + passThroughOnException(): void; + readonly exports: Cloudflare.Exports; + readonly props: Props; + cache?: CacheContext; + readonly access?: CloudflareAccessContext; + tracing: Tracing; + abort(reason?: any): void; +} +type ExportedHandlerFetchHandler = (request: Request>, env: Env, ctx: ExecutionContext) => Response | Promise; +type ExportedHandlerConnectHandler = (socket: Socket, env: Env, ctx: ExecutionContext) => void | Promise; +type ExportedHandlerTailHandler = (events: TraceItem[], env: Env, ctx: ExecutionContext) => void | Promise; +type ExportedHandlerTraceHandler = (traces: TraceItem[], env: Env, ctx: ExecutionContext) => void | Promise; +type ExportedHandlerTailStreamHandler = (event: TailStream.TailEvent, env: Env, ctx: ExecutionContext) => TailStream.TailEventHandlerType | Promise; +type ExportedHandlerScheduledHandler = (controller: ScheduledController, env: Env, ctx: ExecutionContext) => void | Promise; +type ExportedHandlerQueueHandler = (batch: MessageBatch, env: Env, ctx: ExecutionContext) => void | Promise; +type ExportedHandlerTestHandler = (controller: TestController, env: Env, ctx: ExecutionContext) => void | Promise; +interface ExportedHandler { + fetch?: ExportedHandlerFetchHandler; + connect?: ExportedHandlerConnectHandler; + tail?: ExportedHandlerTailHandler; + trace?: ExportedHandlerTraceHandler; + tailStream?: ExportedHandlerTailStreamHandler; + scheduled?: ExportedHandlerScheduledHandler; + test?: ExportedHandlerTestHandler; + email?: EmailExportedHandler; + queue?: ExportedHandlerQueueHandler; +} +interface StructuredSerializeOptions { + transfer?: any[]; +} +declare abstract class Navigator { + sendBeacon(url: string, body?: BodyInit): boolean; + readonly userAgent: string; + readonly hardwareConcurrency: number; + readonly platform: string; + readonly language: string; + readonly languages: string[]; +} +interface AlarmInvocationInfo { + readonly isRetry: boolean; + readonly retryCount: number; + readonly scheduledTime: number; +} +interface Cloudflare { + readonly compatibilityFlags: Record; +} +interface CachePurgeError { + code: number; + message: string; +} +interface CachePurgeResult { + success: boolean; + errors: CachePurgeError[]; +} +interface CachePurgeOptions { + tags?: string[]; + pathPrefixes?: string[]; + purgeEverything?: boolean; +} +interface CacheContext { + purge(options: CachePurgeOptions): Promise; +} +interface CloudflareAccessContext { + readonly aud: string; + getIdentity(): Promise; +} +declare abstract class ColoLocalActorNamespace { + get(actorId: string): Fetcher; +} +interface DurableObject { + fetch(request: Request): Response | Promise; + connect?(socket: Socket): void | Promise; + alarm?(alarmInfo?: AlarmInvocationInfo): void | Promise; + webSocketMessage?(ws: WebSocket, message: string | ArrayBuffer): void | Promise; + webSocketClose?(ws: WebSocket, code: number, reason: string, wasClean: boolean): void | Promise; + webSocketError?(ws: WebSocket, error: unknown): void | Promise; +} +type DurableObjectStub = Fetcher & { + readonly id: DurableObjectId; + readonly name?: string; +}; +interface DurableObjectId { + toString(): string; + equals(other: DurableObjectId): boolean; + readonly name?: string; + readonly jurisdiction?: string; +} +declare abstract class DurableObjectNamespace { + newUniqueId(options?: DurableObjectNamespaceNewUniqueIdOptions): DurableObjectId; + idFromName(name: string): DurableObjectId; + idFromString(id: string): DurableObjectId; + get(id: DurableObjectId, options?: DurableObjectNamespaceGetDurableObjectOptions): DurableObjectStub; + getByName(name: string, options?: DurableObjectNamespaceGetDurableObjectOptions): DurableObjectStub; + jurisdiction(jurisdiction: DurableObjectJurisdiction): DurableObjectNamespace; +} +type DurableObjectJurisdiction = "eu" | "fedramp" | "fedramp-high" | "us"; +interface DurableObjectNamespaceNewUniqueIdOptions { + jurisdiction?: DurableObjectJurisdiction; +} +type DurableObjectLocationHint = "wnam" | "enam" | "sam" | "weur" | "eeur" | "apac" | "apac-ne" | "apac-se" | "oc" | "afr" | "me"; +type DurableObjectRoutingMode = "primary-only"; +interface DurableObjectNamespaceGetDurableObjectOptions { + locationHint?: DurableObjectLocationHint; + routingMode?: DurableObjectRoutingMode; +} +interface DurableObjectClass<_T extends Rpc.DurableObjectBranded | undefined = undefined> { +} +interface DurableObjectState { + waitUntil(promise: Promise): void; + readonly exports: Cloudflare.Exports; + readonly props: Props; + readonly id: DurableObjectId; + readonly storage: DurableObjectStorage; + container?: Container; + facets: DurableObjectFacets; + blockConcurrencyWhile(callback: () => Promise): Promise; + acceptWebSocket(ws: WebSocket, tags?: string[]): void; + getWebSockets(tag?: string): WebSocket[]; + setWebSocketAutoResponse(maybeReqResp?: WebSocketRequestResponsePair): void; + getWebSocketAutoResponse(): WebSocketRequestResponsePair | null; + getWebSocketAutoResponseTimestamp(ws: WebSocket): Date | null; + setHibernatableWebSocketEventTimeout(timeoutMs?: number): void; + getHibernatableWebSocketEventTimeout(): number | null; + getTags(ws: WebSocket): string[]; + abort(reason?: string, options?: DurableObjectAbortOptions): void; +} +interface DurableObjectTransaction { + get(key: string, options?: DurableObjectGetOptions): Promise; + get(keys: string[], options?: DurableObjectGetOptions): Promise>; + list(options?: DurableObjectListOptions): Promise>; + put(key: string, value: T, options?: DurableObjectPutOptions): Promise; + put(entries: Record, options?: DurableObjectPutOptions): Promise; + delete(key: string, options?: DurableObjectPutOptions): Promise; + delete(keys: string[], options?: DurableObjectPutOptions): Promise; + rollback(): void; + getAlarm(options?: DurableObjectGetAlarmOptions): Promise; + setAlarm(scheduledTime: number | Date, options?: DurableObjectSetAlarmOptions): Promise; + deleteAlarm(options?: DurableObjectSetAlarmOptions): Promise; +} +interface DurableObjectStorage { + get(key: string, options?: DurableObjectGetOptions): Promise; + get(keys: string[], options?: DurableObjectGetOptions): Promise>; + list(options?: DurableObjectListOptions): Promise>; + put(key: string, value: T, options?: DurableObjectPutOptions): Promise; + put(entries: Record, options?: DurableObjectPutOptions): Promise; + delete(key: string, options?: DurableObjectPutOptions): Promise; + delete(keys: string[], options?: DurableObjectPutOptions): Promise; + deleteAll(options?: DurableObjectPutOptions): Promise; + transaction(closure: (txn: DurableObjectTransaction) => Promise): Promise; + getAlarm(options?: DurableObjectGetAlarmOptions): Promise; + setAlarm(scheduledTime: number | Date, options?: DurableObjectSetAlarmOptions): Promise; + deleteAlarm(options?: DurableObjectSetAlarmOptions): Promise; + sync(): Promise; + sql: SqlStorage; + kv: SyncKvStorage; + transactionSync(closure: () => T): T; + getCurrentBookmark(): Promise; + getBookmarkForTime(timestamp: number | Date): Promise; + onNextSessionRestoreBookmark(bookmark: string): Promise; +} +interface DurableObjectAbortOptions { + retryAlarm?: boolean; +} +interface DurableObjectListOptions { + start?: string; + startAfter?: string; + end?: string; + prefix?: string; + reverse?: boolean; + limit?: number; + allowConcurrency?: boolean; + noCache?: boolean; +} +interface DurableObjectGetOptions { + allowConcurrency?: boolean; + noCache?: boolean; +} +interface DurableObjectGetAlarmOptions { + allowConcurrency?: boolean; +} +interface DurableObjectPutOptions { + allowConcurrency?: boolean; + allowUnconfirmed?: boolean; + noCache?: boolean; +} +interface DurableObjectSetAlarmOptions { + allowConcurrency?: boolean; + allowUnconfirmed?: boolean; +} +declare class WebSocketRequestResponsePair { + constructor(request: string, response: string); + get request(): string; + get response(): string; +} +interface DurableObjectFacets { + get(name: string, getStartupOptions: () => FacetStartupOptions | Promise>): Fetcher; + abort(name: string, reason: any): void; + delete(name: string): void; + clone(src: string, dst: string): void; +} +interface FacetStartupOptions { + id?: DurableObjectId | string; + class: DurableObjectClass; +} +interface AnalyticsEngineDataset { + writeDataPoint(event?: AnalyticsEngineDataPoint): void; +} +interface AnalyticsEngineDataPoint { + indexes?: ((ArrayBuffer | string) | null)[]; + doubles?: number[]; + blobs?: ((ArrayBuffer | string) | null)[]; +} +/** + * The **`Event`** interface represents an event which takes place on an EventTarget. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event) + */ +declare class Event { + constructor(type: string, init?: EventInit); + /** + * The **`type`** read-only property of the Event interface returns a string containing the event's type. It is set when the event is constructed and is the name commonly used to refer to the specific event, such as click, load, or error. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/type) + */ + get type(): string; + /** + * The **`eventPhase`** read-only property of the Event interface indicates which phase of the event flow is currently being evaluated. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/eventPhase) + */ + get eventPhase(): number; + /** + * The read-only **`composed`** property of the Event interface returns a boolean value which indicates whether or not the event will propagate across the shadow DOM boundary into the standard DOM. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/composed) + */ + get composed(): boolean; + /** + * The **`bubbles`** read-only property of the Event interface indicates whether the event bubbles up through the DOM tree or not. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/bubbles) + */ + get bubbles(): boolean; + /** + * The **`cancelable`** read-only property of the Event interface indicates whether the event can be canceled, and therefore prevented as if the event never happened. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/cancelable) + */ + get cancelable(): boolean; + /** + * The **`defaultPrevented`** read-only property of the Event interface returns a boolean value indicating whether or not the call to Event.preventDefault() canceled the event. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/defaultPrevented) + */ + get defaultPrevented(): boolean; + /** + * The Event property **`returnValue`** indicates whether the default action for this event has been prevented or not. + * @deprecated + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/returnValue) + */ + get returnValue(): boolean; + /** + * The **`currentTarget`** read-only property of the Event interface identifies the element to which the event handler has been attached. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/currentTarget) + */ + get currentTarget(): EventTarget | undefined; + /** + * The read-only **`target`** property of the Event interface is a reference to the object onto which the event was dispatched. It is different from Event.currentTarget when the event handler is called during the bubbling or capturing phase of the event. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/target) + */ + get target(): EventTarget | undefined; + /** + * The deprecated **`Event.srcElement`** is an alias for the Event.target property. Use Event.target instead. + * @deprecated + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/srcElement) + */ + get srcElement(): EventTarget | undefined; + /** + * The **`timeStamp`** read-only property of the Event interface returns the time (in milliseconds) at which the event was created. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/timeStamp) + */ + get timeStamp(): number; + /** + * The **`isTrusted`** read-only property of the Event interface is a boolean value that is true when the event was generated by the user agent (including via user actions and programmatic methods such as HTMLElement.focus()), and false when the event was dispatched via EventTarget.dispatchEvent(). The only exception is the click event, which initializes the isTrusted property to false in user agents. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/isTrusted) + */ + get isTrusted(): boolean; + /** + * The **`cancelBubble`** property of the Event interface is deprecated. Use Event.stopPropagation() instead. Setting its value to true before returning from an event handler prevents propagation of the event. In later implementations, setting this to false does nothing. See Browser compatibility for details. + * @deprecated + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/cancelBubble) + */ + get cancelBubble(): boolean; + /** + * The **`cancelBubble`** property of the Event interface is deprecated. Use Event.stopPropagation() instead. Setting its value to true before returning from an event handler prevents propagation of the event. In later implementations, setting this to false does nothing. See Browser compatibility for details. + * @deprecated + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/cancelBubble) + */ + set cancelBubble(value: boolean); + /** + * The **`stopImmediatePropagation()`** method of the Event interface prevents other listeners of the same event from being called. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/stopImmediatePropagation) + */ + stopImmediatePropagation(): void; + /** + * The **`preventDefault()`** method of the Event interface tells the user agent that the event is being explicitly handled, so its default action, such as page scrolling, link navigation, or pasting text, should not be taken. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/preventDefault) + */ + preventDefault(): void; + /** + * The **`stopPropagation()`** method of the Event interface prevents further propagation of the current event in the capturing and bubbling phases. It does not, however, prevent any default behaviors from occurring; for instance, clicks on links are still processed. If you want to stop those behaviors, see the preventDefault() method. It also does not prevent propagation to other event-handlers of the current element. If you want to stop those, see stopImmediatePropagation(). + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/stopPropagation) + */ + stopPropagation(): void; + /** + * The **`composedPath()`** method of the Event interface returns the event's path which is an array of the objects on which listeners will be invoked. This does not include nodes in shadow trees if the shadow root was created with its ShadowRoot.mode closed. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/composedPath) + */ + composedPath(): EventTarget[]; + static readonly NONE: number; + static readonly CAPTURING_PHASE: number; + static readonly AT_TARGET: number; + static readonly BUBBLING_PHASE: number; +} +interface EventInit { + bubbles?: boolean; + cancelable?: boolean; + composed?: boolean; +} +type EventListener = (event: EventType) => void; +interface EventListenerObject { + handleEvent(event: EventType): void; +} +type EventListenerOrEventListenerObject = EventListener | EventListenerObject; +/** + * The **`EventTarget`** interface is implemented by objects that can receive events and may have listeners for them. In other words, any target of events implements the three methods associated with this interface. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventTarget) + */ +declare class EventTarget = Record> { + constructor(); + /** + * The **`addEventListener()`** method of the EventTarget interface sets up a function that will be called whenever the specified event is delivered to the target. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventTarget/addEventListener) + */ + addEventListener(type: Type, handler: EventListenerOrEventListenerObject, options?: EventTargetAddEventListenerOptions | boolean): void; + /** + * The **`removeEventListener()`** method of the EventTarget interface removes an event listener previously registered with EventTarget.addEventListener() from the target. The event listener to be removed is identified using a combination of the event type, the event listener function itself, and various optional options that may affect the matching process; see Matching event listeners for removal. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventTarget/removeEventListener) + */ + removeEventListener(type: Type, handler: EventListenerOrEventListenerObject, options?: EventTargetEventListenerOptions | boolean): void; + /** + * The **`dispatchEvent()`** method of the EventTarget sends an Event to the object, (synchronously) invoking the affected event listeners in the appropriate order. The normal event processing rules (including the capturing and optional bubbling phase) also apply to events dispatched manually with dispatchEvent(). + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventTarget/dispatchEvent) + */ + dispatchEvent(event: EventMap[keyof EventMap]): boolean; +} +interface EventTargetEventListenerOptions { + capture?: boolean; +} +interface EventTargetAddEventListenerOptions { + capture?: boolean; + passive?: boolean; + once?: boolean; + signal?: AbortSignal; +} +interface EventTargetHandlerObject { + handleEvent: (event: Event) => any | undefined; +} +/** + * The **`AbortController`** interface represents a controller object that allows you to abort one or more Web requests as and when desired. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/AbortController) + */ +declare class AbortController { + constructor(); + /** + * The **`signal`** read-only property of the AbortController interface returns an AbortSignal object instance, which can be used to communicate with/abort an asynchronous operation as desired. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/AbortController/signal) + */ + get signal(): AbortSignal; + /** + * The **`abort()`** method of the AbortController interface aborts an asynchronous operation before it has completed. This is able to abort fetch requests, the consumption of any response bodies, or streams. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/AbortController/abort) + */ + abort(reason?: any): void; +} +/** + * The **`AbortSignal`** interface represents a signal object that allows you to communicate with an asynchronous operation (such as a fetch request) and abort it if required via an AbortController object. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/AbortSignal) + */ +declare abstract class AbortSignal extends EventTarget { + /** + * The **`AbortSignal.abort()`** static method returns an AbortSignal that is already set as aborted (and which does not trigger an abort event). + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/AbortSignal/abort_static) + */ + static abort(reason?: any): AbortSignal; + /** + * The **`AbortSignal.timeout()`** static method returns an AbortSignal that will automatically abort after a specified time. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/AbortSignal/timeout_static) + */ + static timeout(delay: number): AbortSignal; + /** + * The **`AbortSignal.any()`** static method takes an iterable of abort signals and returns an AbortSignal. The returned abort signal is aborted when any of the input iterable abort signals are aborted. The abort reason will be set to the reason of the first signal that is aborted. If any of the given abort signals are already aborted then so will be the returned AbortSignal. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/AbortSignal/any_static) + */ + static any(signals: AbortSignal[]): AbortSignal; + /** + * The **`aborted`** read-only property returns a value that indicates whether the asynchronous operations the signal is communicating with are aborted (true) or not (false). + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/AbortSignal/aborted) + */ + get aborted(): boolean; + /** + * The **`reason`** read-only property returns a JavaScript value that indicates the abort reason. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/AbortSignal/reason) + */ + get reason(): any; + /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/AbortSignal/abort_event) */ + get onabort(): any | null; + /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/AbortSignal/abort_event) */ + set onabort(value: any | null); + /** + * The **`throwIfAborted()`** method throws the signal's abort reason if the signal has been aborted; otherwise it does nothing. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/AbortSignal/throwIfAborted) + */ + throwIfAborted(): void; +} +/** + * The **`Scheduler`** interface of the Prioritized Task Scheduling API provides methods for scheduling prioritized tasks. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Scheduler) + */ +interface Scheduler { + wait(delay: number, maybeOptions?: SchedulerWaitOptions): Promise; +} +interface SchedulerWaitOptions { + signal?: AbortSignal; +} +/** + * The **`ExtendableEvent`** interface extends the lifetime of the install and activate events dispatched on the global scope as part of the service worker lifecycle. This ensures that any functional events (like FetchEvent) are not dispatched until it upgrades database schemas and deletes the outdated cache entries. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ExtendableEvent) + */ +declare abstract class ExtendableEvent extends Event { + /** + * The **`ExtendableEvent.waitUntil()`** method tells the event dispatcher that work is ongoing. It can also be used to detect whether that work was successful. In service workers, waitUntil() tells the browser that work is ongoing until the promise settles, and it shouldn't terminate the service worker if it wants that work to complete. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ExtendableEvent/waitUntil) + */ + waitUntil(promise: Promise): void; +} +/** + * The **`CustomEvent`** interface can be used to attach custom data to an event generated by an application. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CustomEvent) + */ +declare class CustomEvent extends Event { + constructor(type: string, init?: CustomEventCustomEventInit); + /** + * The read-only **`detail`** property of the CustomEvent interface returns any data passed when initializing the event. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CustomEvent/detail) + */ + get detail(): T; +} +interface CustomEventCustomEventInit { + bubbles?: boolean; + cancelable?: boolean; + composed?: boolean; + detail?: any; +} +/** + * The **`Blob`** interface represents a blob, which is a file-like object of immutable, raw data; they can be read as text or binary data, or converted into a ReadableStream so its methods can be used for processing the data. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Blob) + */ +declare class Blob { + constructor(bits?: ((ArrayBuffer | ArrayBufferView) | string | Blob)[], options?: BlobOptions); + /** + * The **`size`** read-only property of the Blob interface returns the size of the Blob or File in bytes. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Blob/size) + */ + get size(): number; + /** + * The **`type`** read-only property of the Blob interface returns the MIME type of the file. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Blob/type) + */ + get type(): string; + /** + * The **`slice()`** method of the Blob interface creates and returns a new Blob object which contains data from a subset of the blob on which it's called. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Blob/slice) + */ + slice(start?: number, end?: number, type?: string): Blob; + /** + * The **`arrayBuffer()`** method of the Blob interface returns a Promise that resolves with the contents of the blob as binary data contained in an ArrayBuffer. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Blob/arrayBuffer) + */ + arrayBuffer(): Promise; + /** + * The **`bytes()`** method of the Blob interface returns a Promise that resolves with a Uint8Array containing the contents of the blob as an array of bytes. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Blob/bytes) + */ + bytes(): Promise; + /** + * The **`text()`** method of the Blob interface returns a Promise that resolves with a string containing the contents of the blob, interpreted as UTF-8. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Blob/text) + */ + text(): Promise; + /** + * The **`stream()`** method of the Blob interface returns a ReadableStream which upon reading returns the data contained within the Blob. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Blob/stream) + */ + stream(): ReadableStream; +} +interface BlobOptions { + type?: string; +} +/** + * The **`File`** interface provides information about files and allows JavaScript in a web page to access their content. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/File) + */ +declare class File extends Blob { + constructor(bits: ((ArrayBuffer | ArrayBufferView) | string | Blob)[] | undefined, name: string, options?: FileOptions); + /** + * The **`name`** read-only property of the File interface returns the name of the file represented by a File object. For security reasons, the path is excluded from this property. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/File/name) + */ + get name(): string; + /** + * The **`lastModified`** read-only property of the File interface provides the last modified date of the file as the number of milliseconds since the Unix epoch (January 1, 1970 at midnight). Files without a known last modified date return the current date. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/File/lastModified) + */ + get lastModified(): number; +} +interface FileOptions { + type?: string; + lastModified?: number; +} +/** +* The Cache API allows fine grained control of reading and writing from the Cloudflare global network cache. +* +* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/cache/) +*/ +declare abstract class CacheStorage { + /** + * The **`open()`** method of the CacheStorage interface returns a Promise that resolves to the Cache object matching the cacheName. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CacheStorage/open) + */ + open(cacheName: string): Promise; + readonly default: Cache; +} +/** +* The Cache API allows fine grained control of reading and writing from the Cloudflare global network cache. +* +* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/cache/) +*/ +declare abstract class Cache { + /* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/cache/#delete) */ + delete(request: RequestInfo | URL, options?: CacheQueryOptions): Promise; + /* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/cache/#match) */ + match(request: RequestInfo | URL, options?: CacheQueryOptions): Promise; + /* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/cache/#put) */ + put(request: RequestInfo | URL, response: Response): Promise; +} +interface CacheQueryOptions { + ignoreMethod?: boolean; +} +/** +* The Web Crypto API provides a set of low-level functions for common cryptographic tasks. +* The Workers runtime implements the full surface of this API, but with some differences in +* the [supported algorithms](https://developers.cloudflare.com/workers/runtime-apis/web-crypto/#supported-algorithms) +* compared to those implemented in most browsers. +* +* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/web-crypto/) +*/ +declare abstract class Crypto { + /** + * The **`Crypto.subtle`** read-only property returns a SubtleCrypto which can then be used to perform low-level cryptographic operations. + * Available only in secure contexts. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Crypto/subtle) + */ + get subtle(): SubtleCrypto; + /** + * The **`Crypto.getRandomValues()`** method lets you get cryptographically strong random values. The array given as the parameter is filled with random numbers (random in its cryptographic meaning). + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Crypto/getRandomValues) + */ + getRandomValues(buffer: T): T; + /** + * The **`randomUUID()`** method of the Crypto interface is used to generate a v4 UUID using a cryptographically secure random number generator. + * Available only in secure contexts. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Crypto/randomUUID) + */ + randomUUID(): string; + DigestStream: typeof DigestStream; +} +/** + * The **`SubtleCrypto`** interface of the Web Crypto API provides a number of low-level cryptographic functions. + * Available only in secure contexts. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto) + */ +declare abstract class SubtleCrypto { + /** + * The **`encrypt()`** method of the SubtleCrypto interface encrypts data. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto/encrypt) + */ + encrypt(algorithm: string | SubtleCryptoEncryptAlgorithm, key: CryptoKey, plainText: ArrayBuffer | ArrayBufferView): Promise; + /** + * The **`decrypt()`** method of the SubtleCrypto interface decrypts some encrypted data. It takes as arguments a key to decrypt with, some optional extra parameters, and the data to decrypt (also known as "ciphertext"). It returns a Promise which will be fulfilled with the decrypted data (also known as "plaintext"). + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto/decrypt) + */ + decrypt(algorithm: string | SubtleCryptoEncryptAlgorithm, key: CryptoKey, cipherText: ArrayBuffer | ArrayBufferView): Promise; + /** + * The **`sign()`** method of the SubtleCrypto interface generates a digital signature. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto/sign) + */ + sign(algorithm: string | SubtleCryptoSignAlgorithm, key: CryptoKey, data: ArrayBuffer | ArrayBufferView): Promise; + /** + * The **`verify()`** method of the SubtleCrypto interface verifies a digital signature. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto/verify) + */ + verify(algorithm: string | SubtleCryptoSignAlgorithm, key: CryptoKey, signature: ArrayBuffer | ArrayBufferView, data: ArrayBuffer | ArrayBufferView): Promise; + /** + * The **`digest()`** method of the SubtleCrypto interface generates a digest of the given data, using the specified hash function. A digest is a short fixed-length value derived from some variable-length input. Cryptographic digests should exhibit collision-resistance, meaning that it's hard to come up with two different inputs that have the same digest value. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto/digest) + */ + digest(algorithm: string | SubtleCryptoHashAlgorithm, data: ArrayBuffer | ArrayBufferView): Promise; + /** + * The **`generateKey()`** method of the SubtleCrypto interface is used to generate a new key (for symmetric algorithms) or key pair (for public-key algorithms). + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto/generateKey) + */ + generateKey(algorithm: string | SubtleCryptoGenerateKeyAlgorithm, extractable: boolean, keyUsages: string[]): Promise; + /** + * The **`deriveKey()`** method of the SubtleCrypto interface can be used to derive a secret key from a master key. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto/deriveKey) + */ + deriveKey(algorithm: string | SubtleCryptoDeriveKeyAlgorithm, baseKey: CryptoKey, derivedKeyAlgorithm: string | SubtleCryptoImportKeyAlgorithm, extractable: boolean, keyUsages: string[]): Promise; + /** + * The **`deriveBits()`** method of the SubtleCrypto interface can be used to derive an array of bits from a base key. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto/deriveBits) + */ + deriveBits(algorithm: string | SubtleCryptoDeriveKeyAlgorithm, baseKey: CryptoKey, length?: number | null): Promise; + /** + * The **`importKey()`** method of the SubtleCrypto interface imports a key: that is, it takes as input a key in an external, portable format and gives you a CryptoKey object that you can use in the Web Crypto API. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto/importKey) + */ + importKey(format: string, keyData: (ArrayBuffer | ArrayBufferView) | JsonWebKey, algorithm: string | SubtleCryptoImportKeyAlgorithm, extractable: boolean, keyUsages: string[]): Promise; + /** + * The **`exportKey()`** method of the SubtleCrypto interface exports a key: that is, it takes as input a CryptoKey object and gives you the key in an external, portable format. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto/exportKey) + */ + exportKey(format: string, key: CryptoKey): Promise; + /** + * The **`wrapKey()`** method of the SubtleCrypto interface "wraps" a key. This means that it exports the key in an external, portable format, then encrypts the exported key. Wrapping a key helps protect it in untrusted environments, such as inside an otherwise unprotected data store or in transmission over an unprotected network. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto/wrapKey) + */ + wrapKey(format: string, key: CryptoKey, wrappingKey: CryptoKey, wrapAlgorithm: string | SubtleCryptoEncryptAlgorithm): Promise; + /** + * The **`unwrapKey()`** method of the SubtleCrypto interface "unwraps" a key. This means that it takes as its input a key that has been exported and then encrypted (also called "wrapped"). It decrypts the key and then imports it, returning a CryptoKey object that can be used in the Web Crypto API. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto/unwrapKey) + */ + unwrapKey(format: string, wrappedKey: ArrayBuffer | ArrayBufferView, unwrappingKey: CryptoKey, unwrapAlgorithm: string | SubtleCryptoEncryptAlgorithm, unwrappedKeyAlgorithm: string | SubtleCryptoImportKeyAlgorithm, extractable: boolean, keyUsages: string[]): Promise; + timingSafeEqual(a: ArrayBuffer | ArrayBufferView, b: ArrayBuffer | ArrayBufferView): boolean; +} +/** + * The **`CryptoKey`** interface of the Web Crypto API represents a cryptographic key obtained from one of the SubtleCrypto methods generateKey(), deriveKey(), importKey(), or unwrapKey(). + * Available only in secure contexts. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CryptoKey) + */ +declare abstract class CryptoKey { + /** + * The read-only **`type`** property of the CryptoKey interface indicates which kind of key is represented by the object. It can have the following values: + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CryptoKey/type) + */ + readonly type: string; + /** + * The read-only **`extractable`** property of the CryptoKey interface indicates whether or not the key may be extracted using SubtleCrypto.exportKey() or SubtleCrypto.wrapKey(). + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CryptoKey/extractable) + */ + readonly extractable: boolean; + /** + * The read-only **`algorithm`** property of the CryptoKey interface returns an object describing the algorithm for which this key can be used, and any associated extra parameters. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CryptoKey/algorithm) + */ + readonly algorithm: CryptoKeyKeyAlgorithm | CryptoKeyAesKeyAlgorithm | CryptoKeyHmacKeyAlgorithm | CryptoKeyRsaKeyAlgorithm | CryptoKeyEllipticKeyAlgorithm | CryptoKeyArbitraryKeyAlgorithm; + /** + * The read-only **`usages`** property of the CryptoKey interface indicates what can be done with the key. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CryptoKey/usages) + */ + readonly usages: string[]; +} +interface CryptoKeyPair { + publicKey: CryptoKey; + privateKey: CryptoKey; +} +interface JsonWebKey { + kty: string; + use?: string; + key_ops?: string[]; + alg?: string; + ext?: boolean; + crv?: string; + x?: string; + y?: string; + d?: string; + n?: string; + e?: string; + p?: string; + q?: string; + dp?: string; + dq?: string; + qi?: string; + oth?: RsaOtherPrimesInfo[]; + k?: string; +} +interface RsaOtherPrimesInfo { + r?: string; + d?: string; + t?: string; +} +interface SubtleCryptoDeriveKeyAlgorithm { + name: string; + salt?: (ArrayBuffer | ArrayBufferView); + iterations?: number; + hash?: (string | SubtleCryptoHashAlgorithm); + $public?: CryptoKey; + info?: (ArrayBuffer | ArrayBufferView); +} +interface SubtleCryptoEncryptAlgorithm { + name: string; + iv?: (ArrayBuffer | ArrayBufferView); + additionalData?: (ArrayBuffer | ArrayBufferView); + tagLength?: number; + counter?: (ArrayBuffer | ArrayBufferView); + length?: number; + label?: (ArrayBuffer | ArrayBufferView); +} +interface SubtleCryptoGenerateKeyAlgorithm { + name: string; + hash?: (string | SubtleCryptoHashAlgorithm); + modulusLength?: number; + publicExponent?: (ArrayBuffer | ArrayBufferView); + length?: number; + namedCurve?: string; +} +interface SubtleCryptoHashAlgorithm { + name: string; +} +interface SubtleCryptoImportKeyAlgorithm { + name: string; + hash?: (string | SubtleCryptoHashAlgorithm); + length?: number; + namedCurve?: string; + compressed?: boolean; +} +interface SubtleCryptoSignAlgorithm { + name: string; + hash?: (string | SubtleCryptoHashAlgorithm); + dataLength?: number; + saltLength?: number; +} +interface CryptoKeyKeyAlgorithm { + name: string; +} +interface CryptoKeyAesKeyAlgorithm { + name: string; + length: number; +} +interface CryptoKeyHmacKeyAlgorithm { + name: string; + hash: CryptoKeyKeyAlgorithm; + length: number; +} +interface CryptoKeyRsaKeyAlgorithm { + name: string; + modulusLength: number; + publicExponent: ArrayBuffer | ArrayBufferView; + hash?: CryptoKeyKeyAlgorithm; +} +interface CryptoKeyEllipticKeyAlgorithm { + name: string; + namedCurve: string; +} +interface CryptoKeyArbitraryKeyAlgorithm { + name: string; + hash?: CryptoKeyKeyAlgorithm; + namedCurve?: string; + length?: number; +} +declare class DigestStream extends WritableStream { + constructor(algorithm: string | SubtleCryptoHashAlgorithm); + readonly digest: Promise; + get bytesWritten(): number | bigint; +} +/** + * The **`TextDecoder`** interface represents a decoder for a specific text encoding, such as UTF-8, ISO-8859-2, or GBK. A decoder takes an array of bytes as input and returns a JavaScript string. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TextDecoder) + */ +declare class TextDecoder { + constructor(label?: string, options?: TextDecoderConstructorOptions); + /** + * The **`TextDecoder.decode()`** method returns a string containing text decoded from the buffer passed as a parameter. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TextDecoder/decode) + */ + decode(input?: (ArrayBuffer | ArrayBufferView), options?: TextDecoderDecodeOptions): string; + get encoding(): string; + get fatal(): boolean; + get ignoreBOM(): boolean; +} +/** + * The **`TextEncoder`** interface enables you to encode a JavaScript string using UTF-8. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TextEncoder) + */ +declare class TextEncoder { + constructor(); + /** + * The **`TextEncoder.encode()`** method takes a string as input, and returns a Uint8Array containing the string encoded using UTF-8. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TextEncoder/encode) + */ + encode(input?: string): Uint8Array; + /** + * The **`TextEncoder.encodeInto()`** method takes a string to encode and a destination Uint8Array to put resulting UTF-8 encoded text into, and returns an object indicating the progress of the encoding. This is potentially more performant than the encode() method — especially when the target buffer is a view into a Wasm heap. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TextEncoder/encodeInto) + */ + encodeInto(input: string, buffer: Uint8Array): TextEncoderEncodeIntoResult; + get encoding(): string; +} +interface TextDecoderConstructorOptions { + fatal: boolean; + ignoreBOM: boolean; +} +interface TextDecoderDecodeOptions { + stream: boolean; +} +interface TextEncoderEncodeIntoResult { + read: number; + written: number; +} +/** + * The **`ErrorEvent`** interface represents events providing information related to errors in scripts or in files. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ErrorEvent) + */ +declare class ErrorEvent extends Event { + constructor(type: string, init?: ErrorEventErrorEventInit); + /** + * The **`filename`** read-only property of the ErrorEvent interface returns a string containing the name of the script file in which the error occurred. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ErrorEvent/filename) + */ + get filename(): string; + /** + * The **`message`** read-only property of the ErrorEvent interface returns a string containing a human-readable error message describing the problem. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ErrorEvent/message) + */ + get message(): string; + /** + * The **`lineno`** read-only property of the ErrorEvent interface returns an integer containing the line number of the script file on which the error occurred. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ErrorEvent/lineno) + */ + get lineno(): number; + /** + * The **`colno`** read-only property of the ErrorEvent interface returns an integer containing the column number of the script file on which the error occurred. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ErrorEvent/colno) + */ + get colno(): number; + /** + * The **`error`** read-only property of the ErrorEvent interface returns a JavaScript value, such as an Error or DOMException, representing the error associated with this event. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ErrorEvent/error) + */ + get error(): any; +} +interface ErrorEventErrorEventInit { + bubbles?: boolean; + cancelable?: boolean; + composed?: boolean; + message?: string; + filename?: string; + lineno?: number; + colno?: number; + error?: any; +} +/** + * The **`MessageEvent`** interface represents a message received by a target object. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessageEvent) + */ +declare class MessageEvent extends Event { + constructor(type: string, initializer?: MessageEventInit); + /** + * The **`data`** read-only property of the MessageEvent interface represents the data sent by the message emitter. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessageEvent/data) + */ + readonly data: any; + /** + * The **`origin`** read-only property of the MessageEvent interface is a string representing the origin of the message emitter. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessageEvent/origin) + */ + readonly origin: string | null; + /** + * The **`lastEventId`** read-only property of the MessageEvent interface is a string representing a unique ID for the event. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessageEvent/lastEventId) + */ + readonly lastEventId: string; + /** + * The **`source`** read-only property of the MessageEvent interface is a MessageEventSource (which can be a WindowProxy, MessagePort, or ServiceWorker object) representing the message emitter. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessageEvent/source) + */ + readonly source: MessagePort | null; + /** + * The **`ports`** read-only property of the MessageEvent interface is an array of MessagePort objects containing all MessagePort objects sent with the message, in order. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessageEvent/ports) + */ + readonly ports: MessagePort[]; +} +interface MessageEventInit { + bubbles?: boolean; + cancelable?: boolean; + composed?: boolean; + data?: any; + origin?: string; + lastEventId?: string; + source?: MessagePort; + ports?: MessagePort[]; +} +/** + * The **`PromiseRejectionEvent`** interface represents events which are sent to the global script context when JavaScript Promises are rejected. These events are particularly useful for telemetry and debugging purposes. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PromiseRejectionEvent) + */ +declare abstract class PromiseRejectionEvent extends Event { + /** + * The PromiseRejectionEvent interface's **`promise`** read-only property indicates the JavaScript Promise which was rejected. You can examine the event's PromiseRejectionEvent.reason property to learn why the promise was rejected. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PromiseRejectionEvent/promise) + */ + readonly promise: Promise; + /** + * The PromiseRejectionEvent **`reason`** read-only property is any JavaScript value or Object which provides the reason passed into Promise.reject(). This in theory provides information about why the promise was rejected. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PromiseRejectionEvent/reason) + */ + readonly reason: any; +} +/** + * The **`FormData`** interface provides a way to construct a set of key/value pairs representing form fields and their values, which can be sent using the fetch(), XMLHttpRequest.send() or navigator.sendBeacon() methods. It uses the same format a form would use if the encoding type were set to "multipart/form-data". + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FormData) + */ +declare class FormData { + constructor(); + /** + * The **`append()`** method of the FormData interface appends a new value onto an existing key inside a FormData object, or adds the key if it does not already exist. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FormData/append) + */ + append(name: string, value: string | Blob): void; + /** + * The **`append()`** method of the FormData interface appends a new value onto an existing key inside a FormData object, or adds the key if it does not already exist. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FormData/append) + */ + append(name: string, value: string): void; + /** + * The **`append()`** method of the FormData interface appends a new value onto an existing key inside a FormData object, or adds the key if it does not already exist. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FormData/append) + */ + append(name: string, value: Blob, filename?: string): void; + /** + * The **`delete()`** method of the FormData interface deletes a key and its value(s) from a FormData object. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FormData/delete) + */ + delete(name: string): void; + /** + * The **`get()`** method of the FormData interface returns the first value associated with a given key from within a FormData object. If you expect multiple values and want all of them, use the getAll() method instead. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FormData/get) + */ + get(name: string): (File | string) | null; + /** + * The **`getAll()`** method of the FormData interface returns all the values associated with a given key from within a FormData object. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FormData/getAll) + */ + getAll(name: string): (File | string)[]; + /** + * The **`has()`** method of the FormData interface returns whether a FormData object contains a certain key. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FormData/has) + */ + has(name: string): boolean; + /** + * The **`set()`** method of the FormData interface sets a new value for an existing key inside a FormData object, or adds the key/value if it does not already exist. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FormData/set) + */ + set(name: string, value: string | Blob): void; + /** + * The **`set()`** method of the FormData interface sets a new value for an existing key inside a FormData object, or adds the key/value if it does not already exist. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FormData/set) + */ + set(name: string, value: string): void; + /** + * The **`set()`** method of the FormData interface sets a new value for an existing key inside a FormData object, or adds the key/value if it does not already exist. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FormData/set) + */ + set(name: string, value: Blob, filename?: string): void; + entries(): IterableIterator<[ + key: string, + value: File | string + ]>; + keys(): IterableIterator; + values(): IterableIterator<(File | string)>; + forEach(callback: (this: This, value: File | string, key: string, parent: FormData) => void, thisArg?: This): void; + [Symbol.iterator](): IterableIterator<[ + key: string, + value: File | string + ]>; +} +interface ContentOptions { + html?: boolean; +} +declare class HTMLRewriter { + constructor(); + on(selector: string, handlers: HTMLRewriterElementContentHandlers): HTMLRewriter; + onDocument(handlers: HTMLRewriterDocumentContentHandlers): HTMLRewriter; + transform(response: Response): Response; +} +interface HTMLRewriterElementContentHandlers { + element?(element: Element): void | Promise; + comments?(comment: Comment): void | Promise; + text?(element: Text): void | Promise; +} +interface HTMLRewriterDocumentContentHandlers { + doctype?(doctype: Doctype): void | Promise; + comments?(comment: Comment): void | Promise; + text?(text: Text): void | Promise; + end?(end: DocumentEnd): void | Promise; +} +interface Doctype { + readonly name: string | null; + readonly publicId: string | null; + readonly systemId: string | null; +} +interface Element { + tagName: string; + readonly attributes: IterableIterator; + readonly removed: boolean; + readonly namespaceURI: string; + getAttribute(name: string): string | null; + hasAttribute(name: string): boolean; + setAttribute(name: string, value: string): Element; + removeAttribute(name: string): Element; + before(content: string | ReadableStream | Response, options?: ContentOptions): Element; + after(content: string | ReadableStream | Response, options?: ContentOptions): Element; + prepend(content: string | ReadableStream | Response, options?: ContentOptions): Element; + append(content: string | ReadableStream | Response, options?: ContentOptions): Element; + replace(content: string | ReadableStream | Response, options?: ContentOptions): Element; + remove(): Element; + removeAndKeepContent(): Element; + setInnerContent(content: string | ReadableStream | Response, options?: ContentOptions): Element; + onEndTag(handler: (tag: EndTag) => void | Promise): void; +} +interface EndTag { + name: string; + before(content: string | ReadableStream | Response, options?: ContentOptions): EndTag; + after(content: string | ReadableStream | Response, options?: ContentOptions): EndTag; + remove(): EndTag; +} +interface Comment { + text: string; + readonly removed: boolean; + before(content: string, options?: ContentOptions): Comment; + after(content: string, options?: ContentOptions): Comment; + replace(content: string, options?: ContentOptions): Comment; + remove(): Comment; +} +interface Text { + readonly text: string; + readonly lastInTextNode: boolean; + readonly removed: boolean; + before(content: string | ReadableStream | Response, options?: ContentOptions): Text; + after(content: string | ReadableStream | Response, options?: ContentOptions): Text; + replace(content: string | ReadableStream | Response, options?: ContentOptions): Text; + remove(): Text; +} +interface DocumentEnd { + append(content: string, options?: ContentOptions): DocumentEnd; +} +/** + * This is the event type for fetch events dispatched on the service worker global scope. It contains information about the fetch, including the request and how the receiver will treat the response. It provides the event.respondWith() method, which allows us to provide a response to this fetch. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FetchEvent) + */ +declare abstract class FetchEvent extends ExtendableEvent { + /** + * The **`request`** read-only property of the FetchEvent interface returns the Request that triggered the event handler. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FetchEvent/request) + */ + readonly request: Request; + /** + * The **`respondWith()`** method of FetchEvent prevents the browser's default fetch handling, and allows you to provide a promise for a Response yourself. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FetchEvent/respondWith) + */ + respondWith(promise: Response | Promise): void; + passThroughOnException(): void; +} +type HeadersInit = Headers | Iterable> | Record; +/** + * The **`Headers`** interface of the Fetch API allows you to perform various actions on HTTP request and response headers. These actions include retrieving, setting, adding to, and removing headers from the list of the request's headers. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Headers) + */ +declare class Headers { + constructor(init?: HeadersInit); + /** + * The **`get()`** method of the Headers interface returns a byte string of all the values of a header within a Headers object with a given name. If the requested header doesn't exist in the Headers object, it returns null. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Headers/get) + */ + get(name: string): string | null; + getAll(name: string): string[]; + /** + * The **`getSetCookie()`** method of the Headers interface returns an array containing the values of all Set-Cookie headers associated with a response. This allows Headers objects to handle having multiple Set-Cookie headers, which wasn't possible prior to its implementation. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Headers/getSetCookie) + */ + getSetCookie(): string[]; + /** + * The **`has()`** method of the Headers interface returns a boolean stating whether a Headers object contains a certain header. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Headers/has) + */ + has(name: string): boolean; + /** + * The **`set()`** method of the Headers interface sets a new value for an existing header inside a Headers object, or adds the header if it does not already exist. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Headers/set) + */ + set(name: string, value: string): void; + /** + * The **`append()`** method of the Headers interface appends a new value onto an existing header inside a Headers object, or adds the header if it does not already exist. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Headers/append) + */ + append(name: string, value: string): void; + /** + * The **`delete()`** method of the Headers interface deletes a header from the current Headers object. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Headers/delete) + */ + delete(name: string): void; + forEach(callback: (this: This, value: string, key: string, parent: Headers) => void, thisArg?: This): void; + entries(): IterableIterator<[ + key: string, + value: string + ]>; + keys(): IterableIterator; + values(): IterableIterator; + [Symbol.iterator](): IterableIterator<[ + key: string, + value: string + ]>; +} +type BodyInit = ReadableStream | string | ArrayBuffer | ArrayBufferView | Blob | URLSearchParams | FormData | Iterable | AsyncIterable; +declare abstract class Body { + /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/body) */ + get body(): ReadableStream | null; + /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/bodyUsed) */ + get bodyUsed(): boolean; + /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/arrayBuffer) */ + arrayBuffer(): Promise; + /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/bytes) */ + bytes(): Promise; + /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/text) */ + text(): Promise; + /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/json) */ + json(): Promise; + /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/formData) */ + formData(): Promise; + /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/blob) */ + blob(): Promise; +} +/** + * The **`Response`** interface of the Fetch API represents the response to a request. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Response) + */ +declare var Response: { + prototype: Response; + new (body?: BodyInit | null, init?: ResponseInit): Response; + error(): Response; + redirect(url: string, status?: number): Response; + json(any: any, maybeInit?: (ResponseInit | Response)): Response; +}; +/** + * The **`Response`** interface of the Fetch API represents the response to a request. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Response) + */ +interface Response extends Body { + /** + * The **`clone()`** method of the Response interface creates a clone of a response object, identical in every way, but stored in a different variable. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Response/clone) + */ + clone(): Response; + /** + * The **`status`** read-only property of the Response interface contains the HTTP status codes of the response. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Response/status) + */ + status: number; + /** + * The **`statusText`** read-only property of the Response interface contains the status message corresponding to the HTTP status code in Response.status. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Response/statusText) + */ + statusText: string; + /** + * The **`headers`** read-only property of the Response interface contains the Headers object associated with the response. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Response/headers) + */ + headers: Headers; + /** + * The **`ok`** read-only property of the Response interface contains a Boolean stating whether the response was successful (status in the range 200-299) or not. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Response/ok) + */ + ok: boolean; + /** + * The **`redirected`** read-only property of the Response interface indicates whether or not the response is the result of a request you made which was redirected. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Response/redirected) + */ + redirected: boolean; + /** + * The **`url`** read-only property of the Response interface contains the URL of the response. The value of the url property will be the final URL obtained after any redirects. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Response/url) + */ + url: string; + webSocket: WebSocket | null; + cf: any | undefined; + /** + * The **`type`** read-only property of the Response interface contains the type of the response. The type determines whether scripts are able to access the response body and headers. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Response/type) + */ + type: "default" | "error"; +} +interface ResponseInit { + status?: number; + statusText?: string; + headers?: HeadersInit; + cf?: any; + webSocket?: (WebSocket | null); + encodeBody?: "automatic" | "manual"; +} +type RequestInfo> = Request | string; +/** + * The **`Request`** interface of the Fetch API represents a resource request. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request) + */ +declare var Request: { + prototype: Request; + new >(input: RequestInfo | URL, init?: RequestInit): Request; +}; +/** + * The **`Request`** interface of the Fetch API represents a resource request. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request) + */ +interface Request> extends Body { + /** + * The **`clone()`** method of the Request interface creates a copy of the current Request object. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/clone) + */ + clone(): Request; + /** + * The **`method`** read-only property of the Request interface contains the request's method (GET, POST, etc.) + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/method) + */ + method: string; + /** + * The **`url`** read-only property of the Request interface contains the URL of the request. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/url) + */ + url: string; + /** + * The **`headers`** read-only property of the Request interface contains the Headers object associated with the request. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/headers) + */ + headers: Headers; + /** + * The **`redirect`** read-only property of the Request interface contains the mode for how redirects are handled. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/redirect) + */ + redirect: string; + fetcher: Fetcher | null; + /** + * The read-only **`signal`** property of the Request interface returns the AbortSignal associated with the request. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/signal) + */ + signal: AbortSignal; + cf?: Cf; + /** + * The **`integrity`** read-only property of the Request interface contains the subresource integrity value of the request. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/integrity) + */ + integrity: string; + /** + * The **`keepalive`** read-only property of the Request interface contains the request's keepalive setting (true or false), which indicates whether the browser will keep the associated request alive if the page that initiated it is unloaded before the request is complete. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/keepalive) + */ + keepalive: boolean; + /** + * The **`cache`** read-only property of the Request interface contains the cache mode of the request. It controls how the request will interact with the browser's HTTP cache. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/cache) + */ + cache?: "no-store" | "no-cache"; +} +interface RequestInit { + /* A string to set request's method. */ + method?: string; + /* A Headers object, an object literal, or an array of two-item arrays to set request's headers. */ + headers?: HeadersInit; + /* A BodyInit object or null to set request's body. */ + body?: BodyInit | null; + /* A string indicating whether request follows redirects, results in an error upon encountering a redirect, or returns the redirect (in an opaque fashion). Sets request's redirect. */ + redirect?: string; + fetcher?: (Fetcher | null); + cf?: Cf; + /* A string indicating how the request will interact with the browser's cache to set request's cache. */ + cache?: "no-store" | "no-cache"; + /* A cryptographic hash of the resource to be fetched by request. Sets request's integrity. */ + integrity?: string; + /* An AbortSignal to set request's signal. */ + signal?: (AbortSignal | null); + encodeResponseBody?: "automatic" | "manual"; +} +type Service Rpc.WorkerEntrypointBranded) | Rpc.WorkerEntrypointBranded | ExportedHandler | undefined = undefined> = T extends new (...args: any[]) => Rpc.WorkerEntrypointBranded ? Fetcher> : T extends Rpc.WorkerEntrypointBranded ? Fetcher : T extends Exclude ? never : Fetcher; +type Fetcher = (T extends Rpc.EntrypointBranded ? Rpc.Provider : unknown) & { + fetch(input: RequestInfo | URL, init?: RequestInit): Promise; + connect(address: SocketAddress | string, options?: SocketOptions): Socket; +}; +interface KVNamespaceListKey { + name: Key; + expiration?: number; + metadata?: Metadata; +} +type KVNamespaceListResult = { + list_complete: false; + keys: KVNamespaceListKey[]; + cursor: string; + cacheStatus: string | null; +} | { + list_complete: true; + keys: KVNamespaceListKey[]; + cacheStatus: string | null; +}; +interface KVNamespace { + get(key: Key, options?: Partial>): Promise; + get(key: Key, type: "text"): Promise; + get(key: Key, type: "json"): Promise; + get(key: Key, type: "arrayBuffer"): Promise; + get(key: Key, type: "stream"): Promise; + get(key: Key, options?: KVNamespaceGetOptions<"text">): Promise; + get(key: Key, options?: KVNamespaceGetOptions<"json">): Promise; + get(key: Key, options?: KVNamespaceGetOptions<"arrayBuffer">): Promise; + get(key: Key, options?: KVNamespaceGetOptions<"stream">): Promise; + get(key: Array, type: "text"): Promise>; + get(key: Array, type: "json"): Promise>; + get(key: Array, options?: Partial>): Promise>; + get(key: Array, options?: KVNamespaceGetOptions<"text">): Promise>; + get(key: Array, options?: KVNamespaceGetOptions<"json">): Promise>; + list(options?: KVNamespaceListOptions): Promise>; + put(key: Key, value: string | ArrayBuffer | ArrayBufferView | ReadableStream, options?: KVNamespacePutOptions): Promise; + getWithMetadata(key: Key, options?: Partial>): Promise>; + getWithMetadata(key: Key, type: "text"): Promise>; + getWithMetadata(key: Key, type: "json"): Promise>; + getWithMetadata(key: Key, type: "arrayBuffer"): Promise>; + getWithMetadata(key: Key, type: "stream"): Promise>; + getWithMetadata(key: Key, options: KVNamespaceGetOptions<"text">): Promise>; + getWithMetadata(key: Key, options: KVNamespaceGetOptions<"json">): Promise>; + getWithMetadata(key: Key, options: KVNamespaceGetOptions<"arrayBuffer">): Promise>; + getWithMetadata(key: Key, options: KVNamespaceGetOptions<"stream">): Promise>; + getWithMetadata(key: Array, type: "text"): Promise>>; + getWithMetadata(key: Array, type: "json"): Promise>>; + getWithMetadata(key: Array, options?: Partial>): Promise>>; + getWithMetadata(key: Array, options?: KVNamespaceGetOptions<"text">): Promise>>; + getWithMetadata(key: Array, options?: KVNamespaceGetOptions<"json">): Promise>>; + delete(key: Key): Promise; +} +interface KVNamespaceListOptions { + limit?: number; + prefix?: (string | null); + cursor?: (string | null); +} +interface KVNamespaceGetOptions { + type: Type; + cacheTtl?: number; +} +interface KVNamespacePutOptions { + expiration?: number; + expirationTtl?: number; + metadata?: (any | null); +} +interface KVNamespaceGetWithMetadataResult { + value: Value | null; + metadata: Metadata | null; + cacheStatus: string | null; +} +type QueueContentType = "text" | "bytes" | "json" | "v8"; +interface Queue { + metrics(): Promise; + send(message: Body, options?: QueueSendOptions): Promise; + sendBatch(messages: Iterable>, options?: QueueSendBatchOptions): Promise; +} +interface QueueSendMetrics { + backlogCount: number; + backlogBytes: number; + oldestMessageTimestamp?: Date; +} +interface QueueSendMetadata { + metrics: QueueSendMetrics; +} +interface QueueSendResponse { + metadata: QueueSendMetadata; +} +interface QueueSendBatchMetrics { + backlogCount: number; + backlogBytes: number; + oldestMessageTimestamp?: Date; +} +interface QueueSendBatchMetadata { + metrics: QueueSendBatchMetrics; +} +interface QueueSendBatchResponse { + metadata: QueueSendBatchMetadata; +} +interface QueueSendOptions { + contentType?: QueueContentType; + delaySeconds?: number; +} +interface QueueSendBatchOptions { + delaySeconds?: number; +} +interface MessageSendRequest { + body: Body; + contentType?: QueueContentType; + delaySeconds?: number; +} +interface QueueMetrics { + backlogCount: number; + backlogBytes: number; + oldestMessageTimestamp?: Date; +} +interface MessageBatchMetrics { + backlogCount: number; + backlogBytes: number; + oldestMessageTimestamp?: Date; +} +interface MessageBatchMetadata { + metrics: MessageBatchMetrics; +} +interface QueueRetryOptions { + delaySeconds?: number; +} +interface Message { + readonly id: string; + readonly timestamp: Date; + readonly body: Body; + readonly attempts: number; + retry(options?: QueueRetryOptions): void; + ack(): void; +} +interface QueueEvent extends ExtendableEvent { + readonly messages: readonly Message[]; + readonly queue: string; + readonly metadata: MessageBatchMetadata; + retryAll(options?: QueueRetryOptions): void; + ackAll(): void; +} +interface MessageBatch { + readonly messages: readonly Message[]; + readonly queue: string; + readonly metadata: MessageBatchMetadata; + retryAll(options?: QueueRetryOptions): void; + ackAll(): void; +} +interface R2Error extends Error { + readonly name: string; + readonly code: number; + readonly message: string; + readonly action: string; + readonly stack: any; +} +interface R2ListOptions { + limit?: number; + prefix?: string; + cursor?: string; + delimiter?: string; + startAfter?: string; + include?: ("httpMetadata" | "customMetadata")[]; +} +interface R2Bucket { + head(key: string): Promise; + get(key: string, options: R2GetOptions & { + onlyIf: R2Conditional | Headers; + }): Promise; + get(key: string, options?: R2GetOptions): Promise; + put(key: string, value: ReadableStream | ArrayBuffer | ArrayBufferView | string | null | Blob, options?: R2PutOptions & { + onlyIf: R2Conditional | Headers; + }): Promise; + put(key: string, value: ReadableStream | ArrayBuffer | ArrayBufferView | string | null | Blob, options?: R2PutOptions): Promise; + createMultipartUpload(key: string, options?: R2MultipartOptions): Promise; + resumeMultipartUpload(key: string, uploadId: string): R2MultipartUpload; + delete(keys: string | string[]): Promise; + list(options?: R2ListOptions): Promise; +} +interface R2MultipartUpload { + readonly key: string; + readonly uploadId: string; + uploadPart(partNumber: number, value: ReadableStream | (ArrayBuffer | ArrayBufferView) | string | Blob, options?: R2UploadPartOptions): Promise; + abort(): Promise; + complete(uploadedParts: R2UploadedPart[]): Promise; +} +interface R2UploadedPart { + partNumber: number; + etag: string; +} +declare abstract class R2Object { + readonly key: string; + readonly version: string; + readonly size: number; + readonly etag: string; + readonly httpEtag: string; + readonly checksums: R2Checksums; + readonly uploaded: Date; + readonly httpMetadata?: R2HTTPMetadata; + readonly customMetadata?: Record; + readonly range?: R2Range; + readonly storageClass: string; + readonly ssecKeyMd5?: string; + writeHttpMetadata(headers: Headers): void; +} +interface R2ObjectBody extends R2Object { + get body(): ReadableStream; + get bodyUsed(): boolean; + arrayBuffer(): Promise; + bytes(): Promise; + text(): Promise; + json(): Promise; + blob(): Promise; +} +type R2Range = { + offset: number; + length?: number; +} | { + offset?: number; + length: number; +} | { + suffix: number; +}; +interface R2Conditional { + etagMatches?: string; + etagDoesNotMatch?: string; + uploadedBefore?: Date; + uploadedAfter?: Date; + secondsGranularity?: boolean; +} +interface R2GetOptions { + onlyIf?: (R2Conditional | Headers); + range?: (R2Range | Headers); + ssecKey?: (ArrayBuffer | string); +} +interface R2PutOptions { + onlyIf?: (R2Conditional | Headers); + httpMetadata?: (R2HTTPMetadata | Headers); + customMetadata?: Record; + md5?: ((ArrayBuffer | ArrayBufferView) | string); + sha1?: ((ArrayBuffer | ArrayBufferView) | string); + sha256?: ((ArrayBuffer | ArrayBufferView) | string); + sha384?: ((ArrayBuffer | ArrayBufferView) | string); + sha512?: ((ArrayBuffer | ArrayBufferView) | string); + storageClass?: string; + ssecKey?: (ArrayBuffer | string); +} +interface R2MultipartOptions { + httpMetadata?: (R2HTTPMetadata | Headers); + customMetadata?: Record; + storageClass?: string; + ssecKey?: (ArrayBuffer | string); +} +interface R2Checksums { + readonly md5?: ArrayBuffer; + readonly sha1?: ArrayBuffer; + readonly sha256?: ArrayBuffer; + readonly sha384?: ArrayBuffer; + readonly sha512?: ArrayBuffer; + toJSON(): R2StringChecksums; +} +interface R2StringChecksums { + md5?: string; + sha1?: string; + sha256?: string; + sha384?: string; + sha512?: string; +} +interface R2HTTPMetadata { + contentType?: string; + contentLanguage?: string; + contentDisposition?: string; + contentEncoding?: string; + cacheControl?: string; + cacheExpiry?: Date; +} +type R2Objects = { + objects: R2Object[]; + delimitedPrefixes: string[]; +} & ({ + truncated: true; + cursor: string; +} | { + truncated: false; +}); +interface R2UploadPartOptions { + ssecKey?: (ArrayBuffer | string); +} +declare abstract class ScheduledEvent extends ExtendableEvent { + readonly scheduledTime: number; + readonly cron: string; + noRetry(): void; +} +interface ScheduledController { + readonly scheduledTime: number; + readonly cron: string; + noRetry(): void; +} +interface QueuingStrategy { + highWaterMark?: (number | bigint); + size?: (chunk: T) => number | bigint; +} +interface UnderlyingSink { + type?: string; + start?: (controller: WritableStreamDefaultController) => void | Promise; + write?: (chunk: W, controller: WritableStreamDefaultController) => void | Promise; + abort?: (reason: any) => void | Promise; + close?: () => void | Promise; +} +interface UnderlyingByteSource { + type: "bytes"; + autoAllocateChunkSize?: number; + start?: (controller: ReadableByteStreamController) => void | Promise; + pull?: (controller: ReadableByteStreamController) => void | Promise; + cancel?: (reason: any) => void | Promise; +} +interface UnderlyingSource { + type?: "" | undefined; + start?: (controller: ReadableStreamDefaultController) => void | Promise; + pull?: (controller: ReadableStreamDefaultController) => void | Promise; + cancel?: (reason: any) => void | Promise; + expectedLength?: (number | bigint); +} +interface Transformer { + readableType?: string; + writableType?: string; + start?: (controller: TransformStreamDefaultController) => void | Promise; + transform?: (chunk: I, controller: TransformStreamDefaultController) => void | Promise; + flush?: (controller: TransformStreamDefaultController) => void | Promise; + cancel?: (reason: any) => void | Promise; + expectedLength?: number; +} +interface StreamPipeOptions { + preventAbort?: boolean; + preventCancel?: boolean; + /** + * Pipes this readable stream to a given writable stream destination. The way in which the piping process behaves under various error conditions can be customized with a number of passed options. It returns a promise that fulfills when the piping process completes successfully, or rejects if any errors were encountered. + * + * Piping a stream will lock it for the duration of the pipe, preventing any other consumer from acquiring a reader. + * + * Errors and closures of the source and destination streams propagate as follows: + * + * An error in this source readable stream will abort destination, unless preventAbort is truthy. The returned promise will be rejected with the source's error, or with any error that occurs during aborting the destination. + * + * An error in destination will cancel this source readable stream, unless preventCancel is truthy. The returned promise will be rejected with the destination's error, or with any error that occurs during canceling the source. + * + * When this source readable stream closes, destination will be closed, unless preventClose is truthy. The returned promise will be fulfilled once this process completes, unless an error is encountered while closing the destination, in which case it will be rejected with that error. + * + * If destination starts out closed or closing, this source readable stream will be canceled, unless preventCancel is true. The returned promise will be rejected with an error indicating piping to a closed stream failed, or with any error that occurs during canceling the source. + * + * The signal option can be set to an AbortSignal to allow aborting an ongoing pipe operation via the corresponding AbortController. In this case, this source readable stream will be canceled, and destination aborted, unless the respective options preventCancel or preventAbort are set. + */ + preventClose?: boolean; + signal?: AbortSignal; +} +type ReadableStreamReadResult = { + done: false; + value: R; +} | { + done: true; + value?: undefined; +}; +/** + * The **`ReadableStream`** interface of the Streams API represents a readable stream of byte data. The Fetch API offers a concrete instance of a ReadableStream through the body property of a Response object. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStream) + */ +interface ReadableStream { + /** + * The **`locked`** read-only property of the ReadableStream interface returns whether or not the readable stream is locked to a reader. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStream/locked) + */ + get locked(): boolean; + /** + * The **`cancel()`** method of the ReadableStream interface returns a Promise that resolves when the stream is canceled. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStream/cancel) + */ + cancel(reason?: any): Promise; + /** + * The **`getReader()`** method of the ReadableStream interface creates a reader and locks the stream to it. While the stream is locked, no other reader can be acquired until this one is released. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStream/getReader) + */ + getReader(): ReadableStreamDefaultReader; + /** + * The **`getReader()`** method of the ReadableStream interface creates a reader and locks the stream to it. While the stream is locked, no other reader can be acquired until this one is released. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStream/getReader) + */ + getReader(options: ReadableStreamGetReaderOptions): ReadableStreamBYOBReader; + /** + * The **`pipeThrough()`** method of the ReadableStream interface provides a chainable way of piping the current stream through a transform stream or any other writable/readable pair. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStream/pipeThrough) + */ + pipeThrough(transform: ReadableWritablePair, options?: StreamPipeOptions): ReadableStream; + /** + * The **`pipeTo()`** method of the ReadableStream interface pipes the current ReadableStream to a given WritableStream and returns a Promise that fulfills when the piping process completes successfully, or rejects if any errors were encountered. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStream/pipeTo) + */ + pipeTo(destination: WritableStream, options?: StreamPipeOptions): Promise; + /** + * The **`tee()`** method of the ReadableStream interface tees the current readable stream, returning a two-element array containing the two resulting branches as new ReadableStream instances. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStream/tee) + */ + tee(): [ + ReadableStream, + ReadableStream + ]; + values(options?: ReadableStreamValuesOptions): AsyncIterableIterator; + [Symbol.asyncIterator](options?: ReadableStreamValuesOptions): AsyncIterableIterator; +} +/** + * The **`ReadableStream`** interface of the Streams API represents a readable stream of byte data. The Fetch API offers a concrete instance of a ReadableStream through the body property of a Response object. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStream) + */ +declare const ReadableStream: { + prototype: ReadableStream; + new (underlyingSource: UnderlyingByteSource, strategy?: QueuingStrategy): ReadableStream; + new (underlyingSource?: UnderlyingSource, strategy?: QueuingStrategy): ReadableStream; +}; +/** + * The **`ReadableStreamDefaultReader`** interface of the Streams API represents a default reader that can be used to read stream data supplied from a network (such as a fetch request). + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamDefaultReader) + */ +declare class ReadableStreamDefaultReader { + constructor(stream: ReadableStream); + get closed(): Promise; + cancel(reason?: any): Promise; + /** + * The **`read()`** method of the ReadableStreamDefaultReader interface returns a Promise providing access to the next chunk in the stream's internal queue. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamDefaultReader/read) + */ + read(): Promise>; + /** + * The **`releaseLock()`** method of the ReadableStreamDefaultReader interface releases the reader's lock on the stream. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamDefaultReader/releaseLock) + */ + releaseLock(): void; +} +/** + * The **`ReadableStreamBYOBReader`** interface of the Streams API defines a reader for a ReadableStream that supports zero-copy reading from an underlying byte source. It is used for efficient copying from underlying sources where the data is delivered as an "anonymous" sequence of bytes, such as files. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamBYOBReader) + */ +declare class ReadableStreamBYOBReader { + constructor(stream: ReadableStream); + get closed(): Promise; + cancel(reason?: any): Promise; + /** + * The **`read()`** method of the ReadableStreamBYOBReader interface is used to read data into a view on a user-supplied buffer from an associated readable byte stream. A request for data will be satisfied from the stream's internal queues if there is any data present. If the stream queues are empty, the request may be supplied as a zero-copy transfer from the underlying byte source. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamBYOBReader/read) + */ + read(view: T): Promise>; + /** + * The **`releaseLock()`** method of the ReadableStreamBYOBReader interface releases the reader's lock on the stream. After the lock is released, the reader is no longer active. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamBYOBReader/releaseLock) + */ + releaseLock(): void; + readAtLeast(minElements: number, view: T): Promise>; +} +interface ReadableStreamBYOBReaderReadableStreamBYOBReaderReadOptions { + min?: number; +} +interface ReadableStreamGetReaderOptions { + /** + * Creates a ReadableStreamBYOBReader and locks the stream to the new reader. + * + * This call behaves the same way as the no-argument variant, except that it only works on readable byte streams, i.e. streams which were constructed specifically with the ability to handle "bring your own buffer" reading. The returned BYOB reader provides the ability to directly read individual chunks from the stream via its read() method, into developer-supplied buffers, allowing more precise control over allocation. + */ + mode: "byob"; +} +/** + * The **`ReadableStreamBYOBRequest`** interface of the Streams API represents a "pull request" for data from an underlying source that will made as a zero-copy transfer to a consumer (bypassing the stream's internal queues). + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamBYOBRequest) + */ +declare abstract class ReadableStreamBYOBRequest { + /** + * The **`view`** getter property of the ReadableStreamBYOBRequest interface returns the current view. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamBYOBRequest/view) + */ + get view(): Uint8Array | null; + /** + * The **`respond()`** method of the ReadableStreamBYOBRequest interface is used to signal to the associated readable byte stream that the specified number of bytes were written into the ReadableStreamBYOBRequest.view. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamBYOBRequest/respond) + */ + respond(bytesWritten: number): void; + /** + * The **`respondWithNewView()`** method of the ReadableStreamBYOBRequest interface specifies a new view that the consumer of the associated readable byte stream should write to instead of ReadableStreamBYOBRequest.view. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamBYOBRequest/respondWithNewView) + */ + respondWithNewView(view: ArrayBuffer | ArrayBufferView): void; + get atLeast(): number | null; +} +/** + * The **`ReadableStreamDefaultController`** interface of the Streams API represents a controller allowing control of a ReadableStream's state and internal queue. Default controllers are for streams that are not byte streams. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamDefaultController) + */ +declare abstract class ReadableStreamDefaultController { + /** + * The **`desiredSize`** read-only property of the ReadableStreamDefaultController interface returns the desired size required to fill the stream's internal queue. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamDefaultController/desiredSize) + */ + get desiredSize(): number | null; + /** + * The **`close()`** method of the ReadableStreamDefaultController interface closes the associated stream. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamDefaultController/close) + */ + close(): void; + /** + * The **`enqueue()`** method of the ReadableStreamDefaultController interface enqueues a given chunk in the associated stream. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamDefaultController/enqueue) + */ + enqueue(chunk?: R): void; + /** + * The **`error()`** method of the ReadableStreamDefaultController interface causes any future interactions with the associated stream to error. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamDefaultController/error) + */ + error(reason: any): void; +} +/** + * The **`ReadableByteStreamController`** interface of the Streams API represents a controller for a readable byte stream. It allows control of the state and internal queue of a ReadableStream with an underlying byte source, and enables efficient zero-copy transfer of data from the underlying source to a consumer when the stream's internal queue is empty. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableByteStreamController) + */ +declare abstract class ReadableByteStreamController { + /** + * The **`byobRequest`** read-only property of the ReadableByteStreamController interface returns the current BYOB request, or null if there are no pending requests. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableByteStreamController/byobRequest) + */ + get byobRequest(): ReadableStreamBYOBRequest | null; + /** + * The **`desiredSize`** read-only property of the ReadableByteStreamController interface returns the number of bytes required to fill the stream's internal queue to its "desired size". + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableByteStreamController/desiredSize) + */ + get desiredSize(): number | null; + /** + * The **`close()`** method of the ReadableByteStreamController interface closes the associated stream. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableByteStreamController/close) + */ + close(): void; + /** + * The **`enqueue()`** method of the ReadableByteStreamController interface enqueues a given chunk on the associated readable byte stream (the chunk is transferred into the stream's internal queues). + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableByteStreamController/enqueue) + */ + enqueue(chunk: ArrayBuffer | ArrayBufferView): void; + /** + * The **`error()`** method of the ReadableByteStreamController interface causes any future interactions with the associated stream to error with the specified reason. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableByteStreamController/error) + */ + error(reason: any): void; +} +/** + * The **`WritableStreamDefaultController`** interface of the Streams API represents a controller allowing control of a WritableStream's state. When constructing a WritableStream, the underlying sink is given a corresponding WritableStreamDefaultController instance to manipulate. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStreamDefaultController) + */ +declare abstract class WritableStreamDefaultController { + /** + * The read-only **`signal`** property of the WritableStreamDefaultController interface returns the AbortSignal associated with the controller. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStreamDefaultController/signal) + */ + get signal(): AbortSignal; + /** + * The **`error()`** method of the WritableStreamDefaultController interface causes any future interactions with the associated stream to error. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStreamDefaultController/error) + */ + error(reason?: any): void; +} +/** + * The **`TransformStreamDefaultController`** interface of the Streams API provides methods to manipulate the associated ReadableStream and WritableStream. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TransformStreamDefaultController) + */ +declare abstract class TransformStreamDefaultController { + /** + * The **`desiredSize`** read-only property of the TransformStreamDefaultController interface returns the desired size to fill the queue of the associated ReadableStream. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TransformStreamDefaultController/desiredSize) + */ + get desiredSize(): number | null; + /** + * The **`enqueue()`** method of the TransformStreamDefaultController interface enqueues the given chunk in the readable side of the stream. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TransformStreamDefaultController/enqueue) + */ + enqueue(chunk?: O): void; + /** + * The **`error()`** method of the TransformStreamDefaultController interface errors both sides of the stream. Any further interactions with it will fail with the given error message, and any chunks in the queue will be discarded. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TransformStreamDefaultController/error) + */ + error(reason: any): void; + /** + * The **`terminate()`** method of the TransformStreamDefaultController interface closes the readable side and errors the writable side of the stream. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TransformStreamDefaultController/terminate) + */ + terminate(): void; +} +interface ReadableWritablePair { + readable: ReadableStream; + /** + * Provides a convenient, chainable way of piping this readable stream through a transform stream (or any other { writable, readable } pair). It simply pipes the stream into the writable side of the supplied pair, and returns the readable side for further use. + * + * Piping a stream will lock it for the duration of the pipe, preventing any other consumer from acquiring a reader. + */ + writable: WritableStream; +} +/** + * The **`WritableStream`** interface of the Streams API provides a standard abstraction for writing streaming data to a destination, known as a sink. This object comes with built-in backpressure and queuing. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStream) + */ +declare class WritableStream { + constructor(underlyingSink?: UnderlyingSink, queuingStrategy?: QueuingStrategy); + /** + * The **`locked`** read-only property of the WritableStream interface returns a boolean indicating whether the WritableStream is locked to a writer. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStream/locked) + */ + get locked(): boolean; + /** + * The **`abort()`** method of the WritableStream interface aborts the stream, signaling that the producer can no longer successfully write to the stream and it is to be immediately moved to an error state, with any queued writes discarded. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStream/abort) + */ + abort(reason?: any): Promise; + /** + * The **`close()`** method of the WritableStream interface closes the associated stream. All chunks written before this method is called are sent before the returned promise is fulfilled. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStream/close) + */ + close(): Promise; + /** + * The **`getWriter()`** method of the WritableStream interface returns a new instance of WritableStreamDefaultWriter and locks the stream to that instance. While the stream is locked, no other writer can be acquired until this one is released. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStream/getWriter) + */ + getWriter(): WritableStreamDefaultWriter; +} +/** + * The **`WritableStreamDefaultWriter`** interface of the Streams API is the object returned by WritableStream.getWriter() and once created locks the writer to the WritableStream ensuring that no other streams can write to the underlying sink. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStreamDefaultWriter) + */ +declare class WritableStreamDefaultWriter { + constructor(stream: WritableStream); + /** + * The **`closed`** read-only property of the WritableStreamDefaultWriter interface returns a Promise that fulfills if the stream becomes closed, or rejects if the stream errors or the writer's lock is released. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStreamDefaultWriter/closed) + */ + get closed(): Promise; + /** + * The **`ready`** read-only property of the WritableStreamDefaultWriter interface returns a Promise that resolves when the desired size of the stream's internal queue transitions from non-positive to positive, signaling that it is no longer applying backpressure. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStreamDefaultWriter/ready) + */ + get ready(): Promise; + /** + * The **`desiredSize`** read-only property of the WritableStreamDefaultWriter interface returns the desired size required to fill the stream's internal queue. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStreamDefaultWriter/desiredSize) + */ + get desiredSize(): number | null; + /** + * The **`abort()`** method of the WritableStreamDefaultWriter interface aborts the stream, signaling that the producer can no longer successfully write to the stream and it is to be immediately moved to an error state, with any queued writes discarded. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStreamDefaultWriter/abort) + */ + abort(reason?: any): Promise; + /** + * The **`close()`** method of the WritableStreamDefaultWriter interface closes the associated writable stream. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStreamDefaultWriter/close) + */ + close(): Promise; + /** + * The **`write()`** method of the WritableStreamDefaultWriter interface writes a passed chunk of data to a WritableStream and its underlying sink, then returns a Promise that resolves to indicate the success or failure of the write operation. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStreamDefaultWriter/write) + */ + write(chunk?: W): Promise; + /** + * The **`releaseLock()`** method of the WritableStreamDefaultWriter interface releases the writer's lock on the corresponding stream. After the lock is released, the writer is no longer active. If the associated stream is errored when the lock is released, the writer will appear errored in the same way from now on; otherwise, the writer will appear closed. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStreamDefaultWriter/releaseLock) + */ + releaseLock(): void; +} +/** + * The **`TransformStream`** interface of the Streams API represents a concrete implementation of the pipe chain transform stream concept. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TransformStream) + */ +declare class TransformStream { + constructor(transformer?: Transformer, writableStrategy?: QueuingStrategy, readableStrategy?: QueuingStrategy); + /** + * The **`readable`** read-only property of the TransformStream interface returns the ReadableStream instance controlled by this TransformStream. This stream emits the transformed output data. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TransformStream/readable) + */ + get readable(): ReadableStream; + /** + * The **`writable`** read-only property of the TransformStream interface returns the WritableStream instance controlled by this TransformStream. This stream accepts input data that will be transformed and emitted to the readable stream. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TransformStream/writable) + */ + get writable(): WritableStream; +} +declare class FixedLengthStream extends IdentityTransformStream { + constructor(expectedLength: number | bigint, queuingStrategy?: IdentityTransformStreamQueuingStrategy); +} +declare class IdentityTransformStream extends TransformStream { + constructor(queuingStrategy?: IdentityTransformStreamQueuingStrategy); +} +interface IdentityTransformStreamQueuingStrategy { + highWaterMark?: (number | bigint); +} +interface ReadableStreamValuesOptions { + preventCancel?: boolean; +} +/** + * The **`CompressionStream`** interface of the Compression Streams API compresses a stream of data. It implements the same shape as a TransformStream, allowing it to be used in ReadableStream.pipeThrough() and similar methods. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CompressionStream) + */ +declare class CompressionStream extends TransformStream { + constructor(format: "gzip" | "deflate" | "deflate-raw"); +} +/** + * The **`DecompressionStream`** interface of the Compression Streams API decompresses a stream of data. It implements the same shape as a TransformStream, allowing it to be used in ReadableStream.pipeThrough() and similar methods. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/DecompressionStream) + */ +declare class DecompressionStream extends TransformStream { + constructor(format: "gzip" | "deflate" | "deflate-raw"); +} +/** + * The **`TextEncoderStream`** interface of the Encoding API converts a stream of strings into bytes in the UTF-8 encoding. It is the streaming equivalent of TextEncoder. It implements the same shape as a TransformStream, allowing it to be used in ReadableStream.pipeThrough() and similar methods. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TextEncoderStream) + */ +declare class TextEncoderStream extends TransformStream { + constructor(); + get encoding(): string; +} +/** + * The **`TextDecoderStream`** interface of the Encoding API converts a stream of text in a binary encoding, such as UTF-8 etc., to a stream of strings. It is the streaming equivalent of TextDecoder. It implements the same shape as a TransformStream, allowing it to be used in ReadableStream.pipeThrough() and similar methods. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TextDecoderStream) + */ +declare class TextDecoderStream extends TransformStream { + constructor(label?: string, options?: TextDecoderStreamTextDecoderStreamInit); + get encoding(): string; + get fatal(): boolean; + get ignoreBOM(): boolean; +} +interface TextDecoderStreamTextDecoderStreamInit { + fatal?: boolean; + ignoreBOM?: boolean; +} +/** + * The **`ByteLengthQueuingStrategy`** interface of the Streams API provides a built-in byte length queuing strategy that can be used when constructing streams. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ByteLengthQueuingStrategy) + */ +declare class ByteLengthQueuingStrategy implements QueuingStrategy { + constructor(init: QueuingStrategyInit); + /** + * The read-only **`ByteLengthQueuingStrategy.highWaterMark`** property returns the total number of bytes that can be contained in the internal queue before backpressure is applied. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ByteLengthQueuingStrategy/highWaterMark) + */ + get highWaterMark(): number; + /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/ByteLengthQueuingStrategy/size) */ + get size(): (chunk?: any) => number; +} +/** + * The **`CountQueuingStrategy`** interface of the Streams API provides a built-in chunk counting queuing strategy that can be used when constructing streams. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CountQueuingStrategy) + */ +declare class CountQueuingStrategy implements QueuingStrategy { + constructor(init: QueuingStrategyInit); + /** + * The read-only **`CountQueuingStrategy.highWaterMark`** property returns the total number of chunks that can be contained in the internal queue before backpressure is applied. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CountQueuingStrategy/highWaterMark) + */ + get highWaterMark(): number; + /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/CountQueuingStrategy/size) */ + get size(): (chunk?: any) => number; +} +interface QueuingStrategyInit { + /** + * Creates a new ByteLengthQueuingStrategy with the provided high water mark. + * + * Note that the provided high water mark will not be validated ahead of time. Instead, if it is negative, NaN, or not a number, the resulting ByteLengthQueuingStrategy will cause the corresponding stream constructor to throw. + */ + highWaterMark: number; +} +interface TracePreviewInfo { + id: string; + slug: string; + name: string; +} +interface ScriptVersion { + id?: string; + tag?: string; + message?: string; +} +declare abstract class TailEvent extends ExtendableEvent { + readonly events: TraceItem[]; + readonly traces: TraceItem[]; +} +interface TraceItem { + readonly event: (TraceItemFetchEventInfo | TraceItemJsRpcEventInfo | TraceItemConnectEventInfo | TraceItemScheduledEventInfo | TraceItemAlarmEventInfo | TraceItemQueueEventInfo | TraceItemEmailEventInfo | TraceItemTailEventInfo | TraceItemCustomEventInfo | TraceItemHibernatableWebSocketEventInfo) | null; + readonly eventTimestamp: number | null; + readonly logs: TraceLog[]; + readonly exceptions: TraceException[]; + readonly diagnosticsChannelEvents: TraceDiagnosticChannelEvent[]; + readonly scriptName: string | null; + readonly entrypoint?: string; + readonly scriptVersion?: ScriptVersion; + readonly dispatchNamespace?: string; + readonly scriptTags?: string[]; + readonly tailAttributes?: Record; + readonly preview?: TracePreviewInfo; + readonly durableObjectId?: string; + readonly outcome: string; + readonly executionModel: string; + readonly truncated: boolean; + readonly cpuTime: number; + readonly wallTime: number; +} +interface TraceItemAlarmEventInfo { + readonly scheduledTime: Date; +} +interface TraceItemConnectEventInfo { +} +interface TraceItemCustomEventInfo { +} +interface TraceItemScheduledEventInfo { + readonly scheduledTime: number; + readonly cron: string; +} +interface TraceItemQueueEventInfo { + readonly queue: string; + readonly batchSize: number; +} +interface TraceItemEmailEventInfo { + readonly mailFrom: string; + readonly rcptTo: string; + readonly rawSize: number; +} +interface TraceItemTailEventInfo { + readonly consumedEvents: TraceItemTailEventInfoTailItem[]; +} +interface TraceItemTailEventInfoTailItem { + readonly scriptName: string | null; +} +interface TraceItemFetchEventInfo { + readonly response?: TraceItemFetchEventInfoResponse; + readonly request: TraceItemFetchEventInfoRequest; +} +interface TraceItemFetchEventInfoRequest { + readonly cf?: any; + readonly headers: Record; + readonly method: string; + readonly url: string; + getUnredacted(): TraceItemFetchEventInfoRequest; +} +interface TraceItemFetchEventInfoResponse { + readonly status: number; +} +interface TraceItemJsRpcEventInfo { + readonly rpcMethod: string; +} +interface TraceItemHibernatableWebSocketEventInfo { + readonly getWebSocketEvent: TraceItemHibernatableWebSocketEventInfoMessage | TraceItemHibernatableWebSocketEventInfoClose | TraceItemHibernatableWebSocketEventInfoError; +} +interface TraceItemHibernatableWebSocketEventInfoMessage { + readonly webSocketEventType: string; +} +interface TraceItemHibernatableWebSocketEventInfoClose { + readonly webSocketEventType: string; + readonly code: number; + readonly wasClean: boolean; +} +interface TraceItemHibernatableWebSocketEventInfoError { + readonly webSocketEventType: string; +} +interface TraceLog { + readonly timestamp: number; + readonly level: string; + readonly message: any; + readonly errorInfo?: (TraceLogErrorInfo | null)[]; +} +interface TraceLogErrorInfo { + name: string; + message: string; + stack?: string; +} +interface TraceException { + readonly timestamp: number; + readonly message: string; + readonly name: string; + readonly stack?: string; +} +interface TraceDiagnosticChannelEvent { + readonly timestamp: number; + readonly channel: string; + readonly message: any; +} +interface TraceMetrics { + readonly cpuTime: number; + readonly wallTime: number; +} +interface UnsafeTraceMetrics { + fromTrace(item: TraceItem): TraceMetrics; +} +/** + * The **`URL`** interface is used to parse, construct, normalize, and encode URLs. It works by providing properties which allow you to easily read and modify the components of a URL. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL) + */ +declare class URL { + constructor(url: string | URL, base?: string | URL); + /** + * The **`origin`** read-only property of the URL interface returns a string containing the Unicode serialization of the origin of the represented URL. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/origin) + */ + get origin(): string; + /** + * The **`href`** property of the URL interface is a string containing the whole URL. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/href) + */ + get href(): string; + /** + * The **`href`** property of the URL interface is a string containing the whole URL. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/href) + */ + set href(value: string); + /** + * The **`protocol`** property of the URL interface is a string containing the protocol or scheme of the URL, including the final ":". + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/protocol) + */ + get protocol(): string; + /** + * The **`protocol`** property of the URL interface is a string containing the protocol or scheme of the URL, including the final ":". + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/protocol) + */ + set protocol(value: string); + /** + * The **`username`** property of the URL interface is a string containing the username component of the URL. If the URL does not have a username, this property contains an empty string, "". + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/username) + */ + get username(): string; + /** + * The **`username`** property of the URL interface is a string containing the username component of the URL. If the URL does not have a username, this property contains an empty string, "". + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/username) + */ + set username(value: string); + /** + * The **`password`** property of the URL interface is a string containing the password component of the URL. If the URL does not have a password, this property contains an empty string, "". + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/password) + */ + get password(): string; + /** + * The **`password`** property of the URL interface is a string containing the password component of the URL. If the URL does not have a password, this property contains an empty string, "". + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/password) + */ + set password(value: string); + /** + * The **`host`** property of the URL interface is a string containing the host, which is the hostname, and then, if the port of the URL is nonempty, a ":", followed by the port of the URL. If the URL does not have a hostname, this property contains an empty string, "". + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/host) + */ + get host(): string; + /** + * The **`host`** property of the URL interface is a string containing the host, which is the hostname, and then, if the port of the URL is nonempty, a ":", followed by the port of the URL. If the URL does not have a hostname, this property contains an empty string, "". + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/host) + */ + set host(value: string); + /** + * The **`hostname`** property of the URL interface is a string containing either the domain name or IP address of the URL. If the URL does not have a hostname, this property contains an empty string, "". IPv4 and IPv6 addresses are normalized, such as stripping leading zeros, and domain names are converted to IDN. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/hostname) + */ + get hostname(): string; + /** + * The **`hostname`** property of the URL interface is a string containing either the domain name or IP address of the URL. If the URL does not have a hostname, this property contains an empty string, "". IPv4 and IPv6 addresses are normalized, such as stripping leading zeros, and domain names are converted to IDN. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/hostname) + */ + set hostname(value: string); + /** + * The **`port`** property of the URL interface is a string containing the port number of the URL. If the port is the default for the protocol (80 for ws: and http:, 443 for wss: and https:, and 21 for ftp:), this property contains an empty string, "". + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/port) + */ + get port(): string; + /** + * The **`port`** property of the URL interface is a string containing the port number of the URL. If the port is the default for the protocol (80 for ws: and http:, 443 for wss: and https:, and 21 for ftp:), this property contains an empty string, "". + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/port) + */ + set port(value: string); + /** + * The **`pathname`** property of the URL interface represents a location in a hierarchical structure. It is a string constructed from a list of path segments, each of which is prefixed by a / character. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/pathname) + */ + get pathname(): string; + /** + * The **`pathname`** property of the URL interface represents a location in a hierarchical structure. It is a string constructed from a list of path segments, each of which is prefixed by a / character. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/pathname) + */ + set pathname(value: string); + /** + * The **`search`** property of the URL interface is a search string, also called a query string, that is a string containing a "?" followed by the parameters of the URL. If the URL does not have a search query, this property contains an empty string, "". + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/search) + */ + get search(): string; + /** + * The **`search`** property of the URL interface is a search string, also called a query string, that is a string containing a "?" followed by the parameters of the URL. If the URL does not have a search query, this property contains an empty string, "". + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/search) + */ + set search(value: string); + /** + * The **`hash`** property of the URL interface is a string containing a "#" followed by the fragment identifier of the URL. If the URL does not have a fragment identifier, this property contains an empty string, "". + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/hash) + */ + get hash(): string; + /** + * The **`hash`** property of the URL interface is a string containing a "#" followed by the fragment identifier of the URL. If the URL does not have a fragment identifier, this property contains an empty string, "". + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/hash) + */ + set hash(value: string); + /** + * The **`searchParams`** read-only property of the URL interface returns a URLSearchParams object allowing access to the GET decoded query arguments contained in the URL. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/searchParams) + */ + get searchParams(): URLSearchParams; + /** + * The **`toJSON()`** method of the URL interface returns a string containing a serialized version of the URL, although in practice it seems to have the same effect as URL.toString(). + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/toJSON) + */ + toJSON(): string; + /*function toString() { [native code] }*/ + toString(): string; + /** + * The **`URL.canParse()`** static method of the URL interface returns a boolean indicating whether or not an absolute URL, or a relative URL combined with a base URL, are parsable and valid. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/canParse_static) + */ + static canParse(url: string, base?: string): boolean; + /** + * The **`URL.parse()`** static method of the URL interface returns a newly created URL object representing the URL defined by the parameters. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/parse_static) + */ + static parse(url: string, base?: string): URL | null; + /** + * The **`createObjectURL()`** static method of the URL interface creates a string containing a blob URL pointing to the object given in the parameter. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/createObjectURL_static) + */ + static createObjectURL(object: File | Blob): string; + /** + * The **`revokeObjectURL()`** static method of the URL interface releases an existing object URL which was previously created by calling URL.createObjectURL(). + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/revokeObjectURL_static) + */ + static revokeObjectURL(object_url: string): void; +} +/** + * The **`URLSearchParams`** interface defines utility methods to work with the query string of a URL. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URLSearchParams) + */ +declare class URLSearchParams { + constructor(init?: (Iterable> | Record | string)); + /** + * The **`size`** read-only property of the URLSearchParams interface indicates the total number of search parameter entries. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URLSearchParams/size) + */ + get size(): number; + /** + * The **`append()`** method of the URLSearchParams interface appends a specified key/value pair as a new search parameter. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URLSearchParams/append) + */ + append(name: string, value: string): void; + /** + * The **`delete()`** method of the URLSearchParams interface deletes specified parameters and their associated value(s) from the list of all search parameters. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URLSearchParams/delete) + */ + delete(name: string, value?: string): void; + /** + * The **`get()`** method of the URLSearchParams interface returns the first value associated to the given search parameter. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URLSearchParams/get) + */ + get(name: string): string | null; + /** + * The **`getAll()`** method of the URLSearchParams interface returns all the values associated with a given search parameter as an array. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URLSearchParams/getAll) + */ + getAll(name: string): string[]; + /** + * The **`has()`** method of the URLSearchParams interface returns a boolean value that indicates whether the specified parameter is in the search parameters. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URLSearchParams/has) + */ + has(name: string, value?: string): boolean; + /** + * The **`set()`** method of the URLSearchParams interface sets the value associated with a given search parameter to the given value. If there were several matching values, this method deletes the others. If the search parameter doesn't exist, this method creates it. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URLSearchParams/set) + */ + set(name: string, value: string): void; + /** + * The **`URLSearchParams.sort()`** method sorts all key/value pairs contained in this object in place and returns undefined. Key/value pairs are sorted by the values of the UTF-16 code units of the keys. This method uses a stable sorting algorithm (i.e., the relative order between key/value pairs with equal keys will be preserved). + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URLSearchParams/sort) + */ + sort(): void; + entries(): IterableIterator<[ + key: string, + value: string + ]>; + keys(): IterableIterator; + values(): IterableIterator; + forEach(callback: (this: This, value: string, key: string, parent: URLSearchParams) => void, thisArg?: This): void; + /*function toString() { [native code] }*/ + toString(): string; + [Symbol.iterator](): IterableIterator<[ + key: string, + value: string + ]>; +} +/** + * The **`URLPattern`** interface of the URL Pattern API matches URLs or parts of URLs against a pattern. The pattern can contain capturing groups that extract parts of the matched URL. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URLPattern) + */ +declare class URLPattern { + constructor(input?: (string | URLPatternInit), baseURL?: (string | URLPatternOptions), patternOptions?: URLPatternOptions); + /** + * The **`protocol`** read-only property of the URLPattern interface is a string containing the pattern used to match the protocol part of a URL. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URLPattern/protocol) + */ + get protocol(): string; + /** + * The **`username`** read-only property of the URLPattern interface is a string containing the pattern used to match the username part of a URL. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URLPattern/username) + */ + get username(): string; + /** + * The **`password`** read-only property of the URLPattern interface is a string containing the pattern used to match the password part of a URL. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URLPattern/password) + */ + get password(): string; + /** + * The **`hostname`** read-only property of the URLPattern interface is a string containing the pattern used to match the hostname part of a URL. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URLPattern/hostname) + */ + get hostname(): string; + /** + * The **`port`** read-only property of the URLPattern interface is a string containing the pattern used to match the port part of a URL. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URLPattern/port) + */ + get port(): string; + /** + * The **`pathname`** read-only property of the URLPattern interface is a string containing the pattern used to match the pathname part of a URL. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URLPattern/pathname) + */ + get pathname(): string; + /** + * The **`search`** read-only property of the URLPattern interface is a string containing the pattern used to match the search part of a URL. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URLPattern/search) + */ + get search(): string; + /** + * The **`hash`** read-only property of the URLPattern interface is a string containing the pattern used to match the fragment part of a URL. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URLPattern/hash) + */ + get hash(): string; + /** + * The **`hasRegExpGroups`** read-only property of the URLPattern interface is a boolean indicating whether or not any of the URLPattern components contain regular expression capturing groups. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URLPattern/hasRegExpGroups) + */ + get hasRegExpGroups(): boolean; + /** + * The **`test()`** method of the URLPattern interface takes a URL string or object of URL parts, and returns a boolean indicating if the given input matches the current pattern. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URLPattern/test) + */ + test(input?: (string | URLPatternInit), baseURL?: string): boolean; + /** + * The **`exec()`** method of the URLPattern interface takes a URL or object of URL parts, and returns either an object containing the results of matching the URL to the pattern, or null if the URL does not match the pattern. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URLPattern/exec) + */ + exec(input?: (string | URLPatternInit), baseURL?: string): URLPatternResult | null; +} +interface URLPatternInit { + protocol?: string; + username?: string; + password?: string; + hostname?: string; + port?: string; + pathname?: string; + search?: string; + hash?: string; + baseURL?: string; +} +interface URLPatternComponentResult { + input: string; + groups: Record; +} +interface URLPatternResult { + inputs: (string | URLPatternInit)[]; + protocol: URLPatternComponentResult; + username: URLPatternComponentResult; + password: URLPatternComponentResult; + hostname: URLPatternComponentResult; + port: URLPatternComponentResult; + pathname: URLPatternComponentResult; + search: URLPatternComponentResult; + hash: URLPatternComponentResult; +} +interface URLPatternOptions { + ignoreCase?: boolean; +} +/** + * A **`CloseEvent`** is sent to clients using WebSockets when the connection is closed. This is delivered to the listener indicated by the WebSocket object's onclose attribute. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CloseEvent) + */ +declare class CloseEvent extends Event { + constructor(type: string, initializer?: CloseEventInit); + /** + * The **`code`** read-only property of the CloseEvent interface returns a WebSocket connection close code indicating the reason the connection was closed. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CloseEvent/code) + */ + readonly code: number; + /** + * The **`reason`** read-only property of the CloseEvent interface returns the WebSocket connection close reason the server gave for closing the connection; that is, a concise human-readable prose explanation for the closure. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CloseEvent/reason) + */ + readonly reason: string; + /** + * The **`wasClean`** read-only property of the CloseEvent interface returns true if the connection closed cleanly. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CloseEvent/wasClean) + */ + readonly wasClean: boolean; +} +interface CloseEventInit { + bubbles?: boolean; + cancelable?: boolean; + composed?: boolean; + code?: number; + reason?: string; + wasClean?: boolean; +} +type WebSocketEventMap = { + close: CloseEvent; + message: MessageEvent; + open: Event; + error: ErrorEvent; +}; +/** + * The **`WebSocket`** object provides the API for creating and managing a WebSocket connection to a server, as well as for sending and receiving data on the connection. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WebSocket) + */ +declare var WebSocket: { + prototype: WebSocket; + new (url: string, protocols?: (string[] | string)): WebSocket; + readonly READY_STATE_CONNECTING: number; + readonly CONNECTING: number; + readonly READY_STATE_OPEN: number; + readonly OPEN: number; + readonly READY_STATE_CLOSING: number; + readonly CLOSING: number; + readonly READY_STATE_CLOSED: number; + readonly CLOSED: number; +}; +/** + * The **`WebSocket`** object provides the API for creating and managing a WebSocket connection to a server, as well as for sending and receiving data on the connection. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WebSocket) + */ +interface WebSocket extends EventTarget { + accept(options?: WebSocketAcceptOptions): void; + /** + * The **`WebSocket.send()`** method enqueues the specified data to be transmitted to the server over the WebSocket connection, increasing the value of bufferedAmount by the number of bytes needed to contain the data. If the data can't be sent (for example, because it needs to be buffered but the buffer is full), the socket is closed automatically. The browser will throw an exception if you call send() when the connection is in the CONNECTING state. If you call send() when the connection is in the CLOSING or CLOSED states, the browser will silently discard the data. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WebSocket/send) + */ + send(message: (ArrayBuffer | ArrayBufferView) | string): void; + /** + * The **`WebSocket.close()`** method closes the WebSocket connection or connection attempt, if any. If the connection is already CLOSED, this method does nothing. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WebSocket/close) + */ + close(code?: number, reason?: string): void; + serializeAttachment(attachment: any): void; + deserializeAttachment(): any | null; + /** + * The **`WebSocket.readyState`** read-only property returns the current state of the WebSocket connection. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WebSocket/readyState) + */ + readyState: number; + /** + * The **`WebSocket.url`** read-only property returns the absolute URL of the WebSocket as resolved by the constructor. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WebSocket/url) + */ + url: string | null; + /** + * The **`WebSocket.protocol`** read-only property returns the name of the sub-protocol the server selected; this will be one of the strings specified in the protocols parameter when creating the WebSocket object, or the empty string if no connection is established. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WebSocket/protocol) + */ + protocol: string | null; + /** + * The **`WebSocket.extensions`** read-only property returns the extensions selected by the server. This is currently only the empty string or a list of extensions as negotiated by the connection. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WebSocket/extensions) + */ + extensions: string | null; + /** + * The **`WebSocket.binaryType`** property controls the type of binary data being received over the WebSocket connection. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WebSocket/binaryType) + */ + binaryType: "blob" | "arraybuffer"; +} +interface WebSocketAcceptOptions { + /** + * When set to `true`, receiving a server-initiated WebSocket Close frame will not + * automatically send a reciprocal Close frame, leaving the connection in a half-open + * state. This is useful for proxying scenarios where you need to coordinate closing + * both sides independently. Defaults to `false` when the + * `no_web_socket_half_open_by_default` compatibility flag is enabled. + */ + allowHalfOpen?: boolean; +} +declare const WebSocketPair: { + new (): { + 0: WebSocket; + 1: WebSocket; + }; +}; +interface SqlStorage { + exec>(query: string, ...bindings: any[]): SqlStorageCursor; + get databaseSize(): number; + Cursor: typeof SqlStorageCursor; + Statement: typeof SqlStorageStatement; +} +declare abstract class SqlStorageStatement { +} +type SqlStorageValue = ArrayBuffer | string | number | null; +declare abstract class SqlStorageCursor> { + next(): { + done?: false; + value: T; + } | { + done: true; + value?: never; + }; + toArray(): T[]; + one(): T; + raw(): IterableIterator; + columnNames: string[]; + get rowsRead(): number; + get rowsWritten(): number; + [Symbol.iterator](): IterableIterator; +} +interface Socket { + get readable(): ReadableStream; + get writable(): WritableStream; + get closed(): Promise; + get opened(): Promise; + get upgraded(): boolean; + get secureTransport(): "on" | "off" | "starttls"; + close(): Promise; + startTls(options?: TlsOptions): Socket; +} +interface SocketOptions { + secureTransport?: string; + allowHalfOpen: boolean; + highWaterMark?: (number | bigint); +} +interface SocketAddress { + hostname: string; + port: number; +} +interface TlsOptions { + expectedServerHostname?: string; +} +interface SocketInfo { + remoteAddress?: string; + localAddress?: string; +} +/** + * The **`EventSource`** interface is web content's interface to server-sent events. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventSource) + */ +declare class EventSource extends EventTarget { + constructor(url: string, init?: EventSourceEventSourceInit); + /** + * The **`close()`** method of the EventSource interface closes the connection, if one is made, and sets the EventSource.readyState attribute to 2 (closed). + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventSource/close) + */ + close(): void; + /** + * The **`url`** read-only property of the EventSource interface returns a string representing the URL of the source. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventSource/url) + */ + get url(): string; + /** + * The **`withCredentials`** read-only property of the EventSource interface returns a boolean value indicating whether the EventSource object was instantiated with CORS credentials set. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventSource/withCredentials) + */ + get withCredentials(): boolean; + /** + * The **`readyState`** read-only property of the EventSource interface returns a number representing the state of the connection. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventSource/readyState) + */ + get readyState(): number; + /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventSource/open_event) */ + get onopen(): any | null; + /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventSource/open_event) */ + set onopen(value: any | null); + /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventSource/message_event) */ + get onmessage(): any | null; + /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventSource/message_event) */ + set onmessage(value: any | null); + /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventSource/error_event) */ + get onerror(): any | null; + /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventSource/error_event) */ + set onerror(value: any | null); + static readonly CONNECTING: number; + static readonly OPEN: number; + static readonly CLOSED: number; + static from(stream: ReadableStream): EventSource; +} +interface EventSourceEventSourceInit { + withCredentials?: boolean; + fetcher?: Fetcher; +} +interface ExecOutput { + readonly stdout: ArrayBuffer; + readonly stderr: ArrayBuffer; + readonly exitCode: number; +} +interface ContainerExecOptions { + cwd?: string; + env?: Record; + user?: string; + signal?: AbortSignal; + pty?: boolean | ContainerExecPtyOptions; + stdin?: ReadableStream | "pipe"; + stdout?: "pipe" | "ignore"; + stderr?: "pipe" | "ignore" | "combined"; +} +interface ContainerExecPtyOptions { + cols?: number; + rows?: number; +} +interface ExecProcess { + readonly stdin: WritableStream | null; + readonly stdout: ReadableStream | null; + readonly stderr: ReadableStream | null; + readonly pid: number; + readonly isPty: boolean; + readonly exitCode: Promise; + output(): Promise; + kill(signal?: number): void; + resize(cols: number, rows: number): void; +} +interface Container { + get running(): boolean; + start(options?: ContainerStartupOptions): void; + monitor(): Promise; + destroy(error?: any): Promise; + signal(signo: number): void; + getTcpPort(port: number): Fetcher; + setInactivityTimeout(durationMs: number | bigint): Promise; + interceptOutboundHttp(addr: string, binding: Fetcher): Promise; + interceptAllOutboundHttp(binding: Fetcher): Promise; + snapshotDirectory(options: ContainerDirectorySnapshotOptions): Promise; + snapshotContainer(options: ContainerSnapshotOptions): Promise; + interceptOutboundHttps(addr: string, binding: Fetcher): Promise; + exec(cmd: string[], options?: ContainerExecOptions): Promise; +} +interface ContainerDirectorySnapshot { + id: string; + size: number; + dir: string; + name?: string; +} +interface ContainerDirectorySnapshotOptions { + dir: string; + name?: string; +} +type ContainerDirectorySnapshotRestoreParams = { + snapshot: ContainerDirectorySnapshot; + mountPoint?: string; +} | { + snapshot?: undefined; + mountPoint: string; +}; +interface ContainerSnapshot { + id: string; + size: number; + name?: string; +} +interface ContainerSnapshotRestoreParams { + id: string; +} +interface ContainerSnapshotOptions { + name?: string; +} +type ContainerStartupOptions = { + entrypoint?: string[]; + enableInternet: boolean; + env?: Record; + instance?: "lite" | "standard-1" | "standard-2" | "standard-3" | "standard-4" | ContainerStartResources; + labels?: Record; + directorySnapshots?: ContainerDirectorySnapshotRestoreParams[]; +} & ({ + image: string; + containerSnapshot?: never; +} | { + image?: never; + containerSnapshot?: ContainerSnapshotRestoreParams; +}); +interface ContainerStartResources { + vcpu: number; + memoryMib: number; + diskMb: number; +} +/** + * The **`MessagePort`** interface of the Channel Messaging API represents one of the two ports of a MessageChannel, allowing messages to be sent from one port and listening out for them arriving at the other. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessagePort) + */ +declare abstract class MessagePort extends EventTarget { + /** + * The **`postMessage()`** method of the MessagePort interface sends a message from the port, and optionally, transfers ownership of objects to other browsing contexts. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessagePort/postMessage) + */ + postMessage(data?: any, options?: (any[] | MessagePortPostMessageOptions)): void; + /** + * The **`close()`** method of the MessagePort interface disconnects the port, so it is no longer active. This stops the flow of messages to that port. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessagePort/close) + */ + close(): void; + /** + * The **`start()`** method of the MessagePort interface starts the sending of messages queued on the port. This method is only needed when using EventTarget.addEventListener; it is implied when using onmessage. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessagePort/start) + */ + start(): void; + get onmessage(): any | null; + set onmessage(value: any | null); +} +/** + * The **`MessageChannel`** interface of the Channel Messaging API allows us to create a new message channel and send data through it via its two MessagePort properties. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessageChannel) + */ +declare class MessageChannel { + constructor(); + /** + * The **`port1`** read-only property of the MessageChannel interface returns the first port of the message channel — the port attached to the context that originated the channel. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessageChannel/port1) + */ + readonly port1: MessagePort; + /** + * The **`port2`** read-only property of the MessageChannel interface returns the second port of the message channel — the port attached to the context at the other end of the channel, which the message is initially sent to. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessageChannel/port2) + */ + readonly port2: MessagePort; +} +interface MessagePortPostMessageOptions { + transfer?: any[]; +} +type LoopbackForExport Rpc.EntrypointBranded) | ExportedHandler | undefined = undefined> = T extends new (...args: any[]) => Rpc.WorkerEntrypointBranded ? LoopbackServiceStub> : T extends new (...args: any[]) => Rpc.DurableObjectBranded ? LoopbackDurableObjectClass> : T extends ExportedHandler ? LoopbackServiceStub : undefined; +type LoopbackServiceStub = Fetcher & (T extends CloudflareWorkersModule.WorkerEntrypoint ? (opts: { + props?: Props; +}) => Fetcher : (opts: { + props?: any; +}) => Fetcher); +type LoopbackDurableObjectClass = DurableObjectClass & (T extends CloudflareWorkersModule.DurableObject ? (opts: { + props?: Props; +}) => DurableObjectClass : (opts: { + props?: any; +}) => DurableObjectClass); +interface LoopbackDurableObjectNamespace extends DurableObjectNamespace { +} +interface LoopbackColoLocalActorNamespace extends ColoLocalActorNamespace { +} +interface SyncKvStorage { + get(key: string): T | undefined; + list(options?: SyncKvListOptions): Iterable<[ + string, + T + ]>; + put(key: string, value: T): void; + delete(key: string): boolean; +} +interface SyncKvListOptions { + start?: string; + startAfter?: string; + end?: string; + prefix?: string; + reverse?: boolean; + limit?: number; +} +interface WorkerStub { + getEntrypoint(name?: string, options?: WorkerStubEntrypointOptions): Fetcher; + getDurableObjectClass(name?: string, options?: WorkerStubEntrypointOptions): DurableObjectClass; +} +interface WorkerStubEntrypointOptions { + props?: any; + limits?: workerdResourceLimits; +} +interface WorkerLoader { + get(name: string | null, getCode: () => WorkerLoaderWorkerCode | Promise): WorkerStub; + load(code: WorkerLoaderWorkerCode): WorkerStub; +} +interface WorkerLoaderModule { + js?: string; + cjs?: string; + text?: string; + data?: ArrayBuffer; + json?: any; + py?: string; + wasm?: ArrayBuffer | ArrayBufferView | WebAssembly.Module; +} +interface WorkerLoaderWorkerCode { + compatibilityDate: string; + compatibilityFlags?: string[]; + allowExperimental?: boolean; + limits?: workerdResourceLimits; + mainModule: string; + modules: Record; + env?: any; + globalOutbound?: (Fetcher | null); + tails?: Fetcher[]; + streamingTails?: Fetcher[]; +} +interface workerdResourceLimits { + cpuMs?: number; + subRequests?: number; +} +/** +* The Workers runtime supports a subset of the Performance API, used to measure timing and performance, +* as well as timing of subrequests and other operations. +* +* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/performance/) +*/ +declare abstract class Performance { + /* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/performance/#performancetimeorigin) */ + get timeOrigin(): number; + /* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/performance/#performancenow) */ + now(): number; + /** + * The **`toJSON()`** method of the Performance interface is a serializer; it returns a JSON representation of the Performance object. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Performance/toJSON) + */ + toJSON(): object; +} +interface Tracing { + enterSpan(name: string, callback: (span: Span, ...args: A) => T, ...args: A): T; + startActiveSpan(name: string, callback: (span: Span, ...args: A) => T, ...args: A): T; + startSpan(name: string): Span; + Span: typeof Span; +} +declare abstract class Span { + get isTraced(): boolean; + setAttribute(key: string, value: boolean | number | string): this; + setAttributes(attributes: Record): this; + end(): void; +} +/** + * Represents the identity of a user authenticated via Cloudflare Access. + * This matches the result of calling /cdn-cgi/access/get-identity. + * + * The exact structure of the returned object depends on the identity provider + * configuration for the Access application. The fields below represent commonly + * available properties, but additional provider-specific fields may be present. + */ +interface CloudflareAccessIdentity extends Record { + /** The user's email address, if available from the identity provider. */ + email?: string; + /** The user's display name. */ + name?: string; + /** The user's unique identifier. */ + user_uuid?: string; + /** The Cloudflare account ID. */ + account_id?: string; + /** Login timestamp (Unix epoch seconds). */ + iat?: number; + /** The user's IP address at authentication time. */ + ip?: string; + /** Authentication methods used (e.g., "pwd"). */ + amr?: string[]; + /** Identity provider information. */ + idp?: { + id: string; + type: string; + }; + /** Geographic information about where the user authenticated. */ + geo?: { + country: string; + }; + /** Group memberships from the identity provider. */ + groups?: Array<{ + id: string; + name: string; + email?: string; + }>; + /** Device posture check results, keyed by check ID. */ + devicePosture?: Record; + /** True if the user connected via Cloudflare WARP. */ + is_warp?: boolean; + /** True if the user is authenticated via Cloudflare Gateway. */ + is_gateway?: boolean; +} +// ============================================================================ +// Agent Memory +// +// Public type surface for user Workers binding to an Agent Memory namespace. +// ============================================================================ +/** Memory type — every memory is classified into exactly one. */ +type AgentMemoryMemoryType = "fact" | "event" | "instruction" | "task"; +/** Search intensity for recall. */ +type AgentMemoryThinkingLevel = "low" | "medium" | "high"; +/** Response verbosity for recall. */ +type AgentMemoryResponseLength = "short" | "medium" | "long"; +/** A conversation message passed to ingest(). */ +interface AgentMemoryMessage { + role: "system" | "user" | "assistant"; + content: string; + /** Optional message timestamp. */ + timestamp?: Date; +} +/** Raw memory content passed to remember(). */ +interface AgentMemoryIncomingMemory { + /** Raw memory content. The service classifies and summarizes automatically. */ + content: string; + /** Optional session identifier to associate with this memory. */ + sessionId?: string | null | undefined; +} +/** A stored memory returned from remember(), get(), and delete(). */ +interface AgentMemoryMemory { + /** Memory ID. */ + id: string; + /** Memory type. */ + type: AgentMemoryMemoryType; + /** Text summary. */ + summary: string; + /** Memory text. */ + content: string; + /** Session that created this memory. */ + sessionId: string | null; + /** Memory creation time. */ + createdAt: Date; + /** Memory last-update time. */ + updatedAt: Date; +} +/** Single entry in a list() response. Same shape as Memory minus full content. */ +type AgentMemoryMemoryListEntry = Omit; +/** A scored memory candidate in a recall result. */ +interface AgentMemoryScoredCandidate { + /** Candidate ID. */ + id: string; + /** Text summary. */ + summary: string; + /** Session that created this candidate, when known. */ + sessionId: string | null; + /** Relevance score (higher is better). Comparable only within a single query. */ + score: number; +} +/** Options for the ingest() method. */ +interface AgentMemoryIngestOptions { + /** Session identifier to associate with memories created during ingestion. */ + sessionId?: string | null | undefined; +} +/** Options for the getSummary() method. */ +interface AgentMemoryGetSummaryOptions { + /** Session identifier to retrieve session summary for. */ + sessionId?: string | null | undefined; +} +/** Response from the getSummary() method. */ +interface AgentMemoryGetSummaryResponse { + /** Markdown summary. */ + summary: string; +} +/** + * Options for the recall() method. + * + * `referenceDate` accepts a Date object, an ISO-8601 date string + * (YYYY-MM-DD), or a full ISO-8601 datetime string. When provided, this + * date is used as "today" for resolving relative time references + * ("how many days ago", "last week") instead of the server's wall-clock time. + */ +interface AgentMemoryRecallOptions { + /** Recall intensity: "low" (default), "medium", or "high". */ + thinkingLevel?: AgentMemoryThinkingLevel; + /** Response verbosity: "short", "medium" (default), or "long". */ + responseLength?: AgentMemoryResponseLength; + /** Temporal anchor for date arithmetic. */ + referenceDate?: Date | string; +} +/** Response from the recall() method. */ +interface AgentMemoryRecallResult { + /** Number of memories retrieved. */ + count: number; + /** LLM-generated answer synthesizing the matching memories. */ + answer: string; + /** Matching memories ranked by relevance. */ + candidates: AgentMemoryScoredCandidate[]; +} +/** + * Options for the list() method. + * + * `cursor` is the opaque continuation token returned by the previous page; + * pass it back unchanged to fetch the next page. `sessionId` and `type` + * are exact-match filters; combining them is allowed. + */ +interface AgentMemoryListMemoriesOptions { + /** Maximum number of memories to return. Default 20, max 500. */ + limit?: number; + /** Opaque cursor from a previous page. */ + cursor?: string; + /** Exact-match session filter. */ + sessionId?: string; + /** Exact-match memory-type filter. */ + type?: AgentMemoryMemoryType; +} +/** Response from the list() method. */ +interface AgentMemoryListMemoriesResult { + memories: AgentMemoryMemoryListEntry[]; + /** Continuation cursor; absent when this page exhausted the result set. */ + cursor?: string; +} +/** + * A single Agent Memory profile, scoped to a profile name. + * + * Returned by {@link AgentMemoryNamespace.getProfile}. + */ +declare abstract class AgentMemoryProfile { + /** + * Retrieve a memory by ID. + * + * @param memoryId - ULID of the memory to retrieve. + * @throws if the memory does not exist. + */ + get(memoryId: string): Promise; + /** + * Delete a memory by ID. + * + * Removes the memory and any source messages linked by the memory's + * source message IDs. + * + * @param memoryId - ULID of the memory to delete. + * @throws if the memory does not exist. + */ + delete(memoryId: string): Promise; + /** + * Store a memory in this profile. The content is automatically classified, + * summarized, and indexed. + * + * @param memory - Raw memory content to persist. + */ + remember(memory: AgentMemoryIncomingMemory): Promise; + /** + * Extract memories from a conversation. + * + * @param messages - Conversation messages to extract memories from. + * @param options - Optional ingest options. + */ + ingest(messages: Iterable, options?: AgentMemoryIngestOptions): Promise; + /** + * Get a profile summary. + * + * @param options - Optional getSummary options. + */ + getSummary(options?: AgentMemoryGetSummaryOptions): Promise; + /** + * Recall memories in this profile. + * + * @param query - Recall query matched against memory content and keywords. + * @param options - Optional recall parameters. + * @returns Matching memories with relevance scores and a synthesized answer. + */ + recall(query: string, options?: AgentMemoryRecallOptions): Promise; + /** + * List active memories in this profile. + * + * Returns a paginated, filterable view of stored memories. Superseded + * versions are excluded. Use the returned `cursor` (when present) to + * fetch the next page. + * + * @param options - Optional pagination and filter options. + */ + list(options?: AgentMemoryListMemoriesOptions): Promise; + /** + * Soft-delete every memory and message in this profile that is tagged + * with `sessionId`. + * + * Idempotent: deleting a sessionId that has no rows is a no-op. + * + * @param sessionId - Session to delete. + */ + deleteSession(sessionId: string): Promise; +} +/** + * Namespace-level Agent Memory binding. + * + * Used as the type of an `env.MEMORY`-style binding backed by the Agent + * Memory product. + * + * @example + * ```ts + * export default { + * async fetch(_request: Request, env: Env): Promise { + * const profile = await env.MEMORY.getProfile("wrangler-e2e"); + * const summary = await profile.getSummary(); + * return Response.json(summary); + * }, + * }; + * ``` + */ +declare abstract class AgentMemoryNamespace { + /** + * Get a memory profile by name. Profiles are isolated by namespace and + * addressed by a compound key (namespaceId:profileName). + * + * @param profileName - Profile name (validated against naming rules). + * @returns RPC target for interacting with the profile. + */ + getProfile(profileName: string): Promise; + /** + * Soft-delete a profile and schedule deferred purge. Marks all + * memories and messages as deleted. + * + * @param profileName - Name of the profile to delete. + */ + deleteProfile(profileName: string): Promise; +} +// ============ AI Search Error Interfaces ============ +interface AiSearchInternalError extends Error { +} +interface AiSearchNotFoundError extends Error { +} +// ============ AI Search Common Types ============ +/** A single message in a conversation-style search or chat request. */ +type AiSearchMessage = { + role: 'system' | 'developer' | 'user' | 'assistant' | 'tool'; + content: string | null; +}; +/** + * Common shape for `ai_search_options` used by both single-instance and multi-instance requests. + * Contains retrieval, query rewrite, reranking, and cache sub-options. + */ +type AiSearchOptions = { + retrieval?: { + /** Which retrieval backend to use. Defaults to the instance's configured index_method. */ + retrieval_type?: 'vector' | 'keyword' | 'hybrid'; + /** Fusion method for combining vector + keyword results. */ + fusion_method?: 'max' | 'rrf'; + /** How keyword terms are combined: "and" = all terms must match, "or" = any term matches. */ + keyword_match_mode?: 'and' | 'or'; + /** Minimum similarity score (0-1) for a result to be included. Default 0.4. */ + match_threshold?: number; + /** Maximum number of results to return (1-50). Default 10. */ + max_num_results?: number; + /** Vectorize metadata filters applied to the search. */ + filters?: VectorizeVectorMetadataFilter; + /** Number of surrounding chunks to include for context (0-3). Default 0. */ + context_expansion?: number; + /** If true, return only item metadata without chunk text. */ + metadata_only?: boolean; + /** If true (default), return empty results on retrieval failure instead of throwing. */ + return_on_failure?: boolean; + /** Boost results by metadata field values. Max 3 entries. */ + boost_by?: Array<{ + field: string; + direction?: 'asc' | 'desc' | 'exists' | 'not_exists'; + }>; + [key: string]: unknown; + }; + query_rewrite?: { + enabled?: boolean; + model?: string; + rewrite_prompt?: string; + [key: string]: unknown; + }; + reranking?: { + enabled?: boolean; + model?: string; + /** Match threshold (0-1, default 0.4) */ + match_threshold?: number; + [key: string]: unknown; + }; + cache?: { + enabled?: boolean; + cache_threshold?: 'super_strict_match' | 'close_enough' | 'flexible_friend' | 'anything_goes'; + }; + [key: string]: unknown; +}; +// ============ AI Search Request Types ============ +/** + * Request body for single-instance search. + * Exactly one of `query` or `messages` must be provided. + */ +type AiSearchSearchRequest = { + /** Simple query string. */ + query: string; + messages?: never; + ai_search_options?: AiSearchOptions; +} | { + query?: never; + /** Conversation-style input. At least one user message with non-empty content is required. */ + messages: AiSearchMessage[]; + ai_search_options?: AiSearchOptions; +}; +type AiSearchChatCompletionsRequest = { + messages: AiSearchMessage[]; + model?: string; + stream?: boolean; + ai_search_options?: AiSearchOptions; + [key: string]: unknown; +}; +// ============ AI Search Multi-Instance Types (Namespace-Scoped) ============ +/** `ai_search_options` shape for multi-instance requests — requires `instance_ids`. */ +type AiSearchMultiSearchOptions = AiSearchOptions & { + /** Instance IDs to search across (1-10). */ + instance_ids: string[]; +}; +/** + * Request for searching across multiple instances within a namespace. + * `ai_search_options` is required and must include `instance_ids`. + * Exactly one of `query` or `messages` must be provided. + */ +type AiSearchMultiSearchRequest = { + /** Simple query string. */ + query: string; + messages?: never; + ai_search_options: AiSearchMultiSearchOptions; +} | { + query?: never; + /** Conversation-style input. */ + messages: AiSearchMessage[]; + ai_search_options: AiSearchMultiSearchOptions; +}; +/** A search result chunk tagged with the instance it originated from. */ +type AiSearchMultiSearchChunk = AiSearchSearchResponse['chunks'][number] & { + instance_id: string; +}; +/** Describes a per-instance error during a multi-instance operation. */ +type AiSearchMultiSearchError = { + instance_id: string; + message: string; +}; +/** Response from a multi-instance search, with chunks tagged by instance and optional partial-failure errors. */ +type AiSearchMultiSearchResponse = { + search_query: string; + chunks: AiSearchMultiSearchChunk[]; + errors?: AiSearchMultiSearchError[]; +}; +/** Request for chat completions across multiple instances within a namespace. `ai_search_options` is required and must include `instance_ids`. */ +type AiSearchMultiChatCompletionsRequest = Omit & { + ai_search_options: AiSearchMultiSearchOptions; +}; +/** Response from multi-instance chat completions, with chunks tagged by instance and optional partial-failure errors. */ +type AiSearchMultiChatCompletionsResponse = Omit & { + chunks: AiSearchMultiSearchChunk[]; + errors?: AiSearchMultiSearchError[]; +}; +// ============ AI Search Response Types ============ +type AiSearchSearchResponse = { + search_query: string; + chunks: Array<{ + id: string; + type: string; + /** Match score (0-1) */ + score: number; + text: string; + item: { + timestamp?: number; + key: string; + metadata?: Record; + }; + scoring_details?: { + /** Keyword match score (0-1) */ + keyword_score?: number; + /** Vector similarity score (0-1) */ + vector_score?: number; + /** Keyword rank position */ + keyword_rank?: number; + /** Vector rank position */ + vector_rank?: number; + /** Reranking model score */ + reranking_score?: number; + /** Fusion method used to combine results */ + fusion_method?: 'rrf' | 'max'; + [key: string]: unknown; + }; + }>; +}; +type AiSearchChatCompletionsResponse = { + id?: string; + object?: string; + model?: string; + choices: Array<{ + index?: number; + message: { + role: 'system' | 'developer' | 'user' | 'assistant' | 'tool'; + content: string | null; + [key: string]: unknown; + }; + [key: string]: unknown; + }>; + chunks: AiSearchSearchResponse['chunks']; + [key: string]: unknown; +}; +type AiSearchStatsResponse = { + queued?: number; + running?: number; + completed?: number; + error?: number; + skipped?: number; + outdated?: number; + last_activity?: string; + /** Storage engine statistics. */ + engine?: { + vectorize?: { + vectorsCount: number; + dimensions: number; + }; + r2?: { + payloadSizeBytes: number; + metadataSizeBytes: number; + objectCount: number; + }; + }; +}; +// ============ AI Search Instance Info Types ============ +type AiSearchInstanceInfo = { + id: string; + type?: 'r2' | 'web-crawler' | string; + source?: string; + source_params?: unknown; + paused?: boolean; + status?: string; + namespace?: string; + created_at?: string; + modified_at?: string; + token_id?: string; + ai_gateway_id?: string; + rewrite_query?: boolean; + reranking?: boolean; + embedding_model?: string; + ai_search_model?: string; + rewrite_model?: string; + reranking_model?: string; + /** @deprecated Use index_method instead. */ + hybrid_search_enabled?: boolean; + /** Controls which storage backends are active. */ + index_method?: { + vector?: boolean; + keyword?: boolean; + }; + /** Fusion method for combining vector and keyword results. */ + fusion_method?: 'max' | 'rrf'; + indexing_options?: { + keyword_tokenizer?: 'porter' | 'trigram'; + } | null; + retrieval_options?: { + keyword_match_mode?: 'and' | 'or'; + boost_by?: Array<{ + field: string; + direction?: 'asc' | 'desc' | 'exists' | 'not_exists'; + }>; + } | null; + chunk?: boolean; + chunk_size?: number; + chunk_overlap?: number; + score_threshold?: number; + max_num_results?: number; + cache?: boolean; + cache_threshold?: 'super_strict_match' | 'close_enough' | 'flexible_friend' | 'anything_goes'; + custom_metadata?: Array<{ + field_name: string; + data_type: 'text' | 'number' | 'boolean' | 'datetime'; + }>; + /** Sync interval in seconds. */ + sync_interval?: 3600 | 7200 | 14400 | 21600 | 43200 | 86400; + metadata?: Record; + [key: string]: unknown; +}; +/** Pagination, search, and ordering parameters for listing instances within a namespace. */ +type AiSearchListInstancesParams = { + page?: number; + per_page?: number; + /** Search instances by ID. */ + search?: string; + /** Field to sort by. */ + order_by?: 'created_at'; + /** Sort direction. */ + order_by_direction?: 'asc' | 'desc'; +}; +type AiSearchListResponse = { + result: AiSearchInstanceInfo[]; + result_info?: { + count: number; + page: number; + per_page: number; + total_count: number; + }; +}; +// ============ AI Search Config Types ============ +type AiSearchConfig = { + /** Instance ID (1-32 chars, pattern: ^[a-z0-9_]+(?:-[a-z0-9_]+)*$) */ + id: string; + /** Instance type. Omit to create with built-in storage. */ + type?: 'r2' | 'web-crawler' | string; + /** Source URL (required for web-crawler type). */ + source?: string; + source_params?: unknown; + /** Token ID (UUID format) */ + token_id?: string; + ai_gateway_id?: string; + /** Enable query rewriting (default false) */ + rewrite_query?: boolean; + /** Enable reranking (default false) */ + reranking?: boolean; + embedding_model?: string; + ai_search_model?: string; + rewrite_model?: string; + reranking_model?: string; + /** @deprecated Use index_method instead. */ + hybrid_search_enabled?: boolean; + /** Controls which storage backends are used during indexing. Defaults to vector-only. */ + index_method?: { + vector?: boolean; + keyword?: boolean; + }; + /** Fusion method for combining vector and keyword results. "rrf" = reciprocal rank fusion (default), "max" = maximum score. */ + fusion_method?: 'max' | 'rrf'; + indexing_options?: { + keyword_tokenizer?: 'porter' | 'trigram'; + } | null; + retrieval_options?: { + keyword_match_mode?: 'and' | 'or'; + boost_by?: Array<{ + field: string; + direction?: 'asc' | 'desc' | 'exists' | 'not_exists'; + }>; + } | null; + chunk?: boolean; + chunk_size?: number; + chunk_overlap?: number; + /** Minimum similarity score (0-1) for a result to be included. */ + score_threshold?: number; + max_num_results?: number; + cache?: boolean; + /** Similarity threshold for cache hits. Stricter = fewer cache hits but higher relevance. */ + cache_threshold?: 'super_strict_match' | 'close_enough' | 'flexible_friend' | 'anything_goes'; + custom_metadata?: Array<{ + field_name: string; + data_type: 'text' | 'number' | 'boolean' | 'datetime'; + }>; + namespace?: string; + /** Sync interval in seconds. 3600=1h, 7200=2h, 14400=4h, 21600=6h, 43200=12h, 86400=24h. */ + sync_interval?: 3600 | 7200 | 14400 | 21600 | 43200 | 86400; + metadata?: Record; + [key: string]: unknown; +}; +// ============ AI Search Item Types ============ +type AiSearchItemInfo = { + id: string; + key: string; + status: 'completed' | 'error' | 'skipped' | 'queued' | 'running' | 'outdated'; + next_action?: 'INDEX' | 'DELETE' | null; + error?: string; + checksum?: string; + namespace?: string; + chunks_count?: number | null; + file_size?: number | null; + source_id?: string | null; + last_seen_at?: string; + created_at?: string; + metadata?: Record; + [key: string]: unknown; +}; +type AiSearchItemContentResult = { + body: ReadableStream; + contentType: string; + filename: string; + size: number; +}; +type AiSearchUploadItemOptions = { + metadata?: Record; +}; +type AiSearchListItemsParams = { + page?: number; + per_page?: number; + /** Search items by key name. */ + search?: string; + /** Sort order for results. */ + sort_by?: 'status' | 'modified_at'; + /** Filter items by processing status. */ + status?: 'queued' | 'running' | 'completed' | 'error' | 'skipped' | 'outdated'; + /** Filter items by source (e.g. "builtin" or "web-crawler:https://example.com"). */ + source?: string; + /** JSON-encoded Vectorize filter for metadata filtering. */ + metadata_filter?: string; + /** Filter items by their unique ID. Returns at most one item. */ + item_id?: string; + /** + * Filter items by their exact key (object key / filename). Keys are unique + * per source, so combine with `source` to disambiguate across data sources. + */ + key?: string; +}; +type AiSearchListItemsResponse = { + result: AiSearchItemInfo[]; + result_info?: { + count: number; + page: number; + per_page: number; + total_count: number; + }; +}; +// ============ AI Search Item Logs Types ============ +type AiSearchItemLogsParams = { + /** Maximum number of log entries to return (1-100, default 50). */ + limit?: number; + /** Opaque cursor for pagination. Pass the `cursor` value from a previous response. */ + cursor?: string; +}; +type AiSearchItemLog = { + timestamp: string; + action: string; + message: string; + fileKey?: string; + chunkCount?: number; + processingTimeMs?: number; + errorType?: string; +}; +/** Paginated response for item processing logs (cursor-based). */ +type AiSearchItemLogsResponse = { + result: AiSearchItemLog[]; + result_info: { + count: number; + per_page: number; + cursor: string | null; + truncated: boolean; + }; +}; +// ============ AI Search Item Chunks Types ============ +type AiSearchItemChunksParams = { + /** Maximum number of chunks to return (1-100, default 20). */ + limit?: number; + /** Offset into the chunks list (default 0). */ + offset?: number; +}; +/** A single indexed chunk belonging to an item, including its text content and byte range. */ +type AiSearchItemChunk = { + id: string; + text: string; + start_byte: number; + end_byte: number; + item?: { + timestamp?: number; + key: string; + metadata?: Record; + }; +}; +/** Paginated response for item chunks (offset-based). */ +type AiSearchItemChunksResponse = { + result: AiSearchItemChunk[]; + result_info: { + count: number; + total: number; + limit: number; + offset: number; + }; +}; +// ============ AI Search Job Types ============ +type AiSearchJobInfo = { + id: string; + source: 'user' | 'schedule'; + description?: string; + last_seen_at?: string; + started_at?: string; + ended_at?: string; + end_reason?: string; +}; +type AiSearchJobLog = { + id: number; + message: string; + message_type: number; + created_at: number; +}; +type AiSearchCreateJobParams = { + description?: string; +}; +type AiSearchListJobsParams = { + page?: number; + per_page?: number; +}; +type AiSearchListJobsResponse = { + result: AiSearchJobInfo[]; + result_info?: { + count: number; + page: number; + per_page: number; + total_count: number; + }; +}; +type AiSearchJobLogsParams = { + page?: number; + per_page?: number; +}; +type AiSearchJobLogsResponse = { + result: AiSearchJobLog[]; + result_info?: { + count: number; + page: number; + per_page: number; + total_count: number; + }; +}; +// ============ AI Search Sub-Service Classes ============ +/** + * Single item service for an AI Search instance. + * Provides info, download, sync, logs, and chunks operations on a specific item. + */ +declare abstract class AiSearchItem { + /** Get metadata about this item. */ + info(): Promise; + /** + * Download the item's content. + * @returns Object with body stream, content type, filename, and size. + */ + download(): Promise; + /** + * Trigger re-indexing of this item. + * @returns The updated item info. + */ + sync(): Promise; + /** + * Retrieve processing logs for this item (cursor-based pagination). + * @param params Optional pagination parameters (limit, cursor). + * @returns Paginated log entries for this item. + */ + logs(params?: AiSearchItemLogsParams): Promise; + /** + * List indexed chunks for this item (offset-based pagination). + * @param params Optional pagination parameters (limit, offset). + * @returns Paginated chunk entries for this item. + */ + chunks(params?: AiSearchItemChunksParams): Promise; +} +/** + * Items collection service for an AI Search instance. + * Provides list, upload, and access to individual items. + */ +declare abstract class AiSearchItems { + /** List items in this instance. */ + list(params?: AiSearchListItemsParams): Promise; + /** + * Upload a file as an item. Behaves as an upsert: if an item with the same + * filename already exists, it is overwritten and re-indexed. + * @param name Filename for the uploaded item. + * @param content File content as a ReadableStream, Blob, or string. + * @param options Optional metadata to attach to the item. + * @returns The created item info. + */ + upload(name: string, content: ReadableStream | Blob | string, options?: AiSearchUploadItemOptions): Promise; + /** + * Upload a file and poll until processing completes. + * Behaves as an upsert: if an item with the same filename already exists, + * it is overwritten and re-indexed. + * @param name Filename for the uploaded item. + * @param content File content as a ReadableStream, Blob, or string. + * @param options Optional metadata and polling configuration. + * @returns The item info after processing completes (or timeout). + */ + uploadAndPoll(name: string, content: ReadableStream | Blob | string, options?: AiSearchUploadItemOptions & { + /** Polling interval in milliseconds (default 1000). */ + pollIntervalMs?: number; + /** Maximum time to wait in milliseconds (default 30000). */ + timeoutMs?: number; + }): Promise; + /** + * Get an item by ID. + * @param itemId The item identifier. + * @returns Item service for info, download, sync, logs, and chunks operations. + */ + get(itemId: string): AiSearchItem; + /** + * Delete an item from the instance. + * @param itemId The item identifier. + */ + delete(itemId: string): Promise; +} +/** + * Single job service for an AI Search instance. + * Provides info, logs, and cancel operations for a specific job. + */ +declare abstract class AiSearchJob { + /** Get metadata about this job. */ + info(): Promise; + /** Get logs for this job. */ + logs(params?: AiSearchJobLogsParams): Promise; + /** + * Cancel a running job. + * @returns The updated job info. + * @throws AiSearchNotFoundError if the job does not exist. + */ + cancel(): Promise; +} +/** + * Jobs collection service for an AI Search instance. + * Provides list, create, and access to individual jobs. + */ +declare abstract class AiSearchJobs { + /** List jobs for this instance. */ + list(params?: AiSearchListJobsParams): Promise; + /** + * Create a new indexing job. + * @param params Optional job parameters. + * @returns The created job info. + */ + create(params?: AiSearchCreateJobParams): Promise; + /** + * Get a job by ID. + * @param jobId The job identifier. + * @returns Job service for info, logs, and cancel operations. + */ + get(jobId: string): AiSearchJob; +} +// ============ AI Search Binding Classes ============ +/** + * Instance-level AI Search service. + * + * Used as: + * - The return type of `AiSearchNamespace.get(name)` (namespace binding) + * - The type of `env.BLOG_SEARCH` (single instance binding via `ai_search`) + * + * Provides search, chat, update, stats, items, and jobs operations. + * + * @example + * ```ts + * // Via namespace binding + * const instance = env.AI_SEARCH.get("blog"); + * const results = await instance.search({ + * query: "How does caching work?", + * }); + * + * // Via single instance binding + * const results = await env.BLOG_SEARCH.search({ + * messages: [{ role: "user", content: "How does caching work?" }], + * }); + * ``` + */ +declare abstract class AiSearchInstance { + /** + * Search the AI Search instance for relevant chunks. + * @param params Search request with query or messages and optional AI search options. + * @returns Search response with matching chunks and search query. + */ + search(params: AiSearchSearchRequest): Promise; + /** + * Generate chat completions with AI Search context (streaming). + * @param params Chat completions request with stream: true. + * @returns ReadableStream of server-sent events. + */ + chatCompletions(params: AiSearchChatCompletionsRequest & { + stream: true; + }): Promise; + /** + * Generate chat completions with AI Search context. + * @param params Chat completions request. + * @returns Chat completion response with choices and RAG chunks. + */ + chatCompletions(params: AiSearchChatCompletionsRequest): Promise; + /** + * Update the instance configuration. + * @param config Partial configuration to update. + * @returns Updated instance info. + */ + update(config: Partial): Promise; + /** Get metadata about this instance. */ + info(): Promise; + /** + * Get instance statistics (item count, indexing status, etc.). + * @returns Statistics with counts per status, last activity time, and engine details. + */ + stats(): Promise; + /** Items collection — list, upload, and manage items in this instance. */ + get items(): AiSearchItems; + /** Jobs collection — list, create, and inspect indexing jobs. */ + get jobs(): AiSearchJobs; +} +/** + * Namespace-level AI Search service. + * + * Used as the type of `env.AI_SEARCH` (namespace binding via `ai_search_namespaces`). + * Scoped to a single namespace. Provides dynamic instance access, creation, deletion, + * and multi-instance search/chat operations. + * + * @example + * ```ts + * // Access an instance within the namespace + * const blog = env.AI_SEARCH.get("blog"); + * const results = await blog.search({ query: "How does caching work?" }); + * + * // List all instances in the namespace + * const instances = await env.AI_SEARCH.list(); + * + * // Create a new instance with built-in storage + * const tenant = await env.AI_SEARCH.create({ id: "tenant-123" }); + * + * // Upload items into the instance + * await tenant.items.upload("doc.pdf", fileContent); + * + * // Search across multiple instances + * const multi = await env.AI_SEARCH.search({ + * query: "caching", + * ai_search_options: { instance_ids: ["blog", "docs"] }, + * }); + * + * // Delete an instance + * await env.AI_SEARCH.delete("tenant-123"); + * ``` + */ +declare abstract class AiSearchNamespace { + /** + * Get an instance by name within the bound namespace. + * @param name Instance name. + * @returns Instance service for search, chat, update, stats, items, and jobs. + */ + get(name: string): AiSearchInstance; + /** + * List instances in the bound namespace. + * @param params Optional pagination, search, and ordering parameters. + * @returns Array of instance metadata with pagination info. + */ + list(params?: AiSearchListInstancesParams): Promise; + /** + * Create a new instance within the bound namespace. + * @param config Instance configuration. Only `id` is required — omit `type` and `source` to create with built-in storage. + * @returns Instance service for the newly created instance. + * + * @example + * ```ts + * // Create with built-in storage (upload items manually) + * const instance = await env.AI_SEARCH.create({ id: "my-search" }); + * + * // Create with web crawler source + * const instance = await env.AI_SEARCH.create({ + * id: "docs-search", + * type: "web-crawler", + * source: "https://developers.cloudflare.com", + * }); + * ``` + */ + create(config: AiSearchConfig): Promise; + /** + * Delete an instance from the bound namespace. + * @param name Instance name to delete. + */ + delete(name: string): Promise; + /** + * Search across multiple instances within the bound namespace. + * Fans out to the specified instance_ids and merges results. + * @param params Search request with required `ai_search_options.instance_ids`. + * @returns Search response with chunks tagged by instance_id and optional partial-failure errors. + */ + search(params: AiSearchMultiSearchRequest): Promise; + /** + * Generate chat completions across multiple instances within the bound namespace (streaming). + * Fans out to the specified instance_ids, merges context, and generates a response. + * @param params Chat completions request with stream: true and required `ai_search_options.instance_ids`. + * @returns ReadableStream of server-sent events. + */ + chatCompletions(params: AiSearchMultiChatCompletionsRequest & { + stream: true; + }): Promise; + /** + * Generate chat completions across multiple instances within the bound namespace. + * Fans out to the specified instance_ids, merges context, and generates a response. + * @param params Chat completions request with required `ai_search_options.instance_ids`. + * @returns Chat completion response with choices, chunks tagged by instance_id, and optional partial-failure errors. + */ + chatCompletions(params: AiSearchMultiChatCompletionsRequest): Promise; +} +type AiImageClassificationInput = { + image: number[]; +}; +type AiImageClassificationOutput = { + score?: number; + label?: string; +}[]; +declare abstract class BaseAiImageClassification { + inputs: AiImageClassificationInput; + postProcessedOutputs: AiImageClassificationOutput; +} +type AiImageToTextInput = { + image: number[]; + prompt?: string; + max_tokens?: number; + temperature?: number; + top_p?: number; + top_k?: number; + seed?: number; + repetition_penalty?: number; + frequency_penalty?: number; + presence_penalty?: number; + raw?: boolean; + messages?: RoleScopedChatInput[]; +}; +type AiImageToTextOutput = { + description: string; +}; +declare abstract class BaseAiImageToText { + inputs: AiImageToTextInput; + postProcessedOutputs: AiImageToTextOutput; +} +type AiImageTextToTextInput = { + image: string; + prompt?: string; + max_tokens?: number; + temperature?: number; + ignore_eos?: boolean; + top_p?: number; + top_k?: number; + seed?: number; + repetition_penalty?: number; + frequency_penalty?: number; + presence_penalty?: number; + raw?: boolean; + messages?: RoleScopedChatInput[]; +}; +type AiImageTextToTextOutput = { + description: string; +}; +declare abstract class BaseAiImageTextToText { + inputs: AiImageTextToTextInput; + postProcessedOutputs: AiImageTextToTextOutput; +} +type AiMultimodalEmbeddingsInput = { + image: string; + text: string[]; +}; +type AiIMultimodalEmbeddingsOutput = { + data: number[][]; + shape: number[]; +}; +declare abstract class BaseAiMultimodalEmbeddings { + inputs: AiImageTextToTextInput; + postProcessedOutputs: AiImageTextToTextOutput; +} +type AiObjectDetectionInput = { + image: number[]; +}; +type AiObjectDetectionOutput = { + score?: number; + label?: string; +}[]; +declare abstract class BaseAiObjectDetection { + inputs: AiObjectDetectionInput; + postProcessedOutputs: AiObjectDetectionOutput; +} +type AiSentenceSimilarityInput = { + source: string; + sentences: string[]; +}; +type AiSentenceSimilarityOutput = number[]; +declare abstract class BaseAiSentenceSimilarity { + inputs: AiSentenceSimilarityInput; + postProcessedOutputs: AiSentenceSimilarityOutput; +} +type AiAutomaticSpeechRecognitionInput = { + audio: number[]; +}; +type AiAutomaticSpeechRecognitionOutput = { + text?: string; + words?: { + word: string; + start: number; + end: number; + }[]; + vtt?: string; +}; +declare abstract class BaseAiAutomaticSpeechRecognition { + inputs: AiAutomaticSpeechRecognitionInput; + postProcessedOutputs: AiAutomaticSpeechRecognitionOutput; +} +type AiSummarizationInput = { + input_text: string; + max_length?: number; +}; +type AiSummarizationOutput = { + summary: string; +}; +declare abstract class BaseAiSummarization { + inputs: AiSummarizationInput; + postProcessedOutputs: AiSummarizationOutput; +} +type AiTextClassificationInput = { + text: string; +}; +type AiTextClassificationOutput = { + score?: number; + label?: string; +}[]; +declare abstract class BaseAiTextClassification { + inputs: AiTextClassificationInput; + postProcessedOutputs: AiTextClassificationOutput; +} +type AiTextEmbeddingsInput = { + text: string | string[]; +}; +type AiTextEmbeddingsOutput = { + shape: number[]; + data: number[][]; +}; +declare abstract class BaseAiTextEmbeddings { + inputs: AiTextEmbeddingsInput; + postProcessedOutputs: AiTextEmbeddingsOutput; +} +type RoleScopedChatInput = { + role: "user" | "assistant" | "system" | "tool" | (string & NonNullable); + content: string; + name?: string; +}; +type AiTextGenerationToolLegacyInput = { + name: string; + description: string; + parameters?: { + type: "object" | (string & NonNullable); + properties: { + [key: string]: { + type: string; + description?: string; + }; + }; + required: string[]; + }; +}; +type AiTextGenerationToolInput = { + type: "function" | (string & NonNullable); + function: { + name: string; + description: string; + parameters?: { + type: "object" | (string & NonNullable); + properties: { + [key: string]: { + type: string; + description?: string; + }; + }; + required: string[]; + }; + }; +}; +type AiTextGenerationFunctionsInput = { + name: string; + code: string; +}; +type AiTextGenerationResponseFormat = { + type: string; + json_schema?: any; +}; +type AiTextGenerationInput = { + prompt?: string; + raw?: boolean; + stream?: boolean; + max_tokens?: number; + temperature?: number; + top_p?: number; + top_k?: number; + seed?: number; + repetition_penalty?: number; + frequency_penalty?: number; + presence_penalty?: number; + messages?: RoleScopedChatInput[]; + response_format?: AiTextGenerationResponseFormat; + tools?: AiTextGenerationToolInput[] | AiTextGenerationToolLegacyInput[] | (object & NonNullable); + functions?: AiTextGenerationFunctionsInput[]; +}; +type AiTextGenerationToolLegacyOutput = { + name: string; + arguments: unknown; +}; +type AiTextGenerationToolOutput = { + id: string; + type: "function"; + function: { + name: string; + arguments: string; + }; +}; +type UsageTags = { + prompt_tokens: number; + completion_tokens: number; + total_tokens: number; +}; +type AiTextGenerationOutput = { + response?: string; + tool_calls?: AiTextGenerationToolLegacyOutput[] & AiTextGenerationToolOutput[]; + usage?: UsageTags; +}; +declare abstract class BaseAiTextGeneration { + inputs: AiTextGenerationInput; + postProcessedOutputs: AiTextGenerationOutput; +} +type AiTextToSpeechInput = { + prompt: string; + lang?: string; +}; +type AiTextToSpeechOutput = Uint8Array | { + audio: string; +}; +declare abstract class BaseAiTextToSpeech { + inputs: AiTextToSpeechInput; + postProcessedOutputs: AiTextToSpeechOutput; +} +type AiTextToImageInput = { + prompt: string; + negative_prompt?: string; + height?: number; + width?: number; + image?: number[]; + image_b64?: string; + mask?: number[]; + num_steps?: number; + strength?: number; + guidance?: number; + seed?: number; +}; +type AiTextToImageOutput = ReadableStream; +declare abstract class BaseAiTextToImage { + inputs: AiTextToImageInput; + postProcessedOutputs: AiTextToImageOutput; +} +type AiTranslationInput = { + text: string; + target_lang: string; + source_lang?: string; +}; +type AiTranslationOutput = { + translated_text?: string; +}; +declare abstract class BaseAiTranslation { + inputs: AiTranslationInput; + postProcessedOutputs: AiTranslationOutput; +} +/** + * Workers AI support for OpenAI's Chat Completions API + */ +type ChatCompletionContentPartText = { + type: "text"; + text: string; +}; +type ChatCompletionContentPartImage = { + type: "image_url"; + image_url: { + url: string; + detail?: "auto" | "low" | "high"; + }; +}; +type ChatCompletionContentPartInputAudio = { + type: "input_audio"; + input_audio: { + /** Base64 encoded audio data. */ + data: string; + format: "wav" | "mp3"; + }; +}; +type ChatCompletionContentPartFile = { + type: "file"; + file: { + /** Base64 encoded file data. */ + file_data?: string; + /** The ID of an uploaded file. */ + file_id?: string; + filename?: string; + }; +}; +type ChatCompletionContentPartRefusal = { + type: "refusal"; + refusal: string; +}; +type ChatCompletionContentPart = ChatCompletionContentPartText | ChatCompletionContentPartImage | ChatCompletionContentPartInputAudio | ChatCompletionContentPartFile; +type FunctionDefinition = { + name: string; + description?: string; + parameters?: Record; + strict?: boolean | null; +}; +type ChatCompletionFunctionTool = { + type: "function"; + function: FunctionDefinition; +}; +type ChatCompletionCustomToolGrammarFormat = { + type: "grammar"; + grammar: { + definition: string; + syntax: "lark" | "regex"; + }; +}; +type ChatCompletionCustomToolTextFormat = { + type: "text"; +}; +type ChatCompletionCustomToolFormat = ChatCompletionCustomToolTextFormat | ChatCompletionCustomToolGrammarFormat; +type ChatCompletionCustomTool = { + type: "custom"; + custom: { + name: string; + description?: string; + format?: ChatCompletionCustomToolFormat; + }; +}; +type ChatCompletionTool = ChatCompletionFunctionTool | ChatCompletionCustomTool; +type ChatCompletionMessageFunctionToolCall = { + id: string; + type: "function"; + function: { + name: string; + /** JSON-encoded arguments string. */ + arguments: string; + }; +}; +type ChatCompletionMessageCustomToolCall = { + id: string; + type: "custom"; + custom: { + name: string; + input: string; + }; +}; +type ChatCompletionMessageToolCall = ChatCompletionMessageFunctionToolCall | ChatCompletionMessageCustomToolCall; +type ChatCompletionToolChoiceFunction = { + type: "function"; + function: { + name: string; + }; +}; +type ChatCompletionToolChoiceCustom = { + type: "custom"; + custom: { + name: string; + }; +}; +type ChatCompletionToolChoiceAllowedTools = { + type: "allowed_tools"; + allowed_tools: { + mode: "auto" | "required"; + tools: Array>; + }; +}; +type ChatCompletionToolChoiceOption = "none" | "auto" | "required" | ChatCompletionToolChoiceFunction | ChatCompletionToolChoiceCustom | ChatCompletionToolChoiceAllowedTools; +type DeveloperMessage = { + role: "developer"; + content: string | Array<{ + type: "text"; + text: string; + }>; + name?: string; +}; +type SystemMessage = { + role: "system"; + content: string | Array<{ + type: "text"; + text: string; + }>; + name?: string; +}; +/** + * Permissive merged content part used inside UserMessage arrays. + * + * Cabidela has a limitation where anyOf/oneOf with enum-based discrimination + * inside nested array items does not correctly match different branches for + * different array elements, so the schema uses a single merged object. + */ +type UserMessageContentPart = { + type: "text" | "image_url" | "input_audio" | "file"; + text?: string; + image_url?: { + url?: string; + detail?: "auto" | "low" | "high"; + }; + input_audio?: { + data?: string; + format?: "wav" | "mp3"; + }; + file?: { + file_data?: string; + file_id?: string; + filename?: string; + }; +}; +type UserMessage = { + role: "user"; + content: string | Array; + name?: string; +}; +type AssistantMessageContentPart = { + type: "text" | "refusal"; + text?: string; + refusal?: string; +}; +type AssistantMessage = { + role: "assistant"; + content?: string | null | Array; + refusal?: string | null; + name?: string; + audio?: { + id: string; + }; + tool_calls?: Array; + function_call?: { + name: string; + arguments: string; + }; +}; +type ToolMessage = { + role: "tool"; + content: string | Array<{ + type: "text"; + text: string; + }>; + tool_call_id: string; +}; +type FunctionMessage = { + role: "function"; + content: string; + name: string; +}; +type ChatCompletionMessageParam = DeveloperMessage | SystemMessage | UserMessage | AssistantMessage | ToolMessage | FunctionMessage; +type ChatCompletionsResponseFormatText = { + type: "text"; +}; +type ChatCompletionsResponseFormatJSONObject = { + type: "json_object"; +}; +type ResponseFormatJSONSchema = { + type: "json_schema"; + json_schema: { + name: string; + description?: string; + schema?: Record; + strict?: boolean | null; + }; +}; +type ResponseFormat = ChatCompletionsResponseFormatText | ChatCompletionsResponseFormatJSONObject | ResponseFormatJSONSchema; +type ChatCompletionsStreamOptions = { + include_usage?: boolean; + include_obfuscation?: boolean; +}; +type PredictionContent = { + type: "content"; + content: string | Array<{ + type: "text"; + text: string; + }>; +}; +type AudioParams = { + voice: string | { + id: string; + }; + format: "wav" | "aac" | "mp3" | "flac" | "opus" | "pcm16"; +}; +type WebSearchUserLocation = { + type: "approximate"; + approximate: { + city?: string; + country?: string; + region?: string; + timezone?: string; + }; +}; +type WebSearchOptions = { + search_context_size?: "low" | "medium" | "high"; + user_location?: WebSearchUserLocation; +}; +type ChatTemplateKwargs = { + /** Whether to enable reasoning, enabled by default. */ + enable_thinking?: boolean; + /** If false, preserves reasoning context between turns. */ + clear_thinking?: boolean; +}; +/** Shared optional properties used by both Prompt and Messages input branches. */ +type ChatCompletionsCommonOptions = { + model?: string; + audio?: AudioParams; + frequency_penalty?: number | null; + logit_bias?: Record | null; + logprobs?: boolean | null; + top_logprobs?: number | null; + max_tokens?: number | null; + max_completion_tokens?: number | null; + metadata?: Record | null; + modalities?: Array<"text" | "audio"> | null; + n?: number | null; + parallel_tool_calls?: boolean; + prediction?: PredictionContent; + presence_penalty?: number | null; + reasoning_effort?: "low" | "medium" | "high" | null; + chat_template_kwargs?: ChatTemplateKwargs; + response_format?: ResponseFormat; + seed?: number | null; + service_tier?: "auto" | "default" | "flex" | "scale" | "priority" | null; + stop?: string | Array | null; + store?: boolean | null; + stream?: boolean | null; + stream_options?: ChatCompletionsStreamOptions; + temperature?: number | null; + tool_choice?: ChatCompletionToolChoiceOption; + tools?: Array; + top_p?: number | null; + user?: string; + web_search_options?: WebSearchOptions; + function_call?: "none" | "auto" | { + name: string; + }; + functions?: Array; +}; +type PromptTokensDetails = { + cached_tokens?: number; + audio_tokens?: number; +}; +type CompletionTokensDetails = { + reasoning_tokens?: number; + audio_tokens?: number; + accepted_prediction_tokens?: number; + rejected_prediction_tokens?: number; +}; +type CompletionUsage = { + prompt_tokens: number; + completion_tokens: number; + total_tokens: number; + prompt_tokens_details?: PromptTokensDetails; + completion_tokens_details?: CompletionTokensDetails; +}; +type ChatCompletionTopLogprob = { + token: string; + logprob: number; + bytes: Array | null; +}; +type ChatCompletionTokenLogprob = { + token: string; + logprob: number; + bytes: Array | null; + top_logprobs: Array; +}; +type ChatCompletionAudio = { + id: string; + /** Base64 encoded audio bytes. */ + data: string; + expires_at: number; + transcript: string; +}; +type ChatCompletionUrlCitation = { + type: "url_citation"; + url_citation: { + url: string; + title: string; + start_index: number; + end_index: number; + }; +}; +type ChatCompletionResponseMessage = { + role: "assistant"; + content: string | null; + refusal: string | null; + annotations?: Array; + audio?: ChatCompletionAudio; + tool_calls?: Array; + function_call?: { + name: string; + arguments: string; + } | null; +}; +type ChatCompletionLogprobs = { + content: Array | null; + refusal?: Array | null; +}; +type ChatCompletionChoice = { + index: number; + message: ChatCompletionResponseMessage; + finish_reason: "stop" | "length" | "tool_calls" | "content_filter" | "function_call"; + logprobs: ChatCompletionLogprobs | null; +}; +type ChatCompletionsMessagesInput = { + messages: Array; +} & ChatCompletionsCommonOptions; +type ChatCompletionsOutput = { + id: string; + object: string; + created: number; + model: string; + choices: Array; + usage?: CompletionUsage; + system_fingerprint?: string | null; + service_tier?: "auto" | "default" | "flex" | "scale" | "priority" | null; +}; +/** + * Workers AI support for OpenAI's Responses API + * Reference: https://github.com/openai/openai-node/blob/master/src/resources/responses/responses.ts + * + * It's a stripped down version from its source. + * It currently supports basic function calling, json mode and accepts images as input. + * + * It does not include types for WebSearch, CodeInterpreter, FileInputs, MCP, CustomTools. + * We plan to add those incrementally as model + platform capabilities evolve. + */ +type ResponsesInput = { + background?: boolean | null; + conversation?: string | ResponseConversationParam | null; + include?: Array | null; + input?: string | ResponseInput; + instructions?: string | null; + max_output_tokens?: number | null; + parallel_tool_calls?: boolean | null; + previous_response_id?: string | null; + prompt_cache_key?: string; + reasoning?: Reasoning | null; + safety_identifier?: string; + service_tier?: "auto" | "default" | "flex" | "scale" | "priority" | null; + stream?: boolean | null; + stream_options?: StreamOptions | null; + temperature?: number | null; + text?: ResponseTextConfig; + tool_choice?: ToolChoiceOptions | ToolChoiceFunction; + tools?: Array; + top_p?: number | null; + truncation?: "auto" | "disabled" | null; +}; +type ResponsesOutput = { + id?: string; + created_at?: number; + output_text?: string; + error?: ResponseError | null; + incomplete_details?: ResponseIncompleteDetails | null; + instructions?: string | Array | null; + object?: "response"; + output?: Array; + parallel_tool_calls?: boolean; + temperature?: number | null; + tool_choice?: ToolChoiceOptions | ToolChoiceFunction; + tools?: Array; + top_p?: number | null; + max_output_tokens?: number | null; + previous_response_id?: string | null; + prompt?: ResponsePrompt | null; + reasoning?: Reasoning | null; + safety_identifier?: string; + service_tier?: "auto" | "default" | "flex" | "scale" | "priority" | null; + status?: ResponseStatus; + text?: ResponseTextConfig; + truncation?: "auto" | "disabled" | null; + usage?: ResponseUsage; +}; +type EasyInputMessage = { + content: string | ResponseInputMessageContentList; + role: "user" | "assistant" | "system" | "developer"; + type?: "message"; +}; +type ResponsesFunctionTool = { + name: string; + parameters: { + [key: string]: unknown; + } | null; + strict: boolean | null; + type: "function"; + description?: string | null; +}; +type ResponseIncompleteDetails = { + reason?: "max_output_tokens" | "content_filter"; +}; +type ResponsePrompt = { + id: string; + variables?: { + [key: string]: string | ResponseInputText | ResponseInputImage; + } | null; + version?: string | null; +}; +type Reasoning = { + effort?: ReasoningEffort | null; + generate_summary?: "auto" | "concise" | "detailed" | null; + summary?: "auto" | "concise" | "detailed" | null; +}; +type ResponseContent = ResponseInputText | ResponseInputImage | ResponseOutputText | ResponseOutputRefusal | ResponseContentReasoningText; +type ResponseContentReasoningText = { + text: string; + type: "reasoning_text"; +}; +type ResponseConversationParam = { + id: string; +}; +type ResponseCreatedEvent = { + response: Response; + sequence_number: number; + type: "response.created"; +}; +type ResponseCustomToolCallOutput = { + call_id: string; + output: string | Array; + type: "custom_tool_call_output"; + id?: string; +}; +type ResponseError = { + code: "server_error" | "rate_limit_exceeded" | "invalid_prompt" | "vector_store_timeout" | "invalid_image" | "invalid_image_format" | "invalid_base64_image" | "invalid_image_url" | "image_too_large" | "image_too_small" | "image_parse_error" | "image_content_policy_violation" | "invalid_image_mode" | "image_file_too_large" | "unsupported_image_media_type" | "empty_image_file" | "failed_to_download_image" | "image_file_not_found"; + message: string; +}; +type ResponseErrorEvent = { + code: string | null; + message: string; + param: string | null; + sequence_number: number; + type: "error"; +}; +type ResponseFailedEvent = { + response: Response; + sequence_number: number; + type: "response.failed"; +}; +type ResponseFormatText = { + type: "text"; +}; +type ResponseFormatJSONObject = { + type: "json_object"; +}; +type ResponseFormatTextConfig = ResponseFormatText | ResponseFormatTextJSONSchemaConfig | ResponseFormatJSONObject; +type ResponseFormatTextJSONSchemaConfig = { + name: string; + schema: { + [key: string]: unknown; + }; + type: "json_schema"; + description?: string; + strict?: boolean | null; +}; +type ResponseFunctionCallArgumentsDeltaEvent = { + delta: string; + item_id: string; + output_index: number; + sequence_number: number; + type: "response.function_call_arguments.delta"; +}; +type ResponseFunctionCallArgumentsDoneEvent = { + arguments: string; + item_id: string; + name: string; + output_index: number; + sequence_number: number; + type: "response.function_call_arguments.done"; +}; +type ResponseFunctionCallOutputItem = ResponseInputTextContent | ResponseInputImageContent; +type ResponseFunctionCallOutputItemList = Array; +type ResponseFunctionToolCall = { + arguments: string; + call_id: string; + name: string; + type: "function_call"; + id?: string; + status?: "in_progress" | "completed" | "incomplete"; +}; +interface ResponseFunctionToolCallItem extends ResponseFunctionToolCall { + id: string; +} +type ResponseFunctionToolCallOutputItem = { + id: string; + call_id: string; + output: string | Array; + type: "function_call_output"; + status?: "in_progress" | "completed" | "incomplete"; +}; +type ResponseIncludable = "message.input_image.image_url" | "message.output_text.logprobs"; +type ResponseIncompleteEvent = { + response: Response; + sequence_number: number; + type: "response.incomplete"; +}; +type ResponseInput = Array; +type ResponseInputContent = ResponseInputText | ResponseInputImage; +type ResponseInputImage = { + detail: "low" | "high" | "auto"; + type: "input_image"; + /** + * Base64 encoded image + */ + image_url?: string | null; +}; +type ResponseInputImageContent = { + type: "input_image"; + detail?: "low" | "high" | "auto" | null; + /** + * Base64 encoded image + */ + image_url?: string | null; +}; +type ResponseInputItem = EasyInputMessage | ResponseInputItemMessage | ResponseOutputMessage | ResponseFunctionToolCall | ResponseInputItemFunctionCallOutput | ResponseReasoningItem; +type ResponseInputItemFunctionCallOutput = { + call_id: string; + output: string | ResponseFunctionCallOutputItemList; + type: "function_call_output"; + id?: string | null; + status?: "in_progress" | "completed" | "incomplete" | null; +}; +type ResponseInputItemMessage = { + content: ResponseInputMessageContentList; + role: "user" | "system" | "developer"; + status?: "in_progress" | "completed" | "incomplete"; + type?: "message"; +}; +type ResponseInputMessageContentList = Array; +type ResponseInputMessageItem = { + id: string; + content: ResponseInputMessageContentList; + role: "user" | "system" | "developer"; + status?: "in_progress" | "completed" | "incomplete"; + type?: "message"; +}; +type ResponseInputText = { + text: string; + type: "input_text"; +}; +type ResponseInputTextContent = { + text: string; + type: "input_text"; +}; +type ResponseItem = ResponseInputMessageItem | ResponseOutputMessage | ResponseFunctionToolCallItem | ResponseFunctionToolCallOutputItem; +type ResponseOutputItem = ResponseOutputMessage | ResponseFunctionToolCall | ResponseReasoningItem; +type ResponseOutputItemAddedEvent = { + item: ResponseOutputItem; + output_index: number; + sequence_number: number; + type: "response.output_item.added"; +}; +type ResponseOutputItemDoneEvent = { + item: ResponseOutputItem; + output_index: number; + sequence_number: number; + type: "response.output_item.done"; +}; +type ResponseOutputMessage = { + id: string; + content: Array; + role: "assistant"; + status: "in_progress" | "completed" | "incomplete"; + type: "message"; +}; +type ResponseOutputRefusal = { + refusal: string; + type: "refusal"; +}; +type ResponseOutputText = { + text: string; + type: "output_text"; + logprobs?: Array; +}; +type ResponseReasoningItem = { + id: string; + summary: Array; + type: "reasoning"; + content?: Array; + encrypted_content?: string | null; + status?: "in_progress" | "completed" | "incomplete"; +}; +type ResponseReasoningSummaryItem = { + text: string; + type: "summary_text"; +}; +type ResponseReasoningContentItem = { + text: string; + type: "reasoning_text"; +}; +type ResponseReasoningTextDeltaEvent = { + content_index: number; + delta: string; + item_id: string; + output_index: number; + sequence_number: number; + type: "response.reasoning_text.delta"; +}; +type ResponseReasoningTextDoneEvent = { + content_index: number; + item_id: string; + output_index: number; + sequence_number: number; + text: string; + type: "response.reasoning_text.done"; +}; +type ResponseRefusalDeltaEvent = { + content_index: number; + delta: string; + item_id: string; + output_index: number; + sequence_number: number; + type: "response.refusal.delta"; +}; +type ResponseRefusalDoneEvent = { + content_index: number; + item_id: string; + output_index: number; + refusal: string; + sequence_number: number; + type: "response.refusal.done"; +}; +type ResponseStatus = "completed" | "failed" | "in_progress" | "cancelled" | "queued" | "incomplete"; +type ResponseStreamEvent = ResponseCompletedEvent | ResponseCreatedEvent | ResponseErrorEvent | ResponseFunctionCallArgumentsDeltaEvent | ResponseFunctionCallArgumentsDoneEvent | ResponseFailedEvent | ResponseIncompleteEvent | ResponseOutputItemAddedEvent | ResponseOutputItemDoneEvent | ResponseReasoningTextDeltaEvent | ResponseReasoningTextDoneEvent | ResponseRefusalDeltaEvent | ResponseRefusalDoneEvent | ResponseTextDeltaEvent | ResponseTextDoneEvent; +type ResponseCompletedEvent = { + response: Response; + sequence_number: number; + type: "response.completed"; +}; +type ResponseTextConfig = { + format?: ResponseFormatTextConfig; + verbosity?: "low" | "medium" | "high" | null; +}; +type ResponseTextDeltaEvent = { + content_index: number; + delta: string; + item_id: string; + logprobs: Array; + output_index: number; + sequence_number: number; + type: "response.output_text.delta"; +}; +type ResponseTextDoneEvent = { + content_index: number; + item_id: string; + logprobs: Array; + output_index: number; + sequence_number: number; + text: string; + type: "response.output_text.done"; +}; +type Logprob = { + token: string; + logprob: number; + top_logprobs?: Array; +}; +type TopLogprob = { + token?: string; + logprob?: number; +}; +type ResponseUsage = { + input_tokens: number; + output_tokens: number; + total_tokens: number; +}; +type Tool = ResponsesFunctionTool; +type ToolChoiceFunction = { + name: string; + type: "function"; +}; +type ToolChoiceOptions = "none"; +type ReasoningEffort = "minimal" | "low" | "medium" | "high" | null; +type StreamOptions = { + include_obfuscation?: boolean; +}; +/** Marks keys from T that aren't in U as optional never */ +type Without = { + [P in Exclude]?: never; +}; +/** Either T or U, but not both (mutually exclusive) */ +type XOR = (T & Without) | (U & Without); +type Ai_Cf_Baai_Bge_Base_En_V1_5_Input = { + text: string | string[]; + /** + * The pooling method used in the embedding process. `cls` pooling will generate more accurate embeddings on larger inputs - however, embeddings created with cls pooling are not compatible with embeddings generated with mean pooling. The default pooling method is `mean` in order for this to not be a breaking change, but we highly suggest using the new `cls` pooling for better accuracy. + */ + pooling?: "mean" | "cls"; +} | { + /** + * Batch of the embeddings requests to run using async-queue + */ + requests: { + text: string | string[]; + /** + * The pooling method used in the embedding process. `cls` pooling will generate more accurate embeddings on larger inputs - however, embeddings created with cls pooling are not compatible with embeddings generated with mean pooling. The default pooling method is `mean` in order for this to not be a breaking change, but we highly suggest using the new `cls` pooling for better accuracy. + */ + pooling?: "mean" | "cls"; + }[]; +}; +type Ai_Cf_Baai_Bge_Base_En_V1_5_Output = { + shape?: number[]; + /** + * Embeddings of the requested text values + */ + data?: number[][]; + /** + * The pooling method used in the embedding process. + */ + pooling?: "mean" | "cls"; +} | Ai_Cf_Baai_Bge_Base_En_V1_5_AsyncResponse; +interface Ai_Cf_Baai_Bge_Base_En_V1_5_AsyncResponse { + /** + * The async request id that can be used to obtain the results. + */ + request_id?: string; +} +declare abstract class Base_Ai_Cf_Baai_Bge_Base_En_V1_5 { + inputs: Ai_Cf_Baai_Bge_Base_En_V1_5_Input; + postProcessedOutputs: Ai_Cf_Baai_Bge_Base_En_V1_5_Output; +} +type Ai_Cf_Openai_Whisper_Input = string | { + /** + * An array of integers that represent the audio data constrained to 8-bit unsigned integer values + */ + audio: number[]; +}; +interface Ai_Cf_Openai_Whisper_Output { + /** + * The transcription + */ + text: string; + word_count?: number; + words?: { + word?: string; + /** + * The second this word begins in the recording + */ + start?: number; + /** + * The ending second when the word completes + */ + end?: number; + }[]; + vtt?: string; +} +declare abstract class Base_Ai_Cf_Openai_Whisper { + inputs: Ai_Cf_Openai_Whisper_Input; + postProcessedOutputs: Ai_Cf_Openai_Whisper_Output; +} +type Ai_Cf_Meta_M2M100_1_2B_Input = { + /** + * The text to be translated + */ + text: string; + /** + * The language code of the source text (e.g., 'en' for English). Defaults to 'en' if not specified + */ + source_lang?: string; + /** + * The language code to translate the text into (e.g., 'es' for Spanish) + */ + target_lang: string; +} | { + /** + * Batch of the embeddings requests to run using async-queue + */ + requests: { + /** + * The text to be translated + */ + text: string; + /** + * The language code of the source text (e.g., 'en' for English). Defaults to 'en' if not specified + */ + source_lang?: string; + /** + * The language code to translate the text into (e.g., 'es' for Spanish) + */ + target_lang: string; + }[]; +}; +type Ai_Cf_Meta_M2M100_1_2B_Output = { + /** + * The translated text in the target language + */ + translated_text?: string; +} | Ai_Cf_Meta_M2M100_1_2B_AsyncResponse; +interface Ai_Cf_Meta_M2M100_1_2B_AsyncResponse { + /** + * The async request id that can be used to obtain the results. + */ + request_id?: string; +} +declare abstract class Base_Ai_Cf_Meta_M2M100_1_2B { + inputs: Ai_Cf_Meta_M2M100_1_2B_Input; + postProcessedOutputs: Ai_Cf_Meta_M2M100_1_2B_Output; +} +type Ai_Cf_Baai_Bge_Small_En_V1_5_Input = { + text: string | string[]; + /** + * The pooling method used in the embedding process. `cls` pooling will generate more accurate embeddings on larger inputs - however, embeddings created with cls pooling are not compatible with embeddings generated with mean pooling. The default pooling method is `mean` in order for this to not be a breaking change, but we highly suggest using the new `cls` pooling for better accuracy. + */ + pooling?: "mean" | "cls"; +} | { + /** + * Batch of the embeddings requests to run using async-queue + */ + requests: { + text: string | string[]; + /** + * The pooling method used in the embedding process. `cls` pooling will generate more accurate embeddings on larger inputs - however, embeddings created with cls pooling are not compatible with embeddings generated with mean pooling. The default pooling method is `mean` in order for this to not be a breaking change, but we highly suggest using the new `cls` pooling for better accuracy. + */ + pooling?: "mean" | "cls"; + }[]; +}; +type Ai_Cf_Baai_Bge_Small_En_V1_5_Output = { + shape?: number[]; + /** + * Embeddings of the requested text values + */ + data?: number[][]; + /** + * The pooling method used in the embedding process. + */ + pooling?: "mean" | "cls"; +} | Ai_Cf_Baai_Bge_Small_En_V1_5_AsyncResponse; +interface Ai_Cf_Baai_Bge_Small_En_V1_5_AsyncResponse { + /** + * The async request id that can be used to obtain the results. + */ + request_id?: string; +} +declare abstract class Base_Ai_Cf_Baai_Bge_Small_En_V1_5 { + inputs: Ai_Cf_Baai_Bge_Small_En_V1_5_Input; + postProcessedOutputs: Ai_Cf_Baai_Bge_Small_En_V1_5_Output; +} +type Ai_Cf_Baai_Bge_Large_En_V1_5_Input = { + text: string | string[]; + /** + * The pooling method used in the embedding process. `cls` pooling will generate more accurate embeddings on larger inputs - however, embeddings created with cls pooling are not compatible with embeddings generated with mean pooling. The default pooling method is `mean` in order for this to not be a breaking change, but we highly suggest using the new `cls` pooling for better accuracy. + */ + pooling?: "mean" | "cls"; +} | { + /** + * Batch of the embeddings requests to run using async-queue + */ + requests: { + text: string | string[]; + /** + * The pooling method used in the embedding process. `cls` pooling will generate more accurate embeddings on larger inputs - however, embeddings created with cls pooling are not compatible with embeddings generated with mean pooling. The default pooling method is `mean` in order for this to not be a breaking change, but we highly suggest using the new `cls` pooling for better accuracy. + */ + pooling?: "mean" | "cls"; + }[]; +}; +type Ai_Cf_Baai_Bge_Large_En_V1_5_Output = { + shape?: number[]; + /** + * Embeddings of the requested text values + */ + data?: number[][]; + /** + * The pooling method used in the embedding process. + */ + pooling?: "mean" | "cls"; +} | Ai_Cf_Baai_Bge_Large_En_V1_5_AsyncResponse; +interface Ai_Cf_Baai_Bge_Large_En_V1_5_AsyncResponse { + /** + * The async request id that can be used to obtain the results. + */ + request_id?: string; +} +declare abstract class Base_Ai_Cf_Baai_Bge_Large_En_V1_5 { + inputs: Ai_Cf_Baai_Bge_Large_En_V1_5_Input; + postProcessedOutputs: Ai_Cf_Baai_Bge_Large_En_V1_5_Output; +} +type Ai_Cf_Unum_Uform_Gen2_Qwen_500M_Input = string | { + /** + * The input text prompt for the model to generate a response. + */ + prompt?: string; + /** + * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. + */ + raw?: boolean; + /** + * Controls the creativity of the AI's responses by adjusting how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; + image: number[] | (string & NonNullable); + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; +}; +interface Ai_Cf_Unum_Uform_Gen2_Qwen_500M_Output { + description?: string; +} +declare abstract class Base_Ai_Cf_Unum_Uform_Gen2_Qwen_500M { + inputs: Ai_Cf_Unum_Uform_Gen2_Qwen_500M_Input; + postProcessedOutputs: Ai_Cf_Unum_Uform_Gen2_Qwen_500M_Output; +} +type Ai_Cf_Openai_Whisper_Tiny_En_Input = string | { + /** + * An array of integers that represent the audio data constrained to 8-bit unsigned integer values + */ + audio: number[]; +}; +interface Ai_Cf_Openai_Whisper_Tiny_En_Output { + /** + * The transcription + */ + text: string; + word_count?: number; + words?: { + word?: string; + /** + * The second this word begins in the recording + */ + start?: number; + /** + * The ending second when the word completes + */ + end?: number; + }[]; + vtt?: string; +} +declare abstract class Base_Ai_Cf_Openai_Whisper_Tiny_En { + inputs: Ai_Cf_Openai_Whisper_Tiny_En_Input; + postProcessedOutputs: Ai_Cf_Openai_Whisper_Tiny_En_Output; +} +interface Ai_Cf_Openai_Whisper_Large_V3_Turbo_Input { + audio: string | { + body?: object; + contentType?: string; + }; + /** + * Supported tasks are 'translate' or 'transcribe'. + */ + task?: string; + /** + * The language of the audio being transcribed or translated. + */ + language?: string; + /** + * Preprocess the audio with a voice activity detection model. + */ + vad_filter?: boolean; + /** + * A text prompt to help provide context to the model on the contents of the audio. + */ + initial_prompt?: string; + /** + * The prefix appended to the beginning of the output of the transcription and can guide the transcription result. + */ + prefix?: string; + /** + * The number of beams to use in beam search decoding. Higher values may improve accuracy at the cost of speed. + */ + beam_size?: number; + /** + * Whether to condition on previous text during transcription. Setting to false may help prevent hallucination loops. + */ + condition_on_previous_text?: boolean; + /** + * Threshold for detecting no-speech segments. Segments with no-speech probability above this value are skipped. + */ + no_speech_threshold?: number; + /** + * Threshold for filtering out segments with high compression ratio, which often indicate repetitive or hallucinated text. + */ + compression_ratio_threshold?: number; + /** + * Threshold for filtering out segments with low average log probability, indicating low confidence. + */ + log_prob_threshold?: number; + /** + * Optional threshold (in seconds) to skip silent periods that may cause hallucinations. + */ + hallucination_silence_threshold?: number; +} +interface Ai_Cf_Openai_Whisper_Large_V3_Turbo_Output { + transcription_info?: { + /** + * The language of the audio being transcribed or translated. + */ + language?: string; + /** + * The confidence level or probability of the detected language being accurate, represented as a decimal between 0 and 1. + */ + language_probability?: number; + /** + * The total duration of the original audio file, in seconds. + */ + duration?: number; + /** + * The duration of the audio after applying Voice Activity Detection (VAD) to remove silent or irrelevant sections, in seconds. + */ + duration_after_vad?: number; + }; + /** + * The complete transcription of the audio. + */ + text: string; + /** + * The total number of words in the transcription. + */ + word_count?: number; + segments?: { + /** + * The starting time of the segment within the audio, in seconds. + */ + start?: number; + /** + * The ending time of the segment within the audio, in seconds. + */ + end?: number; + /** + * The transcription of the segment. + */ + text?: string; + /** + * The temperature used in the decoding process, controlling randomness in predictions. Lower values result in more deterministic outputs. + */ + temperature?: number; + /** + * The average log probability of the predictions for the words in this segment, indicating overall confidence. + */ + avg_logprob?: number; + /** + * The compression ratio of the input to the output, measuring how much the text was compressed during the transcription process. + */ + compression_ratio?: number; + /** + * The probability that the segment contains no speech, represented as a decimal between 0 and 1. + */ + no_speech_prob?: number; + words?: { + /** + * The individual word transcribed from the audio. + */ + word?: string; + /** + * The starting time of the word within the audio, in seconds. + */ + start?: number; + /** + * The ending time of the word within the audio, in seconds. + */ + end?: number; + }[]; + }[]; + /** + * The transcription in WebVTT format, which includes timing and text information for use in subtitles. + */ + vtt?: string; +} +declare abstract class Base_Ai_Cf_Openai_Whisper_Large_V3_Turbo { + inputs: Ai_Cf_Openai_Whisper_Large_V3_Turbo_Input; + postProcessedOutputs: Ai_Cf_Openai_Whisper_Large_V3_Turbo_Output; +} +type Ai_Cf_Baai_Bge_M3_Input = Ai_Cf_Baai_Bge_M3_Input_QueryAnd_Contexts | Ai_Cf_Baai_Bge_M3_Input_Embedding | { + /** + * Batch of the embeddings requests to run using async-queue + */ + requests: (Ai_Cf_Baai_Bge_M3_Input_QueryAnd_Contexts_1 | Ai_Cf_Baai_Bge_M3_Input_Embedding_1)[]; +}; +interface Ai_Cf_Baai_Bge_M3_Input_QueryAnd_Contexts { + /** + * A query you wish to perform against the provided contexts. If no query is provided the model with respond with embeddings for contexts + */ + query?: string; + /** + * List of provided contexts. Note that the index in this array is important, as the response will refer to it. + */ + contexts: { + /** + * One of the provided context content + */ + text?: string; + }[]; + /** + * When provided with too long context should the model error out or truncate the context to fit? + */ + truncate_inputs?: boolean; +} +interface Ai_Cf_Baai_Bge_M3_Input_Embedding { + text: string | string[]; + /** + * When provided with too long context should the model error out or truncate the context to fit? + */ + truncate_inputs?: boolean; +} +interface Ai_Cf_Baai_Bge_M3_Input_QueryAnd_Contexts_1 { + /** + * A query you wish to perform against the provided contexts. If no query is provided the model with respond with embeddings for contexts + */ + query?: string; + /** + * List of provided contexts. Note that the index in this array is important, as the response will refer to it. + */ + contexts: { + /** + * One of the provided context content + */ + text?: string; + }[]; + /** + * When provided with too long context should the model error out or truncate the context to fit? + */ + truncate_inputs?: boolean; +} +interface Ai_Cf_Baai_Bge_M3_Input_Embedding_1 { + text: string | string[]; + /** + * When provided with too long context should the model error out or truncate the context to fit? + */ + truncate_inputs?: boolean; +} +type Ai_Cf_Baai_Bge_M3_Output = Ai_Cf_Baai_Bge_M3_Output_Query | Ai_Cf_Baai_Bge_M3_Output_EmbeddingFor_Contexts | Ai_Cf_Baai_Bge_M3_Output_Embedding | Ai_Cf_Baai_Bge_M3_AsyncResponse; +interface Ai_Cf_Baai_Bge_M3_Output_Query { + response?: { + /** + * Index of the context in the request + */ + id?: number; + /** + * Score of the context under the index. + */ + score?: number; + }[]; +} +interface Ai_Cf_Baai_Bge_M3_Output_EmbeddingFor_Contexts { + response?: number[][]; + shape?: number[]; + /** + * The pooling method used in the embedding process. + */ + pooling?: "mean" | "cls"; +} +interface Ai_Cf_Baai_Bge_M3_Output_Embedding { + shape?: number[]; + /** + * Embeddings of the requested text values + */ + data?: number[][]; + /** + * The pooling method used in the embedding process. + */ + pooling?: "mean" | "cls"; +} +interface Ai_Cf_Baai_Bge_M3_AsyncResponse { + /** + * The async request id that can be used to obtain the results. + */ + request_id?: string; +} +declare abstract class Base_Ai_Cf_Baai_Bge_M3 { + inputs: Ai_Cf_Baai_Bge_M3_Input; + postProcessedOutputs: Ai_Cf_Baai_Bge_M3_Output; +} +interface Ai_Cf_Black_Forest_Labs_Flux_1_Schnell_Input { + /** + * A text description of the image you want to generate. + */ + prompt: string; + /** + * The number of diffusion steps; higher values can improve quality but take longer. + */ + steps?: number; +} +interface Ai_Cf_Black_Forest_Labs_Flux_1_Schnell_Output { + /** + * The generated image in Base64 format. + */ + image?: string; +} +declare abstract class Base_Ai_Cf_Black_Forest_Labs_Flux_1_Schnell { + inputs: Ai_Cf_Black_Forest_Labs_Flux_1_Schnell_Input; + postProcessedOutputs: Ai_Cf_Black_Forest_Labs_Flux_1_Schnell_Output; +} +type Ai_Cf_Meta_Llama_3_2_11B_Vision_Instruct_Input = Ai_Cf_Meta_Llama_3_2_11B_Vision_Instruct_Prompt | Ai_Cf_Meta_Llama_3_2_11B_Vision_Instruct_Messages; +interface Ai_Cf_Meta_Llama_3_2_11B_Vision_Instruct_Prompt { + /** + * The input text prompt for the model to generate a response. + */ + prompt: string; + image?: number[] | (string & NonNullable); + /** + * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. + */ + raw?: boolean; + /** + * If true, the response will be streamed back incrementally using SSE, Server Sent Events. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; + /** + * Name of the LoRA (Low-Rank Adaptation) model to fine-tune the base model. + */ + lora?: string; +} +interface Ai_Cf_Meta_Llama_3_2_11B_Vision_Instruct_Messages { + /** + * An array of message objects representing the conversation history. + */ + messages: { + /** + * The role of the message sender (e.g., 'user', 'assistant', 'system', 'tool'). + */ + role?: string; + /** + * The tool call id. If you don't know what to put here you can fall back to 000000001 + */ + tool_call_id?: string; + content?: string | { + /** + * Type of the content provided + */ + type?: string; + text?: string; + image_url?: { + /** + * image uri with data (e.g. data:image/jpeg;base64,/9j/...). HTTP URL will not be accepted + */ + url?: string; + }; + }[] | { + /** + * Type of the content provided + */ + type?: string; + text?: string; + image_url?: { + /** + * image uri with data (e.g. data:image/jpeg;base64,/9j/...). HTTP URL will not be accepted + */ + url?: string; + }; + }; + }[]; + image?: number[] | (string & NonNullable); + functions?: { + name: string; + code: string; + }[]; + /** + * A list of tools available for the assistant to use. + */ + tools?: ({ + /** + * The name of the tool. More descriptive the better. + */ + name: string; + /** + * A brief description of what the tool does. + */ + description: string; + /** + * Schema defining the parameters accepted by the tool. + */ + parameters: { + /** + * The type of the parameters object (usually 'object'). + */ + type: string; + /** + * List of required parameter names. + */ + required?: string[]; + /** + * Definitions of each parameter. + */ + properties: { + [k: string]: { + /** + * The data type of the parameter. + */ + type: string; + /** + * A description of the expected parameter. + */ + description: string; + }; + }; + }; + } | { + /** + * Specifies the type of tool (e.g., 'function'). + */ + type: string; + /** + * Details of the function tool. + */ + function: { + /** + * The name of the function. + */ + name: string; + /** + * A brief description of what the function does. + */ + description: string; + /** + * Schema defining the parameters accepted by the function. + */ + parameters: { + /** + * The type of the parameters object (usually 'object'). + */ + type: string; + /** + * List of required parameter names. + */ + required?: string[]; + /** + * Definitions of each parameter. + */ + properties: { + [k: string]: { + /** + * The data type of the parameter. + */ + type: string; + /** + * A description of the expected parameter. + */ + description: string; + }; + }; + }; + }; + })[]; + /** + * If true, the response will be streamed back incrementally. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Controls the creativity of the AI's responses by adjusting how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; +} +type Ai_Cf_Meta_Llama_3_2_11B_Vision_Instruct_Output = { + /** + * The generated text response from the model + */ + response?: string; + /** + * An array of tool calls requests made during the response generation + */ + tool_calls?: { + /** + * The arguments passed to be passed to the tool call request + */ + arguments?: object; + /** + * The name of the tool to be called + */ + name?: string; + }[]; +}; +declare abstract class Base_Ai_Cf_Meta_Llama_3_2_11B_Vision_Instruct { + inputs: Ai_Cf_Meta_Llama_3_2_11B_Vision_Instruct_Input; + postProcessedOutputs: Ai_Cf_Meta_Llama_3_2_11B_Vision_Instruct_Output; +} +type Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_Input = Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_Prompt | Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_Messages | Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_Async_Batch; +interface Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_Prompt { + /** + * The input text prompt for the model to generate a response. + */ + prompt: string; + /** + * Name of the LoRA (Low-Rank Adaptation) model to fine-tune the base model. + */ + lora?: string; + response_format?: Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_JSON_Mode; + /** + * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. + */ + raw?: boolean; + /** + * If true, the response will be streamed back incrementally using SSE, Server Sent Events. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; +} +interface Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_JSON_Mode { + type?: "json_object" | "json_schema"; + json_schema?: unknown; +} +interface Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_Messages { + /** + * An array of message objects representing the conversation history. + */ + messages: { + /** + * The role of the message sender (e.g., 'user', 'assistant', 'system', 'tool'). + */ + role: string; + content: string | { + /** + * Type of the content (text) + */ + type?: string; + /** + * Text content + */ + text?: string; + }[]; + }[]; + functions?: { + name: string; + code: string; + }[]; + /** + * A list of tools available for the assistant to use. + */ + tools?: ({ + /** + * The name of the tool. More descriptive the better. + */ + name: string; + /** + * A brief description of what the tool does. + */ + description: string; + /** + * Schema defining the parameters accepted by the tool. + */ + parameters: { + /** + * The type of the parameters object (usually 'object'). + */ + type: string; + /** + * List of required parameter names. + */ + required?: string[]; + /** + * Definitions of each parameter. + */ + properties: { + [k: string]: { + /** + * The data type of the parameter. + */ + type: string; + /** + * A description of the expected parameter. + */ + description: string; + }; + }; + }; + } | { + /** + * Specifies the type of tool (e.g., 'function'). + */ + type: string; + /** + * Details of the function tool. + */ + function: { + /** + * The name of the function. + */ + name: string; + /** + * A brief description of what the function does. + */ + description: string; + /** + * Schema defining the parameters accepted by the function. + */ + parameters: { + /** + * The type of the parameters object (usually 'object'). + */ + type: string; + /** + * List of required parameter names. + */ + required?: string[]; + /** + * Definitions of each parameter. + */ + properties: { + [k: string]: { + /** + * The data type of the parameter. + */ + type: string; + /** + * A description of the expected parameter. + */ + description: string; + }; + }; + }; + }; + })[]; + response_format?: Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_JSON_Mode_1; + /** + * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. + */ + raw?: boolean; + /** + * If true, the response will be streamed back incrementally using SSE, Server Sent Events. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; +} +interface Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_JSON_Mode_1 { + type?: "json_object" | "json_schema"; + json_schema?: unknown; +} +interface Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_Async_Batch { + requests?: { + /** + * User-supplied reference. This field will be present in the response as well it can be used to reference the request and response. It's NOT validated to be unique. + */ + external_reference?: string; + /** + * Prompt for the text generation model + */ + prompt?: string; + /** + * If true, the response will be streamed back incrementally using SSE, Server Sent Events. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; + response_format?: Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_JSON_Mode_2; + }[]; +} +interface Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_JSON_Mode_2 { + type?: "json_object" | "json_schema"; + json_schema?: unknown; +} +type Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_Output = { + /** + * The generated text response from the model + */ + response: string; + /** + * Usage statistics for the inference request + */ + usage?: { + /** + * Total number of tokens in input + */ + prompt_tokens?: number; + /** + * Total number of tokens in output + */ + completion_tokens?: number; + /** + * Total number of input and output tokens + */ + total_tokens?: number; + }; + /** + * An array of tool calls requests made during the response generation + */ + tool_calls?: { + /** + * The arguments passed to be passed to the tool call request + */ + arguments?: object; + /** + * The name of the tool to be called + */ + name?: string; + }[]; +} | string | Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_AsyncResponse; +interface Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_AsyncResponse { + /** + * The async request id that can be used to obtain the results. + */ + request_id?: string; +} +declare abstract class Base_Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast { + inputs: Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_Input; + postProcessedOutputs: Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_Output; +} +interface Ai_Cf_Meta_Llama_Guard_3_8B_Input { + /** + * An array of message objects representing the conversation history. + */ + messages: { + /** + * The role of the message sender must alternate between 'user' and 'assistant'. + */ + role: "user" | "assistant"; + /** + * The content of the message as a string. + */ + content: string; + }[]; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Dictate the output format of the generated response. + */ + response_format?: { + /** + * Set to json_object to process and output generated text as JSON. + */ + type?: string; + }; +} +interface Ai_Cf_Meta_Llama_Guard_3_8B_Output { + response?: string | { + /** + * Whether the conversation is safe or not. + */ + safe?: boolean; + /** + * A list of what hazard categories predicted for the conversation, if the conversation is deemed unsafe. + */ + categories?: string[]; + }; + /** + * Usage statistics for the inference request + */ + usage?: { + /** + * Total number of tokens in input + */ + prompt_tokens?: number; + /** + * Total number of tokens in output + */ + completion_tokens?: number; + /** + * Total number of input and output tokens + */ + total_tokens?: number; + }; +} +declare abstract class Base_Ai_Cf_Meta_Llama_Guard_3_8B { + inputs: Ai_Cf_Meta_Llama_Guard_3_8B_Input; + postProcessedOutputs: Ai_Cf_Meta_Llama_Guard_3_8B_Output; +} +interface Ai_Cf_Baai_Bge_Reranker_Base_Input { + /** + * A query you wish to perform against the provided contexts. + */ + /** + * Number of returned results starting with the best score. + */ + top_k?: number; + /** + * List of provided contexts. Note that the index in this array is important, as the response will refer to it. + */ + contexts: { + /** + * One of the provided context content + */ + text?: string; + }[]; +} +interface Ai_Cf_Baai_Bge_Reranker_Base_Output { + response?: { + /** + * Index of the context in the request + */ + id?: number; + /** + * Score of the context under the index. + */ + score?: number; + }[]; +} +declare abstract class Base_Ai_Cf_Baai_Bge_Reranker_Base { + inputs: Ai_Cf_Baai_Bge_Reranker_Base_Input; + postProcessedOutputs: Ai_Cf_Baai_Bge_Reranker_Base_Output; +} +type Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct_Input = Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct_Prompt | Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct_Messages; +interface Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct_Prompt { + /** + * The input text prompt for the model to generate a response. + */ + prompt: string; + /** + * Name of the LoRA (Low-Rank Adaptation) model to fine-tune the base model. + */ + lora?: string; + response_format?: Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct_JSON_Mode; + /** + * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. + */ + raw?: boolean; + /** + * If true, the response will be streamed back incrementally using SSE, Server Sent Events. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; +} +interface Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct_JSON_Mode { + type?: "json_object" | "json_schema"; + json_schema?: unknown; +} +interface Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct_Messages { + /** + * An array of message objects representing the conversation history. + */ + messages: { + /** + * The role of the message sender (e.g., 'user', 'assistant', 'system', 'tool'). + */ + role: string; + /** + * The content of the message as a string. + */ + content: string; + }[]; + functions?: { + name: string; + code: string; + }[]; + /** + * A list of tools available for the assistant to use. + */ + tools?: ({ + /** + * The name of the tool. More descriptive the better. + */ + name: string; + /** + * A brief description of what the tool does. + */ + description: string; + /** + * Schema defining the parameters accepted by the tool. + */ + parameters: { + /** + * The type of the parameters object (usually 'object'). + */ + type: string; + /** + * List of required parameter names. + */ + required?: string[]; + /** + * Definitions of each parameter. + */ + properties: { + [k: string]: { + /** + * The data type of the parameter. + */ + type: string; + /** + * A description of the expected parameter. + */ + description: string; + }; + }; + }; + } | { + /** + * Specifies the type of tool (e.g., 'function'). + */ + type: string; + /** + * Details of the function tool. + */ + function: { + /** + * The name of the function. + */ + name: string; + /** + * A brief description of what the function does. + */ + description: string; + /** + * Schema defining the parameters accepted by the function. + */ + parameters: { + /** + * The type of the parameters object (usually 'object'). + */ + type: string; + /** + * List of required parameter names. + */ + required?: string[]; + /** + * Definitions of each parameter. + */ + properties: { + [k: string]: { + /** + * The data type of the parameter. + */ + type: string; + /** + * A description of the expected parameter. + */ + description: string; + }; + }; + }; + }; + })[]; + response_format?: Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct_JSON_Mode_1; + /** + * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. + */ + raw?: boolean; + /** + * If true, the response will be streamed back incrementally using SSE, Server Sent Events. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; +} +interface Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct_JSON_Mode_1 { + type?: "json_object" | "json_schema"; + json_schema?: unknown; +} +type Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct_Output = { + /** + * The generated text response from the model + */ + response: string; + /** + * Usage statistics for the inference request + */ + usage?: { + /** + * Total number of tokens in input + */ + prompt_tokens?: number; + /** + * Total number of tokens in output + */ + completion_tokens?: number; + /** + * Total number of input and output tokens + */ + total_tokens?: number; + }; + /** + * An array of tool calls requests made during the response generation + */ + tool_calls?: { + /** + * The arguments passed to be passed to the tool call request + */ + arguments?: object; + /** + * The name of the tool to be called + */ + name?: string; + }[]; +}; +declare abstract class Base_Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct { + inputs: Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct_Input; + postProcessedOutputs: Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct_Output; +} +type Ai_Cf_Qwen_Qwq_32B_Input = Ai_Cf_Qwen_Qwq_32B_Prompt | Ai_Cf_Qwen_Qwq_32B_Messages; +interface Ai_Cf_Qwen_Qwq_32B_Prompt { + /** + * The input text prompt for the model to generate a response. + */ + prompt: string; + /** + * JSON schema that should be fulfilled for the response. + */ + guided_json?: object; + /** + * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. + */ + raw?: boolean; + /** + * If true, the response will be streamed back incrementally using SSE, Server Sent Events. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; +} +interface Ai_Cf_Qwen_Qwq_32B_Messages { + /** + * An array of message objects representing the conversation history. + */ + messages: { + /** + * The role of the message sender (e.g., 'user', 'assistant', 'system', 'tool'). + */ + role?: string; + /** + * The tool call id. If you don't know what to put here you can fall back to 000000001 + */ + tool_call_id?: string; + content?: string | { + /** + * Type of the content provided + */ + type?: string; + text?: string; + image_url?: { + /** + * image uri with data (e.g. data:image/jpeg;base64,/9j/...). HTTP URL will not be accepted + */ + url?: string; + }; + }[] | { + /** + * Type of the content provided + */ + type?: string; + text?: string; + image_url?: { + /** + * image uri with data (e.g. data:image/jpeg;base64,/9j/...). HTTP URL will not be accepted + */ + url?: string; + }; + }; + }[]; + functions?: { + name: string; + code: string; + }[]; + /** + * A list of tools available for the assistant to use. + */ + tools?: ({ + /** + * The name of the tool. More descriptive the better. + */ + name: string; + /** + * A brief description of what the tool does. + */ + description: string; + /** + * Schema defining the parameters accepted by the tool. + */ + parameters: { + /** + * The type of the parameters object (usually 'object'). + */ + type: string; + /** + * List of required parameter names. + */ + required?: string[]; + /** + * Definitions of each parameter. + */ + properties: { + [k: string]: { + /** + * The data type of the parameter. + */ + type: string; + /** + * A description of the expected parameter. + */ + description: string; + }; + }; + }; + } | { + /** + * Specifies the type of tool (e.g., 'function'). + */ + type: string; + /** + * Details of the function tool. + */ + function: { + /** + * The name of the function. + */ + name: string; + /** + * A brief description of what the function does. + */ + description: string; + /** + * Schema defining the parameters accepted by the function. + */ + parameters: { + /** + * The type of the parameters object (usually 'object'). + */ + type: string; + /** + * List of required parameter names. + */ + required?: string[]; + /** + * Definitions of each parameter. + */ + properties: { + [k: string]: { + /** + * The data type of the parameter. + */ + type: string; + /** + * A description of the expected parameter. + */ + description: string; + }; + }; + }; + }; + })[]; + /** + * JSON schema that should be fufilled for the response. + */ + guided_json?: object; + /** + * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. + */ + raw?: boolean; + /** + * If true, the response will be streamed back incrementally using SSE, Server Sent Events. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; +} +type Ai_Cf_Qwen_Qwq_32B_Output = { + /** + * The generated text response from the model + */ + response: string; + /** + * Usage statistics for the inference request + */ + usage?: { + /** + * Total number of tokens in input + */ + prompt_tokens?: number; + /** + * Total number of tokens in output + */ + completion_tokens?: number; + /** + * Total number of input and output tokens + */ + total_tokens?: number; + }; + /** + * An array of tool calls requests made during the response generation + */ + tool_calls?: { + /** + * The arguments passed to be passed to the tool call request + */ + arguments?: object; + /** + * The name of the tool to be called + */ + name?: string; + }[]; +}; +declare abstract class Base_Ai_Cf_Qwen_Qwq_32B { + inputs: Ai_Cf_Qwen_Qwq_32B_Input; + postProcessedOutputs: Ai_Cf_Qwen_Qwq_32B_Output; +} +type Ai_Cf_Mistralai_Mistral_Small_3_1_24B_Instruct_Input = Ai_Cf_Mistralai_Mistral_Small_3_1_24B_Instruct_Prompt | Ai_Cf_Mistralai_Mistral_Small_3_1_24B_Instruct_Messages; +interface Ai_Cf_Mistralai_Mistral_Small_3_1_24B_Instruct_Prompt { + /** + * The input text prompt for the model to generate a response. + */ + prompt: string; + /** + * JSON schema that should be fulfilled for the response. + */ + guided_json?: object; + /** + * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. + */ + raw?: boolean; + /** + * If true, the response will be streamed back incrementally using SSE, Server Sent Events. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; +} +interface Ai_Cf_Mistralai_Mistral_Small_3_1_24B_Instruct_Messages { + /** + * An array of message objects representing the conversation history. + */ + messages: { + /** + * The role of the message sender (e.g., 'user', 'assistant', 'system', 'tool'). + */ + role?: string; + /** + * The tool call id. Must be supplied for tool calls for Mistral-3. If you don't know what to put here you can fall back to 000000001 + */ + tool_call_id?: string; + content?: string | { + /** + * Type of the content provided + */ + type?: string; + text?: string; + image_url?: { + /** + * image uri with data (e.g. data:image/jpeg;base64,/9j/...). HTTP URL will not be accepted + */ + url?: string; + }; + }[] | { + /** + * Type of the content provided + */ + type?: string; + text?: string; + image_url?: { + /** + * image uri with data (e.g. data:image/jpeg;base64,/9j/...). HTTP URL will not be accepted + */ + url?: string; + }; + }; + }[]; + functions?: { + name: string; + code: string; + }[]; + /** + * A list of tools available for the assistant to use. + */ + tools?: ({ + /** + * The name of the tool. More descriptive the better. + */ + name: string; + /** + * A brief description of what the tool does. + */ + description: string; + /** + * Schema defining the parameters accepted by the tool. + */ + parameters: { + /** + * The type of the parameters object (usually 'object'). + */ + type: string; + /** + * List of required parameter names. + */ + required?: string[]; + /** + * Definitions of each parameter. + */ + properties: { + [k: string]: { + /** + * The data type of the parameter. + */ + type: string; + /** + * A description of the expected parameter. + */ + description: string; + }; + }; + }; + } | { + /** + * Specifies the type of tool (e.g., 'function'). + */ + type: string; + /** + * Details of the function tool. + */ + function: { + /** + * The name of the function. + */ + name: string; + /** + * A brief description of what the function does. + */ + description: string; + /** + * Schema defining the parameters accepted by the function. + */ + parameters: { + /** + * The type of the parameters object (usually 'object'). + */ + type: string; + /** + * List of required parameter names. + */ + required?: string[]; + /** + * Definitions of each parameter. + */ + properties: { + [k: string]: { + /** + * The data type of the parameter. + */ + type: string; + /** + * A description of the expected parameter. + */ + description: string; + }; + }; + }; + }; + })[]; + /** + * JSON schema that should be fufilled for the response. + */ + guided_json?: object; + /** + * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. + */ + raw?: boolean; + /** + * If true, the response will be streamed back incrementally using SSE, Server Sent Events. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; +} +type Ai_Cf_Mistralai_Mistral_Small_3_1_24B_Instruct_Output = { + /** + * The generated text response from the model + */ + response: string; + /** + * Usage statistics for the inference request + */ + usage?: { + /** + * Total number of tokens in input + */ + prompt_tokens?: number; + /** + * Total number of tokens in output + */ + completion_tokens?: number; + /** + * Total number of input and output tokens + */ + total_tokens?: number; + }; + /** + * An array of tool calls requests made during the response generation + */ + tool_calls?: { + /** + * The arguments passed to be passed to the tool call request + */ + arguments?: object; + /** + * The name of the tool to be called + */ + name?: string; + }[]; +}; +declare abstract class Base_Ai_Cf_Mistralai_Mistral_Small_3_1_24B_Instruct { + inputs: Ai_Cf_Mistralai_Mistral_Small_3_1_24B_Instruct_Input; + postProcessedOutputs: Ai_Cf_Mistralai_Mistral_Small_3_1_24B_Instruct_Output; +} +type Ai_Cf_Google_Gemma_3_12B_It_Input = Ai_Cf_Google_Gemma_3_12B_It_Prompt | Ai_Cf_Google_Gemma_3_12B_It_Messages; +interface Ai_Cf_Google_Gemma_3_12B_It_Prompt { + /** + * The input text prompt for the model to generate a response. + */ + prompt: string; + /** + * JSON schema that should be fufilled for the response. + */ + guided_json?: object; + /** + * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. + */ + raw?: boolean; + /** + * If true, the response will be streamed back incrementally using SSE, Server Sent Events. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; +} +interface Ai_Cf_Google_Gemma_3_12B_It_Messages { + /** + * An array of message objects representing the conversation history. + */ + messages: { + /** + * The role of the message sender (e.g., 'user', 'assistant', 'system', 'tool'). + */ + role?: string; + content?: string | { + /** + * Type of the content provided + */ + type?: string; + text?: string; + image_url?: { + /** + * image uri with data (e.g. data:image/jpeg;base64,/9j/...). HTTP URL will not be accepted + */ + url?: string; + }; + }[]; + }[]; + functions?: { + name: string; + code: string; + }[]; + /** + * A list of tools available for the assistant to use. + */ + tools?: ({ + /** + * The name of the tool. More descriptive the better. + */ + name: string; + /** + * A brief description of what the tool does. + */ + description: string; + /** + * Schema defining the parameters accepted by the tool. + */ + parameters: { + /** + * The type of the parameters object (usually 'object'). + */ + type: string; + /** + * List of required parameter names. + */ + required?: string[]; + /** + * Definitions of each parameter. + */ + properties: { + [k: string]: { + /** + * The data type of the parameter. + */ + type: string; + /** + * A description of the expected parameter. + */ + description: string; + }; + }; + }; + } | { + /** + * Specifies the type of tool (e.g., 'function'). + */ + type: string; + /** + * Details of the function tool. + */ + function: { + /** + * The name of the function. + */ + name: string; + /** + * A brief description of what the function does. + */ + description: string; + /** + * Schema defining the parameters accepted by the function. + */ + parameters: { + /** + * The type of the parameters object (usually 'object'). + */ + type: string; + /** + * List of required parameter names. + */ + required?: string[]; + /** + * Definitions of each parameter. + */ + properties: { + [k: string]: { + /** + * The data type of the parameter. + */ + type: string; + /** + * A description of the expected parameter. + */ + description: string; + }; + }; + }; + }; + })[]; + /** + * JSON schema that should be fufilled for the response. + */ + guided_json?: object; + /** + * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. + */ + raw?: boolean; + /** + * If true, the response will be streamed back incrementally using SSE, Server Sent Events. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; +} +type Ai_Cf_Google_Gemma_3_12B_It_Output = { + /** + * The generated text response from the model + */ + response: string; + /** + * Usage statistics for the inference request + */ + usage?: { + /** + * Total number of tokens in input + */ + prompt_tokens?: number; + /** + * Total number of tokens in output + */ + completion_tokens?: number; + /** + * Total number of input and output tokens + */ + total_tokens?: number; + }; + /** + * An array of tool calls requests made during the response generation + */ + tool_calls?: { + /** + * The arguments passed to be passed to the tool call request + */ + arguments?: object; + /** + * The name of the tool to be called + */ + name?: string; + }[]; +}; +declare abstract class Base_Ai_Cf_Google_Gemma_3_12B_It { + inputs: Ai_Cf_Google_Gemma_3_12B_It_Input; + postProcessedOutputs: Ai_Cf_Google_Gemma_3_12B_It_Output; +} +type Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Input = Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Prompt | Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Messages | Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Async_Batch; +interface Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Prompt { + /** + * The input text prompt for the model to generate a response. + */ + prompt: string; + /** + * JSON schema that should be fulfilled for the response. + */ + guided_json?: object; + response_format?: Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_JSON_Mode; + /** + * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. + */ + raw?: boolean; + /** + * If true, the response will be streamed back incrementally using SSE, Server Sent Events. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; +} +interface Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_JSON_Mode { + type?: "json_object" | "json_schema"; + json_schema?: unknown; +} +interface Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Messages { + /** + * An array of message objects representing the conversation history. + */ + messages: { + /** + * The role of the message sender (e.g., 'user', 'assistant', 'system', 'tool'). + */ + role?: string; + /** + * The tool call id. If you don't know what to put here you can fall back to 000000001 + */ + tool_call_id?: string; + content?: string | { + /** + * Type of the content provided + */ + type?: string; + text?: string; + image_url?: { + /** + * image uri with data (e.g. data:image/jpeg;base64,/9j/...). HTTP URL will not be accepted + */ + url?: string; + }; + }[] | { + /** + * Type of the content provided + */ + type?: string; + text?: string; + image_url?: { + /** + * image uri with data (e.g. data:image/jpeg;base64,/9j/...). HTTP URL will not be accepted + */ + url?: string; + }; + }; + }[]; + functions?: { + name: string; + code: string; + }[]; + /** + * A list of tools available for the assistant to use. + */ + tools?: ({ + /** + * The name of the tool. More descriptive the better. + */ + name: string; + /** + * A brief description of what the tool does. + */ + description: string; + /** + * Schema defining the parameters accepted by the tool. + */ + parameters: { + /** + * The type of the parameters object (usually 'object'). + */ + type: string; + /** + * List of required parameter names. + */ + required?: string[]; + /** + * Definitions of each parameter. + */ + properties: { + [k: string]: { + /** + * The data type of the parameter. + */ + type: string; + /** + * A description of the expected parameter. + */ + description: string; + }; + }; + }; + } | { + /** + * Specifies the type of tool (e.g., 'function'). + */ + type: string; + /** + * Details of the function tool. + */ + function: { + /** + * The name of the function. + */ + name: string; + /** + * A brief description of what the function does. + */ + description: string; + /** + * Schema defining the parameters accepted by the function. + */ + parameters: { + /** + * The type of the parameters object (usually 'object'). + */ + type: string; + /** + * List of required parameter names. + */ + required?: string[]; + /** + * Definitions of each parameter. + */ + properties: { + [k: string]: { + /** + * The data type of the parameter. + */ + type: string; + /** + * A description of the expected parameter. + */ + description: string; + }; + }; + }; + }; + })[]; + response_format?: Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_JSON_Mode; + /** + * JSON schema that should be fufilled for the response. + */ + guided_json?: object; + /** + * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. + */ + raw?: boolean; + /** + * If true, the response will be streamed back incrementally using SSE, Server Sent Events. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; +} +interface Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Async_Batch { + requests: (Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Prompt_Inner | Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Messages_Inner)[]; +} +interface Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Prompt_Inner { + /** + * The input text prompt for the model to generate a response. + */ + prompt: string; + /** + * JSON schema that should be fulfilled for the response. + */ + guided_json?: object; + response_format?: Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_JSON_Mode; + /** + * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. + */ + raw?: boolean; + /** + * If true, the response will be streamed back incrementally using SSE, Server Sent Events. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; +} +interface Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Messages_Inner { + /** + * An array of message objects representing the conversation history. + */ + messages: { + /** + * The role of the message sender (e.g., 'user', 'assistant', 'system', 'tool'). + */ + role?: string; + /** + * The tool call id. If you don't know what to put here you can fall back to 000000001 + */ + tool_call_id?: string; + content?: string | { + /** + * Type of the content provided + */ + type?: string; + text?: string; + image_url?: { + /** + * image uri with data (e.g. data:image/jpeg;base64,/9j/...). HTTP URL will not be accepted + */ + url?: string; + }; + }[] | { + /** + * Type of the content provided + */ + type?: string; + text?: string; + image_url?: { + /** + * image uri with data (e.g. data:image/jpeg;base64,/9j/...). HTTP URL will not be accepted + */ + url?: string; + }; + }; + }[]; + functions?: { + name: string; + code: string; + }[]; + /** + * A list of tools available for the assistant to use. + */ + tools?: ({ + /** + * The name of the tool. More descriptive the better. + */ + name: string; + /** + * A brief description of what the tool does. + */ + description: string; + /** + * Schema defining the parameters accepted by the tool. + */ + parameters: { + /** + * The type of the parameters object (usually 'object'). + */ + type: string; + /** + * List of required parameter names. + */ + required?: string[]; + /** + * Definitions of each parameter. + */ + properties: { + [k: string]: { + /** + * The data type of the parameter. + */ + type: string; + /** + * A description of the expected parameter. + */ + description: string; + }; + }; + }; + } | { + /** + * Specifies the type of tool (e.g., 'function'). + */ + type: string; + /** + * Details of the function tool. + */ + function: { + /** + * The name of the function. + */ + name: string; + /** + * A brief description of what the function does. + */ + description: string; + /** + * Schema defining the parameters accepted by the function. + */ + parameters: { + /** + * The type of the parameters object (usually 'object'). + */ + type: string; + /** + * List of required parameter names. + */ + required?: string[]; + /** + * Definitions of each parameter. + */ + properties: { + [k: string]: { + /** + * The data type of the parameter. + */ + type: string; + /** + * A description of the expected parameter. + */ + description: string; + }; + }; + }; + }; + })[]; + response_format?: Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_JSON_Mode; + /** + * JSON schema that should be fufilled for the response. + */ + guided_json?: object; + /** + * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. + */ + raw?: boolean; + /** + * If true, the response will be streamed back incrementally using SSE, Server Sent Events. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; +} +type Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Output = { + /** + * The generated text response from the model + */ + response: string; + /** + * Usage statistics for the inference request + */ + usage?: { + /** + * Total number of tokens in input + */ + prompt_tokens?: number; + /** + * Total number of tokens in output + */ + completion_tokens?: number; + /** + * Total number of input and output tokens + */ + total_tokens?: number; + }; + /** + * An array of tool calls requests made during the response generation + */ + tool_calls?: { + /** + * The tool call id. + */ + id?: string; + /** + * Specifies the type of tool (e.g., 'function'). + */ + type?: string; + /** + * Details of the function tool. + */ + function?: { + /** + * The name of the tool to be called + */ + name?: string; + /** + * The arguments passed to be passed to the tool call request + */ + arguments?: object; + }; + }[]; +}; +declare abstract class Base_Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct { + inputs: Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Input; + postProcessedOutputs: Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Output; +} +type Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Input = Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Prompt | Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Messages | Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Async_Batch; +interface Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Prompt { + /** + * The input text prompt for the model to generate a response. + */ + prompt: string; + /** + * Name of the LoRA (Low-Rank Adaptation) model to fine-tune the base model. + */ + lora?: string; + response_format?: Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_JSON_Mode; + /** + * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. + */ + raw?: boolean; + /** + * If true, the response will be streamed back incrementally using SSE, Server Sent Events. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; +} +interface Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_JSON_Mode { + type?: "json_object" | "json_schema"; + json_schema?: unknown; +} +interface Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Messages { + /** + * An array of message objects representing the conversation history. + */ + messages: { + /** + * The role of the message sender (e.g., 'user', 'assistant', 'system', 'tool'). + */ + role: string; + content: string | { + /** + * Type of the content (text) + */ + type?: string; + /** + * Text content + */ + text?: string; + }[]; + }[]; + functions?: { + name: string; + code: string; + }[]; + /** + * A list of tools available for the assistant to use. + */ + tools?: ({ + /** + * The name of the tool. More descriptive the better. + */ + name: string; + /** + * A brief description of what the tool does. + */ + description: string; + /** + * Schema defining the parameters accepted by the tool. + */ + parameters: { + /** + * The type of the parameters object (usually 'object'). + */ + type: string; + /** + * List of required parameter names. + */ + required?: string[]; + /** + * Definitions of each parameter. + */ + properties: { + [k: string]: { + /** + * The data type of the parameter. + */ + type: string; + /** + * A description of the expected parameter. + */ + description: string; + }; + }; + }; + } | { + /** + * Specifies the type of tool (e.g., 'function'). + */ + type: string; + /** + * Details of the function tool. + */ + function: { + /** + * The name of the function. + */ + name: string; + /** + * A brief description of what the function does. + */ + description: string; + /** + * Schema defining the parameters accepted by the function. + */ + parameters: { + /** + * The type of the parameters object (usually 'object'). + */ + type: string; + /** + * List of required parameter names. + */ + required?: string[]; + /** + * Definitions of each parameter. + */ + properties: { + [k: string]: { + /** + * The data type of the parameter. + */ + type: string; + /** + * A description of the expected parameter. + */ + description: string; + }; + }; + }; + }; + })[]; + response_format?: Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_JSON_Mode_1; + /** + * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. + */ + raw?: boolean; + /** + * If true, the response will be streamed back incrementally using SSE, Server Sent Events. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; +} +interface Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_JSON_Mode_1 { + type?: "json_object" | "json_schema"; + json_schema?: unknown; +} +interface Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Async_Batch { + requests: (Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Prompt_1 | Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Messages_1)[]; +} +interface Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Prompt_1 { + /** + * The input text prompt for the model to generate a response. + */ + prompt: string; + /** + * Name of the LoRA (Low-Rank Adaptation) model to fine-tune the base model. + */ + lora?: string; + response_format?: Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_JSON_Mode_2; + /** + * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. + */ + raw?: boolean; + /** + * If true, the response will be streamed back incrementally using SSE, Server Sent Events. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; +} +interface Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_JSON_Mode_2 { + type?: "json_object" | "json_schema"; + json_schema?: unknown; +} +interface Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Messages_1 { + /** + * An array of message objects representing the conversation history. + */ + messages: { + /** + * The role of the message sender (e.g., 'user', 'assistant', 'system', 'tool'). + */ + role: string; + content: string | { + /** + * Type of the content (text) + */ + type?: string; + /** + * Text content + */ + text?: string; + }[]; + }[]; + functions?: { + name: string; + code: string; + }[]; + /** + * A list of tools available for the assistant to use. + */ + tools?: ({ + /** + * The name of the tool. More descriptive the better. + */ + name: string; + /** + * A brief description of what the tool does. + */ + description: string; + /** + * Schema defining the parameters accepted by the tool. + */ + parameters: { + /** + * The type of the parameters object (usually 'object'). + */ + type: string; + /** + * List of required parameter names. + */ + required?: string[]; + /** + * Definitions of each parameter. + */ + properties: { + [k: string]: { + /** + * The data type of the parameter. + */ + type: string; + /** + * A description of the expected parameter. + */ + description: string; + }; + }; + }; + } | { + /** + * Specifies the type of tool (e.g., 'function'). + */ + type: string; + /** + * Details of the function tool. + */ + function: { + /** + * The name of the function. + */ + name: string; + /** + * A brief description of what the function does. + */ + description: string; + /** + * Schema defining the parameters accepted by the function. + */ + parameters: { + /** + * The type of the parameters object (usually 'object'). + */ + type: string; + /** + * List of required parameter names. + */ + required?: string[]; + /** + * Definitions of each parameter. + */ + properties: { + [k: string]: { + /** + * The data type of the parameter. + */ + type: string; + /** + * A description of the expected parameter. + */ + description: string; + }; + }; + }; + }; + })[]; + response_format?: Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_JSON_Mode_3; + /** + * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. + */ + raw?: boolean; + /** + * If true, the response will be streamed back incrementally using SSE, Server Sent Events. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; +} +interface Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_JSON_Mode_3 { + type?: "json_object" | "json_schema"; + json_schema?: unknown; +} +type Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Output = Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Chat_Completion_Response | Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Text_Completion_Response | string | Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_AsyncResponse; +interface Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Chat_Completion_Response { + /** + * Unique identifier for the completion + */ + id?: string; + /** + * Object type identifier + */ + object?: "chat.completion"; + /** + * Unix timestamp of when the completion was created + */ + created?: number; + /** + * Model used for the completion + */ + model?: string; + /** + * List of completion choices + */ + choices?: { + /** + * Index of the choice in the list + */ + index?: number; + /** + * The message generated by the model + */ + message?: { + /** + * Role of the message author + */ + role: string; + /** + * The content of the message + */ + content: string; + /** + * Internal reasoning content (if available) + */ + reasoning_content?: string; + /** + * Tool calls made by the assistant + */ + tool_calls?: { + /** + * Unique identifier for the tool call + */ + id: string; + /** + * Type of tool call + */ + type: "function"; + function: { + /** + * Name of the function to call + */ + name: string; + /** + * JSON string of arguments for the function + */ + arguments: string; + }; + }[]; + }; + /** + * Reason why the model stopped generating + */ + finish_reason?: string; + /** + * Stop reason (may be null) + */ + stop_reason?: string | null; + /** + * Log probabilities (if requested) + */ + logprobs?: {} | null; + }[]; + /** + * Usage statistics for the inference request + */ + usage?: { + /** + * Total number of tokens in input + */ + prompt_tokens?: number; + /** + * Total number of tokens in output + */ + completion_tokens?: number; + /** + * Total number of input and output tokens + */ + total_tokens?: number; + }; + /** + * Log probabilities for the prompt (if requested) + */ + prompt_logprobs?: {} | null; +} +interface Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Text_Completion_Response { + /** + * Unique identifier for the completion + */ + id?: string; + /** + * Object type identifier + */ + object?: "text_completion"; + /** + * Unix timestamp of when the completion was created + */ + created?: number; + /** + * Model used for the completion + */ + model?: string; + /** + * List of completion choices + */ + choices?: { + /** + * Index of the choice in the list + */ + index: number; + /** + * The generated text completion + */ + text: string; + /** + * Reason why the model stopped generating + */ + finish_reason: string; + /** + * Stop reason (may be null) + */ + stop_reason?: string | null; + /** + * Log probabilities (if requested) + */ + logprobs?: {} | null; + /** + * Log probabilities for the prompt (if requested) + */ + prompt_logprobs?: {} | null; + }[]; + /** + * Usage statistics for the inference request + */ + usage?: { + /** + * Total number of tokens in input + */ + prompt_tokens?: number; + /** + * Total number of tokens in output + */ + completion_tokens?: number; + /** + * Total number of input and output tokens + */ + total_tokens?: number; + }; +} +interface Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_AsyncResponse { + /** + * The async request id that can be used to obtain the results. + */ + request_id?: string; +} +declare abstract class Base_Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8 { + inputs: Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Input; + postProcessedOutputs: Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Output; +} +interface Ai_Cf_Deepgram_Nova_3_Input { + audio: { + body: object; + contentType: string; + }; + /** + * Sets how the model will interpret strings submitted to the custom_topic param. When strict, the model will only return topics submitted using the custom_topic param. When extended, the model will return its own detected topics in addition to those submitted using the custom_topic param. + */ + custom_topic_mode?: "extended" | "strict"; + /** + * Custom topics you want the model to detect within your input audio or text if present Submit up to 100 + */ + custom_topic?: string; + /** + * Sets how the model will interpret intents submitted to the custom_intent param. When strict, the model will only return intents submitted using the custom_intent param. When extended, the model will return its own detected intents in addition those submitted using the custom_intents param + */ + custom_intent_mode?: "extended" | "strict"; + /** + * Custom intents you want the model to detect within your input audio if present + */ + custom_intent?: string; + /** + * Identifies and extracts key entities from content in submitted audio + */ + detect_entities?: boolean; + /** + * Identifies the dominant language spoken in submitted audio + */ + detect_language?: boolean; + /** + * Recognize speaker changes. Each word in the transcript will be assigned a speaker number starting at 0 + */ + diarize?: boolean; + /** + * Identify and extract key entities from content in submitted audio + */ + dictation?: boolean; + /** + * Specify the expected encoding of your submitted audio + */ + encoding?: "linear16" | "flac" | "mulaw" | "amr-nb" | "amr-wb" | "opus" | "speex" | "g729"; + /** + * Arbitrary key-value pairs that are attached to the API response for usage in downstream processing + */ + extra?: string; + /** + * Filler Words can help transcribe interruptions in your audio, like 'uh' and 'um' + */ + filler_words?: boolean; + /** + * Key term prompting can boost or suppress specialized terminology and brands. + */ + keyterm?: string; + /** + * Keywords can boost or suppress specialized terminology and brands. + */ + keywords?: string; + /** + * The BCP-47 language tag that hints at the primary spoken language. Depending on the Model and API endpoint you choose only certain languages are available. + */ + language?: string; + /** + * Spoken measurements will be converted to their corresponding abbreviations. + */ + measurements?: boolean; + /** + * Opts out requests from the Deepgram Model Improvement Program. Refer to our Docs for pricing impacts before setting this to true. https://dpgr.am/deepgram-mip. + */ + mip_opt_out?: boolean; + /** + * Mode of operation for the model representing broad area of topic that will be talked about in the supplied audio + */ + mode?: "general" | "medical" | "finance"; + /** + * Transcribe each audio channel independently. + */ + multichannel?: boolean; + /** + * Numerals converts numbers from written format to numerical format. + */ + numerals?: boolean; + /** + * Splits audio into paragraphs to improve transcript readability. + */ + paragraphs?: boolean; + /** + * Profanity Filter looks for recognized profanity and converts it to the nearest recognized non-profane word or removes it from the transcript completely. + */ + profanity_filter?: boolean; + /** + * Add punctuation and capitalization to the transcript. + */ + punctuate?: boolean; + /** + * Redaction removes sensitive information from your transcripts. + */ + redact?: string; + /** + * Search for terms or phrases in submitted audio and replaces them. + */ + replace?: string; + /** + * Search for terms or phrases in submitted audio. + */ + search?: string; + /** + * Recognizes the sentiment throughout a transcript or text. + */ + sentiment?: boolean; + /** + * Apply formatting to transcript output. When set to true, additional formatting will be applied to transcripts to improve readability. + */ + smart_format?: boolean; + /** + * Detect topics throughout a transcript or text. + */ + topics?: boolean; + /** + * Segments speech into meaningful semantic units. + */ + utterances?: boolean; + /** + * Seconds to wait before detecting a pause between words in submitted audio. + */ + utt_split?: number; + /** + * The number of channels in the submitted audio + */ + channels?: number; + /** + * Specifies whether the streaming endpoint should provide ongoing transcription updates as more audio is received. When set to true, the endpoint sends continuous updates, meaning transcription results may evolve over time. Note: Supported only for webosockets. + */ + interim_results?: boolean; + /** + * Indicates how long model will wait to detect whether a speaker has finished speaking or pauses for a significant period of time. When set to a value, the streaming endpoint immediately finalizes the transcription for the processed time range and returns the transcript with a speech_final parameter set to true. Can also be set to false to disable endpointing + */ + endpointing?: string; + /** + * Indicates that speech has started. You'll begin receiving Speech Started messages upon speech starting. Note: Supported only for webosockets. + */ + vad_events?: boolean; + /** + * Indicates how long model will wait to send an UtteranceEnd message after a word has been transcribed. Use with interim_results. Note: Supported only for webosockets. + */ + utterance_end_ms?: boolean; +} +interface Ai_Cf_Deepgram_Nova_3_Output { + results?: { + channels?: { + alternatives?: { + confidence?: number; + transcript?: string; + words?: { + confidence?: number; + end?: number; + start?: number; + word?: string; + }[]; + }[]; + }[]; + summary?: { + result?: string; + short?: string; + }; + sentiments?: { + segments?: { + text?: string; + start_word?: number; + end_word?: number; + sentiment?: string; + sentiment_score?: number; + }[]; + average?: { + sentiment?: string; + sentiment_score?: number; + }; + }; + }; +} +declare abstract class Base_Ai_Cf_Deepgram_Nova_3 { + inputs: Ai_Cf_Deepgram_Nova_3_Input; + postProcessedOutputs: Ai_Cf_Deepgram_Nova_3_Output; +} +interface Ai_Cf_Qwen_Qwen3_Embedding_0_6B_Input { + queries?: string | string[]; + /** + * Optional instruction for the task + */ + instruction?: string; + documents?: string | string[]; + text?: string | string[]; +} +interface Ai_Cf_Qwen_Qwen3_Embedding_0_6B_Output { + data?: number[][]; + shape?: number[]; +} +declare abstract class Base_Ai_Cf_Qwen_Qwen3_Embedding_0_6B { + inputs: Ai_Cf_Qwen_Qwen3_Embedding_0_6B_Input; + postProcessedOutputs: Ai_Cf_Qwen_Qwen3_Embedding_0_6B_Output; +} +type Ai_Cf_Pipecat_Ai_Smart_Turn_V2_Input = { + /** + * readable stream with audio data and content-type specified for that data + */ + audio: { + body: object; + contentType: string; + }; + /** + * type of data PCM data that's sent to the inference server as raw array + */ + dtype?: "uint8" | "float32" | "float64"; +} | { + /** + * base64 encoded audio data + */ + audio: string; + /** + * type of data PCM data that's sent to the inference server as raw array + */ + dtype?: "uint8" | "float32" | "float64"; +}; +interface Ai_Cf_Pipecat_Ai_Smart_Turn_V2_Output { + /** + * if true, end-of-turn was detected + */ + is_complete?: boolean; + /** + * probability of the end-of-turn detection + */ + probability?: number; +} +declare abstract class Base_Ai_Cf_Pipecat_Ai_Smart_Turn_V2 { + inputs: Ai_Cf_Pipecat_Ai_Smart_Turn_V2_Input; + postProcessedOutputs: Ai_Cf_Pipecat_Ai_Smart_Turn_V2_Output; +} +declare abstract class Base_Ai_Cf_Openai_Gpt_Oss_120B { + inputs: XOR; + postProcessedOutputs: XOR; +} +declare abstract class Base_Ai_Cf_Openai_Gpt_Oss_20B { + inputs: XOR; + postProcessedOutputs: XOR; +} +interface Ai_Cf_Leonardo_Phoenix_1_0_Input { + /** + * A text description of the image you want to generate. + */ + prompt: string; + /** + * Controls how closely the generated image should adhere to the prompt; higher values make the image more aligned with the prompt + */ + guidance?: number; + /** + * Random seed for reproducibility of the image generation + */ + seed?: number; + /** + * The height of the generated image in pixels + */ + height?: number; + /** + * The width of the generated image in pixels + */ + width?: number; + /** + * The number of diffusion steps; higher values can improve quality but take longer + */ + num_steps?: number; + /** + * Specify what to exclude from the generated images + */ + negative_prompt?: string; +} +/** + * The generated image in JPEG format + */ +type Ai_Cf_Leonardo_Phoenix_1_0_Output = string; +declare abstract class Base_Ai_Cf_Leonardo_Phoenix_1_0 { + inputs: Ai_Cf_Leonardo_Phoenix_1_0_Input; + postProcessedOutputs: Ai_Cf_Leonardo_Phoenix_1_0_Output; +} +interface Ai_Cf_Leonardo_Lucid_Origin_Input { + /** + * A text description of the image you want to generate. + */ + prompt: string; + /** + * Controls how closely the generated image should adhere to the prompt; higher values make the image more aligned with the prompt + */ + guidance?: number; + /** + * Random seed for reproducibility of the image generation + */ + seed?: number; + /** + * The height of the generated image in pixels + */ + height?: number; + /** + * The width of the generated image in pixels + */ + width?: number; + /** + * The number of diffusion steps; higher values can improve quality but take longer + */ + num_steps?: number; + /** + * The number of diffusion steps; higher values can improve quality but take longer + */ + steps?: number; +} +interface Ai_Cf_Leonardo_Lucid_Origin_Output { + /** + * The generated image in Base64 format. + */ + image?: string; +} +declare abstract class Base_Ai_Cf_Leonardo_Lucid_Origin { + inputs: Ai_Cf_Leonardo_Lucid_Origin_Input; + postProcessedOutputs: Ai_Cf_Leonardo_Lucid_Origin_Output; +} +interface Ai_Cf_Deepgram_Aura_1_Input { + /** + * Speaker used to produce the audio. + */ + speaker?: "angus" | "asteria" | "arcas" | "orion" | "orpheus" | "athena" | "luna" | "zeus" | "perseus" | "helios" | "hera" | "stella"; + /** + * Encoding of the output audio. + */ + encoding?: "linear16" | "flac" | "mulaw" | "alaw" | "mp3" | "opus" | "aac"; + /** + * Container specifies the file format wrapper for the output audio. The available options depend on the encoding type.. + */ + container?: "none" | "wav" | "ogg"; + /** + * The text content to be converted to speech + */ + text: string; + /** + * Sample Rate specifies the sample rate for the output audio. Based on the encoding, different sample rates are supported. For some encodings, the sample rate is not configurable + */ + sample_rate?: number; + /** + * The bitrate of the audio in bits per second. Choose from predefined ranges or specific values based on the encoding type. + */ + bit_rate?: number; +} +/** + * The generated audio in MP3 format + */ +type Ai_Cf_Deepgram_Aura_1_Output = string; +declare abstract class Base_Ai_Cf_Deepgram_Aura_1 { + inputs: Ai_Cf_Deepgram_Aura_1_Input; + postProcessedOutputs: Ai_Cf_Deepgram_Aura_1_Output; +} +interface Ai_Cf_Ai4Bharat_Indictrans2_En_Indic_1B_Input { + /** + * Input text to translate. Can be a single string or a list of strings. + */ + text: string | string[]; + /** + * Target langauge to translate to + */ + target_language: "asm_Beng" | "awa_Deva" | "ben_Beng" | "bho_Deva" | "brx_Deva" | "doi_Deva" | "eng_Latn" | "gom_Deva" | "gon_Deva" | "guj_Gujr" | "hin_Deva" | "hne_Deva" | "kan_Knda" | "kas_Arab" | "kas_Deva" | "kha_Latn" | "lus_Latn" | "mag_Deva" | "mai_Deva" | "mal_Mlym" | "mar_Deva" | "mni_Beng" | "mni_Mtei" | "npi_Deva" | "ory_Orya" | "pan_Guru" | "san_Deva" | "sat_Olck" | "snd_Arab" | "snd_Deva" | "tam_Taml" | "tel_Telu" | "urd_Arab" | "unr_Deva"; +} +interface Ai_Cf_Ai4Bharat_Indictrans2_En_Indic_1B_Output { + /** + * Translated texts + */ + translations: string[]; +} +declare abstract class Base_Ai_Cf_Ai4Bharat_Indictrans2_En_Indic_1B { + inputs: Ai_Cf_Ai4Bharat_Indictrans2_En_Indic_1B_Input; + postProcessedOutputs: Ai_Cf_Ai4Bharat_Indictrans2_En_Indic_1B_Output; +} +type Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Input = Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Prompt | Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Messages | Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Async_Batch; +interface Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Prompt { + /** + * The input text prompt for the model to generate a response. + */ + prompt: string; + /** + * Name of the LoRA (Low-Rank Adaptation) model to fine-tune the base model. + */ + lora?: string; + response_format?: Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_JSON_Mode; + /** + * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. + */ + raw?: boolean; + /** + * If true, the response will be streamed back incrementally using SSE, Server Sent Events. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; +} +interface Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_JSON_Mode { + type?: "json_object" | "json_schema"; + json_schema?: unknown; +} +interface Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Messages { + /** + * An array of message objects representing the conversation history. + */ + messages: { + /** + * The role of the message sender (e.g., 'user', 'assistant', 'system', 'tool'). + */ + role: string; + content: string | { + /** + * Type of the content (text) + */ + type?: string; + /** + * Text content + */ + text?: string; + }[]; + }[]; + functions?: { + name: string; + code: string; + }[]; + /** + * A list of tools available for the assistant to use. + */ + tools?: ({ + /** + * The name of the tool. More descriptive the better. + */ + name: string; + /** + * A brief description of what the tool does. + */ + description: string; + /** + * Schema defining the parameters accepted by the tool. + */ + parameters: { + /** + * The type of the parameters object (usually 'object'). + */ + type: string; + /** + * List of required parameter names. + */ + required?: string[]; + /** + * Definitions of each parameter. + */ + properties: { + [k: string]: { + /** + * The data type of the parameter. + */ + type: string; + /** + * A description of the expected parameter. + */ + description: string; + }; + }; + }; + } | { + /** + * Specifies the type of tool (e.g., 'function'). + */ + type: string; + /** + * Details of the function tool. + */ + function: { + /** + * The name of the function. + */ + name: string; + /** + * A brief description of what the function does. + */ + description: string; + /** + * Schema defining the parameters accepted by the function. + */ + parameters: { + /** + * The type of the parameters object (usually 'object'). + */ + type: string; + /** + * List of required parameter names. + */ + required?: string[]; + /** + * Definitions of each parameter. + */ + properties: { + [k: string]: { + /** + * The data type of the parameter. + */ + type: string; + /** + * A description of the expected parameter. + */ + description: string; + }; + }; + }; + }; + })[]; + response_format?: Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_JSON_Mode_1; + /** + * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. + */ + raw?: boolean; + /** + * If true, the response will be streamed back incrementally using SSE, Server Sent Events. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; +} +interface Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_JSON_Mode_1 { + type?: "json_object" | "json_schema"; + json_schema?: unknown; +} +interface Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Async_Batch { + requests: (Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Prompt_1 | Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Messages_1)[]; +} +interface Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Prompt_1 { + /** + * The input text prompt for the model to generate a response. + */ + prompt: string; + /** + * Name of the LoRA (Low-Rank Adaptation) model to fine-tune the base model. + */ + lora?: string; + response_format?: Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_JSON_Mode_2; + /** + * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. + */ + raw?: boolean; + /** + * If true, the response will be streamed back incrementally using SSE, Server Sent Events. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; +} +interface Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_JSON_Mode_2 { + type?: "json_object" | "json_schema"; + json_schema?: unknown; +} +interface Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Messages_1 { + /** + * An array of message objects representing the conversation history. + */ + messages: { + /** + * The role of the message sender (e.g., 'user', 'assistant', 'system', 'tool'). + */ + role: string; + content: string | { + /** + * Type of the content (text) + */ + type?: string; + /** + * Text content + */ + text?: string; + }[]; + }[]; + functions?: { + name: string; + code: string; + }[]; + /** + * A list of tools available for the assistant to use. + */ + tools?: ({ + /** + * The name of the tool. More descriptive the better. + */ + name: string; + /** + * A brief description of what the tool does. + */ + description: string; + /** + * Schema defining the parameters accepted by the tool. + */ + parameters: { + /** + * The type of the parameters object (usually 'object'). + */ + type: string; + /** + * List of required parameter names. + */ + required?: string[]; + /** + * Definitions of each parameter. + */ + properties: { + [k: string]: { + /** + * The data type of the parameter. + */ + type: string; + /** + * A description of the expected parameter. + */ + description: string; + }; + }; + }; + } | { + /** + * Specifies the type of tool (e.g., 'function'). + */ + type: string; + /** + * Details of the function tool. + */ + function: { + /** + * The name of the function. + */ + name: string; + /** + * A brief description of what the function does. + */ + description: string; + /** + * Schema defining the parameters accepted by the function. + */ + parameters: { + /** + * The type of the parameters object (usually 'object'). + */ + type: string; + /** + * List of required parameter names. + */ + required?: string[]; + /** + * Definitions of each parameter. + */ + properties: { + [k: string]: { + /** + * The data type of the parameter. + */ + type: string; + /** + * A description of the expected parameter. + */ + description: string; + }; + }; + }; + }; + })[]; + response_format?: Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_JSON_Mode_3; + /** + * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. + */ + raw?: boolean; + /** + * If true, the response will be streamed back incrementally using SSE, Server Sent Events. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; +} +interface Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_JSON_Mode_3 { + type?: "json_object" | "json_schema"; + json_schema?: unknown; +} +type Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Output = Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Chat_Completion_Response | Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Text_Completion_Response | string | Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_AsyncResponse; +interface Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Chat_Completion_Response { + /** + * Unique identifier for the completion + */ + id?: string; + /** + * Object type identifier + */ + object?: "chat.completion"; + /** + * Unix timestamp of when the completion was created + */ + created?: number; + /** + * Model used for the completion + */ + model?: string; + /** + * List of completion choices + */ + choices?: { + /** + * Index of the choice in the list + */ + index?: number; + /** + * The message generated by the model + */ + message?: { + /** + * Role of the message author + */ + role: string; + /** + * The content of the message + */ + content: string; + /** + * Internal reasoning content (if available) + */ + reasoning_content?: string; + /** + * Tool calls made by the assistant + */ + tool_calls?: { + /** + * Unique identifier for the tool call + */ + id: string; + /** + * Type of tool call + */ + type: "function"; + function: { + /** + * Name of the function to call + */ + name: string; + /** + * JSON string of arguments for the function + */ + arguments: string; + }; + }[]; + }; + /** + * Reason why the model stopped generating + */ + finish_reason?: string; + /** + * Stop reason (may be null) + */ + stop_reason?: string | null; + /** + * Log probabilities (if requested) + */ + logprobs?: {} | null; + }[]; + /** + * Usage statistics for the inference request + */ + usage?: { + /** + * Total number of tokens in input + */ + prompt_tokens?: number; + /** + * Total number of tokens in output + */ + completion_tokens?: number; + /** + * Total number of input and output tokens + */ + total_tokens?: number; + }; + /** + * Log probabilities for the prompt (if requested) + */ + prompt_logprobs?: {} | null; +} +interface Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Text_Completion_Response { + /** + * Unique identifier for the completion + */ + id?: string; + /** + * Object type identifier + */ + object?: "text_completion"; + /** + * Unix timestamp of when the completion was created + */ + created?: number; + /** + * Model used for the completion + */ + model?: string; + /** + * List of completion choices + */ + choices?: { + /** + * Index of the choice in the list + */ + index: number; + /** + * The generated text completion + */ + text: string; + /** + * Reason why the model stopped generating + */ + finish_reason: string; + /** + * Stop reason (may be null) + */ + stop_reason?: string | null; + /** + * Log probabilities (if requested) + */ + logprobs?: {} | null; + /** + * Log probabilities for the prompt (if requested) + */ + prompt_logprobs?: {} | null; + }[]; + /** + * Usage statistics for the inference request + */ + usage?: { + /** + * Total number of tokens in input + */ + prompt_tokens?: number; + /** + * Total number of tokens in output + */ + completion_tokens?: number; + /** + * Total number of input and output tokens + */ + total_tokens?: number; + }; +} +interface Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_AsyncResponse { + /** + * The async request id that can be used to obtain the results. + */ + request_id?: string; +} +declare abstract class Base_Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It { + inputs: Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Input; + postProcessedOutputs: Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Output; +} +interface Ai_Cf_Pfnet_Plamo_Embedding_1B_Input { + /** + * Input text to embed. Can be a single string or a list of strings. + */ + text: string | string[]; +} +interface Ai_Cf_Pfnet_Plamo_Embedding_1B_Output { + /** + * Embedding vectors, where each vector is a list of floats. + */ + data: number[][]; + /** + * Shape of the embedding data as [number_of_embeddings, embedding_dimension]. + * + * @minItems 2 + * @maxItems 2 + */ + shape: [ + number, + number + ]; +} +declare abstract class Base_Ai_Cf_Pfnet_Plamo_Embedding_1B { + inputs: Ai_Cf_Pfnet_Plamo_Embedding_1B_Input; + postProcessedOutputs: Ai_Cf_Pfnet_Plamo_Embedding_1B_Output; +} +interface Ai_Cf_Deepgram_Flux_Input { + /** + * Encoding of the audio stream. Currently only supports raw signed little-endian 16-bit PCM. + */ + encoding: "linear16"; + /** + * Sample rate of the audio stream in Hz. + */ + sample_rate: string; + /** + * End-of-turn confidence required to fire an eager end-of-turn event. When set, enables EagerEndOfTurn and TurnResumed events. Valid Values 0.3 - 0.9. + */ + eager_eot_threshold?: string; + /** + * End-of-turn confidence required to finish a turn. Valid Values 0.5 - 0.9. + */ + eot_threshold?: string; + /** + * A turn will be finished when this much time has passed after speech, regardless of EOT confidence. + */ + eot_timeout_ms?: string; + /** + * Keyterm prompting can improve recognition of specialized terminology. Pass multiple keyterm query parameters to boost multiple keyterms. + */ + keyterm?: string; + /** + * Opts out requests from the Deepgram Model Improvement Program. Refer to Deepgram Docs for pricing impacts before setting this to true. https://dpgr.am/deepgram-mip + */ + mip_opt_out?: "true" | "false"; + /** + * Label your requests for the purpose of identification during usage reporting + */ + tag?: string; +} +/** + * Output will be returned as websocket messages. + */ +interface Ai_Cf_Deepgram_Flux_Output { + /** + * The unique identifier of the request (uuid) + */ + request_id?: string; + /** + * Starts at 0 and increments for each message the server sends to the client. + */ + sequence_id?: number; + /** + * The type of event being reported. + */ + event?: "Update" | "StartOfTurn" | "EagerEndOfTurn" | "TurnResumed" | "EndOfTurn"; + /** + * The index of the current turn + */ + turn_index?: number; + /** + * Start time in seconds of the audio range that was transcribed + */ + audio_window_start?: number; + /** + * End time in seconds of the audio range that was transcribed + */ + audio_window_end?: number; + /** + * Text that was said over the course of the current turn + */ + transcript?: string; + /** + * The words in the transcript + */ + words?: { + /** + * The individual punctuated, properly-cased word from the transcript + */ + word: string; + /** + * Confidence that this word was transcribed correctly + */ + confidence: number; + }[]; + /** + * Confidence that no more speech is coming in this turn + */ + end_of_turn_confidence?: number; +} +declare abstract class Base_Ai_Cf_Deepgram_Flux { + inputs: Ai_Cf_Deepgram_Flux_Input; + postProcessedOutputs: Ai_Cf_Deepgram_Flux_Output; +} +interface Ai_Cf_Deepgram_Aura_2_En_Input { + /** + * Speaker used to produce the audio. + */ + speaker?: "amalthea" | "andromeda" | "apollo" | "arcas" | "aries" | "asteria" | "athena" | "atlas" | "aurora" | "callista" | "cora" | "cordelia" | "delia" | "draco" | "electra" | "harmonia" | "helena" | "hera" | "hermes" | "hyperion" | "iris" | "janus" | "juno" | "jupiter" | "luna" | "mars" | "minerva" | "neptune" | "odysseus" | "ophelia" | "orion" | "orpheus" | "pandora" | "phoebe" | "pluto" | "saturn" | "thalia" | "theia" | "vesta" | "zeus"; + /** + * Encoding of the output audio. + */ + encoding?: "linear16" | "flac" | "mulaw" | "alaw" | "mp3" | "opus" | "aac"; + /** + * Container specifies the file format wrapper for the output audio. The available options depend on the encoding type.. + */ + container?: "none" | "wav" | "ogg"; + /** + * The text content to be converted to speech + */ + text: string; + /** + * Sample Rate specifies the sample rate for the output audio. Based on the encoding, different sample rates are supported. For some encodings, the sample rate is not configurable + */ + sample_rate?: number; + /** + * The bitrate of the audio in bits per second. Choose from predefined ranges or specific values based on the encoding type. + */ + bit_rate?: number; +} +/** + * The generated audio in MP3 format + */ +type Ai_Cf_Deepgram_Aura_2_En_Output = string; +declare abstract class Base_Ai_Cf_Deepgram_Aura_2_En { + inputs: Ai_Cf_Deepgram_Aura_2_En_Input; + postProcessedOutputs: Ai_Cf_Deepgram_Aura_2_En_Output; +} +interface Ai_Cf_Deepgram_Aura_2_Es_Input { + /** + * Speaker used to produce the audio. + */ + speaker?: "sirio" | "nestor" | "carina" | "celeste" | "alvaro" | "diana" | "aquila" | "selena" | "estrella" | "javier"; + /** + * Encoding of the output audio. + */ + encoding?: "linear16" | "flac" | "mulaw" | "alaw" | "mp3" | "opus" | "aac"; + /** + * Container specifies the file format wrapper for the output audio. The available options depend on the encoding type.. + */ + container?: "none" | "wav" | "ogg"; + /** + * The text content to be converted to speech + */ + text: string; + /** + * Sample Rate specifies the sample rate for the output audio. Based on the encoding, different sample rates are supported. For some encodings, the sample rate is not configurable + */ + sample_rate?: number; + /** + * The bitrate of the audio in bits per second. Choose from predefined ranges or specific values based on the encoding type. + */ + bit_rate?: number; +} +/** + * The generated audio in MP3 format + */ +type Ai_Cf_Deepgram_Aura_2_Es_Output = string; +declare abstract class Base_Ai_Cf_Deepgram_Aura_2_Es { + inputs: Ai_Cf_Deepgram_Aura_2_Es_Input; + postProcessedOutputs: Ai_Cf_Deepgram_Aura_2_Es_Output; +} +interface Ai_Cf_Black_Forest_Labs_Flux_2_Dev_Input { + multipart: { + body?: object; + contentType?: string; + }; +} +interface Ai_Cf_Black_Forest_Labs_Flux_2_Dev_Output { + /** + * Generated image as Base64 string. + */ + image?: string; +} +declare abstract class Base_Ai_Cf_Black_Forest_Labs_Flux_2_Dev { + inputs: Ai_Cf_Black_Forest_Labs_Flux_2_Dev_Input; + postProcessedOutputs: Ai_Cf_Black_Forest_Labs_Flux_2_Dev_Output; +} +interface Ai_Cf_Black_Forest_Labs_Flux_2_Klein_4B_Input { + multipart: { + body?: object; + contentType?: string; + }; +} +interface Ai_Cf_Black_Forest_Labs_Flux_2_Klein_4B_Output { + /** + * Generated image as Base64 string. + */ + image?: string; +} +declare abstract class Base_Ai_Cf_Black_Forest_Labs_Flux_2_Klein_4B { + inputs: Ai_Cf_Black_Forest_Labs_Flux_2_Klein_4B_Input; + postProcessedOutputs: Ai_Cf_Black_Forest_Labs_Flux_2_Klein_4B_Output; +} +interface Ai_Cf_Black_Forest_Labs_Flux_2_Klein_9B_Input { + multipart: { + body?: object; + contentType?: string; + }; +} +interface Ai_Cf_Black_Forest_Labs_Flux_2_Klein_9B_Output { + /** + * Generated image as Base64 string. + */ + image?: string; +} +declare abstract class Base_Ai_Cf_Black_Forest_Labs_Flux_2_Klein_9B { + inputs: Ai_Cf_Black_Forest_Labs_Flux_2_Klein_9B_Input; + postProcessedOutputs: Ai_Cf_Black_Forest_Labs_Flux_2_Klein_9B_Output; +} +declare abstract class Base_Ai_Cf_Zai_Org_Glm_4_7_Flash { + inputs: ChatCompletionsInput; + postProcessedOutputs: ChatCompletionsOutput; +} +declare abstract class Base_Ai_Cf_Moonshotai_Kimi_K2_5 { + inputs: ChatCompletionsInput; + postProcessedOutputs: ChatCompletionsOutput; +} +declare abstract class Base_Ai_Cf_Moonshotai_Kimi_K2_6 { + inputs: ChatCompletionsInput; + postProcessedOutputs: ChatCompletionsOutput; +} +declare abstract class Base_Ai_Cf_Nvidia_Nemotron_3_120B_A12B { + inputs: ChatCompletionsInput; + postProcessedOutputs: ChatCompletionsOutput; +} +declare abstract class Base_Ai_Cf_Google_Gemma_4_26B_A4B_IT { + inputs: ChatCompletionsInput; + postProcessedOutputs: ChatCompletionsOutput; +} +interface AiModels { + "@cf/huggingface/distilbert-sst-2-int8": BaseAiTextClassification; + "@cf/stabilityai/stable-diffusion-xl-base-1.0": BaseAiTextToImage; + "@cf/runwayml/stable-diffusion-v1-5-inpainting": BaseAiTextToImage; + "@cf/runwayml/stable-diffusion-v1-5-img2img": BaseAiTextToImage; + "@cf/lykon/dreamshaper-8-lcm": BaseAiTextToImage; + "@cf/bytedance/stable-diffusion-xl-lightning": BaseAiTextToImage; + "@cf/myshell-ai/melotts": BaseAiTextToSpeech; + "@cf/google/embeddinggemma-300m": BaseAiTextEmbeddings; + "@cf/microsoft/resnet-50": BaseAiImageClassification; + "@cf/meta/llama-2-7b-chat-int8": BaseAiTextGeneration; + "@cf/mistral/mistral-7b-instruct-v0.1": BaseAiTextGeneration; + "@cf/meta/llama-2-7b-chat-fp16": BaseAiTextGeneration; + "@hf/thebloke/llama-2-13b-chat-awq": BaseAiTextGeneration; + "@hf/thebloke/mistral-7b-instruct-v0.1-awq": BaseAiTextGeneration; + "@hf/thebloke/zephyr-7b-beta-awq": BaseAiTextGeneration; + "@hf/thebloke/openhermes-2.5-mistral-7b-awq": BaseAiTextGeneration; + "@hf/thebloke/neural-chat-7b-v3-1-awq": BaseAiTextGeneration; + "@hf/thebloke/deepseek-coder-6.7b-base-awq": BaseAiTextGeneration; + "@hf/thebloke/deepseek-coder-6.7b-instruct-awq": BaseAiTextGeneration; + "@cf/deepseek-ai/deepseek-math-7b-instruct": BaseAiTextGeneration; + "@cf/defog/sqlcoder-7b-2": BaseAiTextGeneration; + "@cf/openchat/openchat-3.5-0106": BaseAiTextGeneration; + "@cf/tiiuae/falcon-7b-instruct": BaseAiTextGeneration; + "@cf/thebloke/discolm-german-7b-v1-awq": BaseAiTextGeneration; + "@cf/qwen/qwen1.5-0.5b-chat": BaseAiTextGeneration; + "@cf/qwen/qwen1.5-7b-chat-awq": BaseAiTextGeneration; + "@cf/qwen/qwen1.5-14b-chat-awq": BaseAiTextGeneration; + "@cf/tinyllama/tinyllama-1.1b-chat-v1.0": BaseAiTextGeneration; + "@cf/microsoft/phi-2": BaseAiTextGeneration; + "@cf/qwen/qwen1.5-1.8b-chat": BaseAiTextGeneration; + "@cf/mistral/mistral-7b-instruct-v0.2-lora": BaseAiTextGeneration; + "@hf/nousresearch/hermes-2-pro-mistral-7b": BaseAiTextGeneration; + "@hf/nexusflow/starling-lm-7b-beta": BaseAiTextGeneration; + "@hf/google/gemma-7b-it": BaseAiTextGeneration; + "@cf/meta-llama/llama-2-7b-chat-hf-lora": BaseAiTextGeneration; + "@cf/google/gemma-2b-it-lora": BaseAiTextGeneration; + "@cf/google/gemma-7b-it-lora": BaseAiTextGeneration; + "@hf/mistral/mistral-7b-instruct-v0.2": BaseAiTextGeneration; + "@cf/meta/llama-3-8b-instruct": BaseAiTextGeneration; + "@cf/fblgit/una-cybertron-7b-v2-bf16": BaseAiTextGeneration; + "@cf/meta/llama-3-8b-instruct-awq": BaseAiTextGeneration; + "@cf/meta/llama-3.1-8b-instruct-fp8": BaseAiTextGeneration; + "@cf/meta/llama-3.1-8b-instruct-awq": BaseAiTextGeneration; + "@cf/meta/llama-3.2-3b-instruct": BaseAiTextGeneration; + "@cf/meta/llama-3.2-1b-instruct": BaseAiTextGeneration; + "@cf/deepseek-ai/deepseek-r1-distill-qwen-32b": BaseAiTextGeneration; + "@cf/ibm-granite/granite-4.0-h-micro": BaseAiTextGeneration; + "@cf/facebook/bart-large-cnn": BaseAiSummarization; + "@cf/llava-hf/llava-1.5-7b-hf": BaseAiImageToText; + "@cf/baai/bge-base-en-v1.5": Base_Ai_Cf_Baai_Bge_Base_En_V1_5; + "@cf/openai/whisper": Base_Ai_Cf_Openai_Whisper; + "@cf/meta/m2m100-1.2b": Base_Ai_Cf_Meta_M2M100_1_2B; + "@cf/baai/bge-small-en-v1.5": Base_Ai_Cf_Baai_Bge_Small_En_V1_5; + "@cf/baai/bge-large-en-v1.5": Base_Ai_Cf_Baai_Bge_Large_En_V1_5; + "@cf/unum/uform-gen2-qwen-500m": Base_Ai_Cf_Unum_Uform_Gen2_Qwen_500M; + "@cf/openai/whisper-tiny-en": Base_Ai_Cf_Openai_Whisper_Tiny_En; + "@cf/openai/whisper-large-v3-turbo": Base_Ai_Cf_Openai_Whisper_Large_V3_Turbo; + "@cf/baai/bge-m3": Base_Ai_Cf_Baai_Bge_M3; + "@cf/black-forest-labs/flux-1-schnell": Base_Ai_Cf_Black_Forest_Labs_Flux_1_Schnell; + "@cf/meta/llama-3.2-11b-vision-instruct": Base_Ai_Cf_Meta_Llama_3_2_11B_Vision_Instruct; + "@cf/meta/llama-3.3-70b-instruct-fp8-fast": Base_Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast; + "@cf/meta/llama-guard-3-8b": Base_Ai_Cf_Meta_Llama_Guard_3_8B; + "@cf/baai/bge-reranker-base": Base_Ai_Cf_Baai_Bge_Reranker_Base; + "@cf/qwen/qwen2.5-coder-32b-instruct": Base_Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct; + "@cf/qwen/qwq-32b": Base_Ai_Cf_Qwen_Qwq_32B; + "@cf/mistralai/mistral-small-3.1-24b-instruct": Base_Ai_Cf_Mistralai_Mistral_Small_3_1_24B_Instruct; + "@cf/google/gemma-3-12b-it": Base_Ai_Cf_Google_Gemma_3_12B_It; + "@cf/meta/llama-4-scout-17b-16e-instruct": Base_Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct; + "@cf/qwen/qwen3-30b-a3b-fp8": Base_Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8; + "@cf/deepgram/nova-3": Base_Ai_Cf_Deepgram_Nova_3; + "@cf/qwen/qwen3-embedding-0.6b": Base_Ai_Cf_Qwen_Qwen3_Embedding_0_6B; + "@cf/pipecat-ai/smart-turn-v2": Base_Ai_Cf_Pipecat_Ai_Smart_Turn_V2; + "@cf/openai/gpt-oss-120b": Base_Ai_Cf_Openai_Gpt_Oss_120B; + "@cf/openai/gpt-oss-20b": Base_Ai_Cf_Openai_Gpt_Oss_20B; + "@cf/leonardo/phoenix-1.0": Base_Ai_Cf_Leonardo_Phoenix_1_0; + "@cf/leonardo/lucid-origin": Base_Ai_Cf_Leonardo_Lucid_Origin; + "@cf/deepgram/aura-1": Base_Ai_Cf_Deepgram_Aura_1; + "@cf/ai4bharat/indictrans2-en-indic-1B": Base_Ai_Cf_Ai4Bharat_Indictrans2_En_Indic_1B; + "@cf/aisingapore/gemma-sea-lion-v4-27b-it": Base_Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It; + "@cf/pfnet/plamo-embedding-1b": Base_Ai_Cf_Pfnet_Plamo_Embedding_1B; + "@cf/deepgram/flux": Base_Ai_Cf_Deepgram_Flux; + "@cf/deepgram/aura-2-en": Base_Ai_Cf_Deepgram_Aura_2_En; + "@cf/deepgram/aura-2-es": Base_Ai_Cf_Deepgram_Aura_2_Es; + "@cf/black-forest-labs/flux-2-dev": Base_Ai_Cf_Black_Forest_Labs_Flux_2_Dev; + "@cf/black-forest-labs/flux-2-klein-4b": Base_Ai_Cf_Black_Forest_Labs_Flux_2_Klein_4B; + "@cf/black-forest-labs/flux-2-klein-9b": Base_Ai_Cf_Black_Forest_Labs_Flux_2_Klein_9B; + "@cf/zai-org/glm-4.7-flash": Base_Ai_Cf_Zai_Org_Glm_4_7_Flash; + "@cf/moonshotai/kimi-k2.5": Base_Ai_Cf_Moonshotai_Kimi_K2_5; + "@cf/moonshotai/kimi-k2.6": Base_Ai_Cf_Moonshotai_Kimi_K2_6; + "@cf/nvidia/nemotron-3-120b-a12b": Base_Ai_Cf_Nvidia_Nemotron_3_120B_A12B; + "@cf/google/gemma-4-26b-a4b-it": Base_Ai_Cf_Google_Gemma_4_26B_A4B_IT; +} +type AiOptions = { + /** + * Send requests as an asynchronous batch job, only works for supported models + * https://developers.cloudflare.com/workers-ai/features/batch-api + */ + queueRequest?: boolean; + /** + * Establish websocket connections, only works for supported models + */ + websocket?: boolean; + /** + * Tag your requests to group and view them in Cloudflare dashboard. + * + * Rules: + * Tags must only contain letters, numbers, and the symbols: : - . / @ + * Each tag can have maximum 50 characters. + * Maximum 5 tags are allowed each request. + * Duplicate tags will removed. + */ + tags?: string[]; + gateway?: GatewayOptions; + returnRawResponse?: boolean; + prefix?: string; + extraHeaders?: object; + signal?: AbortSignal; +}; +type AiModelsSearchParams = { + author?: string; + hide_experimental?: boolean; + page?: number; + per_page?: number; + search?: string; + source?: number; + task?: string; +}; +type AiModelsSearchObject = { + id: string; + source: number; + name: string; + description: string; + task: { + id: string; + name: string; + description: string; + }; + tags: string[]; + properties: { + property_id: string; + value: string; + }[]; +}; +type ChatCompletionsBase = ChatCompletionsMessagesInput; +type ChatCompletionsInput = ChatCompletionsMessagesInput; +interface InferenceUpstreamError extends Error { +} +interface AiInternalError extends Error { +} +type AiModelListType = Record; +type AiAsyncBatchResponse = { + request_id: string; +}; +declare abstract class Ai { + aiGatewayLogId: string | null; + gateway(gatewayId: string): AiGateway; + /** + * @deprecated Use the standalone `ai_search_namespaces` or `ai_search` Workers bindings instead. + * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ + */ + aiSearch(): AiSearchNamespace; + /** + * @deprecated AutoRAG has been replaced by AI Search. + * Use the standalone `ai_search_namespaces` or `ai_search` Workers bindings instead. + * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ + * + * @param autoragId Instance ID + */ + autorag(autoragId: string): AutoRAG; + // Batch request + run(model: Name, inputs: { + requests: AiModelList[Name]['inputs'][]; + }, options: AiOptions & { + queueRequest: true; + }): Promise; + // Raw response + run(model: Name, inputs: AiModelList[Name]['inputs'], options: AiOptions & { + returnRawResponse: true; + }): Promise; + // WebSocket + run(model: Name, inputs: AiModelList[Name]['inputs'], options: AiOptions & { + websocket: true; + }): Promise; + // Streaming + run(model: Name, inputs: AiModelList[Name]['inputs'] & { + stream: true; + }, options?: AiOptions): Promise; + // Normal (default) - known model + run(model: Name, inputs: AiModelList[Name]['inputs'], options?: AiOptions): Promise; + // Unknown model (fallback). + // + // The `Exclude<..., keyof AiModelList>` constraint forces TypeScript to + // route any model name that is a literal key of `AiModelList` to one of + // the known-model overloads above (so input/output mismatches surface as + // type errors rather than silently falling back to `Record`). + // Names that aren't in `AiModelList` — e.g. third-party gateway models + // like `"google/nano-banana"` — still hit this overload. + run(model: Model extends keyof AiModelList ? never : Model, inputs: Record, options?: AiOptions): Promise>; + models(params?: AiModelsSearchParams): Promise; + toMarkdown(): ToMarkdownService; + toMarkdown(files: MarkdownDocument[], options?: ConversionRequestOptions): Promise; + toMarkdown(files: MarkdownDocument, options?: ConversionRequestOptions): Promise; +} +type GatewayRetries = { + maxAttempts?: 1 | 2 | 3 | 4 | 5; + retryDelayMs?: number; + backoff?: 'constant' | 'linear' | 'exponential'; +}; +type GatewayOptions = { + id: string; + cacheKey?: string; + cacheTtl?: number; + skipCache?: boolean; + metadata?: Record; + collectLog?: boolean; + eventId?: string; + requestTimeoutMs?: number; + retries?: GatewayRetries; +}; +type UniversalGatewayOptions = Exclude & { + /** + ** @deprecated + */ + id?: string; +}; +type AiGatewayPatchLog = { + score?: number | null; + feedback?: -1 | 1 | null; + metadata?: Record | null; +}; +type AiGatewayLog = { + id: string; + provider: string; + model: string; + model_type?: string; + path: string; + duration: number; + request_type?: string; + request_content_type?: string; + status_code: number; + response_content_type?: string; + success: boolean; + cached: boolean; + tokens_in?: number; + tokens_out?: number; + metadata?: Record; + step?: number; + cost?: number; + custom_cost?: boolean; + request_size: number; + request_head?: string; + request_head_complete: boolean; + response_size: number; + response_head?: string; + response_head_complete: boolean; + created_at: Date; +}; +type AIGatewayProviders = 'workers-ai' | 'anthropic' | 'aws-bedrock' | 'azure-openai' | 'google-vertex-ai' | 'huggingface' | 'openai' | 'perplexity-ai' | 'replicate' | 'groq' | 'cohere' | 'google-ai-studio' | 'mistral' | 'grok' | 'openrouter' | 'deepseek' | 'cerebras' | 'cartesia' | 'elevenlabs' | 'adobe-firefly'; +type AIGatewayHeaders = { + 'cf-aig-metadata': Record | string; + 'cf-aig-custom-cost': { + per_token_in?: number; + per_token_out?: number; + } | { + total_cost?: number; + } | string; + 'cf-aig-cache-ttl': number | string; + 'cf-aig-skip-cache': boolean | string; + 'cf-aig-cache-key': string; + 'cf-aig-event-id': string; + 'cf-aig-request-timeout': number | string; + 'cf-aig-max-attempts': number | string; + 'cf-aig-retry-delay': number | string; + 'cf-aig-backoff': string; + 'cf-aig-collect-log': boolean | string; + Authorization: string; + 'Content-Type': string; + [key: string]: string | number | boolean | object; +}; +type AIGatewayUniversalRequest = { + provider: AIGatewayProviders | string; // eslint-disable-line + endpoint: string; + headers: Partial; + query: unknown; +}; +interface AiGatewayInternalError extends Error { +} +interface AiGatewayLogNotFound extends Error { +} +declare abstract class AiGateway { + patchLog(logId: string, data: AiGatewayPatchLog): Promise; + getLog(logId: string): Promise; + run(data: AIGatewayUniversalRequest | AIGatewayUniversalRequest[], options?: { + gateway?: UniversalGatewayOptions; + extraHeaders?: object; + signal?: AbortSignal; + }): Promise; + getUrl(provider?: AIGatewayProviders | string): Promise; // eslint-disable-line +} +// Copyright (c) 2022-2025 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 +/** + * Artifacts — Git-compatible file storage on Cloudflare Workers. + * + * Provides programmatic access to create, manage, and fork repositories, + * and to issue and revoke scoped access tokens. + */ +/** Information about a repository. */ +interface ArtifactsRepoInfo { + /** Unique repository ID. */ + id: string; + /** Repository name. */ + name: string; + /** Repository description, or null if not set. */ + description: string | null; + /** Default branch name (e.g. "main"). */ + defaultBranch: string; + /** ISO 8601 creation timestamp. */ + createdAt: string; + /** ISO 8601 last-updated timestamp. */ + updatedAt: string; + /** ISO 8601 timestamp of the last push, or null if never pushed. */ + lastPushAt: string | null; + /** Fork source (e.g. "github:owner/repo", "artifacts:namespace/repo"), or null if not a fork. */ + source: string | null; + /** Whether the repository is read-only. */ + readOnly: boolean; + /** HTTPS git remote URL. */ + remote: string; +} +/** Result of creating a repository — includes the initial access token. */ +interface ArtifactsCreateRepoResult { + /** Unique repository ID. */ + id: string; + /** Repository name. */ + name: string; + /** Repository description, or null if not set. */ + description: string | null; + /** Default branch name. */ + defaultBranch: string; + /** HTTPS git remote URL. */ + remote: string; + /** Plaintext access token (only returned at creation time). */ + token: string; + /** ISO 8601 token expiry timestamp. */ + tokenExpiresAt: string; +} +/** Paginated list of repositories. */ +interface ArtifactsRepoListResult { + /** Repositories in this page (without the `remote` field). */ + repos: Omit[]; + /** Total number of repositories in the namespace. */ + total: number; + /** Cursor for the next page, if there are more results. */ + cursor?: string; +} +/** Result of creating an access token. */ +interface ArtifactsCreateTokenResult { + /** Unique token ID. */ + id: string; + /** Plaintext token (only returned at creation time). */ + plaintext: string; + /** Token scope: "read" or "write". */ + scope: 'read' | 'write'; + /** ISO 8601 token expiry timestamp. */ + expiresAt: string; +} +/** Token metadata (no plaintext). */ +interface ArtifactsTokenInfo { + /** Unique token ID. */ + id: string; + /** Token scope: "read" or "write". */ + scope: 'read' | 'write'; + /** Token state: "active", "expired", or "revoked". */ + state: 'active' | 'expired' | 'revoked'; + /** ISO 8601 creation timestamp. */ + createdAt: string; + /** ISO 8601 expiry timestamp. */ + expiresAt: string; +} +/** Paginated list of tokens for a repository. */ +interface ArtifactsTokenListResult { + /** Tokens in this page. */ + tokens: ArtifactsTokenInfo[]; + /** Total number of tokens for the repository. */ + total: number; +} +/** + * Handle for a single repository. Returned by Artifacts.get(). + * + * Methods may throw `ArtifactsError` with code `INTERNAL_ERROR` if an unexpected service error occurs. + */ +interface ArtifactsRepo extends ArtifactsRepoInfo { + /** + * Create an access token for this repo. + * @param scope Token scope: "write" (default) or "read". + * @param ttl Time-to-live in seconds (default 86400, min 60, max 31536000). + * @throws {ArtifactsError} with code `INVALID_TTL` if ttl is out of range. + */ + createToken(scope?: 'write' | 'read', ttl?: number): Promise; + /** List tokens for this repo (metadata only, no plaintext). */ + listTokens(): Promise; + /** + * Revoke a token by plaintext or ID. + * @param tokenOrId Plaintext token or token ID. + * @returns true if revoked, false if not found. + * @throws {ArtifactsError} with code `INVALID_INPUT` if tokenOrId is empty. + */ + revokeToken(tokenOrId: string): Promise; + // ── Fork ── + /** + * Fork this repo to a new repo. + * @param name Target repository name. + * @param opts Optional: description, readOnly flag, defaultBranchOnly (default true). + * @throws {ArtifactsError} with code `INVALID_REPO_NAME` if name is invalid. + * @throws {ArtifactsError} with code `ALREADY_EXISTS` if the target repo already exists. + * @throws {ArtifactsError} with code `FORK_IN_PROGRESS` if a fork is already running. + */ + fork(name: string, opts?: { + description?: string; + readOnly?: boolean; + defaultBranchOnly?: boolean; + }): Promise; +} +// ── Error types ────────────────────────────────────────────────────────────── +/** + * Error codes returned by Artifacts binding operations. + * + * Each code maps to a numeric code available on `ArtifactsError.numericCode`. + */ +type ArtifactsErrorCode = 'ALREADY_EXISTS' | 'NOT_FOUND' | 'IMPORT_IN_PROGRESS' | 'FORK_IN_PROGRESS' | 'INVALID_INPUT' | 'INVALID_REPO_NAME' | 'INVALID_TTL' | 'INVALID_URL' | 'REMOTE_AUTH_REQUIRED' | 'UPSTREAM_UNAVAILABLE' | 'MEMORY_LIMIT' | 'INTERNAL_ERROR'; +/** + * Error thrown by Artifacts binding operations. + * + * Uses a string `.code` discriminator following the Cloudflare platform + * convention (StreamError, ImagesError, etc.). The `.numericCode` matches + * the REST API `errors[].code` values. + */ +interface ArtifactsError extends Error { + readonly name: 'ArtifactsError'; + /** String error code for programmatic matching. */ + readonly code: ArtifactsErrorCode; + /** Numeric error code matching the REST API. */ + readonly numericCode: number; +} +// ── Binding ────────────────────────────────────────────────────────────────── +/** + * Artifacts binding — namespace-level operations. + * + * Methods may throw `ArtifactsError` with code `INTERNAL_ERROR` if an unexpected service error occurs. + */ +interface Artifacts { + /** + * Create a new repository with an initial access token. + * @param name Repository name (alphanumeric, dots, hyphens, underscores). + * @param opts Optional: readOnly flag, description, default branch name. + * @returns Repo metadata with initial token. + * @throws {ArtifactsError} with code `INVALID_REPO_NAME` if name is invalid. + * @throws {ArtifactsError} with code `ALREADY_EXISTS` if the repo already exists. + */ + create(name: string, opts?: { + readOnly?: boolean; + description?: string; + setDefaultBranch?: string; + }): Promise; + /** + * Get a handle to an existing repository. + * @param name Repository name. + * @returns Repo handle. + * @throws {ArtifactsError} with code `NOT_FOUND` if the repo does not exist. + * @throws {ArtifactsError} with code `IMPORT_IN_PROGRESS` if the repo is still importing. + * @throws {ArtifactsError} with code `FORK_IN_PROGRESS` if the repo is still forking. + */ + get(name: string): Promise; + /** + * Import a repository from an external git remote. + * @param params Source URL and optional branch/depth, plus target name and options. + * @returns Repo metadata with initial token. + * @throws {ArtifactsError} with code `INVALID_REPO_NAME` if the target name is invalid. + * @throws {ArtifactsError} with code `INVALID_INPUT` if the source URL is not valid HTTPS. + * @throws {ArtifactsError} with code `INVALID_URL` if the source URL does not point to a git repository. + * @throws {ArtifactsError} with code `REMOTE_AUTH_REQUIRED` if the remote requires authentication. + * @throws {ArtifactsError} with code `NOT_FOUND` if the remote repository does not exist. + * @throws {ArtifactsError} with code `UPSTREAM_UNAVAILABLE` if the remote cannot be reached. + * @throws {ArtifactsError} with code `MEMORY_LIMIT` if the import exceeds service memory limits. + * @throws {ArtifactsError} with code `ALREADY_EXISTS` if the target repo already exists. + */ + import(params: { + source: { + url: string; + branch?: string; + depth?: number; + }; + target: { + name: string; + opts?: { + description?: string; + readOnly?: boolean; + }; + }; + }): Promise; + /** + * List repositories with cursor-based pagination. + * @param opts Optional: limit (1–200, default 50), cursor for next page. + */ + list(opts?: { + limit?: number; + cursor?: string; + }): Promise; + /** + * Delete a repository and all associated tokens. + * @param name Repository name. + * @returns true if deleted, false if not found. + * @throws {ArtifactsError} with code `INVALID_REPO_NAME` if name is invalid. + */ + delete(name: string): Promise; +} +/** + * @deprecated Use the standalone AI Search Workers binding instead. + * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ + */ +interface AutoRAGInternalError extends Error { +} +/** + * @deprecated Use the standalone AI Search Workers binding instead. + * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ + */ +interface AutoRAGNotFoundError extends Error { +} +/** + * @deprecated Use the standalone AI Search Workers binding instead. + * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ + */ +interface AutoRAGUnauthorizedError extends Error { +} +/** + * @deprecated Use the standalone AI Search Workers binding instead. + * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ + */ +interface AutoRAGNameNotSetError extends Error { +} +type ComparisonFilter = { + key: string; + type: 'eq' | 'ne' | 'gt' | 'gte' | 'lt' | 'lte'; + value: string | number | boolean; +}; +type CompoundFilter = { + type: 'and' | 'or'; + filters: ComparisonFilter[]; +}; +/** + * @deprecated Use the standalone AI Search Workers binding instead. + * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ + */ +type AutoRagSearchRequest = { + query: string; + filters?: CompoundFilter | ComparisonFilter; + max_num_results?: number; + ranking_options?: { + ranker?: string; + score_threshold?: number; + }; + reranking?: { + enabled?: boolean; + model?: string; + }; + rewrite_query?: boolean; +}; +/** + * @deprecated Use the standalone AI Search Workers binding instead. + * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ + */ +type AutoRagAiSearchRequest = AutoRagSearchRequest & { + stream?: boolean; + system_prompt?: string; +}; +/** + * @deprecated Use the standalone AI Search Workers binding instead. + * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ + */ +type AutoRagAiSearchRequestStreaming = Omit & { + stream: true; +}; +/** + * @deprecated Use the standalone AI Search Workers binding instead. + * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ + */ +type AutoRagSearchResponse = { + object: 'vector_store.search_results.page'; + search_query: string; + data: { + file_id: string; + filename: string; + score: number; + attributes: Record; + content: { + type: 'text'; + text: string; + }[]; + }[]; + has_more: boolean; + next_page: string | null; +}; +/** + * @deprecated Use the standalone AI Search Workers binding instead. + * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ + */ +type AutoRagListResponse = { + id: string; + enable: boolean; + type: string; + source: string; + vectorize_name: string; + paused: boolean; + status: string; +}[]; +/** + * @deprecated Use the standalone AI Search Workers binding instead. + * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ + */ +type AutoRagAiSearchResponse = AutoRagSearchResponse & { + response: string; +}; +/** + * @deprecated Use the standalone AI Search Workers binding instead. + * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ + */ +declare abstract class AutoRAG { + /** + * @deprecated Use the standalone AI Search Workers binding instead. + * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ + */ + list(): Promise; + /** + * @deprecated Use the standalone AI Search Workers binding instead. + * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ + */ + search(params: AutoRagSearchRequest): Promise; + /** + * @deprecated Use the standalone AI Search Workers binding instead. + * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ + */ + aiSearch(params: AutoRagAiSearchRequestStreaming): Promise; + /** + * @deprecated Use the standalone AI Search Workers binding instead. + * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ + */ + aiSearch(params: AutoRagAiSearchRequest): Promise; + /** + * @deprecated Use the standalone AI Search Workers binding instead. + * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ + */ + aiSearch(params: AutoRagAiSearchRequest): Promise; +} +type BrowserRunLifecycleEvent = 'load' | 'domcontentloaded' | 'networkidle0' | 'networkidle2'; +type BrowserRunResourceType = 'document' | 'stylesheet' | 'image' | 'media' | 'font' | 'script' | 'texttrack' | 'xhr' | 'fetch' | 'prefetch' | 'eventsource' | 'websocket' | 'manifest' | 'signedexchange' | 'ping' | 'cspviolationreport' | 'preflight' | 'other'; +/** Options fields shared by all quick actions. */ +interface BrowserRunBaseOptions { + /** Adds ` + + diff --git a/examples/microduck-world/web/package.json b/examples/microduck-world/web/package.json new file mode 100644 index 0000000000..9325c400ff --- /dev/null +++ b/examples/microduck-world/web/package.json @@ -0,0 +1,20 @@ +{ + "name": "microduck-world-web", + "private": true, + "type": "module", + "dependencies": { + "react": "19.2.0", + "react-dom": "19.2.0", + "three": "0.180.0" + }, + "devDependencies": { + "@types/react": "19.2.0", + "@types/react-dom": "19.2.0", + "@types/three": "0.180.0", + "@types/node": "24.10.1", + "@vitejs/plugin-react": "5.1.0", + "vite": "7.1.0", + "vitest": "3.2.4", + "typescript": "5.9.2" + } +} diff --git a/examples/microduck-world/web/public/chakra-petch-license.txt b/examples/microduck-world/web/public/chakra-petch-license.txt new file mode 100644 index 0000000000..4b4c452c97 --- /dev/null +++ b/examples/microduck-world/web/public/chakra-petch-license.txt @@ -0,0 +1,93 @@ +Copyright 2018 The Chakra Petch Project Authors (https://github.com/m4rc1e/Chakra-Petch.git) + +This Font Software is licensed under the SIL Open Font License, Version 1.1. +This license is copied below, and is also available with a FAQ at: +http://scripts.sil.org/OFL + + +----------------------------------------------------------- +SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007 +----------------------------------------------------------- + +PREAMBLE +The goals of the Open Font License (OFL) are to stimulate worldwide +development of collaborative font projects, to support the font creation +efforts of academic and linguistic communities, and to provide a free and +open framework in which fonts may be shared and improved in partnership +with others. + +The OFL allows the licensed fonts to be used, studied, modified and +redistributed freely as long as they are not sold by themselves. The +fonts, including any derivative works, can be bundled, embedded, +redistributed and/or sold with any software provided that any reserved +names are not used by derivative works. The fonts and derivatives, +however, cannot be released under any other type of license. The +requirement for fonts to remain under this license does not apply +to any document created using the fonts or their derivatives. + +DEFINITIONS +"Font Software" refers to the set of files released by the Copyright +Holder(s) under this license and clearly marked as such. This may +include source files, build scripts and documentation. + +"Reserved Font Name" refers to any names specified as such after the +copyright statement(s). + +"Original Version" refers to the collection of Font Software components as +distributed by the Copyright Holder(s). + +"Modified Version" refers to any derivative made by adding to, deleting, +or substituting -- in part or in whole -- any of the components of the +Original Version, by changing formats or by porting the Font Software to a +new environment. + +"Author" refers to any designer, engineer, programmer, technical +writer or other person who contributed to the Font Software. + +PERMISSION & CONDITIONS +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Font Software, to use, study, copy, merge, embed, modify, +redistribute, and sell modified and unmodified copies of the Font +Software, subject to the following conditions: + +1) Neither the Font Software nor any of its individual components, +in Original or Modified Versions, may be sold by itself. + +2) Original or Modified Versions of the Font Software may be bundled, +redistributed and/or sold with any software, provided that each copy +contains the above copyright notice and this license. These can be +included either as stand-alone text files, human-readable headers or +in the appropriate machine-readable metadata fields within text or +binary files as long as those fields can be easily viewed by the user. + +3) No Modified Version of the Font Software may use the Reserved Font +Name(s) unless explicit written permission is granted by the corresponding +Copyright Holder. This restriction only applies to the primary font name as +presented to the users. + +4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font +Software shall not be used to promote, endorse or advertise any +Modified Version, except to acknowledge the contribution(s) of the +Copyright Holder(s) and the Author(s) or with their explicit written +permission. + +5) The Font Software, modified or unmodified, in part or in whole, +must be distributed entirely under this license, and must not be +distributed under any other license. The requirement for fonts to +remain under this license does not apply to any document created +using the Font Software. + +TERMINATION +This license becomes null and void if any of the above conditions are +not met. + +DISCLAIMER +THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE +COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL +DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM +OTHER DEALINGS IN THE FONT SOFTWARE. diff --git a/examples/microduck-world/web/public/duck-preview.json b/examples/microduck-world/web/public/duck-preview.json new file mode 100644 index 0000000000..2c27722116 --- /dev/null +++ b/examples/microduck-world/web/public/duck-preview.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8eba4351bff2c4ce4154af3d116bd98cc8643f32987b1ee591faf05b7fc74beb +size 149713 diff --git a/examples/microduck-world/web/public/social/duck-world-v1.jpg b/examples/microduck-world/web/public/social/duck-world-v1.jpg new file mode 100644 index 0000000000..5c5307b561 --- /dev/null +++ b/examples/microduck-world/web/public/social/duck-world-v1.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e2ce55ddc5b68f8dc237a456f83b1359b6f5635639f5cc471d2db08d5cb8630 +size 55326 diff --git a/examples/microduck-world/web/public/social/duck-world-v2.jpg b/examples/microduck-world/web/public/social/duck-world-v2.jpg new file mode 100644 index 0000000000..20e4095751 --- /dev/null +++ b/examples/microduck-world/web/public/social/duck-world-v2.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17b311ab559cf3f6b5259c90ca0586609ea958c52ce06c1e01dd417e1921c3b3 +size 102129 diff --git a/examples/microduck-world/web/src/BallCameraPanel.tsx b/examples/microduck-world/web/src/BallCameraPanel.tsx new file mode 100644 index 0000000000..89d8ac8d58 --- /dev/null +++ b/examples/microduck-world/web/src/BallCameraPanel.tsx @@ -0,0 +1,161 @@ +import { PanelFrame } from "./cockpit/PanelFrame.tsx"; +import { useContext, useEffect, useState, useRef } from "react"; +import type { PanelProps } from "@dimos/cockpit/panels/registry.tsx"; +import { useOptionalSlot } from "@dimos/cockpit/panels/hooks.ts"; +import { ViewerSession } from "./viewerSession.ts"; +import { PlayerDirectory } from "./players.ts"; +import { playerName, robotIds } from "./roster.ts"; +import { FrameRate } from "./frameRate.ts"; +import styles from "./ballCamera.module.css"; + +type CameraFrame = { + robot: string; + generation: string; + ts: number; + width: number; + height: number; + status: string; + image: string; + boxes: { xyxy: [number, number, number, number]; confidence: number }[]; +}; +export function readCameraFrame(value: unknown): CameraFrame | null { + if (!value || typeof value !== "object") return null; + const v = value as CameraFrame; + if ( + !robotIds.some((id) => id === v.robot) || typeof v.generation !== "string" || + !Number.isFinite(v.ts) || !Number.isInteger(v.width) || v.width < 1 || v.width > 1920 || + !Number.isInteger(v.height) || v.height < 1 || v.height > 1080 || typeof v.image !== "string" || + !/^data:image\/jpeg;base64,[A-Za-z0-9+/=]+$/.test(v.image) || v.image.length > 500_000 || + !Array.isArray(v.boxes) || v.boxes.length > 100 + ) return null; + if ( + !v.boxes.every((b) => + Array.isArray(b.xyxy) && b.xyxy.length === 4 && b.xyxy.every(Number.isFinite) && + Number.isFinite(b.confidence) && b.confidence >= 0 && b.confidence <= 1 && b.xyxy[0] >= 0 && + b.xyxy[1] >= 0 && b.xyxy[2] <= v.width && b.xyxy[3] <= v.height && b.xyxy[2] >= b.xyxy[0] && + b.xyxy[3] >= b.xyxy[1] + ) + ) return null; + return v; +} + +export function BallCameraPanel({ spec, store }: PanelProps) { + const session = useContext(ViewerSession); + const players = useContext(PlayerDirectory); + const [robot, setRobot] = useState(String(spec.params?.robot ?? "duck1")); + const visible = true; + const selectable = spec.params?.selectable === true; + const channel = selectable ? `${robot}_ball_camera` : spec.channels[0]; + const slot = useOptionalSlot(store, channel); + const latest = readCameraFrame(slot?.value); + const [frame, setFrame] = useState(null); + const rate = useRef(new FrameRate()); + const [fps, setFps] = useState(0); + useEffect(() => { rate.current.reset(); setFps(0); }, [robot]); + const [now, setNow] = useState(Date.now()); + useEffect(() => { + const timer = setInterval(() => setNow(Date.now()), 250); + return () => clearInterval(timer); + }, []); + useEffect(() => { + if (visible && selectable && channel) return session?.subscribe(channel, () => {}); + }, [session, channel, selectable, visible]); + useEffect(() => { + if (!visible || !latest || latest.robot !== robot) { + setFrame(null); + return; + } + let canceled = false; + const image = new Image(); + image.src = latest.image; + image.decode().then(() => { + if (!canceled) { + setFrame(latest); + const measured = rate.current.draw(performance.now()); + if (measured !== null) setFps(measured); + } + }).catch(() => {}); + return () => { + canceled = true; + }; + }, [slot, robot, visible]); + const player = players.find((p) => p.id === robot); + const fresh = frame && frame.robot === robot && frame.generation === player?.generation && + now - frame.ts * 1000 < 2000 && now - frame.ts * 1000 > -2000; + return ( + + {selectable && ( + + )} +}> +
+ {visible && ( + <> +
+ {fresh ? fps.toFixed(1) : "0"} FPS + {fresh + ? ( + <> + {`${player?.displayName + + {frame.boxes.map((box, i) => ( + + + + ball {Math.round(box.confidence * 100)}% + + + ))} + + + ) + : ( +

+ {player?.generation + ? "Waiting for a fresh camera frame…" + : "This duck's camera starts when a player joins."} +

+ )} +
+

+ {fresh + ? frame.status !== "ready" + ? "Camera connected. Detector unavailable." + : frame.boxes.length + ? `${frame.boxes.length} ball${frame.boxes.length === 1 ? "" : "s"} detected` + : "No ball detected in this frame" + : "Camera waiting"} +

+ + )} +
+
+ ); +} diff --git a/examples/microduck-world/web/src/LobbyApp.tsx b/examples/microduck-world/web/src/LobbyApp.tsx new file mode 100644 index 0000000000..ae3e34abe7 --- /dev/null +++ b/examples/microduck-world/web/src/LobbyApp.tsx @@ -0,0 +1,662 @@ +import { type CSSProperties, useEffect, useRef, useState } from "react"; +import { connect, type Session } from "@dimos/sdk"; +import { useStatus } from "@dimos/sdk/react"; +import { Workspace } from "./cockpit/Workspace.tsx"; +import { App } from "@dimos/cockpit/App.tsx"; +import { cockpitDecoders, installAutoSubscriptions } from "@dimos/cockpit/subscriptions.ts"; +import { transportForSession } from "./publicTransport.ts"; +import { BallCameraPanel } from "./BallCameraPanel.tsx"; +import { WorldPanel } from "./WorldPanel.tsx"; +import { ViewerSession } from "./viewerSession.ts"; +import { duckColors, RoomPreview } from "./RoomPreview.tsx"; +import { useLobbyPreview } from "./lobbyPreview.ts"; +import { HoverPreview } from "./hoverPreview.tsx"; +import { playerName, robotIds, roster, teams } from "./roster.ts"; +import styles from "./lobby.module.css"; +import { PlayerDirectory, type PlayerIdentity } from "./players.ts"; + +interface Slot extends PlayerIdentity { + id: string; + occupied: boolean; + connected: boolean; + mine: boolean; + reconnectSeconds: number; +} +interface Entry { + token: string; + role: "observe" | "visitor" | "host"; + robot: string; + runtime: string; + displayName: string | null; + slots: Slot[]; + graceSeconds: number; +} +class LobbyError extends Error { + constructor(message: string, readonly status: number) { + super(message); + } +} +async function request( + action: string, + token?: string, + robot?: string, + displayName?: string, +): Promise { + const response = await fetch("/api/lobby", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ action, token, robot, displayName }), + signal: AbortSignal.timeout(8000), + }); + const data = await response.json(); + if (!response.ok) { + throw new LobbyError(data.error ?? "Could not connect to the world.", response.status); + } + return data; +} + +function useWorldSession(entry: Entry | null): Session | null { + const key = entry ? `${entry.token}:${entry.runtime}:${entry.role}` : ""; + const [current, setCurrent] = useState< + { key: string; session: Session } | null + >(null); + useEffect(() => { + if (!entry) return; + const session = connect({ + url: `${location.origin}/sessions/${entry.token}`, + robot: entry.runtime, + decoders: cockpitDecoders, + }, transportForSession(() => { + // Replacement is intentional ownership transfer, not a network failure. + session.close(); + globalThis.dispatchEvent(new Event("microduck-session-replaced")); + })); + const unsubscribe = entry.role === "observe" + ? session.subscribe("world_state", () => {}) + : installAutoSubscriptions(session); + setCurrent({ key, session }); + return () => { + unsubscribe(); + session.close(); + }; + // A poll returns a new Entry but does not change the connection identity. + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [key]); + return current?.key === key ? current.session : null; +} + +function World({ entry, session, preview }: { + entry: Entry; + session: Session; + preview: ReturnType; +}) { + const status = useStatus(session); + if (!status.manifest) { + return ( +
+ +

+ {entry.role === "observe" + ? "Opening the world…" + : `Getting ${entry.displayName || playerName(entry.robot)} ready…`} +

+

+ {entry.role === "visitor" + ? "Preparing your camera, controls and private agent." + : "Connecting you to the football club."} +

+
+ ); + } + return ( + + + + {entry.role === "observe" + ? ( +
+
+ +
+ +
+
+ +
+ ) + : } +
+
+
+ ); +} + +interface AuthState { + required: boolean; + configured?: boolean; + user: { id: string; login: string; preferredName: string } | null; +} +function signIn(robot?: string) { + sessionStorage.setItem("world-join-after-login", robot ?? "observe"); + location.assign("/auth/login"); +} + +export function LobbyApp() { + const [auth, setAuth] = useState(null); + const [nickname, setNickname] = useState( + sessionStorage.getItem("world-display-name") ?? "", + ); + const [hovered, setHovered] = useState(null); + const [joining, setJoining] = useState(null); + const nameDialog = useRef(null); + const nameInput = useRef(null); + const [portraits, setPortraits] = useState>({}); + const [entry, setEntry] = useState(null); + const [entered, setEntered] = useState( + sessionStorage.getItem("world-entered") === "yes", + ); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(""); + const revision = useRef(0); + const pending = useRef(false); + const session = useWorldSession(auth?.required ? (entered && auth.user ? entry : null) : entry); + const preview = useLobbyPreview( + !entered || entry?.role === "observe" ? session : null, + ); + useEffect(() => { + let disposed = false; + let retry: ReturnType; + const open = async () => { + try { + const authResponse = await fetch("/api/auth", { signal: AbortSignal.timeout(8000) }); + const identity: AuthState = authResponse.status === 404 + ? { required: false, user: null } + : authResponse.ok + ? await authResponse.json() + : (() => { + throw new Error("Sign-in unavailable"); + })(); + if (disposed) return; + setAuth(identity); + const previous = sessionStorage.getItem("world-github-id"); + if (identity.required && previous !== identity.user?.id) { + sessionStorage.removeItem("world-ticket"); + sessionStorage.removeItem("world-entered"); + setEntered(false); + if (identity.user) sessionStorage.setItem("world-github-id", identity.user.id); + else sessionStorage.removeItem("world-github-id"); + } + if (identity.required && !identity.user) { + const response = await fetch("/api/lobby", { signal: AbortSignal.timeout(8000) }); + if (!response.ok) throw new Error("World offline"); + const state = await response.json(); + if (!disposed) { + setEntry({ ...state, token: "", role: "observe", robot: "world", runtime: "world" }); + setError(""); + } + return; + } + const data = await request("create", sessionStorage.getItem("world-ticket") ?? undefined); + if (disposed) return; + sessionStorage.setItem("world-ticket", data.token); + const afterLogin = sessionStorage.getItem("world-join-after-login"); + if (data.role !== "observe") { + setEntered(true); + sessionStorage.setItem("world-entered", "yes"); + } else if (afterLogin && identity.user) { + sessionStorage.removeItem("world-join-after-login"); + if (afterLogin === "observe") { + setEntered(true); + sessionStorage.setItem("world-entered", "yes"); + } else { + setNickname( + identity.user.preferredName || sessionStorage.getItem("world-display-name") || "", + ); + setJoining(afterLogin); + } + } else if (new URLSearchParams(location.search).has("host")) { + setEntered(false); + sessionStorage.removeItem("world-entered"); + setJoining("duck1"); + } + setEntry(data); + setError(""); + } catch { + if (!disposed) { + setError("The world is reconnecting. Trying again…"); + retry = setTimeout(open, 3000); + } + } + }; + void open(); + return () => { + disposed = true; + clearTimeout(retry); + }; + }, []); + useEffect(() => { + if (!entry) return; + let disposed = false; + const poll = setInterval(() => { + if (pending.current) return; + const version = revision.current; + const update: Promise = auth?.required && !auth.user + ? fetch("/api/lobby").then(async (r) => { + if (!r.ok) throw new Error("World offline"); + return { + ...await r.json(), + token: "", + role: "observe", + robot: "world", + runtime: "world", + }; + }) + : request("create", entry.token); + update.then((data) => { + if (disposed || version !== revision.current) return; + sessionStorage.setItem("world-ticket", data.token); + if ( + data.token !== entry.token || + (entry.role !== "observe" && data.role === "observe") + ) { + setEntered(false); + sessionStorage.removeItem("world-entered"); + setError("Your session ended. Choose a duck or watch the world."); + } + setEntry(data); + }).catch((error) => { + if (disposed || version !== revision.current) return; + if (auth?.required && error instanceof LobbyError && error.status === 401) { + sessionStorage.removeItem("world-ticket"); + sessionStorage.removeItem("world-entered"); + setEntered(false); + setJoining(null); + setAuth({ ...auth, user: null }); + setError("Your sign-in expired. Sign in with GitHub to join again."); + } + }); + }, 3000); + return () => { + disposed = true; + clearInterval(poll); + }; + }, [entry?.token, entry?.role, auth]); + useEffect(() => { + const dialog = nameDialog.current; + if (joining && dialog && !dialog.open) { + dialog.showModal(); + nameInput.current?.focus(); + nameInput.current?.select(); + } else if (!joining && dialog?.open) dialog.close(); + }, [joining]); + const choose = async (robot?: string, returnToLobby = false) => { + if (auth?.required && !auth.user) { + signIn(robot); + return; + } + if (!entry || pending.current) return; + pending.current = true; + revision.current++; + setBusy(true); + setError(""); + try { + const next = await request( + robot ? "join" : "observe", + entry.token, + robot, + robot ? nickname : undefined, + ); + if (robot && next.displayName) { + setNickname(next.displayName); + sessionStorage.setItem("world-display-name", next.displayName); + } + setEntry(next); + setJoining(null); + setEntered(!returnToLobby); + if (returnToLobby) sessionStorage.removeItem("world-entered"); + else sessionStorage.setItem("world-entered", "yes"); + if (new URLSearchParams(location.search).has("host")) { + history.replaceState(null, "", location.pathname); + } + } catch (e) { + setError(e instanceof Error ? e.message : "Could not enter."); + } finally { + pending.current = false; + setBusy(false); + } + }; + const joiningSlot = entry?.slots.find((slot) => slot.id === joining); + const joiningUnavailable = !!joiningSlot?.occupied && !joiningSlot.mine; + const occupied = entry?.slots.filter((s) => s.occupied).length ?? 0; + return ( +
+
+
+ DIMENSIONAL +
+
+ {entry ? `${occupied} / 6 ducks occupied` : "Connecting…"} +
+ {auth?.user && ( +
+ @{auth.user.login} + +
+ )} + {entered && entry && ( +
+ + {entry.role === "observe" ? "Spectator" : entry.displayName && + entry.displayName !== playerName(entry.robot) + ? `${entry.displayName} · ${playerName(entry.robot)}` + : playerName(entry.robot)} + + +
+ )} +
+ {error && !joining &&
{error}
} + {entered && entry + ? ( +
+ {session + ? + : ( +
+ Connecting to the world… +
+ )} +
+ ) + : ( +
+
+

+ Welcome to a new DIMENSION +

+

+ Each microduck has its own context, the world is a shared MuJoCo environment. +

+
+ +
+
+ {teams.map((team) => ( + + {team === "red" ? "Red team" : "Blue team"} + {robotIds.filter((id) => + roster[id].team === team && + !entry?.slots.find((s) => s.id === id)?.occupied + ).length} / 3 available + + ))} +
+

+ {auth?.required && !auth.user + ? "Sign in with GitHub to play or watch. Choose a duck to continue." + : "Choose your duck. Meet your team beside the midfield touchline."} +

+
+
+ {robotIds.map( + (id) => { + const player = roster[id]; + const color = duckColors[id]; + const slot = entry?.slots.find((s) => s.id === id); + const unavailable = !!slot?.occupied && !slot.mine; + const availability = !entry + ? "Connecting" + : slot?.mine + ? "Your duck" + : unavailable + ? slot.connected ? "In use" : `Reconnecting · ${slot.reconnectSeconds}s` + : "Available"; + return ( +
{ + if (event.pointerType !== "touch") setHovered(id); + }} + onPointerLeave={(event) => { + if (event.pointerType !== "touch") setHovered(null); + }} + onFocusCapture={(event) => { + if ( + (event.target as HTMLElement).matches( + ":focus-visible", + ) + ) setHovered(id); + }} + onBlurCapture={(event) => { + if ( + !event.currentTarget.contains(event.relatedTarget) + ) setHovered(null); + }} + > +
+ + {player.team === "red" ? "R" : "B"}0{player.number} + + + + {availability} + +
+
+
+
+

{player.name}

+ {slot?.displayName &&

{slot.displayName}

} + + PITCH + LOCKER ROOMS KNOWN + + +
+
+ ); + }, + )} +
+ +
+ + W + A + S + D to walk · Talk to your agent to explore + + + Your discoveries stay private. A new player starts fresh. + +
+
+ )} + { + if (busy) event.preventDefault(); + else setJoining(null); + }} + > +
{ + event.preventDefault(); + if (joining && !joiningUnavailable) void choose(joining); + }} + > + MEET YOUR TEAM +

Join as {joining ? playerName(joining) : "a player"}

+

What should the other players call you?

+ + { + setNickname(event.target.value); + sessionStorage.setItem("world-display-name", event.target.value); + }} + /> +

+ Shown above your duck. Leave blank to use its team number. +

+ {(error || joiningUnavailable) && ( +

+ {joiningUnavailable ? "This duck was just taken. Choose another duck." : error} +

+ )} +
+ + +
+
+
+
+ ); +} diff --git a/examples/microduck-world/web/src/RoomPreview.tsx b/examples/microduck-world/web/src/RoomPreview.tsx new file mode 100644 index 0000000000..ef2f4e9235 --- /dev/null +++ b/examples/microduck-world/web/src/RoomPreview.tsx @@ -0,0 +1,110 @@ +import places from "../../assets/scenes/apartment/places.json"; +import appearance from "../../assets/scenes/apartment/viewer.json"; +import roster from "../../assets/scenes/apartment/multiplayer.json"; +import type { WorldDefinition, WorldSnapshot } from "./worldModel.ts"; + +export const roomNames = Object.keys(places.rooms); +export const objectCount = Object.keys(places.objects).length; +export const duckColors: Record = roster.colors; +const rooms = Object.values(places.rooms); +const xmin = Math.min(...rooms.map((r) => r.bounds[0])); +const xmax = Math.max(...rooms.map((r) => r.bounds[1])); +const ymin = Math.min(...rooms.map((r) => r.bounds[2])); +const ymax = Math.max(...rooms.map((r) => r.bounds[3])); +const scale = 216 / Math.max(xmax - xmin, ymax - ymin); +const x = (value: number) => 126 + (value - (xmin + xmax) / 2) * scale; +const y = (value: number) => 126 - (value - (ymin + ymax) / 2) * scale; + +/** Human spectator metadata; this is never published into a robot's sensor streams. */ +export function RoomPreview({ model, snapshot }: { + model: WorldDefinition | null; + snapshot: WorldSnapshot | null; +}) { + return ( + + + {Object.entries(places.rooms).map(([name, room]) => { + const [left, right, bottom, top] = room.bounds; + return ( + + + + {name.includes("corridor") || name === "player_tunnel" + ? "" + : name === "red_lockers" + ? "RED" + : name === "blue_lockers" + ? "BLUE" + : name === "football" + ? "PITCH" + : name[0].toUpperCase() + name.slice(1)} + + + ); + })} + {Object.entries(places.objects).map(([name, point]) => ( + + {name.replaceAll("_", " ")} + + ))} + {(model?.actors ?? []).filter((actor) => + snapshot?.actors?.some((a) => a.id === actor.id && a.active) + ).map((actor) => { + const pose = snapshot?.poses[model!.bodyIds.indexOf(actor.focusBody)]; + if (!pose) return null; + return ( + + + + {actor.id.replace("duck", "")} + + {actor.id.replace("duck", "Duck ")} + + ); + })} + + ); +} diff --git a/examples/microduck-world/web/src/WorldControls.tsx b/examples/microduck-world/web/src/WorldControls.tsx new file mode 100644 index 0000000000..fe9a21717a --- /dev/null +++ b/examples/microduck-world/web/src/WorldControls.tsx @@ -0,0 +1,97 @@ +import { useEffect, useState } from "react"; +import { ControlPanel } from "@dimos/cockpit/panels/ControlPanel.tsx"; +import { useOptionalSlot } from "@dimos/cockpit/panels/hooks.ts"; +import { paramChannel } from "@dimos/cockpit/panels/panelParams.ts"; +import type { PanelProps } from "@dimos/cockpit/panels/registry.tsx"; +import { txReasonText } from "@dimos/cockpit/panels/txReason.ts"; +import { readSnapshot } from "./worldModel.ts"; +import styles from "./controls.module.css"; + +/** Extend the stock strip without copying its policy or mode controls. */ +export function WorldControls(props: PanelProps) { + const { spec, store, teleop } = props; + const command = paramChannel(spec, "command", 3); + const policy = useOptionalSlot(store, paramChannel(spec, "policies", 1))?.value; + const count = policy && typeof policy === "object" && "respawns" in policy && + typeof policy.respawns === "number" + ? policy.respawns + : null; + const world = readSnapshot(useOptionalSlot(store, "world_state")?.value); + const balls = [["football_ball_1", "Ball 1"], ["football_ball_2", "Ball 2"], ["football_ball_3", "Ball 3"], ["ball", "Benchmark ball"]]; + const [dropping, setDropping] = useState>({}); + useEffect(() => { + setDropping(old => { + const entries = Object.entries(old).filter(([ball, pending]) => { + if ((world?.football?.drops?.[ball] ?? 0) > pending.count) return false; + return true; + }); + return entries.length === Object.keys(old).length ? old : Object.fromEntries(entries); + }); + }, [world]); + useEffect(() => { + const entries = Object.values(dropping); + if (!entries.length) return; + const timer = setTimeout(() => { + setError("Ball reset timed out. Try again."); + setDropping(old => Object.fromEntries(Object.entries(old).filter(([, value]) => Date.now() - value.at < 6000))); + }, Math.max(1, Math.min(...entries.map(value => value.at + 6000 - Date.now())))); + return () => clearTimeout(timer); + }, [dropping]); + const drop = (ball: string) => { + if (!teleop || !command) return; + const result = teleop.tx(command, { name: "drop_ball", args: { ball } }); + if (!result.ok) { setError(txReasonText(result.reason)); return; } + setError(""); + setDropping(old => ({ ...old, [ball]: { count: world?.football?.drops?.[ball] ?? 0, at: Date.now() } })); + }; + const [pending, setPending] = useState(null); + const [error, setError] = useState(""); + + useEffect(() => { + if (pending === null) return; + if (count !== null && count > pending) { + setPending(null); + return; + } + const timer = setTimeout(() => { + setPending(null); + setError("Waiting for a clear midfield spawn or a connection."); + }, 6000); + return () => clearTimeout(timer); + }, [count, pending]); + + const respawn = () => { + if (!teleop || !command || count === null) return; + const result = teleop.tx(command, { name: "respawn", args: {} }); + if (!result.ok) { + setError(txReasonText(result.reason)); + return; + } + setError(""); + setPending(count); + }; + + return ( +
+ +
+ + Simulation reset + {error && {error}} +
+
+ Drop at midfield · 2 m +
{balls.map(([ball, label]) => )}
+
+
+ ); +} diff --git a/examples/microduck-world/web/src/WorldPanel.tsx b/examples/microduck-world/web/src/WorldPanel.tsx new file mode 100644 index 0000000000..10d707b07a --- /dev/null +++ b/examples/microduck-world/web/src/WorldPanel.tsx @@ -0,0 +1,332 @@ +import { useContext, useEffect, useRef, useState } from "react"; +import { useStoreChannel } from "@dimos/sdk/react"; +import { Badge, type DrawHealth, PanelFrame } from "./cockpit/PanelFrame.tsx"; +import type { PanelProps } from "@dimos/cockpit/panels/registry.tsx"; +import { startVideoSink, VIDEO_STALE_MS } from "@dimos/cockpit/panels/VideoPanel.tsx"; +import { type CameraMode, WorldRenderer } from "./worldRenderer.ts"; +import { fetchDefinition, readSnapshot, type WorldSnapshot } from "./worldModel.ts"; +import { ViewerSession } from "./viewerSession.ts"; +import styles from "./world.module.css"; +import { PlayerDirectory } from "./players.ts"; +import { playerName } from "./roster.ts"; + +export function WorldPanel({ spec, store }: PanelProps) { + const session = useContext(ViewerSession); + const players = useContext(PlayerDirectory); + const [showNames, setShowNames] = useState( + localStorage.getItem("world-show-names") !== "false", + ); + const [followed, setFollowed] = useState(""); + const host = useRef(null); + const jpegCanvas = useRef(null); + const viewer = useRef(null); + const health = useRef({ lastDrawOkAtMs: Date.now(), failures: 0 }).current; + const [mode, setMode] = useState("explore"); + const [backend, setBackend] = useState<"three" | "jpeg">("three"); + const [fps, setFps] = useState(0); + const [score, setScore] = useState<[number, number] | null>(null); + const [status, setStatus] = useState("Waiting for the world…"); + const [ready, setReady] = useState(false); + const view = spec.params?.view === "pov" ? "pov" : "world"; + const prefix = view === "pov" ? "duck3d" : "world3d"; + const ch = spec.channels[0]; + const jpegCh = String(spec.params?.jpeg ?? ""); + const { stats } = useStoreChannel(store, ch); + const { slot: jpegSlot } = useStoreChannel(store, jpegCh); + + useEffect(() => { + const element = host.current; + if (!element || !ch) return; + let renderer: WorldRenderer; + try { + renderer = new WorldRenderer( + element, + setMode, + view, + setFps, + String(spec.params?.robot ?? "duck1"), + ); + } catch { + setStatus( + "3D graphics are unavailable. Enable hardware acceleration or select MuJoCo JPEG.", + ); + return; + } + viewer.current = renderer; + let modelUrl = ""; + let loadedUrl = ""; + let latest: WorldSnapshot | null = null; + let disposed = false; + const contextLost = (event: Event) => { + event.preventDefault(); + renderer.setActive(false); + setReady(false); + setStatus( + "The graphics connection was lost. Reload or select MuJoCo JPEG.", + ); + }; + renderer.renderer.domElement.addEventListener( + "webglcontextlost", + contextLost, + ); + const ingest = () => { + const snapshot = readSnapshot(store.get(ch)?.value); + if (!snapshot) return; + latest = snapshot; + if (snapshot.football) { + const next = snapshot.football.scores; + setScore((old) => old?.[0] === next[0] && old?.[1] === next[1] ? old : next); + } + if (snapshot.model !== modelUrl) { + modelUrl = snapshot.model; + loadedUrl = ""; + setReady(false); + setStatus("Loading the world…"); + const requestedUrl = modelUrl; + fetchDefinition(requestedUrl).then((model) => { + if (disposed || requestedUrl !== modelUrl) return; + renderer.load(model); + loadedUrl = requestedUrl; + if (latest?.model === requestedUrl) renderer.push(latest); + setReady(true); + }).catch((error: unknown) => { + if (disposed || requestedUrl !== modelUrl) return; + setReady(false); + setStatus( + error instanceof Error ? error.message : "The world could not load. Reload to retry.", + ); + }); + } else if (loadedUrl === modelUrl) { + try { + renderer.push(snapshot); + } catch (error) { + setReady(false); + setStatus( + error instanceof Error ? error.message : "Invalid world update.", + ); + } + } + }; + const unsubscribe = store.subscribe(ch, ingest); + ingest(); + return () => { + disposed = true; + unsubscribe(); + renderer.renderer.domElement.removeEventListener( + "webglcontextlost", + contextLost, + ); + renderer.dispose(); + viewer.current = null; + }; + }, [ch, store, view]); + + useEffect(() => { + viewer.current?.setActive(backend === "three"); + if (backend !== "jpeg" || !session || !jpegCanvas.current || !jpegCh) { + return; + } + let release: (() => void) | null = null; + const visibility = () => { + if (document.hidden) { + release?.(); + release = null; + } else if (!release) release = session.subscribe(jpegCh, () => {}); + }; + visibility(); + document.addEventListener("visibilitychange", visibility); + const stopSink = startVideoSink(store, jpegCh, jpegCanvas.current, health); + return () => { + document.removeEventListener("visibilitychange", visibility); + release?.(); + stopSink(); + }; + }, [backend, session, jpegCh, store, health]); + + useEffect(() => { + viewer.current?.setPlayers(players, showNames); + }, [players, showNames, ready]); + + const switchBackend = (value: "three" | "jpeg") => { + if (value === "jpeg" && view === "world") viewer.current?.follow(); + setBackend(value); + }; + const fpsBadge = (backend === "three" + ? ( + + {fps.toFixed(0)} FPS + + ) + : ( + + )); + return ( + + {view !== "pov" && fpsBadge} + + + } + > +
+ {view === "pov" &&
{fpsBadge}
} + + + ); +} diff --git a/examples/microduck-world/web/src/ballCamera.module.css b/examples/microduck-world/web/src/ballCamera.module.css new file mode 100644 index 0000000000..5b8cf22907 --- /dev/null +++ b/examples/microduck-world/web/src/ballCamera.module.css @@ -0,0 +1,18 @@ +.panel { background: #10181b; color: #e7f0ef; min-width: 0; padding: 12px; border: 1px solid #324247; border-radius: 10px; } +.header { display: flex; align-items: center; justify-content: space-between; gap: 12px; font-size: 13px; } +.header button, .selector select { color: inherit; background: #1b282d; border: 1px solid #4a6068; border-radius: 5px; padding: 5px 8px; cursor: pointer; } +.header button:focus-visible, .selector select:focus-visible { outline: 2px solid #96cde8; outline-offset: 2px; } +.selector { display: flex; justify-content: space-between; align-items: center; gap: 8px; font-size: 12px; margin-top: 10px; } +.picture { position: relative; aspect-ratio: 16 / 9; margin-top: 10px; background: #080d10; overflow: hidden; border-radius: 5px; } +.picture img, .picture svg { position: absolute; inset: 0; width: 100%; height: 100%; } +.picture rect { fill: none; stroke: #a8ff81; stroke-width: 2; } +.picture text { fill: #a8ff81; font: bold 14px system-ui; stroke: #112514; stroke-width: 3px; paint-order: stroke; } +.picture p { padding: 20px; font-size: 12px; line-height: 1.5; color: #a5b4b9; } +.status { font-size: 11px; margin: 8px 0 0; color: #b4c8cb; } + +.panel {height:100%;width:100%;display:flex;flex-direction:column;padding:0;border:0;border-radius:0;background:var(--mw-panel,#14171a);color:var(--mw-text,#d7dde3)} +.picture{flex:1;min-height:80px;margin:0;border-radius:0}.status{padding:5px 8px;margin:0;color:var(--mw-muted,#8b949e)}.selector{padding:5px 8px;margin:0} + +.panel{height:auto}.picture{aspect-ratio:1;flex:none;width:100%;height:auto}.picture img{object-fit:cover}.picture svg{width:100%;height:100%} + +.picture{aspect-ratio:16 / 9}.picture img{object-fit:contain} diff --git a/examples/microduck-world/web/src/cockpit/Badge.tsx b/examples/microduck-world/web/src/cockpit/Badge.tsx new file mode 100644 index 0000000000..336c823612 --- /dev/null +++ b/examples/microduck-world/web/src/cockpit/Badge.tsx @@ -0,0 +1,56 @@ +import type { ChannelStore } from "@dimos/sdk"; +import { useStoreChannel } from "@dimos/sdk/react"; +import styles from "@dimos/cockpit/layout/PanelFrame.module.css"; +/** Sink-side draw diagnostics, mutated in place and sampled by the badge. */ +export interface DrawHealth { + /** Browser ms of the last successful draw, stamped at sink start before the first. */ + lastDrawOkAtMs: number; + /** Consecutive failed decode-or-draw attempts since the last success. */ + failures: number; +} + +/** Hz/staleness readout for a canvas panel's primary channel. Re-rendered on + * the 500 ms UI tick via useChannel; `health` is mutated by the sink at draw + * rate and simply sampled here (intended coupling). */ +export function Badge({ store, ch, health, staleMs, unit, testId }: { + store: ChannelStore; + ch: string; + health: DrawHealth; + staleMs: number; + unit: string; + testId: string; +}) { + const { stats } = useStoreChannel(store, ch); + let text: string; + let error = false; + let stale = false; + if (stats.frames === 0) { + // Nothing ever arrived; a corrupt first frame is an error, not "waiting". + text = "waiting"; + } else if (stats.decodeFailing || health.failures > 0) { + // A single bad frame trips this; the next success clears it. + text = "decode failing"; + error = true; + } else if (stats.ageMs !== null && stats.ageMs > staleMs) { + text = `stale ${(stats.ageMs / 1000).toFixed(1)} s`; + stale = true; + } else if (stats.lastFrameAtMs - health.lastDrawOkAtMs > staleMs) { + // Frames arrive but nothing draws (e.g. a decoder that never settles); + // both operands are browser milliseconds. + text = "stalled"; + stale = true; + } else { + text = `${stats.hz.toFixed(1)} ${unit}`; + } + return ( + + {text} + + ); +} diff --git a/examples/microduck-world/web/src/cockpit/ChatPanel.tsx b/examples/microduck-world/web/src/cockpit/ChatPanel.tsx new file mode 100644 index 0000000000..d3591eed10 --- /dev/null +++ b/examples/microduck-world/web/src/cockpit/ChatPanel.tsx @@ -0,0 +1,284 @@ +// Adapted from DimOS 536679f, Apache-2.0. Application-specific presentation. +// Agent chat panel: the humancli transcript in a browser. Rows come from the +// per-channel ChatLog (direct store path, so lines land as they arrive); +// the idle flag and mode ride the UI tick. Typed lines go out through the +// session's generic tx on the panel's input channel and stay "pending" until +// the agent loop echoes the HumanMessage back on the chat channel. +// +// Key events never leave the textarea (stopPropagation on keydown/keyup): +// WASD typed here must not reach the teleop pad, and no global shortcuts +// are registered by this panel. + +import { + type KeyboardEvent, + type UIEvent, + useLayoutEffect, + useMemo, + useRef, + useState, + useSyncExternalStore, +} from "react"; +import { displayText, inlineTokens } from "./terminalText.ts"; +import { useStatus } from "@dimos/sdk/react"; +import { PanelFrame } from "./PanelFrame.tsx"; +import { formatClock, type Row, rowPrefix } from "@dimos/cockpit/panels/chatFold.ts"; +import { chatLogFor, type PendingLine } from "@dimos/cockpit/panels/chatLog.ts"; +import styles from "./humancli.module.css"; +import { useOptionalSlot } from "@dimos/cockpit/panels/hooks.ts"; +import { paramChannel, readString } from "@dimos/cockpit/panels/panelParams.ts"; +import type { PanelProps } from "@dimos/cockpit/panels/registry.tsx"; +import { txReasonText } from "@dimos/cockpit/panels/txReason.ts"; + +/** Bridge cap for human_input text (relay_bridge_module._CHAT_IN_MAX_CHARS). */ +const HUMANCLI_LOGO = " \u2587\u2587\u2587\u2587\u2587\u2587\u2557 \u2587\u2587\u2557\u2587\u2587\u2587\u2557 \u2587\u2587\u2587\u2557\u2587\u2587\u2587\u2587\u2587\u2587\u2587\u2557\u2587\u2587\u2587\u2557 \u2587\u2587\u2557\u2587\u2587\u2587\u2587\u2587\u2587\u2587\u2557\u2587\u2587\u2557 \u2587\u2587\u2587\u2587\u2587\u2587\u2557 \u2587\u2587\u2587\u2557 \u2587\u2587\u2557 \u2587\u2587\u2587\u2587\u2587\u2557 \u2587\u2587\u2557\n \u2587\u2587\u2554\u2550\u2550\u2587\u2587\u2557\u2587\u2587\u2551\u2587\u2587\u2587\u2587\u2557 \u2587\u2587\u2587\u2587\u2551\u2587\u2587\u2554\u2550\u2550\u2550\u2550\u255d\u2587\u2587\u2587\u2587\u2557 \u2587\u2587\u2551\u2587\u2587\u2554\u2550\u2550\u2550\u2550\u255d\u2587\u2587\u2551\u2587\u2587\u2554\u2550\u2550\u2550\u2587\u2587\u2557\u2587\u2587\u2587\u2587\u2557 \u2587\u2587\u2551\u2587\u2587\u2554\u2550\u2550\u2587\u2587\u2557\u2587\u2587\u2551\n \u2587\u2587\u2551 \u2587\u2587\u2551\u2587\u2587\u2551\u2587\u2587\u2554\u2587\u2587\u2587\u2587\u2554\u2587\u2587\u2551\u2587\u2587\u2587\u2587\u2587\u2557 \u2587\u2587\u2554\u2587\u2587\u2557 \u2587\u2587\u2551\u2587\u2587\u2587\u2587\u2587\u2587\u2587\u2557\u2587\u2587\u2551\u2587\u2587\u2551 \u2587\u2587\u2551\u2587\u2587\u2554\u2587\u2587\u2557 \u2587\u2587\u2551\u2587\u2587\u2587\u2587\u2587\u2587\u2587\u2551\u2587\u2587\u2551\n \u2587\u2587\u2551 \u2587\u2587\u2551\u2587\u2587\u2551\u2587\u2587\u2551\u255a\u2587\u2587\u2554\u255d\u2587\u2587\u2551\u2587\u2587\u2554\u2550\u2550\u255d \u2587\u2587\u2551\u255a\u2587\u2587\u2557\u2587\u2587\u2551\u255a\u2550\u2550\u2550\u2550\u2587\u2587\u2551\u2587\u2587\u2551\u2587\u2587\u2551 \u2587\u2587\u2551\u2587\u2587\u2551\u255a\u2587\u2587\u2557\u2587\u2587\u2551\u2587\u2587\u2554\u2550\u2550\u2587\u2587\u2551\u2587\u2587\u2551\n \u2587\u2587\u2587\u2587\u2587\u2587\u2554\u255d\u2587\u2587\u2551\u2587\u2587\u2551 \u255a\u2550\u255d \u2587\u2587\u2551\u2587\u2587\u2587\u2587\u2587\u2587\u2587\u2557\u2587\u2587\u2551 \u255a\u2587\u2587\u2587\u2587\u2551\u2587\u2587\u2587\u2587\u2587\u2587\u2587\u2551\u2587\u2587\u2551\u255a\u2587\u2587\u2587\u2587\u2587\u2587\u2554\u255d\u2587\u2587\u2551 \u255a\u2587\u2587\u2587\u2587\u2551\u2587\u2587\u2551 \u2587\u2587\u2551\u2587\u2587\u2587\u2587\u2587\u2587\u2587\u2557\n \u255a\u2550\u2550\u2550\u2550\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u2550\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d \u255a\u2550\u255d \u255a\u2550\u2550\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d"; + +export const CHAT_INPUT_MAX_CHARS = 900; +export const AGENT_MODE_NOTICE = "Switch to Agent mode to talk to the duck"; +export const THINKING_TEXT = "◌ agent thinking"; +/** An empty transcript means nothing has been said yet, not that the agent is + * still coming up - "waiting for the agent..." read as a stuck spinner. */ +export const EMPTY_TEXT = "Ask the duck something"; +/** Scroll slack under which the transcript still counts as "at the bottom". */ +const STICK_SLACK_PX = 8; + +export interface ChatChannels { + chat: string; + idle: string | undefined; + mode: string | undefined; + input: string | undefined; +} + +export function chatChannels(spec: PanelProps["spec"]): ChatChannels | null { + const chat = paramChannel(spec, "chat", 0); + if (chat === undefined) return null; + return { + chat, + idle: paramChannel(spec, "idle", 1), + mode: paramChannel(spec, "mode", 2), + input: paramChannel(spec, "input", 3), + }; +} + +function readFlag(v: unknown): boolean | null { + if (typeof v !== "object" || v === null) return null; + const value = (v as Record).value; + return typeof value === "boolean" ? value : null; +} + +function readMode(v: unknown): string | null { + if (typeof v !== "object" || v === null) return null; + return readString((v as Record).mode); +} + +export function ChatPanel({ spec, store, teleop }: PanelProps) { + const chans = chatChannels(spec); + if (chans === null) { + return ( + + chat panel {spec.id}: no channel bound + + ); + } + return ; +} + +function ChatView(props: PanelProps & { chans: ChatChannels }) { + return props.teleop ? : ; +} +function ConnectedChat(props: PanelProps & { chans: ChatChannels; teleop: NonNullable }) { + const status = useStatus(props.teleop); + const control = status.manifest?.panels.find(p => p.kind === "control"); + const command = control ? paramChannel(control, "command", 3) : undefined; + return ; +} +function ChatContent({ spec, store, teleop, chans, command }: PanelProps & { chans: ChatChannels; command?: string }) { + const log = chatLogFor(store, chans.chat); + const { rows, pending } = useSyncExternalStore(log.subscribe, log.getSnapshot); + const idleSlot = useOptionalSlot(store, chans.idle); + const modeSlot = useOptionalSlot(store, chans.mode); + const idle = readFlag(idleSlot?.value); + const mode = readMode(modeSlot?.value); + const thinking = idle === false; + // humancli stamps the spinner with the time it appeared. + const thinkingSince = useMemo(() => (thinking ? Date.now() / 1000 : 0), [thinking]); + const wrongMode = mode !== null && mode !== "agent"; + + const inputRef = useRef(null); + const [editing, setEditing] = useState(false); + const [draft, setDraft] = useState(""); + const [error, setError] = useState(null); + const listRef = useRef(null); + const stick = useRef(true); + + useLayoutEffect(() => { + const el = listRef.current; + if (el !== null && stick.current) el.scrollTop = el.scrollHeight; + }, [rows, pending, thinking]); + + const onScroll = (e: UIEvent): void => { + const el = e.currentTarget; + stick.current = el.scrollHeight - el.scrollTop - el.clientHeight <= STICK_SLACK_PX; + }; + + const canCompose = teleop !== undefined && chans.input !== undefined && spec.params.read_only !== true; + const canSend = teleop !== undefined && chans.input !== undefined && !wrongMode && spec.params.read_only !== true; + + const transmit = (text: string): boolean => { + if (!canSend || teleop === undefined || chans.input === undefined) { + setError("no send path bound"); + return false; + } + const result = teleop.tx(chans.input, { text }); + if (!result.ok) { + setError(txReasonText(result.reason)); + return false; + } + setError(null); + return true; + }; + + const send = (): void => { + const text = draft.trim(); + if (text === "") return; + if (text.length > CHAT_INPUT_MAX_CHARS) { + setError(`message too long (max ${CHAT_INPUT_MAX_CHARS} chars)`); + return; + } + if (transmit(text)) { + log.addPending(text); + setDraft(""); + stick.current = true; + } + }; + + const retry = (line: PendingLine): void => { + if (transmit(line.text)) log.resent(line.key); + }; + + const onKeyDown = (e: KeyboardEvent): void => { + e.stopPropagation(); + if (e.key === "Enter" && !e.shiftKey) { + e.preventDefault(); + if (canSend) send(); + } + }; + const onKeyUp = (e: KeyboardEvent): void => { + e.stopPropagation(); + }; + + + const selectAgent = (target: EventTarget) => { + if (!canCompose || (target as HTMLElement).closest("button,select,[data-panel-action]")) return; + if (wrongMode && !editing && teleop && command) { + const result = teleop.tx(command, { name: "set_mode", args: { mode: "agent" } }); + setError(result.ok ? null : txReasonText(result.reason)); + if (!result.ok) return; + } + setEditing(true); + }; + return ( +
selectAgent(e.target)} + onClick={e => { if (!(e.target as HTMLElement).closest("button,select,[data-panel-action]")) inputRef.current?.focus({ preventScroll: true }); }} + onBlur={e => { if (!e.currentTarget.contains(e.relatedTarget as Node | null)) setEditing(false); }}> + +
+
+ line.length)) * 6} ${HUMANCLI_LOGO.split("\n").length * 12}`} role="img" aria-label="DIMENSIONAL" preserveAspectRatio="xMinYMin meet"> + {HUMANCLI_LOGO.split("\n").map((line, i) => {line})} + + {rows.map((row) => )} + {pending.map((line) => ( +
+ + {rowPrefix(formatClock(line.sentAt / 1000), "human")} + + + {line.text} + {line.status === "failed" && ( + + not delivered · + + + )} + +
+ ))} + {thinking && ( +
+ {rowPrefix(formatClock(thinkingSince), "")} + {THINKING_TEXT} +
+ )} + {rows.length === 0 && pending.length === 0 && !thinking && ( + {EMPTY_TEXT} + )} +
+
+