diff --git a/examples/02_remote_agent_server/17_convo_with_agent_sandbox_server.py b/examples/02_remote_agent_server/17_convo_with_agent_sandbox_server.py new file mode 100644 index 0000000000..167bf61171 --- /dev/null +++ b/examples/02_remote_agent_server/17_convo_with_agent_sandbox_server.py @@ -0,0 +1,91 @@ +"""Run a conversation whose agent server lives in a kubernetes-sigs/agent-sandbox pod. + +Prerequisites (see agent_sandbox_deploy/README.md for a full kind walkthrough): + 1. A Kubernetes cluster (kind / minikube / GKE) with the agent-sandbox controller + and extensions installed, reachable via your kubeconfig. + 2. The agent-server SandboxTemplate + SandboxWarmPool applied: + kubectl apply -f agent_sandbox_deploy/sandboxtemplate.yaml + kubectl apply -f agent_sandbox_deploy/sandboxwarmpool.yaml + 3. pip install openhands-workspace[agent-sandbox] + 4. export LLM_API_KEY=... (and optionally LLM_MODEL, LLM_BASE_URL) + +The LLM is called from inside the pod (the agent runs on the agent server), so the +cluster needs egress to your LLM endpoint. +""" + +import os +import time + +from pydantic import SecretStr + +from openhands.sdk import ( + LLM, + Conversation, + RemoteConversation, + get_logger, +) +from openhands.tools.preset.default import get_default_agent +from openhands.workspace import AgentSandboxWorkspace + + +logger = get_logger(__name__) + +# 1) LLM configuration +api_key = os.getenv("LLM_API_KEY") +assert api_key is not None, "LLM_API_KEY environment variable is not set." + +llm = LLM( + usage_id="agent", + model=os.getenv("LLM_MODEL", "gpt-5.5"), + base_url=os.getenv("LLM_BASE_URL"), + api_key=SecretStr(api_key), +) + +# 2) Claim a pod from the warm pool. With a pre-warmed pool this returns in well +# under a second; the workspace connects to the pod's agent server via +# kubectl port-forward (the default 'port_forward' connection mode). +warmpool = os.getenv("AGENT_SANDBOX_WARMPOOL", "openhands-pool") +namespace = os.getenv("AGENT_SANDBOX_NAMESPACE", "default") +logger.info(f"Claiming a sandbox from warm pool {warmpool!r} in {namespace!r}...") +with AgentSandboxWorkspace( + warmpool=warmpool, + namespace=namespace, + # Safety net: auto-delete the claim after 30 minutes if something leaks it. + shutdown_after_seconds=30 * 60, +) as workspace: + # 3) Create the agent + agent = get_default_agent(llm=llm, cli_mode=True) + + # 4) Sanity-check the workspace with a direct command + result = workspace.execute_command("echo 'Hello from agent-sandbox!' && pwd") + logger.info(f"Command exit code: {result.exit_code}") + logger.info(f"Output: {result.stdout}") + + # 5) Run a conversation + conversation = Conversation(agent=agent, workspace=workspace) + assert isinstance(conversation, RemoteConversation) + try: + logger.info(f"Conversation ID: {conversation.state.id}") + conversation.send_message( + "Read the current repo and write 3 facts about the project into FACTS.txt." + ) + conversation.run() + logger.info(f"Agent status: {conversation.state.execution_status}") + + # 6) Demonstrate native pause/resume. The pod is suspended (operatingMode + # -> Suspended) and its PVC is retained, then resumed for a follow-up. + logger.info("Pausing the sandbox (operatingMode -> Suspended)...") + workspace.pause() + time.sleep(3) + logger.info("Resuming the sandbox (operatingMode -> Running)...") + workspace.resume() + + conversation.send_message("Great! Now append a 4th fact to FACTS.txt.") + conversation.run() + logger.info("Second task completed after resume.") + + cost = conversation.conversation_stats.get_combined_metrics().accumulated_cost + print(f"EXAMPLE_COST: {cost}") + finally: + print("\nCleaning up conversation...") + conversation.close() diff --git a/examples/02_remote_agent_server/agent_sandbox_deploy/README.md b/examples/02_remote_agent_server/agent_sandbox_deploy/README.md new file mode 100644 index 0000000000..c04c2c1935 --- /dev/null +++ b/examples/02_remote_agent_server/agent_sandbox_deploy/README.md @@ -0,0 +1,71 @@ +# Running the agent server in agent-sandbox on a local kind cluster + +End-to-end walkthrough for +[`17_convo_with_agent_sandbox_server.py`](../17_convo_with_agent_sandbox_server.py) +using [kind](https://kind.sigs.k8s.io/). The same steps work on minikube or a cloud +cluster such as GKE. Only the cluster-creation step differs. + +## 1. Create a cluster + +```bash +kind create cluster --name openhands +``` + +## 2. Install the agent-sandbox controller + extensions + +Pick a release from + and apply the core + +extensions manifests: + +```bash +export VERSION="vX.Y.Z" +kubectl apply -f https://github.com/kubernetes-sigs/agent-sandbox/releases/download/${VERSION}/manifest.yaml +kubectl apply -f https://github.com/kubernetes-sigs/agent-sandbox/releases/download/${VERSION}/extensions.yaml +``` + +Wait for the controller to be ready: + +```bash +kubectl -n agent-sandbox-system rollout status deploy --timeout=120s +``` + +## 3. Apply the agent-server template + warm pool + +```bash +kubectl apply -f sandboxtemplate.yaml +kubectl apply -f sandboxwarmpool.yaml +``` + +The first pull of `ghcr.io/openhands/agent-server` can take a minute. Watch the pool +fill up (pods become `Ready` once the agent server passes its `/health` probe): + +```bash +kubectl get pods -w +``` + +## 4. Install the client and run the example + +```bash +pip install openhands-workspace[agent-sandbox] + +export LLM_API_KEY=... # your LLM key (called from inside the pod) +export LLM_MODEL=... # optional, e.g. a hosted model id +python ../17_convo_with_agent_sandbox_server.py +``` + +You should see: a sub-second claim from the warm pool, a command run in the pod, the +agent editing files, then a **pause** (pod suspended, PVC retained) and **resume** +(same conversation continues). + +## Notes + +- **LLM egress.** The agent runs on the agent server *inside* the pod, so the LLM API + is called from the pod. kind allows outbound internet by default. To use a local + Ollama, point `LLM_BASE_URL` at an address reachable from the pod and (on kind) + ensure the container network can reach your host. +- **Cold start vs warm pool.** With `replicas: 0` the pool becomes on-demand and each + claim creates a fresh pod (full cold start). Increase `replicas` to keep ready pods. +- **Strong isolation.** Uncomment `runtimeClassName: gvisor` in `sandboxtemplate.yaml` + (requires a gVisor-enabled node pool) to sandbox untrusted agent code at the kernel + boundary. +- **Cleanup.** `kind delete cluster --name openhands`. diff --git a/examples/02_remote_agent_server/agent_sandbox_deploy/sandboxtemplate.yaml b/examples/02_remote_agent_server/agent_sandbox_deploy/sandboxtemplate.yaml new file mode 100644 index 0000000000..13a1275f8a --- /dev/null +++ b/examples/02_remote_agent_server/agent_sandbox_deploy/sandboxtemplate.yaml @@ -0,0 +1,48 @@ +--- +# SandboxTemplate that runs the OpenHands agent server. See the canonical copy and +# docs at openhands-workspace/openhands/workspace/agent_sandbox/. +apiVersion: extensions.agents.x-k8s.io/v1beta1 +kind: SandboxTemplate +metadata: + name: openhands-agent-server + namespace: default +spec: + volumeClaimTemplates: + - metadata: + name: workspace-volume + spec: + accessModes: [ReadWriteOnce] + resources: + requests: + storage: 5Gi + podTemplate: + spec: + # runtimeClassName: gvisor # uncomment for strong isolation of untrusted code + securityContext: + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + containers: + - name: agent-server + image: ghcr.io/openhands/agent-server:1.42.1-python + args: [--host, 0.0.0.0, --port, '8000'] + workingDir: /workspace + ports: + - containerPort: 8000 + readinessProbe: + httpGet: + path: /health + port: 8000 + initialDelaySeconds: 3 + periodSeconds: 3 + failureThreshold: 40 + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: '2' + memory: 2Gi + volumeMounts: + - name: workspace-volume + mountPath: /workspace diff --git a/examples/02_remote_agent_server/agent_sandbox_deploy/sandboxwarmpool.yaml b/examples/02_remote_agent_server/agent_sandbox_deploy/sandboxwarmpool.yaml new file mode 100644 index 0000000000..2aafafebf7 --- /dev/null +++ b/examples/02_remote_agent_server/agent_sandbox_deploy/sandboxwarmpool.yaml @@ -0,0 +1,11 @@ +--- +# Pre-warms agent-server pods so AgentSandboxWorkspace claims are near-instant. +apiVersion: extensions.agents.x-k8s.io/v1beta1 +kind: SandboxWarmPool +metadata: + name: openhands-pool + namespace: default +spec: + replicas: 2 + sandboxTemplateRef: + name: openhands-agent-server diff --git a/openhands-workspace/openhands/workspace/__init__.py b/openhands-workspace/openhands/workspace/__init__.py index f35bd857fe..b5e8ca3b2e 100644 --- a/openhands-workspace/openhands/workspace/__init__.py +++ b/openhands-workspace/openhands/workspace/__init__.py @@ -4,6 +4,7 @@ from openhands.sdk.workspace import PlatformType, TargetType +from .agent_sandbox import AgentSandboxWorkspace from .apptainer import ApptainerWorkspace from .cloud import ( CloneResult, @@ -21,6 +22,7 @@ __all__ = [ "APIRemoteWorkspace", + "AgentSandboxWorkspace", "ApptainerWorkspace", "CloneResult", "DockerDevWorkspace", diff --git a/openhands-workspace/openhands/workspace/agent_sandbox/README.md b/openhands-workspace/openhands/workspace/agent_sandbox/README.md new file mode 100644 index 0000000000..e2d191a822 --- /dev/null +++ b/openhands-workspace/openhands/workspace/agent_sandbox/README.md @@ -0,0 +1,86 @@ +# AgentSandboxWorkspace + +Run the OpenHands agent server inside a Kubernetes pod managed by +[`kubernetes-sigs/agent-sandbox`](https://github.com/kubernetes-sigs/agent-sandbox). + +The pod is claimed from a `SandboxWarmPool`, so a pre-warmed pool gives sub-second +starts instead of paying a container cold-start per conversation. It adds native +pause/resume (via the Sandbox `operatingMode`) and, with a persistent volume, +workspace state that survives a suspend. gVisor / Kata isolation is available by +setting a `runtimeClass` on the `SandboxTemplate`. + +It is a drop-in `RemoteWorkspace`, so it works anywhere a `DockerWorkspace` does, +from a laptop kind or minikube cluster to a cloud cluster such as GKE. + +## Install + +```bash +pip install openhands-workspace[agent-sandbox] +``` + +This pulls in the [`k8s-agent-sandbox`](https://pypi.org/project/k8s-agent-sandbox/) +client used to create and manage the sandbox. + +## Prerequisites + +1. A Kubernetes cluster reachable via your kubeconfig, with the agent-sandbox + controller + extensions installed + ([install guide](https://github.com/kubernetes-sigs/agent-sandbox#installation)). +2. A `SandboxTemplate` running the agent server, and a `SandboxWarmPool` that + references it. Ready-to-apply manifests live in [`deploy/`](deploy/): + + ```bash + kubectl apply -f deploy/sandboxtemplate.yaml + kubectl apply -f deploy/sandboxwarmpool.yaml + ``` + +## Usage + +```python +from openhands.sdk import Conversation +from openhands.tools.preset.default import get_default_agent +from openhands.workspace import AgentSandboxWorkspace + +with AgentSandboxWorkspace(warmpool="openhands-pool", namespace="default") as workspace: + result = workspace.execute_command("echo hello && pwd") + print(result.stdout) + + conversation = Conversation(agent=get_default_agent(llm=llm), workspace=workspace) + conversation.send_message("Write 3 facts about this repo into FACTS.txt.") + conversation.run() +``` + +### Connection modes + +* `connection="port_forward"` (default) spawns `kubectl port-forward` to the pod. + This works from a laptop against local kind or minikube, and against any cluster + your kubeconfig can reach. `host_port` picks a free local port for you. +* `connection="direct"` means you supply `host` yourself: a `sandbox-router` or + Gateway URL, or the in-cluster DNS name when OpenHands runs in the cluster. No + port-forward is started. + +### Testing + +See [`TESTING.md`](TESTING.md) for an end-to-end walkthrough: unit tests, a keyless +workspace smoke test, and a full agent run against a local Ollama model (no API key). + +### Pause / resume + +```python +workspace.pause() # operatingMode -> Suspended; pod terminated, PVC retained +workspace.resume() # operatingMode -> Running; reconnects to the agent server +``` + +### Where configuration lives + +Most knobs belong in Kubernetes, not on the Python constructor: + +- CPU, memory, image, `runtimeClass` and the security context belong in + `SandboxTemplate.spec.podTemplate`. +- Pre-warming and pool size belong in `SandboxWarmPool.spec.replicas`. +- Network egress allow-listing belongs in the template's `NetworkPolicy`. +- Persistent volumes belong in the template's `volumeClaimTemplates`. + +Python-side knobs: `warmpool` (required), `namespace`, `server_port`, `connection`, +`host_port`, `kube_context`, `sandbox_ready_timeout`, `health_check_timeout`, +`shutdown_after_seconds`, `labels`, `pod_labels`, `pod_annotations`. diff --git a/openhands-workspace/openhands/workspace/agent_sandbox/TESTING.md b/openhands-workspace/openhands/workspace/agent_sandbox/TESTING.md new file mode 100644 index 0000000000..ff28cbc861 --- /dev/null +++ b/openhands-workspace/openhands/workspace/agent_sandbox/TESTING.md @@ -0,0 +1,192 @@ +# Testing `AgentSandboxWorkspace` + +Three tiers, from no cluster at all to a full agent run. Every one of them runs +locally without an API key. The steps are cluster-agnostic, so they work on kind, +minikube, or any cluster your `kubectl` can reach. + +1. **Unit tests**, which need no cluster. +2. **Keyless workspace smoke test**, which needs a real cluster but no LLM. It + proves the integration on its own: claim, exec, pause/resume and persistence. +3. **Full agent e2e**, which needs a real cluster plus either a local Ollama model + (no key) or a hosted LLM key. + +## Prerequisites + +- A Kubernetes cluster (kind/minikube/cloud) reachable via your kubeconfig, with the + [agent-sandbox controller + extensions](https://github.com/kubernetes-sigs/agent-sandbox#installation) + installed. +- `kubectl`, and for a local Ollama run, [`ollama`](https://ollama.com) (or use the + in-cluster Ollama manifest below). +- The package with its optional client: + + ```bash + pip install openhands-workspace[agent-sandbox] + ``` + +## 1. Unit tests (no cluster) + +```bash +uv run pytest tests/workspace/test_agent_sandbox_workspace.py -v +``` + +These mock the `k8s-agent-sandbox` client, so no cluster or credentials are needed. + +## 2. Cluster setup (shared by tiers 2 and 3) + +Install the controller (pick a release from +): + +```bash +export VERSION="v0.5.2" +kubectl apply -f "https://github.com/kubernetes-sigs/agent-sandbox/releases/download/${VERSION}/sandbox-with-extensions.yaml" +kubectl -n agent-sandbox-system rollout status deploy --timeout=180s +``` + +Apply the agent-server `SandboxTemplate` and `SandboxWarmPool` (from +[`../deploy/`](../deploy/)): + +```bash +kubectl apply -f ../deploy/sandboxtemplate.yaml +kubectl apply -f ../deploy/sandboxwarmpool.yaml +kubectl get pods -w # wait for the warm pool pods to become Ready +``` + +The first pull of `ghcr.io/openhands/agent-server` can take a minute. On kind you can +pre-load it to avoid an in-cluster pull: + +```bash +docker pull ghcr.io/openhands/agent-server:1.42.1-python +kind load docker-image ghcr.io/openhands/agent-server:1.42.1-python --name +``` + +## 3. Keyless workspace smoke test (no LLM) + +This validates the whole integration without an LLM, and it works against the +**secure default template** because nothing ever leaves the pod, so no network +policy change is needed. + +```bash +python testing/smoke_test.py +``` + +Expected: a sub-second claim from the warm pool, a command run in the pod, then a +`pause()` / `resume()` where a file written before the pause is still present after +(persistent-volume state survives the suspend), and a clean teardown. + +## 4. Full agent e2e + +An OpenHands agent runs *inside* the sandbox pod and calls the LLM from there, so the +**sandbox pod must be able to reach the LLM**: + +- **Hosted LLM (public API):** the secure default template already allows public + egress, so there is nothing to change. +- **Local / in-cluster LLM:** the default template's `NetworkPolicy` blocks private + ranges (cluster and host IPs). For a test, relax it: + + ```bash + kubectl patch sandboxtemplate openhands-agent-server --type merge \ + -p '{"spec":{"networkPolicyManagement":"Unmanaged"}}' + # recreate the pool so new pods pick up the change: + kubectl patch sandboxwarmpool openhands-pool --type merge -p '{"spec":{"replicas":0}}' + kubectl patch sandboxwarmpool openhands-pool --type merge -p '{"spec":{"replicas":1}}' + ``` + + `Unmanaged` drops the policy entirely, which is fine for a throwaway test cluster + but not for real use. For anything beyond a local test, keep the policy **Managed** + and allow only what the agent needs. See the commented `networkPolicy` block in + [`deploy/sandboxtemplate.yaml`](deploy/sandboxtemplate.yaml) for a copy-pasteable + scoped-egress rule (DNS + your LLM endpoint). + +### 4a. No key, using local Ollama + +Deploy the in-cluster Ollama and pull a tool-capable model: + +```bash +kubectl apply -f testing/ollama.yaml +kubectl rollout status deploy/ollama --timeout=180s +kubectl exec deploy/ollama -- ollama pull qwen2.5 +``` + +> **Model choice matters.** Use a model that returns *structured* tool calls: +> `qwen2.5` (7b) and `llama3.1:8b` work. `qwen2.5-coder` and the `:3b` variants +> return tool calls as plain **text**, so the agent never acts. Quick check: +> +> ```bash +> curl -s http://:11434/api/chat -d '{"model":"qwen2.5","stream":false, +> "messages":[{"role":"user","content":"call run_bash to make hello.txt"}], +> "tools":[{"type":"function","function":{"name":"run_bash","parameters": +> {"type":"object","properties":{"command":{"type":"string"}}}}}]}' | python3 -c \ +> 'import sys,json;print("tool_calls:",json.load(sys.stdin)["message"].get("tool_calls"))' +> ``` +> +> A non-`null` `tool_calls` means the model is usable. + +Run it, pointing at the Ollama Service: + +```bash +export OLLAMA_URL="http://$(kubectl get svc ollama -o jsonpath='{.spec.clusterIP}'):11434" +python testing/agent_ollama_example.py +``` + +Notes baked into the example: +- `reasoning_effort="none"`, because qwen2.5 has no "thinking" mode and the request + is otherwise rejected with `does not support thinking`. +- A **minimal terminal-only agent**, because the default multi-tool agent overwhelms + models this size: they plan, think and finish without ever executing. +- Small models on CPU are slow (up to minutes per turn); the example sets a generous + timeout. + +### 4b. With a hosted key + +Any `RemoteWorkspace`-style usage works; construct the workspace and hand it to a +`Conversation`: + +```python +from pydantic import SecretStr +from openhands.sdk import LLM, Conversation +from openhands.tools.preset.default import get_default_agent +from openhands.workspace import AgentSandboxWorkspace + +llm = LLM(usage_id="agent", model="", api_key=SecretStr("")) +with AgentSandboxWorkspace(warmpool="openhands-pool") as ws: + conv = Conversation(agent=get_default_agent(llm=llm), workspace=ws) + conv.send_message("Write 3 facts about this repo into FACTS.txt.") + conv.run() +``` + +A capable hosted model can drive the full default agent; that's why 4b uses +`get_default_agent` while 4a uses the minimal agent. + +## What success looks like + +- A `SandboxClaim`, `Sandbox`, and pod appear while a test runs: + + ```bash + kubectl get sandboxclaim,sandbox,pods + ``` + +- Smoke test: the marker file survives `pause()`/`resume()`. +- Agent e2e: the agent emits a `terminal` action, and `/workspace/hello.txt` ends up + with the expected content. +- On exit the workspace deletes the `SandboxClaim` (and its pod) automatically. + +## Cleanup + +```bash +kubectl delete -f testing/ollama.yaml --ignore-not-found +kubectl delete -f ../deploy/sandboxwarmpool.yaml -f ../deploy/sandboxtemplate.yaml --ignore-not-found +# kind: kind delete cluster --name +``` + +## Troubleshooting + +| Symptom | Cause / fix | +|---|---| +| Agent connects but the LLM call times out; nothing reaches Ollama | The sandbox pod can't reach the LLM. The default `NetworkPolicy` blocks private ranges (cluster ClusterIPs, host IPs). Add a scoped egress rule (see the commented block in `deploy/sandboxtemplate.yaml`), or `networkPolicyManagement: Unmanaged` for a throwaway test cluster. Public/hosted APIs are already allowed. | +| Agent replies with text and `finish`es without doing anything (no `ActionEvent`) | The model returns tool calls as text. Use `qwen2.5` (7b) or `llama3.1:8b`; avoid `qwen2.5-coder` and `:3b`. Verify with the raw `/api/chat` probe above. | +| `litellm ... "" does not support thinking` | Set `reasoning_effort="none"` on the `LLM` (qwen2.5 has no thinking mode). | +| Agent plans/thinks/finishes but never runs the command | The default 7-tool agent is too heavy for a small model. Use a minimal terminal-only agent (as in `agent_ollama_example.py`), or a larger/hosted model. | +| `model '' not found` after editing the Ollama Deployment | The model store is an `emptyDir`; editing the Deployment restarts the pod and wipes it. `kubectl exec deploy/ollama -- ollama pull ` again (or use a PVC). | +| `resume()` fails with a port or "network namespace is closed" error | Transient churn right after resume; the workspace retries with a fresh local port. If you see it persist, raise `health_check_timeout`. | +| Claim never becomes Ready | First agent-server image pull is slow; pre-load it (see step 2) or raise `sandbox_ready_timeout`. | +| `AgentSandboxWorkspace requires the 'agent-sandbox' extra` | `pip install openhands-workspace[agent-sandbox]`. | diff --git a/openhands-workspace/openhands/workspace/agent_sandbox/__init__.py b/openhands-workspace/openhands/workspace/agent_sandbox/__init__.py new file mode 100644 index 0000000000..9a7dd8b8d6 --- /dev/null +++ b/openhands-workspace/openhands/workspace/agent_sandbox/__init__.py @@ -0,0 +1,6 @@ +"""agent-sandbox (Kubernetes) workspace backend.""" + +from openhands.workspace.agent_sandbox.workspace import AgentSandboxWorkspace + + +__all__ = ["AgentSandboxWorkspace"] diff --git a/openhands-workspace/openhands/workspace/agent_sandbox/deploy/sandboxtemplate.yaml b/openhands-workspace/openhands/workspace/agent_sandbox/deploy/sandboxtemplate.yaml new file mode 100644 index 0000000000..6ce5e2012f --- /dev/null +++ b/openhands-workspace/openhands/workspace/agent_sandbox/deploy/sandboxtemplate.yaml @@ -0,0 +1,75 @@ +--- +# SandboxTemplate that runs the OpenHands agent server. +# +# The agent server image is a standalone server that listens on a port and speaks +# the same HTTP API the DockerWorkspace / APIRemoteWorkspace connect to, so it drops +# straight into an agent-sandbox pod. A readiness probe on /health makes the Sandbox +# reach its Ready condition only once the agent server is actually accepting requests. +apiVersion: extensions.agents.x-k8s.io/v1beta1 +kind: SandboxTemplate +metadata: + name: openhands-agent-server + namespace: default +spec: + # The controller-managed NetworkPolicy (the default) allows public egress but + # blocks private ranges, so a sandbox cannot reach an in-cluster or on-host LLM. + # Rather than switching the template to `networkPolicyManagement: Unmanaged`, + # keep it Managed and allow just what the agent needs, e.g.: + # + # networkPolicy: + # egress: + # # DNS + # - to: + # - namespaceSelector: + # matchLabels: {kubernetes.io/metadata.name: kube-system} + # ports: [{protocol: UDP, port: 53}, {protocol: TCP, port: 53}] + # # In-cluster LLM (adjust selector/namespace/port to your deployment) + # - to: + # - podSelector: + # matchLabels: {app: ollama} + # ports: [{protocol: TCP, port: 11434}] + # # Hosted LLM APIs over HTTPS + # - ports: [{protocol: TCP, port: 443}] + # + # A persistent volume mounted at /workspace so the workspace (and anything the + # agent writes) survives a suspend/resume cycle. + volumeClaimTemplates: + - metadata: + name: workspace-volume + spec: + accessModes: [ReadWriteOnce] + resources: + requests: + storage: 5Gi + podTemplate: + spec: + # runtimeClassName: gvisor # uncomment for strong isolation of untrusted code + securityContext: + # The agent-server image runs as uid/gid 10001; make the PVC group-writable. + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + containers: + - name: agent-server + image: ghcr.io/openhands/agent-server:1.42.1-python + args: [--host, 0.0.0.0, --port, '8000'] + workingDir: /workspace + ports: + - containerPort: 8000 + readinessProbe: + httpGet: + path: /health + port: 8000 + initialDelaySeconds: 3 + periodSeconds: 3 + failureThreshold: 40 + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: '2' + memory: 2Gi + volumeMounts: + - name: workspace-volume + mountPath: /workspace diff --git a/openhands-workspace/openhands/workspace/agent_sandbox/deploy/sandboxwarmpool.yaml b/openhands-workspace/openhands/workspace/agent_sandbox/deploy/sandboxwarmpool.yaml new file mode 100644 index 0000000000..4b7e1926cc --- /dev/null +++ b/openhands-workspace/openhands/workspace/agent_sandbox/deploy/sandboxwarmpool.yaml @@ -0,0 +1,13 @@ +--- +# Pre-warms agent-server pods so AgentSandboxWorkspace claims are near-instant. +# Set replicas to how many ready pods you want kept available; 0 turns this into a +# plain on-demand pool (a pod is created per claim, paying full cold-start). +apiVersion: extensions.agents.x-k8s.io/v1beta1 +kind: SandboxWarmPool +metadata: + name: openhands-pool + namespace: default +spec: + replicas: 2 + sandboxTemplateRef: + name: openhands-agent-server diff --git a/openhands-workspace/openhands/workspace/agent_sandbox/testing/agent_ollama_example.py b/openhands-workspace/openhands/workspace/agent_sandbox/testing/agent_ollama_example.py new file mode 100644 index 0000000000..7f42fb83a1 --- /dev/null +++ b/openhands-workspace/openhands/workspace/agent_sandbox/testing/agent_ollama_example.py @@ -0,0 +1,75 @@ +"""Full-agent e2e for AgentSandboxWorkspace with a local Ollama model (no API key). + +Runs an OpenHands agent inside an agent-sandbox pod, driven by a local Ollama model, +and has it create a file with a bash command. + +Two things make this work reliably with a small local model: + * a model that emits structured tool calls. qwen2.5:7b and llama3.1:8b do; + qwen2.5-coder and the :3b variants return tool calls as plain text, which the + agent cannot act on; + * a minimal, terminal-only agent. The default multi-tool agent overwhelms models + at this size, and they plan, think and finish without ever executing. + +Prerequisites (see TESTING.md): a cluster with the agent-sandbox controller and a +warm pool, an Ollama endpoint the *sandbox pod* can reach (in-cluster Service, or any +reachable host), and: + + pip install openhands-workspace[agent-sandbox] + +Env: + OLLAMA_URL required, e.g. http://:11434 + OLLAMA_MODEL default: qwen2.5 + AGENT_SANDBOX_WARMPOOL default: openhands-pool + AGENT_SANDBOX_NAMESPACE default: default +""" + +import os + +from pydantic import SecretStr + +from openhands.sdk import LLM, Agent, Conversation +from openhands.sdk.tool import Tool +from openhands.tools.preset.default import register_default_tools +from openhands.tools.terminal import TerminalTool +from openhands.workspace import AgentSandboxWorkspace + + +WARMPOOL = os.environ.get("AGENT_SANDBOX_WARMPOOL", "openhands-pool") +NAMESPACE = os.environ.get("AGENT_SANDBOX_NAMESPACE", "default") + + +def main() -> None: + llm = LLM( + usage_id="agent", + model="ollama_chat/" + os.environ.get("OLLAMA_MODEL", "qwen2.5"), + base_url=os.environ["OLLAMA_URL"], + api_key=SecretStr("ollama"), # ignored by Ollama + reasoning_effort="none", # qwen2.5 has no "thinking" mode + num_retries=8, + timeout=900, # small models on CPU can be slow per turn + ) + register_default_tools(enable_browser=False) + + with AgentSandboxWorkspace(warmpool=WARMPOOL, namespace=NAMESPACE) as ws: + print("connected:", ws.host) + agent = Agent( + llm=llm, + tools=[Tool(name=TerminalTool.name)], + system_prompt_kwargs={"cli_mode": True}, + ) + conv = Conversation(agent=agent, workspace=ws) + conv.send_message( + "Run this exact bash command: echo 'hi from the agent' > hello.txt" + ) + conv.run() + print("agent status:", conv.state.execution_status) + + out = ws.execute_command("cat /workspace/hello.txt") + print(f"hello.txt (exit={out.exit_code}): {out.stdout!r}") + assert out.exit_code == 0 and "hi from the agent" in out.stdout, out + + print("AGENT E2E PASSED") + + +if __name__ == "__main__": + main() diff --git a/openhands-workspace/openhands/workspace/agent_sandbox/testing/ollama.yaml b/openhands-workspace/openhands/workspace/agent_sandbox/testing/ollama.yaml new file mode 100644 index 0000000000..5063b15378 --- /dev/null +++ b/openhands-workspace/openhands/workspace/agent_sandbox/testing/ollama.yaml @@ -0,0 +1,47 @@ +--- +# In-cluster Ollama for the no-API-key agent e2e (see TESTING.md). +# Portable across clusters: the agent server reaches it by Service DNS / ClusterIP, +# so no host networking is involved. Models live in an emptyDir, so `ollama pull` +# again if the pod restarts. CPU-only; size the node/model for your cluster. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ollama + namespace: default +spec: + replicas: 1 + selector: + matchLabels: {app: ollama} + template: + metadata: + labels: {app: ollama} + spec: + containers: + - name: ollama + image: ollama/ollama:latest + ports: [containerPort: 11434] + env: + - {name: OLLAMA_HOST, value: 0.0.0.0:11434} + - {name: OLLAMA_KEEP_ALIVE, value: '-1'} + readinessProbe: + httpGet: {path: /api/tags, port: 11434} + initialDelaySeconds: 5 + periodSeconds: 5 + resources: + requests: {cpu: '1', memory: 4Gi} + limits: {memory: 8Gi} + volumeMounts: + - {name: models, mountPath: /root/.ollama} + volumes: + - name: models + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: ollama + namespace: default +spec: + selector: {app: ollama} + ports: + - {port: 11434, targetPort: 11434} diff --git a/openhands-workspace/openhands/workspace/agent_sandbox/testing/smoke_test.py b/openhands-workspace/openhands/workspace/agent_sandbox/testing/smoke_test.py new file mode 100644 index 0000000000..de8934df8e --- /dev/null +++ b/openhands-workspace/openhands/workspace/agent_sandbox/testing/smoke_test.py @@ -0,0 +1,55 @@ +"""Keyless smoke test for AgentSandboxWorkspace against a real cluster (no LLM). + +Validates the integration end to end without needing an LLM: claim a pod from the +warm pool, run a command inside it, pause/resume, and confirm the workspace state +survives the suspend. Works against the secure default template (no network policy +changes needed) because it never leaves the pod to reach an LLM. + +Prerequisites (see TESTING.md): a cluster reachable via your kubeconfig with the +agent-sandbox controller installed and a warm pool of agent-server pods. Then: + + pip install openhands-workspace[agent-sandbox] + python smoke_test.py + +Env (optional): + AGENT_SANDBOX_WARMPOOL warm pool name (default: openhands-pool) + AGENT_SANDBOX_NAMESPACE namespace (default: default) +""" + +import os +import time + +from openhands.workspace import AgentSandboxWorkspace + + +WARMPOOL = os.environ.get("AGENT_SANDBOX_WARMPOOL", "openhands-pool") +NAMESPACE = os.environ.get("AGENT_SANDBOX_NAMESPACE", "default") + + +def main() -> None: + t0 = time.time() + with AgentSandboxWorkspace(warmpool=WARMPOOL, namespace=NAMESPACE) as ws: + print(f"claimed + connected in {time.time() - t0:.1f}s at {ws.host}") + + result = ws.execute_command("echo hello-from-pod && whoami && pwd && uname -m") + assert result.exit_code == 0, result + print("command output:\n" + result.stdout) + + # Write a file, then suspend and resume the sandbox. + ws.execute_command("echo persisted-across-suspend > /workspace/marker.txt") + print("pause() -> Suspended ...") + ws.pause() + time.sleep(3) + print("resume() -> Running ...") + ws.resume() + + # With a persistent volume the file is still there after the resume. + out = ws.execute_command("cat /workspace/marker.txt") + assert out.stdout.strip() == "persisted-across-suspend", out + print(f"state survived suspend/resume: {out.stdout!r}") + + print("SMOKE TEST PASSED") + + +if __name__ == "__main__": + main() diff --git a/openhands-workspace/openhands/workspace/agent_sandbox/workspace.py b/openhands-workspace/openhands/workspace/agent_sandbox/workspace.py new file mode 100644 index 0000000000..9eaadbb6c6 --- /dev/null +++ b/openhands-workspace/openhands/workspace/agent_sandbox/workspace.py @@ -0,0 +1,434 @@ +"""agent-sandbox (Kubernetes) based remote workspace implementation. + +Runs the OpenHands agent server inside a pod managed by the +`kubernetes-sigs/agent-sandbox `_ +controller. The pod is claimed from a ``SandboxWarmPool`` (sub-second when the pool +is pre-warmed), which removes the container cold-start latency that the Docker and +hosted-runtime backends pay per conversation. + +Compared with the other remote backends this one adds: + +* **Warm pools**: ``SandboxWarmPool`` hands out an already-running pod on claim. +* **Native pause/resume**: ``pause()`` and ``resume()`` flip the Sandbox + ``spec.operatingMode`` between ``Suspended`` and ``Running``. With a persistent + volume the workspace state survives the suspend. +* **Strong isolation**: the pod can run under gVisor or Kata via a ``runtimeClass`` + set on the ``SandboxTemplate``, which is an infrastructure choice rather than a + Python one. + +Requires the optional dependency:: + + pip install openhands-workspace[agent-sandbox] +""" + +import os +import signal +import subprocess +import sys +import threading +import time +from typing import Any, Literal +from urllib.request import urlopen + +from pydantic import Field, PrivateAttr + +from openhands.sdk.logger import get_logger +from openhands.sdk.workspace import RemoteWorkspace +from openhands.workspace.docker.workspace import ( + check_port_available, + find_available_tcp_port, +) + + +logger = get_logger(__name__) + + +class AgentSandboxWorkspace(RemoteWorkspace): + """Remote workspace backed by a kubernetes-sigs/agent-sandbox Sandbox pod. + + Claims a pod running the OpenHands agent server from a ``SandboxWarmPool``, + connects to it over HTTP, and manages its lifecycle (pause / resume / delete) + through the agent-sandbox custom resources. + + Two connection modes are supported: + + * ``port_forward`` (default) spawns ``kubectl port-forward`` to the pod, so it + works from a laptop against a local kind or minikube cluster, and against any + cluster your kubeconfig can reach. + * ``direct`` means you supply ``host`` yourself, such as a ``sandbox-router`` or + Gateway URL, or the in-cluster DNS name when OpenHands runs in the cluster. + No port-forward is started. + + Example: + with AgentSandboxWorkspace(warmpool="openhands-pool") as workspace: + result = workspace.execute_command("ls -la") + """ + + # Override parent fields with defaults + working_dir: str = Field( + default="/workspace", + description="Working directory inside the sandbox pod.", + ) + host: str = Field( + default="", + description=( + "Agent server URL. Set automatically in 'port_forward' mode; must be " + "provided by the caller in 'direct' mode." + ), + ) + + # agent-sandbox configuration + warmpool: str = Field( + description="Name of the SandboxWarmPool to claim the pod from.", + ) + namespace: str = Field( + default="default", + description="Kubernetes namespace holding the warm pool and the sandbox pod.", + ) + server_port: int = Field( + default=8000, + description="Port the agent server listens on inside the pod.", + ) + connection: Literal["port_forward", "direct"] = Field( + default="port_forward", + description="How to reach the agent server: local kubectl port-forward, or a " + "caller-provided 'host' URL.", + ) + host_port: int | None = Field( + default=None, + description="Local port for port-forward. If None, an available port is used.", + ) + kube_context: str | None = Field( + default=None, + description="kubectl context to use for port-forward (defaults to current).", + ) + sandbox_ready_timeout: int = Field( + default=180, + description="Seconds to wait for the Sandbox to reach the Ready condition.", + ) + health_check_timeout: float = Field( + default=120.0, + gt=0.0, + description="Seconds to wait for the agent server /health endpoint to pass.", + ) + shutdown_after_seconds: int | None = Field( + default=None, + description="Optional TTL; the controller auto-deletes the claim after this " + "many seconds (a safety net against leaked sandboxes).", + ) + labels: dict[str, str] | None = Field( + default=None, + description="Kubernetes labels to attach to the SandboxClaim object.", + ) + pod_labels: dict[str, str] | None = Field( + default=None, + description="Labels stamped onto the running pod (readable via the Downward " + "API from inside the sandbox).", + ) + pod_annotations: dict[str, str] | None = Field( + default=None, + description="Annotations stamped onto the running pod.", + ) + detach_logs: bool = Field( + default=True, + description="Whether to stream port-forward output in the background.", + ) + + _sandbox: Any = PrivateAttr(default=None) # k8s_agent_sandbox.Sandbox handle + _sb_client: Any = PrivateAttr(default=None) # k8s_agent_sandbox.SandboxClient + _pf_process: subprocess.Popen[str] | None = PrivateAttr(default=None) + _logs_thread: threading.Thread | None = PrivateAttr(default=None) + _stop_logs: threading.Event = PrivateAttr(default_factory=threading.Event) + # The caller's explicit host_port, if any. `host_port` itself tracks the port + # currently in use, so it cannot double as the preference across reconnects. + _preferred_host_port: int | None = PrivateAttr(default=None) + + def model_post_init(self, context: Any) -> None: + """Claim a sandbox pod, connect to the agent server, and initialize.""" + # Validate connection config here (not in a model_validator) to match the + # sibling backends and avoid Pydantic validator/post-init ordering surprises. + if self.connection == "direct" and not self.host: + raise ValueError( + "connection='direct' requires 'host' to be set to the agent-server URL." + ) + + self._preferred_host_port = self.host_port + + try: + import k8s_agent_sandbox # type: ignore[import-not-found] + except ImportError as e: + raise ImportError( + "AgentSandboxWorkspace requires the 'agent-sandbox' extra. Install " + "with: pip install openhands-workspace[agent-sandbox]" + ) from e + + # 1) Claim a pod from the warm pool (blocks until the Sandbox is Ready). + self._sb_client = k8s_agent_sandbox.SandboxClient() + logger.info( + "Claiming a sandbox from warm pool %r in namespace %r...", + self.warmpool, + self.namespace, + ) + self._sandbox = self._sb_client.create_sandbox( + warmpool=self.warmpool, + namespace=self.namespace, + sandbox_ready_timeout=self.sandbox_ready_timeout, + labels=self.labels, + shutdown_after_seconds=self.shutdown_after_seconds, + pod_labels=self.pod_labels, + pod_annotations=self.pod_annotations, + ) + logger.info( + "Sandbox %r is ready (claim %r).", + self._sandbox.sandbox_id, + self._sandbox.claim_name, + ) + + # Everything past this point must clean up the claim on failure: the + # constructor raising means the caller never gets an object to close, so + # the claim and its pod would leak until GC, or forever when + # shutdown_after_seconds is left at its default of None. + try: + # 2) Establish the connection to the agent server and wait for health. + if self.connection == "port_forward": + self._connect_port_forward_with_retry( + preferred_port=self._preferred_host_port + ) + else: + # 'direct': self.host was provided by the caller. + self._wait_for_health(timeout=self.health_check_timeout) + logger.info("agent-sandbox workspace is ready at %s", self.host) + + # 3) Initialize the parent RemoteWorkspace against the agent server URL. + super().model_post_init(context) + except Exception: + self.cleanup() + raise + + def _connect_port_forward_with_retry( + self, attempts: int = 8, *, preferred_port: int | None = None + ) -> None: + """Start a port-forward and wait for health, retrying transient failures. + + Right after a resume the pod's network namespace can still be churning, so + kubectl port-forward may drop with "network namespace ... is closed". A + dead forward exits immediately, so the retries back off progressively -- + otherwise the whole budget is spent in a few seconds, well before a + freshly resumed pod is forwardable. + + ``preferred_port`` is only honored on the first attempt; pass None (the + default, used on resume) to always take a freshly allocated port, since + the previous session's port may still be in TIME_WAIT. + """ + last_error: Exception | None = None + for i in range(attempts): + # Honor a caller-provided port on the first try; auto-pick on retries. + self.host_port = preferred_port if i == 0 else None + try: + self._start_port_forward() + self._wait_for_health(timeout=self.health_check_timeout) + return + except Exception as e: + last_error = e + logger.warning( + "Agent server not reachable (attempt %d/%d): %s", + i + 1, + attempts, + e, + ) + self._stop_port_forward() + time.sleep(min(2 * (i + 1), 10)) + raise RuntimeError( + f"Could not reach agent server after {attempts} attempts: {last_error}" + ) + + def _resolve_pod_name(self) -> str: + """Read the current pod name from the live Sandbox object. + + The pod name can change across a suspend/resume cycle, so this always + re-reads it rather than using ``Sandbox.get_pod_name()``, which caches the + first value for the lifetime of the handle. Uses only public client API; + if ``k8s_agent_sandbox`` grows a public refresh (e.g. + ``get_pod_name(refresh=True)``), this can defer to it. + """ + from k8s_agent_sandbox.constants import ( # type: ignore[import-not-found] + POD_NAME_ANNOTATION, + ) + + sandbox_object = ( + self._sb_client.k8s_helper.get_sandbox( + self._sandbox.sandbox_id, self.namespace + ) + or {} + ) + annotations = (sandbox_object.get("metadata") or {}).get("annotations") or {} + return annotations.get(POD_NAME_ANNOTATION) or self._sandbox.sandbox_id + + def _start_port_forward(self) -> None: + """Start (or restart) kubectl port-forward to the sandbox pod.""" + if self.host_port is None: + self.host_port = find_available_tcp_port() + elif not check_port_available(self.host_port): + raise RuntimeError(f"Port {self.host_port} is not available") + + pod = self._resolve_pod_name() + + cmd = ["kubectl", "port-forward"] + if self.kube_context: + cmd += ["--context", self.kube_context] + cmd += [ + "-n", + self.namespace, + f"pod/{pod}", + f"{self.host_port}:{self.server_port}", + ] + logger.info("Starting port-forward: %s", " ".join(cmd)) + self._stop_logs = threading.Event() + self._pf_process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + start_new_session=True, + ) + object.__setattr__(self, "host", f"http://127.0.0.1:{self.host_port}") + + if self.detach_logs: + self._logs_thread = threading.Thread(target=self._stream_logs, daemon=True) + self._logs_thread.start() + + def _stream_logs(self) -> None: + """Stream port-forward output to stdout in the background.""" + if not self._pf_process or not self._pf_process.stdout: + return + try: + for line in iter(self._pf_process.stdout.readline, ""): + if self._stop_logs.is_set(): + break + if line: + sys.stdout.write(f"[PORT-FORWARD] {line}") + sys.stdout.flush() + except Exception as e: + sys.stderr.write(f"Error streaming port-forward logs: {e}\n") + finally: + self._stop_logs.set() + + def _wait_for_health(self, *, timeout: float) -> None: + """Wait for the agent server /health endpoint to return success.""" + start = time.time() + health_url = f"{self.host.rstrip('/')}/health" + while time.time() - start < timeout: + try: + with urlopen(health_url, timeout=1.0) as resp: + if 200 <= getattr(resp, "status", 200) < 300: + return + except Exception: + pass + if self._pf_process and self._pf_process.poll() is not None: + raise RuntimeError( + "kubectl port-forward exited unexpectedly with code " + f"{self._pf_process.returncode}" + ) + time.sleep(1) + raise RuntimeError("agent server failed to become healthy in time") + + def _patch_operating_mode(self, mode: str) -> None: + """Patch the Sandbox spec.operatingMode ('Running' or 'Suspended'). + + TODO: agent-sandbox #1160 (claim-level idle lifecycle) and #1296 + (traffic-triggered resume) will make this a claim-level concern; once they + land, pause/resume should move to the claim API instead of patching the + Sandbox directly. + """ + from k8s_agent_sandbox.constants import ( # type: ignore[import-not-found] + SANDBOX_API_GROUP, + SANDBOX_API_VERSION, + SANDBOX_PLURAL_NAME, + ) + + self._sb_client.k8s_helper.custom_objects_api.patch_namespaced_custom_object( + group=SANDBOX_API_GROUP, + version=SANDBOX_API_VERSION, + namespace=self.namespace, + plural=SANDBOX_PLURAL_NAME, + name=self._sandbox.sandbox_id, + body={"spec": {"operatingMode": mode}}, + ) + + def pause(self) -> None: + """Suspend the sandbox to conserve resources. + + Sets the Sandbox ``operatingMode`` to ``Suspended``; the controller + terminates the pod while keeping any persistent volume. Resume with + ``resume()``. + """ + if self._sandbox is None: + raise RuntimeError("Cannot pause: no active sandbox") + logger.info("Suspending sandbox %r...", self._sandbox.sandbox_id) + self._stop_port_forward() + self._patch_operating_mode("Suspended") + + def resume(self) -> None: + """Resume a suspended sandbox and reconnect to the agent server.""" + if self._sandbox is None: + raise RuntimeError("Cannot resume: no active sandbox") + logger.info("Resuming sandbox %r...", self._sandbox.sandbox_id) + self._patch_operating_mode("Running") + self._sb_client.k8s_helper.wait_for_sandbox_ready( + self._sandbox.sandbox_id, self.namespace, self.sandbox_ready_timeout + ) + if self.connection == "port_forward": + # Reconnect on a fresh local port (the old one may be in TIME_WAIT) and + # rebuild the HTTP client so it targets the new host URL. + self._connect_port_forward_with_retry(preferred_port=None) + self.reset_client() + else: + self._wait_for_health(timeout=self.health_check_timeout) + logger.info("Sandbox %r resumed at %s", self._sandbox.sandbox_id, self.host) + + def _stop_port_forward(self) -> None: + """Stop the kubectl port-forward subprocess if running.""" + if self._pf_process is None: + return + self._stop_logs.set() + if self._logs_thread and self._logs_thread.is_alive(): + self._logs_thread.join(timeout=2) + try: + os.killpg(os.getpgid(self._pf_process.pid), signal.SIGTERM) + self._pf_process.wait(timeout=5) + except Exception: + try: + os.killpg(os.getpgid(self._pf_process.pid), signal.SIGKILL) + self._pf_process.wait(timeout=2) + except Exception: + pass + self._pf_process = None + + def __enter__(self) -> "AgentSandboxWorkspace": + """Context manager entry - returns the workspace itself.""" + return self + + def __exit__(self, exc_type, exc_val, exc_tb) -> None: # type: ignore[no-untyped-def] + """Context manager exit - tears down the sandbox.""" + self.cleanup() + + def __del__(self) -> None: + """Best-effort cleanup when the workspace is garbage collected.""" + try: + if getattr(self, "__pydantic_private__", None) is not None: + self.cleanup() + except Exception: + # Never raise from __del__ (e.g. during interpreter shutdown). + pass + + def cleanup(self) -> None: + """Stop the port-forward and delete the SandboxClaim (idempotent).""" + self._stop_port_forward() + sandbox = getattr(self, "_sandbox", None) + if sandbox is not None: + try: + logger.info("Terminating sandbox %r...", sandbox.sandbox_id) + sandbox.terminate() + except Exception as e: + logger.warning("Error terminating sandbox: %s", e) + self._sandbox = None diff --git a/openhands-workspace/pyproject.toml b/openhands-workspace/pyproject.toml index 375ab4ae31..4cd45be557 100644 --- a/openhands-workspace/pyproject.toml +++ b/openhands-workspace/pyproject.toml @@ -10,6 +10,12 @@ dependencies = [ "pydantic>=2.11.7", ] +[project.optional-dependencies] +# AgentSandboxWorkspace: run the agent server in a kubernetes-sigs/agent-sandbox pod. +agent-sandbox = [ + "k8s-agent-sandbox>=0.5.0", +] + [project.urls] Source = "https://github.com/OpenHands/software-agent-sdk" Homepage = "https://github.com/OpenHands/software-agent-sdk" diff --git a/tests/workspace/test_agent_sandbox_workspace.py b/tests/workspace/test_agent_sandbox_workspace.py new file mode 100644 index 0000000000..65b5f4e607 --- /dev/null +++ b/tests/workspace/test_agent_sandbox_workspace.py @@ -0,0 +1,161 @@ +"""Tests for AgentSandboxWorkspace (no cluster required).""" + +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest + + +def test_agent_sandbox_workspace_import(): + """AgentSandboxWorkspace can be imported from the package.""" + from openhands.workspace import AgentSandboxWorkspace + + assert AgentSandboxWorkspace is not None + + +def test_agent_sandbox_workspace_inheritance(): + """AgentSandboxWorkspace is a RemoteWorkspace.""" + from openhands.sdk.workspace import RemoteWorkspace + from openhands.workspace import AgentSandboxWorkspace + + assert issubclass(AgentSandboxWorkspace, RemoteWorkspace) + + +def test_agent_sandbox_workspace_fields(): + """The key agent-sandbox knobs are exposed as fields.""" + from openhands.workspace import AgentSandboxWorkspace + + for field in ("warmpool", "namespace", "server_port", "connection", "host_port"): + assert field in AgentSandboxWorkspace.model_fields + + +def test_direct_connection_requires_host(): + """connection='direct' without a host is rejected before any k8s call.""" + from openhands.workspace import AgentSandboxWorkspace + + with pytest.raises(ValueError, match="requires 'host'"): + AgentSandboxWorkspace(warmpool="pool", connection="direct") + + +@pytest.fixture +def k8s_mocks(): + """Install a fake k8s_agent_sandbox module for the duration of the test.""" + fake_handle = MagicMock() + fake_handle.sandbox_id = "sandbox-abc" + fake_handle.claim_name = "sandbox-claim-abc" + fake_handle.get_pod_name.return_value = "sandbox-abc-pod" + + fake_client = MagicMock() + fake_client.create_sandbox.return_value = fake_handle + + fake_module = MagicMock() + fake_module.SandboxClient.return_value = fake_client + + # `_patch_operating_mode` does `from k8s_agent_sandbox.constants import ...`. + fake_constants = MagicMock( + SANDBOX_API_GROUP="agents.x-k8s.io", + SANDBOX_API_VERSION="v1beta1", + SANDBOX_PLURAL_NAME="sandboxes", + ) + + with patch.dict( + "sys.modules", + { + "k8s_agent_sandbox": fake_module, + "k8s_agent_sandbox.constants": fake_constants, + }, + ): + yield SimpleNamespace(client=fake_client, handle=fake_handle) + + +def _make_ws(**overrides): + """Construct a workspace with connection + parent init mocked out.""" + from openhands.workspace import AgentSandboxWorkspace + + kwargs = {"warmpool": "openhands-pool", "detach_logs": False} + kwargs.update(overrides) + with ( + patch.object(AgentSandboxWorkspace, "_start_port_forward"), + patch.object(AgentSandboxWorkspace, "_wait_for_health"), + # RemoteWorkspace.model_post_init would open a real HTTP client; skip it. + patch( + "openhands.sdk.workspace.remote.base.RemoteWorkspace.model_post_init", + return_value=None, + ), + ): + return AgentSandboxWorkspace(**kwargs) + + +def test_claims_from_warmpool_on_init(k8s_mocks): + """model_post_init claims a sandbox from the configured warm pool.""" + _make_ws(namespace="ns1", shutdown_after_seconds=600) + + k8s_mocks.client.create_sandbox.assert_called_once() + call = k8s_mocks.client.create_sandbox.call_args + assert call.kwargs["warmpool"] == "openhands-pool" + assert call.kwargs["namespace"] == "ns1" + assert call.kwargs["shutdown_after_seconds"] == 600 + + +def test_pause_and_resume_flip_operating_mode(k8s_mocks): + """pause()/resume() patch the Sandbox operatingMode.""" + ws = _make_ws() + patch_call = ( + k8s_mocks.client.k8s_helper.custom_objects_api.patch_namespaced_custom_object + ) + + with patch.object(type(ws), "_stop_port_forward"): + ws.pause() + assert patch_call.call_args.kwargs["body"] == { + "spec": {"operatingMode": "Suspended"} + } + + with ( + patch.object(type(ws), "_start_port_forward"), + patch.object(type(ws), "_wait_for_health"), + ): + ws.resume() + assert patch_call.call_args.kwargs["body"] == {"spec": {"operatingMode": "Running"}} + + +def test_failed_connect_terminates_claim(k8s_mocks): + """A failure after the claim is created must not leak the sandbox.""" + from openhands.workspace import AgentSandboxWorkspace + + with ( + patch.object( + AgentSandboxWorkspace, + "_connect_port_forward_with_retry", + side_effect=RuntimeError("boom"), + ), + patch.object(AgentSandboxWorkspace, "_stop_port_forward"), + pytest.raises(RuntimeError, match="boom"), + ): + AgentSandboxWorkspace(warmpool="openhands-pool", detach_logs=False) + + k8s_mocks.handle.terminate.assert_called_once() + + +def test_resume_reconnects_on_a_fresh_port(k8s_mocks): + """resume() must not reuse the previous session's local port.""" + ws = _make_ws(host_port=45000) + + with ( + patch.object(type(ws), "_stop_port_forward"), + patch.object(type(ws), "_connect_port_forward_with_retry") as connect, + patch.object(type(ws), "reset_client"), + ): + ws.resume() + + assert connect.call_args.kwargs["preferred_port"] is None + + +def test_cleanup_terminates_sandbox(k8s_mocks): + """cleanup() deletes the claim via the handle and is idempotent.""" + ws = _make_ws() + + with patch.object(type(ws), "_stop_port_forward"): + ws.cleanup() + ws.cleanup() # second call is a no-op + + k8s_mocks.handle.terminate.assert_called_once() diff --git a/uv.lock b/uv.lock index 749d8a8187..7591040f89 100644 --- a/uv.lock +++ b/uv.lock @@ -970,6 +970,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/66/dd/f95350e853a4468ec37478414fc04ae2d61dad7a947b3015c3dcc51a09b9/docutils-0.22.2-py3-none-any.whl", hash = "sha256:b0e98d679283fc3bb0ead8a5da7f501baa632654e7056e9c5846842213d674d8", size = 632667, upload-time = "2025-09-20T17:55:43.052Z" }, ] +[[package]] +name = "durationpy" +version = "0.10" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9d/a4/e44218c2b394e31a6dd0d6b095c4e1f32d0be54c2a4b250032d717647bab/durationpy-0.10.tar.gz", hash = "sha256:1fa6893409a6e739c9c72334fc65cca1f355dbdd93405d30f726deb5bde42fba", size = 3335, upload-time = "2025-05-17T13:52:37.26Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b0/0d/9feae160378a3553fa9a339b0e9c1a048e147a4127210e286ef18b730f03/durationpy-0.10-py3-none-any.whl", hash = "sha256:3b41e1b601234296b4fb368338fdcd3e13e0b4fb5b67345948f4f2bf9868b286", size = 3922, upload-time = "2025-05-17T13:52:36.463Z" }, +] + [[package]] name = "email-validator" version = "2.3.0" @@ -2052,6 +2061,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, ] +[[package]] +name = "k8s-agent-sandbox" +version = "0.5.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "kubernetes" }, + { name = "prometheus-client" }, + { name = "pydantic" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4a/85/d4bbd50df3a9ef7bfe43f6c3b1e17cb1cd933594b3c8e343b86c6c385c97/k8s_agent_sandbox-0.5.1.tar.gz", hash = "sha256:fb9873c114d206e88d6375c7d3c459785a80322a0937c13919682bef92e1bf6c", size = 125524, upload-time = "2026-07-09T23:35:15.181Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/72/61419c29f5e7525ecb6a87a0c3bcdee586f3e0ea11da24b2fb34681f5066/k8s_agent_sandbox-0.5.1-py3-none-any.whl", hash = "sha256:a9916d8f8adc95dded087fb22048ef1891074e02380f471e72631c99fc1a13ad", size = 77258, upload-time = "2026-07-09T23:35:13.811Z" }, +] + [[package]] name = "keyring" version = "25.7.0" @@ -2069,6 +2093,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/db/e655086b7f3a705df045bf0933bdd9c2f79bb3c97bfef1384598bb79a217/keyring-25.7.0-py3-none-any.whl", hash = "sha256:be4a0b195f149690c166e850609a477c532ddbfbaed96a404d4e43f8d5e2689f", size = 39160, upload-time = "2025-11-16T16:26:08.402Z" }, ] +[[package]] +name = "kubernetes" +version = "36.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, + { name = "certifi" }, + { name = "durationpy" }, + { name = "python-dateutil" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "requests-oauthlib" }, + { name = "six" }, + { name = "urllib3" }, + { name = "websocket-client" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bf/59/dc635e4e9afb3884bc5c57f14fe23783e4c04601aa20b835ac75c41d1625/kubernetes-36.0.0.tar.gz", hash = "sha256:027b606bb8032e6c6464a53236bdd9bd9a94c237e1063bc45a303c25b304ced9", size = 2346728, upload-time = "2026-05-20T20:44:24.28Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cd/d2/6f99ca9c7eb961dfdd45b9643101399a8ee20922c662c362c91e9cc7e832/kubernetes-36.0.0-py2.py3-none-any.whl", hash = "sha256:a766433357ec9f90db7565cccf52e28e7fca40b0ef366c80a6022adbc0ac0425", size = 4660469, upload-time = "2026-05-20T20:44:20.893Z" }, +] + [[package]] name = "libtmux" version = "0.53.0" @@ -2860,12 +2905,19 @@ dependencies = [ { name = "pydantic" }, ] +[package.optional-dependencies] +agent-sandbox = [ + { name = "k8s-agent-sandbox" }, +] + [package.metadata] requires-dist = [ + { name = "k8s-agent-sandbox", marker = "extra == 'agent-sandbox'", specifier = ">=0.5.0" }, { name = "openhands-agent-server", editable = "openhands-agent-server" }, { name = "openhands-sdk", editable = "openhands-sdk" }, { name = "pydantic", specifier = ">=2.11.7" }, ] +provides-extras = ["agent-sandbox"] [[package]] name = "opentelemetry-api" @@ -3269,6 +3321,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5b/a5/987a405322d78a73b66e39e4a90e4ef156fd7141bf71df987e50717c321b/pre_commit-4.3.0-py2.py3-none-any.whl", hash = "sha256:2b0747ad7e6e967169136edffee14c16e148a778a54e4f967921aa1ebf2308d8", size = 220965, upload-time = "2025-08-09T18:56:13.192Z" }, ] +[[package]] +name = "prometheus-client" +version = "0.25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1b/fb/d9aa83ffe43ce1f19e557c0971d04b90561b0cfd50762aafb01968285553/prometheus_client-0.25.0.tar.gz", hash = "sha256:5e373b75c31afb3c86f1a52fa1ad470c9aace18082d39ec0d2f918d11cc9ba28", size = 86035, upload-time = "2026-04-09T19:53:42.359Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8d/9b/d4b1e644385499c8346fa9b622a3f030dce14cd6ef8a1871c221a17a67e7/prometheus_client-0.25.0-py3-none-any.whl", hash = "sha256:d5aec89e349a6ec230805d0df882f3807f74fd6c1a2fa86864e3c2279059fed1", size = 64154, upload-time = "2026-04-09T19:53:41.324Z" }, +] + [[package]] name = "prompt-toolkit" version = "3.0.52" @@ -7472,6 +7533,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/af/b5/123f13c975e9f27ab9c0770f514345bd406d0e8d3b7a0723af9d43f710af/wcwidth-0.2.14-py2.py3-none-any.whl", hash = "sha256:a7bb560c8aee30f9957e5f9895805edd20602f2d7f720186dfd906e82b4982e1", size = 37286, upload-time = "2025-09-22T16:29:51.641Z" }, ] +[[package]] +name = "websocket-client" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2c/41/aa4bf9664e4cda14c3b39865b12251e8e7d239f4cd0e3cc1b6c2ccde25c1/websocket_client-1.9.0.tar.gz", hash = "sha256:9e813624b6eb619999a97dc7958469217c3176312b3a16a4bd1bc7e08a46ec98", size = 70576, upload-time = "2025-10-07T21:16:36.495Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/34/db/b10e48aa8fff7407e67470363eac595018441cf32d5e1001567a7aeba5d2/websocket_client-1.9.0-py3-none-any.whl", hash = "sha256:af248a825037ef591efbf6ed20cc5faa03d3b47b9e5a2230a529eeee1c1fc3ef", size = 82616, upload-time = "2025-10-07T21:16:34.951Z" }, +] + [[package]] name = "websockets" version = "15.0.1"