From 777d18946c3b3aea73c0fe55b6e247542dee1114 Mon Sep 17 00:00:00 2001 From: "druks-operator[bot]" <284423593+druks-operator[bot]@users.noreply.github.com> Date: Tue, 18 Aug 2026 06:26:30 +0000 Subject: [PATCH 1/2] Classify transient sandbox provisioning failures for in-run retry Route transient drukbox control-plane failures through the existing bounded in-run retry so slow provider windows recover instead of dead-ending a run. - Add HarnessSandboxProvisioningError (code="sandbox_provisioning"), inheriting HarnessSandboxError's transient retry schedule. - Translate SandboxProvisioningError/SandboxUnavailableError from create_host and SandboxUnavailableError from get_host at the shared client boundary, preserving the SDK exception as the cause; fatal SDK errors pass through. - Reclassify acquire-time SSH/helper reachability failures as provisioning after the existing rollback; cancellation and local/programming errors propagate unchanged. Co-Authored-By: Claude Opus 4.8 --- backend/druks/harnesses/exceptions.py | 13 ++ backend/druks/sandbox/client.py | 65 +++++++-- backend/tests/test_agents.py | 180 ++++++++++++++++++++++++ backend/tests/test_run_state.py | 19 +++ backend/tests/test_sandbox_lifecycle.py | 133 ++++++++++++++++- 5 files changed, 399 insertions(+), 11 deletions(-) diff --git a/backend/druks/harnesses/exceptions.py b/backend/druks/harnesses/exceptions.py index ed5b18dc..232e4da7 100644 --- a/backend/druks/harnesses/exceptions.py +++ b/backend/druks/harnesses/exceptions.py @@ -62,6 +62,19 @@ class HarnessSandboxError(HarnessError): retry_delays = (60, 300) +class HarnessSandboxProvisioningError(HarnessSandboxError): + """A control-plane provisioning or transport failure while creating or + re-attaching a host, or reaching a freshly created one. + + Transient on the same schedule as :class:`HarnessSandboxError` (it + inherits ``retry``/``retry_delays``), but carries its own ``code`` so an + exhausted retry records ``sandbox_provisioning`` rather than the generic + ``sandbox`` classification — the failure taxonomy can then name the most + transient failure class in the system instead of leaving it blank.""" + + code = "sandbox_provisioning" + + class OAuthTokenError(Exception): """No usable subscription credential is available. diff --git a/backend/druks/sandbox/client.py b/backend/druks/sandbox/client.py index 01de2728..b4090d65 100644 --- a/backend/druks/sandbox/client.py +++ b/backend/druks/sandbox/client.py @@ -5,18 +5,21 @@ from datetime import UTC, datetime, timedelta from pathlib import Path +import asyncssh from drukbox_sdk import SandboxAPI, SandboxHost from drukbox_sdk.exceptions import ( SandboxAPIError, SandboxNotFoundError, + SandboxProvisioningError, SandboxUnavailableError, ) from uuid_utils import uuid7 +from druks.harnesses.exceptions import HarnessSandboxProvisioningError from druks.settings import load_settings from .constants import SANDBOX_HOST_LEASE_SECONDS -from .exceptions import HostGone, SandboxUnreachable +from .exceptions import HostGone, SandboxError, SandboxUnreachable from .host import Sandbox from .layout import get_helper_script_path, get_remote_home @@ -24,6 +27,20 @@ _DRUKS_SANDBOX_LOCAL_SCRIPT = Path(__file__).parent / "druks-sandbox.sh" +# A freshly created host that we can't reach or set up over SSH is a +# provisioning failure in the same sense as a create-time control-plane one: +# the VM was never usable. These are the shapes such reachability/setup +# failures take — SSH connect/auth/channel errors (asyncssh), socket-level +# failures and connect timeouts (OSError/TimeoutError), and the sandbox +# layer's own SandboxUnreachable/SandboxError. Cancellation and unrelated +# local/programming bugs are deliberately excluded so they propagate untouched. +_ACQUIRE_SETUP_REACHABILITY_ERRORS = ( + SandboxError, + asyncssh.Error, + OSError, + TimeoutError, +) + class Client: """Ambient client for the drukbox control plane. @@ -80,13 +97,25 @@ async def acquire( # Fixed lease: drukbox reaps the host when this lapses, so a run whose # worker dies frees its VM without a druks-side reconciler. expires_at = datetime.now(UTC) + timedelta(seconds=SANDBOX_HOST_LEASE_SECONDS) - record = await api.create_host( - expires_at=expires_at, - env=sandbox_env, - idempotency_key=key, - image=image or None, - provider=provider, - ) + try: + record = await api.create_host( + expires_at=expires_at, + env=sandbox_env, + idempotency_key=key, + image=image or None, + provider=provider, + ) + except (SandboxProvisioningError, SandboxUnavailableError) as exc: + # Transient control-plane failures — a 502 the service raises + # when the provider/Tailscale/keyscan step fails, or a + # transport/503 SandboxUnavailableError. Classify them into the + # in-run retry path so a slow provider window recovers instead + # of dead-ending the run. Fatal SDK errors (auth, validation, + # conflict, not-found, generic response) are subclasses of the + # untouched SandboxAPIError base and fall through unretried. + raise HarnessSandboxProvisioningError( + f"sandbox host provisioning failed: {exc}" + ) from exc logger.info("sandbox host created id=%s", record.id) key_path = settings.sandbox_keys_dir / record.id if record.private_key: @@ -96,11 +125,21 @@ async def acquire( sandbox = Sandbox(record=record) try: await _upload_helper_script(sandbox) - except BaseException: + except BaseException as error: # Caller never sees this host; release rather than orphan. await sandbox.aclose() await self._best_effort_delete(api, record.id) key_path.unlink(missing_ok=True) + # A fresh VM that never became reachable/usable during SSH + + # helper setup is a provisioning failure just like a create-time + # one, so classify it into the same transient-retry path — the + # raw error would otherwise bypass agent retry and record an + # empty failure code. Cancellation and unrelated + # local/programming errors keep their own type and propagate. + if isinstance(error, _ACQUIRE_SETUP_REACHABILITY_ERRORS): + raise HarnessSandboxProvisioningError( + f"sandbox host {record.id} unreachable during setup: {error}" + ) from error raise try: yield sandbox @@ -139,6 +178,14 @@ async def attach(self, *, host_id: str) -> AsyncIterator[Sandbox]: raise HostGone( f"sandbox host {host_id} no longer exists", ) from exc + except SandboxUnavailableError as exc: + # Transport/503 while looking up an existing host — transient, + # so classify it the same as a create-time control-plane + # failure and let the in-run retry re-attach once the service + # recovers. + raise HarnessSandboxProvisioningError( + f"sandbox host {host_id} lookup failed: {exc}" + ) from exc sandbox = Sandbox(record=record) try: yield sandbox diff --git a/backend/tests/test_agents.py b/backend/tests/test_agents.py index bcb637b1..502d02ce 100644 --- a/backend/tests/test_agents.py +++ b/backend/tests/test_agents.py @@ -580,6 +580,186 @@ async def run_agent(_self, **_context): durable_sleep.assert_not_awaited() +def _flaky_ephemeral(sandbox, keys, *, failures): + """An ephemeral() that raises each error in ``failures`` in turn (recording + the idempotency key each attempt presents) then yields ``sandbox``. Used to + simulate a transient provisioning failure at acquire time that recovers.""" + attempts = 0 + + @asynccontextmanager + async def _fake(self, *, idempotency_key=None, **_kwargs): + nonlocal attempts + keys.append(idempotency_key) + index = attempts + attempts += 1 + if index < len(failures): + raise failures[index] + yield sandbox + + return _fake + + +async def test_provisioning_failure_recovers_through_durable_retry( + druks_db, tmp_path, monkeypatch, current_run, _inline_agent_steps +): + """A transient provisioning failure at acquire time is retried by the + body-level durable path with backoff; a later attempt succeeds, and every + attempt for the one logical acquire presents the same ephemeral key.""" + from druks.harnesses.exceptions import HarnessSandboxProvisioningError + + sandbox = _patch_runtime(monkeypatch, tmp_path, {"ok": True}) + keys: list[str | None] = [] + monkeypatch.setattr( + "druks.sandbox.client.Client.ephemeral", + _flaky_ephemeral( + sandbox, keys, failures=[HarnessSandboxProvisioningError("exe.dev create timed out")] + ), + ) + monkeypatch.setattr(agents.random, "uniform", lambda _low, _high: 1.0) + current_run._reap_run = AsyncMock() + checkpoints, sleep = _inline_agent_steps + + result = await DUMMY_AGENT() + + assert result == DummyOutput(ok=True) + # First delay off the schedule HarnessSandboxProvisioningError inherits (60, 300). + assert [awaited.args[0] for awaited in sleep.await_args_list] == [60.0] + # Each attempt re-enters _run with the same workflow/agent identity → same key. + assert keys == ["wf-9:dummy", "wf-9:dummy"] + # A 60s wait is under the reap-before threshold, so the (never-acquired) VM + # isn't reaped between attempts. + current_run._reap_run.assert_not_awaited() + # Both attempts are separate durable steps under the agent's step name. + assert [options["name"] for options in checkpoints] == ["test.agent.dummy"] * 2 + + +async def test_provisioning_failure_recovers_in_step_retry( + druks_db, tmp_path, monkeypatch, current_run +): + """An agent invoked inside an enclosing @step retries the same transient + provisioning failure in memory (plain asyncio.sleep, no durable sleep) and + recovers, holding the ephemeral key stable across attempts.""" + from druks.harnesses.exceptions import HarnessSandboxProvisioningError + from druks.workflows import _in_step + + sandbox = _patch_runtime(monkeypatch, tmp_path, {"ok": True}) + keys: list[str | None] = [] + monkeypatch.setattr( + "druks.sandbox.client.Client.ephemeral", + _flaky_ephemeral( + sandbox, keys, failures=[HarnessSandboxProvisioningError("exe.dev create timed out")] + ), + ) + monkeypatch.setattr(agents.random, "uniform", lambda _low, _high: 1.0) + memory_sleep = AsyncMock() + durable_sleep = AsyncMock() + monkeypatch.setattr(agents.asyncio, "sleep", memory_sleep) + monkeypatch.setattr(agents.DBOS, "sleep_async", durable_sleep) + + token = _in_step.set(True) + try: + result = await DUMMY_AGENT() + finally: + _in_step.reset(token) + + assert result == DummyOutput(ok=True) + memory_sleep.assert_awaited_once_with(60.0) + durable_sleep.assert_not_awaited() + assert keys == ["wf-9:dummy", "wf-9:dummy"] + + +async def test_reused_host_retry_presents_a_stable_idempotency_key(monkeypatch, current_run): + """A warm workflow that reuses its sandbox re-provisions the host on the + same logical acquire with an unchanged idempotency key, so an ambiguous + create timeout resolves to the existing host instead of leaking a duplicate.""" + from druks.harnesses.exceptions import HarnessSandboxProvisioningError + + current_run.steps_reuse_sandbox = True + keys: list[str | None] = [] + host = MagicMock() + host.id = "warm-host" + host.expires_at = None + attempts = 0 + + async def fake_provision(self, *, idempotency_key=None, **_kwargs): + nonlocal attempts + keys.append(idempotency_key) + index = attempts + attempts += 1 + if index == 0: + raise HarnessSandboxProvisioningError("provider slow") + return host + + monkeypatch.setattr("druks.sandbox.client.Client.provision", fake_provision) + + with pytest.raises(HarnessSandboxProvisioningError): + await current_run._ensure_host() + host_id = await current_run._ensure_host() + + assert host_id == "warm-host" + assert keys == ["wf-9:sandbox", "wf-9:sandbox"] + + +async def test_provisioning_failure_exhausts_retries_with_classified_code( + druks_db, tmp_path, monkeypatch, current_run, _inline_agent_steps +): + """When transient provisioning retries exhaust, the propagated error still + carries ``code=sandbox_provisioning`` — the classification survives to the + run's failure record instead of falling back to an empty code.""" + from druks.harnesses.exceptions import HarnessSandboxProvisioningError + + _patch_runtime(monkeypatch, tmp_path, {"ok": True}) + failures = [HarnessSandboxProvisioningError(f"provider down {i}") for i in range(3)] + monkeypatch.setattr( + "druks.sandbox.client.Client.ephemeral", _flaky_ephemeral(None, [], failures=failures) + ) + monkeypatch.setattr(agents.random, "uniform", lambda _low, _high: 1.0) + current_run._reap_run = AsyncMock() + _, sleep = _inline_agent_steps + + with pytest.raises(HarnessSandboxProvisioningError) as excinfo: + await DUMMY_AGENT() + + assert excinfo.value is failures[-1] + assert excinfo.value.code == "sandbox_provisioning" + # One sleep per inherited delay; past the schedule the failure stands. + assert [awaited.args[0] for awaited in sleep.await_args_list] == [60.0, 300.0] + # The 300s wait is over the reap-before threshold, so the run is reaped once. + assert current_run._reap_run.await_count == 1 + + +async def test_fatal_sdk_error_does_not_trigger_agent_retry( + druks_db, tmp_path, monkeypatch, current_run, _inline_agent_steps +): + """A non-transient SDK failure surfacing from acquire is not a HarnessError, + so it propagates on the first attempt with no retry sleep — auth/validation + failures can't recover without changed inputs.""" + from drukbox_sdk.exceptions import SandboxValidationError + + _patch_runtime(monkeypatch, tmp_path, {"ok": True}) + fatal = SandboxValidationError("bad image") + attempts = 0 + + @asynccontextmanager + async def fatal_ephemeral(self, *, idempotency_key=None, **_kwargs): + nonlocal attempts + attempts += 1 + raise fatal + yield # pragma: no cover + + monkeypatch.setattr("druks.sandbox.client.Client.ephemeral", fatal_ephemeral) + current_run._reap_run = AsyncMock() + _, sleep = _inline_agent_steps + + with pytest.raises(SandboxValidationError) as excinfo: + await DUMMY_AGENT() + + assert excinfo.value is fatal + assert attempts == 1 + sleep.assert_not_awaited() + current_run._reap_run.assert_not_awaited() + + async def test_recovery_supersedes_the_orphaned_running_call(druks_db): """A worker crash leaves a RUNNING row; the recovered step re-runs with a fresh id and abandons the orphan, so the timeline shows one live step.""" diff --git a/backend/tests/test_run_state.py b/backend/tests/test_run_state.py index 648d421d..7206870e 100644 --- a/backend/tests/test_run_state.py +++ b/backend/tests/test_run_state.py @@ -289,6 +289,25 @@ async def body() -> None: assert Run.get(run.id).failure_code == "overloaded" +@pytest.mark.asyncio +async def test_an_exhausted_provisioning_failure_stamps_its_code(druks_db, _inline_steps): + # An exhausted transient provisioning failure records the classified + # ``sandbox_provisioning`` code rather than the empty string a raw drukbox + # SDK exception used to leave behind — so the dashboard/taxonomy can name it. + from druks.harnesses.exceptions import HarnessSandboxProvisioningError + + item, run = _item_and_run(druks_db, "running") + + async def body() -> None: + raise HarnessSandboxProvisioningError("exe.dev VM creation timed out") + + with pytest.raises(HarnessSandboxProvisioningError): + await _execute_run(run.id, run.kind, {"type": "work_item", "id": item.id}, None, body) + + ambient_session().expire_all() + assert Run.get(run.id).failure_code == "sandbox_provisioning" + + @pytest.mark.asyncio async def test_a_foreign_code_never_becomes_the_failure_code(druks_db, _inline_steps): """``code`` is a common attribute name — asyncssh's is an int — so only diff --git a/backend/tests/test_sandbox_lifecycle.py b/backend/tests/test_sandbox_lifecycle.py index d5e81c03..fffb2516 100644 --- a/backend/tests/test_sandbox_lifecycle.py +++ b/backend/tests/test_sandbox_lifecycle.py @@ -5,13 +5,19 @@ import pytest from drukbox_sdk import SandboxHost as SandboxHostRecord -from drukbox_sdk.exceptions import SandboxNotFoundError +from drukbox_sdk.exceptions import ( + SandboxAuthError, + SandboxNotFoundError, + SandboxProvisioningError, + SandboxUnavailableError, +) +from druks.harnesses.exceptions import HarnessSandboxProvisioningError, Retry from druks.sandbox import credentials as creds_module from druks.sandbox import layout, repo from druks.sandbox.client import sandbox_client from druks.sandbox.constants import SANDBOX_HOST_LEASE_SECONDS from druks.sandbox.datastructures import Credentials -from druks.sandbox.exceptions import ExecFailed, HostGone +from druks.sandbox.exceptions import ExecFailed, HostGone, SandboxUnreachable from druks.sandbox.host import ExecResult @@ -80,6 +86,7 @@ class _FakeAPI: deleted_ids: list[str] = field(default_factory=list) get_host_responses: list[SandboxHostRecord] = field(default_factory=list) create_record: SandboxHostRecord | None = None + create_raises: Exception | None = None delete_raises: Exception | None = None # When set, every get_host call raises this exception instead of # returning from get_host_responses. Used by the attach() tests @@ -98,6 +105,8 @@ async def create_host( ) -> SandboxHostRecord: self.created_envs.append(env) self.created_expires_at.append(expires_at) + if self.create_raises is not None: + raise self.create_raises assert self.create_record is not None, "test forgot to set create_record" return self.create_record @@ -486,6 +495,126 @@ async def _fake_upload(sandbox: Any) -> None: ) +@pytest.mark.parametrize( + "sdk_error", + [ + pytest.param(SandboxProvisioningError("provider timed out"), id="provisioning"), + pytest.param(SandboxUnavailableError("exe.dev transport failed"), id="unavailable"), + ], +) +async def test_acquire_translates_transient_create_failures( + sdk_error: Exception, + patched_real_sandbox: list[_FakeSandbox], + patched_sandbox_api: list[_FakeAPI], +): + """A transient control-plane failure from ``create_host`` (a 502 + provisioning error or a transport/503 unavailable error) is translated at + the client boundary into the classified, transient-retryable harness error + with the SDK exception preserved as the cause.""" + api = _FakeAPI(create_raises=sdk_error) + patched_sandbox_api.append(api) + + with pytest.raises(HarnessSandboxProvisioningError) as excinfo: + async with sandbox_client.acquire(): + pass + + error = excinfo.value + assert error.code == "sandbox_provisioning" + assert error.retry is Retry.TRANSIENT + # The schedule is inherited from HarnessSandboxError, not re-declared. + assert error.retry_delays == (60, 300) + assert error.__cause__ is sdk_error + + +async def test_acquire_passes_through_fatal_create_failure( + patched_real_sandbox: list[_FakeSandbox], + patched_sandbox_api: list[_FakeAPI], +): + """A non-transient SDK failure (auth, and the other ``SandboxAPIError`` + subclasses) is left untouched — it must not become a provisioning failure + and so must not enter the transient retry path.""" + fatal = SandboxAuthError("token revoked") + api = _FakeAPI(create_raises=fatal) + patched_sandbox_api.append(api) + + with pytest.raises(SandboxAuthError) as excinfo: + async with sandbox_client.acquire(): + pass + + assert excinfo.value is fatal + # No host was created, so nothing to roll back. + assert api.deleted_ids == [] + + +async def test_acquire_classifies_setup_reachability_failure_after_rollback( + patched_real_sandbox: list[_FakeSandbox], + patched_sandbox_api: list[_FakeAPI], +): + """A freshly created host that never becomes reachable/usable during SSH + + helper setup is rolled back (host deleted, key removed) and re-raised as a + classified provisioning failure carrying the original reachability error as + its cause — so the run retries instead of dead-ending on an empty code.""" + api = _FakeAPI(create_record=_record(status="active")) + patched_sandbox_api.append(api) + unreachable = SandboxUnreachable("failed to write /root/.gitconfig") + + async def _fake_upload(sandbox: Any) -> None: + raise unreachable + + with pytest.MonkeyPatch.context() as mp: + mp.setattr("druks.sandbox.client._upload_helper_script", _fake_upload) + with pytest.raises(HarnessSandboxProvisioningError) as excinfo: + async with sandbox_client.acquire(): + pass + + assert excinfo.value.code == "sandbox_provisioning" + assert excinfo.value.__cause__ is unreachable + # Rollback still happened despite the reclassification. + assert api.deleted_ids == ["host-xyz"] + + +async def test_acquire_setup_cancellation_propagates_unclassified( + patched_real_sandbox: list[_FakeSandbox], + patched_sandbox_api: list[_FakeAPI], +): + """Cancellation during setup rolls the host back but is never reclassified + as a provisioning failure — the run is being torn down, not retried.""" + import asyncio + + api = _FakeAPI(create_record=_record(status="active")) + patched_sandbox_api.append(api) + + async def _fake_upload(sandbox: Any) -> None: + raise asyncio.CancelledError + + with pytest.MonkeyPatch.context() as mp: + mp.setattr("druks.sandbox.client._upload_helper_script", _fake_upload) + with pytest.raises(asyncio.CancelledError): + async with sandbox_client.acquire(): + pass + + assert api.deleted_ids == ["host-xyz"] + + +async def test_attach_translates_unavailable_lookup_failure( + patched_real_sandbox: list[_FakeSandbox], + patched_sandbox_api: list[_FakeAPI], +): + """A transport/503 while looking up an existing host is classified as a + provisioning failure (transient retry re-attaches once the service + recovers), distinct from a 404 which stays ``HostGone``.""" + unavailable = SandboxUnavailableError("exe.dev transport failed") + api = _FakeAPI(create_record=None, get_host_raises=unavailable) + patched_sandbox_api.append(api) + + with pytest.raises(HarnessSandboxProvisioningError) as excinfo: + async with sandbox_client.attach(host_id="host-xyz"): + pass + + assert excinfo.value.code == "sandbox_provisioning" + assert excinfo.value.__cause__ is unavailable + + async def test_attach_returns_sandbox( patched_real_sandbox: list[_FakeSandbox], patched_sandbox_api: list[_FakeAPI], From 067ff785be4bfa1e353f9b8ec37b0166bd98efed Mon Sep 17 00:00:00 2001 From: "druks-operator[bot]" <284423593+druks-operator[bot]@users.noreply.github.com> Date: Wed, 19 Aug 2026 05:45:47 +0000 Subject: [PATCH 2/2] Apply reviewer style edits: split acquire rollback into two except clauses, trim comment and docstring Co-Authored-By: commonzenpython@gmail.com --- backend/druks/harnesses/exceptions.py | 12 ++++------- backend/druks/sandbox/client.py | 30 ++++++++++----------------- 2 files changed, 15 insertions(+), 27 deletions(-) diff --git a/backend/druks/harnesses/exceptions.py b/backend/druks/harnesses/exceptions.py index 232e4da7..70dfc169 100644 --- a/backend/druks/harnesses/exceptions.py +++ b/backend/druks/harnesses/exceptions.py @@ -63,14 +63,10 @@ class HarnessSandboxError(HarnessError): class HarnessSandboxProvisioningError(HarnessSandboxError): - """A control-plane provisioning or transport failure while creating or - re-attaching a host, or reaching a freshly created one. - - Transient on the same schedule as :class:`HarnessSandboxError` (it - inherits ``retry``/``retry_delays``), but carries its own ``code`` so an - exhausted retry records ``sandbox_provisioning`` rather than the generic - ``sandbox`` classification — the failure taxonomy can then name the most - transient failure class in the system instead of leaving it blank.""" + """Provisioning or transport failure during host creation, attach, or SSH setup. + + Inherits the transient retry schedule from HarnessSandboxError; uses + ``sandbox_provisioning`` so exhausted retries record a named failure code.""" code = "sandbox_provisioning" diff --git a/backend/druks/sandbox/client.py b/backend/druks/sandbox/client.py index b4090d65..ce2ea276 100644 --- a/backend/druks/sandbox/client.py +++ b/backend/druks/sandbox/client.py @@ -27,13 +27,8 @@ _DRUKS_SANDBOX_LOCAL_SCRIPT = Path(__file__).parent / "druks-sandbox.sh" -# A freshly created host that we can't reach or set up over SSH is a -# provisioning failure in the same sense as a create-time control-plane one: -# the VM was never usable. These are the shapes such reachability/setup -# failures take — SSH connect/auth/channel errors (asyncssh), socket-level -# failures and connect timeouts (OSError/TimeoutError), and the sandbox -# layer's own SandboxUnreachable/SandboxError. Cancellation and unrelated -# local/programming bugs are deliberately excluded so they propagate untouched. +# SSH/socket/sandbox errors that mean a fresh VM never became usable. +# CancelledError and programming errors are excluded so they propagate unchanged. _ACQUIRE_SETUP_REACHABILITY_ERRORS = ( SandboxError, asyncssh.Error, @@ -125,21 +120,18 @@ async def acquire( sandbox = Sandbox(record=record) try: await _upload_helper_script(sandbox) - except BaseException as error: - # Caller never sees this host; release rather than orphan. + except _ACQUIRE_SETUP_REACHABILITY_ERRORS as error: + # Fresh VM never became usable; roll back and classify as provisioning. + await sandbox.aclose() + await self._best_effort_delete(api, record.id) + key_path.unlink(missing_ok=True) + raise HarnessSandboxProvisioningError( + f"sandbox host {record.id} unreachable during setup: {error}" + ) from error + except BaseException: await sandbox.aclose() await self._best_effort_delete(api, record.id) key_path.unlink(missing_ok=True) - # A fresh VM that never became reachable/usable during SSH + - # helper setup is a provisioning failure just like a create-time - # one, so classify it into the same transient-retry path — the - # raw error would otherwise bypass agent retry and record an - # empty failure code. Cancellation and unrelated - # local/programming errors keep their own type and propagate. - if isinstance(error, _ACQUIRE_SETUP_REACHABILITY_ERRORS): - raise HarnessSandboxProvisioningError( - f"sandbox host {record.id} unreachable during setup: {error}" - ) from error raise try: yield sandbox