[AGE-4000] fix(runner): rebuild agent mounts before STS credentials expire

api/oss/src/core/mounts/service.py

@@ -39,6 +39,7 @@
     MountStorageUnavailable,
 )
 from oss.src.core.shared.dtos import Reference, Windowing
+from oss.src.utils.env import env
 from oss.src.utils.logging import get_module_logger
 
 log = get_module_logger(__name__)
@@ -54,10 +55,6 @@
 # The single session-bound mount: the agent's durable working directory.
 _SESSION_CWD_NAME = "cwd"
 
-# Default TTL (seconds) for signed mount credentials. Covers the mount lifetime for a
-# turn; geesefs holds the creds without refresh, so a turn outliving this hits ExpiredToken.
-_CREDENTIALS_TTL_SECONDS = 3600
-
 
 def _slugify(value: str) -> str:
     return sub(r"[^a-z0-9]+", "-", value.lower()).strip("-")
@@ -608,12 +605,13 @@ async def sign_mount_credentials(
         *,
         project_id: UUID,
         mount_id: UUID,
-        duration_seconds: int = _CREDENTIALS_TTL_SECONDS,
     ) -> MountCredentials:
         """Mint short-lived, prefix-scoped credentials for one mount.
 
         The master key signs the STS request API-side and never leaves; the returned
         key pair + session token are scoped to this mount's prefix and expire in minutes.
+        geesefs holds the credentials without refresh, so a turn outliving the TTL hits
+        ExpiredToken.
         """
         if self.mounts_store is None:
             raise MountStorageUnavailable()
@@ -626,7 +624,7 @@ async def sign_mount_credentials(
         creds = await self.mounts_store.sign_temp_credentials(
             bucket=bucket,
             prefix=prefix,
-            duration_seconds=duration_seconds,
+            duration_seconds=env.mounts.credentials_ttl_seconds,
         )
         return MountCredentials(
             endpoint=self.mounts_store.endpoint_url,

api/oss/src/core/store/storage.py

@@ -20,12 +20,23 @@
 # STS responses are SOAP-ish XML under the 2011-06-15 namespace; strip it for tag lookups.
 _STS_NS = "{https://sts.amazonaws.com/doc/2011-06-15/}"
 
+# Mirrors the `maxSessionLength: 12h` we configure in the compose SeaweedFS IAM config; the two
+# must stay in sync.
+_SEAWEEDFS_MAX_SESSION_SECONDS = 43200
+
+# AWS GetFederationToken accepts DurationSeconds in [900, 129600].
+_AWS_MAX_FEDERATION_SECONDS = 129600
+
 
 def _parse_sts_credentials(xml_text: str) -> Credentials:
     """Parse an STS XML response (`AssumeRoleWithWebIdentity`) into a miniopy Credentials.
 
     Returns the scoped access/secret/session triple plus expiry; raises if the response
     carried no Credentials block (e.g. an STS error document slipped past the status check).
+
+    Fails closed on a missing or unparsable `Expiration`: the runner reads a missing expiry as
+    "never expires", so one malformed response would silently disable the reuse bound for the
+    lifetime of a pooled environment.
     """
     root = ElementTree.fromstring(xml_text)
     creds_el = root.find(f".//{_STS_NS}Credentials")
@@ -36,13 +47,15 @@ def _text(tag: str) -> Optional[str]:
         el = creds_el.find(f"{_STS_NS}{tag}")
         return el.text if el is not None else None
 
-    expiration = None
     raw_exp = _text("Expiration")
-    if raw_exp:
-        try:
-            expiration = datetime.fromisoformat(raw_exp.replace("Z", "+00:00"))
-        except ValueError:
-            expiration = None
+    if not raw_exp:
+        raise MountStorageUnavailable("STS response carried no credential expiry.")
+    try:
+        expiration = datetime.fromisoformat(raw_exp.replace("Z", "+00:00"))
+    except ValueError as exc:
+        raise MountStorageUnavailable(
+            "STS response carried an unparsable credential expiry."
+        ) from exc
 
     return Credentials(
         access_key=_text("AccessKeyId"),
@@ -229,14 +242,21 @@ async def _sign_web_identity(
         """SeaweedFS path: unauthenticated `AssumeRoleWithWebIdentity` against the store endpoint."""
         host, secure = self._host_secure()
         endpoint = f"{'https' if secure else 'http'}://{host}"
+        # SeaweedFS validates DurationSeconds in [900, 43200], but the EFFECTIVE lifetime is
+        # min(DurationSeconds, web-identity token expiry, maxSessionLength). The token-expiry cap
+        # has no floor, so minting the token at the capped TTL is what lets a sub-900 TTL take
+        # effect (the QA lever); DurationSeconds only has to stay inside the validated range.
+        capped = min(duration_seconds, _SEAWEEDFS_MAX_SESSION_SECONDS)
         body = urlencode(
             {
                 "Action": "AssumeRoleWithWebIdentity",
                 "Version": "2011-06-15",
                 "RoleArn": webidentity.role_arn(),
                 "RoleSessionName": webidentity.STORE_SUBJECT,
-                "WebIdentityToken": webidentity.mint_web_identity_token(),
-                "DurationSeconds": str(duration_seconds),
+                "WebIdentityToken": webidentity.mint_web_identity_token(
+                    ttl_seconds=capped
+                ),
+                "DurationSeconds": str(max(capped, 900)),
                 "Policy": scope_policy,
             }
         )
@@ -261,9 +281,9 @@ async def _sign_federation_token(
         """Remote-S3 path: SigV4-signed `GetFederationToken` against the store's STS endpoint.
 
         The STS endpoint defaults to the S3 endpoint (MinIO co-locates STS there); AWS splits
-        it onto `sts.<region>.amazonaws.com`, set via AGENTA_STORE_STS_ENDPOINT_URL. Duration is
-        clamped to STS's 900s floor. The request is signed with the store's master keys; the
-        returned credentials carry only the inline `Policy`'s permissions.
+        it onto `sts.<region>.amazonaws.com`, set via AGENTA_STORE_STS_ENDPOINT_URL. Duration
+        is clamped into STS's [900, 129600] range. The request is signed with the store's
+        master keys; the returned credentials carry only the inline `Policy`'s permissions.
         """
         endpoint = (self._sts_endpoint_url or self._endpoint_url or "").rstrip("/")
         if not endpoint:
@@ -275,7 +295,10 @@ async def _sign_federation_token(
                 "Action": "GetFederationToken",
                 "Version": "2011-06-15",
                 "Name": webidentity.STORE_SUBJECT,
-                "DurationSeconds": str(max(duration_seconds, 900)),
+                # AWS rejects anything outside its own range; clamp rather than fail the sign.
+                "DurationSeconds": str(
+                    min(max(duration_seconds, 900), _AWS_MAX_FEDERATION_SECONDS)
+                ),
                 "Policy": scope_policy,
             }
         )

api/oss/src/utils/env.py

@@ -1117,6 +1117,15 @@ def enabled(self) -> bool:
 class MountsConfig(BaseModel):
     """Mounts-domain config. Store credentials live in StoreConfig."""
 
+    # Lifetime of signed mount credentials, in seconds. The store backends clamp it to their
+    # own STS bounds.
+    credentials_ttl_seconds: int = Field(
+        default_factory=lambda: (
+            _parse_optional_positive_int_env("AGENTA_MOUNTS_CREDENTIALS_TTL_SECONDS")
+            or 3600
+        )
+    )
+
     model_config = ConfigDict(extra="ignore")
 
 

api/oss/tests/pytest/unit/test_mounts_injection.py

@@ -12,8 +12,10 @@
 """
 
 from typing import Dict, Optional
+from urllib.parse import parse_qs
 from uuid import uuid4, uuid5, NAMESPACE_DNS
 
+import jwt
 import pytest
 
 from oss.src.core.mounts.dtos import Mount, MountCreate
@@ -409,7 +411,6 @@ async def test_remote_s3_sts_defaults_to_s3_endpoint(self, monkeypatch):
         )
         assert cap.url == "http://minio:9000/"
         assert "Action=GetFederationToken" in cap.data
-        assert "DurationSeconds=900" in cap.data
 
     async def test_seaweedfs_uses_web_identity_against_endpoint(self, monkeypatch):
         import oss.src.core.store.storage as storage_mod
@@ -436,3 +437,148 @@ async def test_seaweedfs_uses_web_identity_against_endpoint(self, monkeypatch):
         assert "Action=AssumeRoleWithWebIdentity" in cap.data
         assert "WebIdentityToken=" in cap.data
         assert "Authorization" not in (cap.headers or {})
+
+
+# ---------------------------------------------------------------------------
+# Credential lifetime: STS duration clamps + web-identity token lifetime
+# ---------------------------------------------------------------------------
+
+
+def _seaweedfs_store():
+    from oss.src.core.store.storage import ObjectStore
+
+    return ObjectStore(
+        endpoint_url="http://seaweedfs:8333",
+        access_key=_MASTER_KEY,
+        secret_key=_MASTER_SECRET,
+        signing_key="dummy-signing-key",
+    )
+
+
+def _remote_s3_store():
+    from oss.src.core.store.storage import ObjectStore
+
+    return ObjectStore(
+        endpoint_url="https://s3.eu-central-1.amazonaws.com",
+        sts_endpoint_url="https://sts.eu-central-1.amazonaws.com",
+        access_key=_MASTER_KEY,
+        secret_key=_MASTER_SECRET,
+        region="eu-central-1",
+    )
+
+
+async def _capture_sign(
+    monkeypatch, *, store, duration_seconds: int, xml: str = _STS_XML
+):
+    """Sign against a captured POST; returns the request body fields as parsed form data."""
+    import oss.src.core.store.storage as storage_mod
+
+    cap = _CapturingPost(xml)
+    monkeypatch.setattr(storage_mod.aiohttp, "ClientSession", lambda: cap)
+    await store.sign_temp_credentials(
+        bucket=_BUCKET, prefix="mounts/p/m", duration_seconds=duration_seconds
+    )
+    return parse_qs(cap.data)
+
+
+def _token_lifetime_seconds(token: str) -> int:
+    claims = jwt.decode(token, options={"verify_signature": False})
+    return claims["exp"] - claims["iat"]
+
+
+@pytest.mark.asyncio
+class TestStsDurations:
+    @pytest.mark.parametrize(
+        "ttl, expected_duration, expected_token_lifetime",
+        [
+            # The requested hour, honestly: the token no longer caps the session at 15 minutes.
+            (3600, 3600, 3600),
+            # Sub-900 is a hard SeaweedFS request error, so DurationSeconds is floored and the
+            # short lifetime is delivered through the token expiry, which has no floor.
+            (120, 900, 120),
+            # Both capped at SeaweedFS's 12h maxSessionLength.
+            (90000, 43200, 43200),
+        ],
+    )
+    async def test_seaweedfs_clamps_duration_and_matches_token_lifetime(
+        self, monkeypatch, ttl, expected_duration, expected_token_lifetime
+    ):
+        fields = await _capture_sign(
+            monkeypatch, store=_seaweedfs_store(), duration_seconds=ttl
+        )
+
+        assert fields["Action"] == ["AssumeRoleWithWebIdentity"]
+        assert fields["DurationSeconds"] == [str(expected_duration)]
+        assert (
+            _token_lifetime_seconds(fields["WebIdentityToken"][0])
+            == expected_token_lifetime
+        )
+
+    @pytest.mark.parametrize(
+        "ttl, expected_duration",
+        [(60, 900), (3600, 3600), (200000, 129600)],
+    )
+    async def test_federation_token_duration_clamped_to_aws_range(
+        self, monkeypatch, ttl, expected_duration
+    ):
+        fields = await _capture_sign(
+            monkeypatch, store=_remote_s3_store(), duration_seconds=ttl
+        )
+
+        assert fields["Action"] == ["GetFederationToken"]
+        assert fields["DurationSeconds"] == [str(expected_duration)]
+
+
+_STS_XML_NO_EXPIRATION = _STS_XML.replace(
+    "<Expiration>2026-06-30T08:00:00Z</Expiration>", ""
+)
+
+_STS_XML_BAD_EXPIRATION = _STS_XML.replace(
+    "<Expiration>2026-06-30T08:00:00Z</Expiration>",
+    "<Expiration>not-a-date</Expiration>",
+)
+
+
+@pytest.mark.asyncio
+class TestStsExpiryFailsClosed:
+    @pytest.mark.parametrize("xml", [_STS_XML_NO_EXPIRATION, _STS_XML_BAD_EXPIRATION])
+    @pytest.mark.parametrize("store_factory", [_seaweedfs_store, _remote_s3_store])
+    async def test_sign_refuses_credentials_without_a_usable_expiry(
+        self, monkeypatch, store_factory, xml
+    ):
+        # The runner reads a missing expiry as "never expires", so an unusable one must not
+        # reach it as a signed credential.
+        with pytest.raises(MountStorageUnavailable):
+            await _capture_sign(
+                monkeypatch,
+                store=store_factory(),
+                duration_seconds=3600,
+                xml=xml,
+            )
+
+
+# ---------------------------------------------------------------------------
+# TTL configuration (env)
+# ---------------------------------------------------------------------------
+
+
+class TestMountsCredentialsTtlConfig:
+    @pytest.mark.parametrize(
+        "value, expected",
+        [
+            (None, 3600),
+            ("120", 120),
+            # An unset compose passthrough delivers an empty string, not an absent variable.
+            ("", 3600),
+        ],
+    )
+    def test_credentials_ttl_reads_the_env_at_instantiation(
+        self, monkeypatch, value, expected
+    ):
+        from oss.src.utils.env import MountsConfig
+
+        if value is None:
+            monkeypatch.delenv("AGENTA_MOUNTS_CREDENTIALS_TTL_SECONDS", raising=False)
+        else:
+            monkeypatch.setenv("AGENTA_MOUNTS_CREDENTIALS_TTL_SECONDS", value)
+        assert MountsConfig().credentials_ttl_seconds == expected

docs/design/fix-sts-mount-expiry/README.md

@@ -0,0 +1,58 @@
+# Fix: agent mounts return EACCES after STS credentials expire
+
+Planning workspace for GitHub issue Agenta-AI/agenta#5516. Agent file mounts go
+"Permission denied" for many minutes at a time because the temporary store credentials
+behind them expire and nothing reacts. This workspace holds the investigation and the
+proposed fix.
+
+## What each file answers
+
+| File | Question it answers |
+| --- | --- |
+| [context.md](context.md) | What breaks for the user today, and why this work exists. |
+| [research.md](research.md) | How credentials flow end to end, what actually expires and when, where credentials are installed into mounts, which refresh mechanisms geesefs supports, and why the fix is safe on both SeaweedFS and real AWS. |
+| [plan.md](plan.md) | The chosen fix, the exact code changes per file, the test plan, and the live QA plan. |
+| [decisions.md](decisions.md) | Every decision the fix embeds, its trade-off, and how to back it out. |
+| [status.md](status.md) | Where this work stands right now. |
+
+## Domain nouns, one sentence each
+
+- **Runner**: the Node sidecar (`services/runner/`) that executes one agent turn per
+  HTTP `/run` request.
+- **Sandbox**: the isolated environment a turn runs in; "local" means the runner's own
+  host, "remote" means a Daytona cloud VM.
+- **Turn**: one user-message-to-agent-reply exchange; the runner's unit of work.
+- **Turn boundary**: the moment between two turns, when the runner decides whether to
+  reuse a live environment or build a fresh one.
+- **Session pool**: the runner's in-memory table of live environments parked between
+  turns so the next turn can continue without a rebuild.
+- **Mount**: a directory in the sandbox whose contents live in the object store, so
+  files survive sandbox teardown.
+- **Durable agent volume**: the per-agent mount (`agent-files/`) shared across all
+  sessions of one agent; long-lived by design.
+- **Session scratch**: the per-session working directory mount (`cwd`); re-signed at
+  every turn boundary.
+- **Mount credentials**: the temporary, prefix-scoped S3 key, secret, and session
+  token a mount's geesefs daemon holds; they carry an expiry and cannot be renewed
+  in place.
+- **Installed lease**: the expiry of the credentials a live environment's daemons
+  are actually running on, as opposed to the expiry of credentials signed later and
+  never given to them.
+- **geesefs**: the FUSE filesystem binary that presents an S3 prefix as a local
+  directory; each mount is one geesefs daemon process.
+- **S3**: the object-storage HTTP protocol; Agenta speaks it to whichever store backs
+  the deployment.
+- **SeaweedFS**: the S3-compatible store bundled with self-hosted Agenta.
+- **STS**: the "Security Token Service" protocol for minting temporary, scoped
+  S3 credentials; both SeaweedFS and AWS implement it.
+- **EACCES**: the "Permission denied" error code; geesefs returns it for every file
+  operation once the store answers 403.
+- **ENOTCONN**: the "Transport endpoint is not connected" error code; the kernel
+  returns it when a FUSE daemon has died but its mount point is still registered.
+
+## Related work
+
+- `docs/design/mount-file-viewer/`: the mounts file-listing UI; shares the same sign
+  endpoints, no credential-lifetime logic.
+- Draft PR #5247 and design issue #5215 introduced the durable agent mount this bug
+  hits hardest; its design workspace lives on that PR's branch, not on main.