fix: wait for HTTPS authority startup admission

docs/guides/native-candidate.md

@@ -1392,6 +1392,19 @@ Services without managed values use ordinary exec. The native live qualification
 of this fresh-delivery path remains separate from its unit and transport tests.
 
 
+Managed HTTPS authority startup uses the same five-second provider-lock admission
+budget as ordinary pool startup. This lets a brief concurrent graph inspection
+finish before admission. Pool identity, socket ownership and certificate admission
+are checked only after the lock is acquired; expiry refuses with `provider_busy`
+and no authority is published. Serving and cleanup are never replayed.
+
+An authority child that exits before readiness can supply a bounded structured
+native error code without exposing its stderr. Malformed, oversized or missing
+output leaves the cause unknown. The `authority-ready` diagnostic now covers only
+readiness; later socket-identity, permission-port and Caddyfile failures have
+separate stages. These classifications do not acknowledge cleanup or permit an
+owner to be adopted or replaced.
+
 A detached HTTPS helper that fails during startup may retain a generation-bound
 `startup-failure.json` beside its owner configuration. The CLI reports only the
 reviewed startup stage and an allowlisted native error code; child output, paths
@@ -1401,3 +1414,12 @@ This record is diagnostic evidence only: it does not acknowledge retirement or
 permit a replacement owner. A missing record (including a helper crash before
 publication) leaves the cause unknown. Preserve the retained owner and finalization
 records for inspection; do not delete them to force another startup.
+
+When an HTTPS owner refuses an exact lease release, it now attempts a bounded
+failure response containing only the release stage and a reviewed native error
+code. The client verifies the complete lease identity and rejects extra or
+noncanonical protocol data. This response is not a release acknowledgement and
+never triggers a retry: transport loss still leaves the outcome unconfirmed.
+For example, an observed `graph-verification: provider_busy` would locate the
+refusal without exposing native stderr; it would not prove cleanup completed or
+permit another owner to start. Preserve the original records while investigating.

packages/runtime-core/src/provider/hostname_authority/managed.rs

@@ -1,7 +1,7 @@
 //! Explicit foreground authority bound to the candidate pool's lifetime.
-use super::super::{state, status};
+use super::super::{lifecycle, state, status};
 use crate::{Candidate, CandidateError};
-use std::path::PathBuf;
+use std::{path::PathBuf, time::Duration};
 fn socket(owner: &state::Owner) -> PathBuf {
     PathBuf::from(format!(
         "/private/tmp/hka-{}-{}/route.sock",
@@ -29,7 +29,16 @@ pub fn serve_with_certificate_limit(
     c: &Candidate,
     limit: Option<usize>,
 ) -> Result<(), CandidateError> {
-    let lock = state::Lock::acquire(&c.state_root.join("run/smolvm"))?;
+    serve_with_startup_wait(c, limit, lifecycle::STARTUP_LEASE_WAIT)
+}
+fn serve_with_startup_wait(
+    c: &Candidate,
+    limit: Option<usize>,
+    wait: Duration,
+) -> Result<(), CandidateError> {
+    // Graph inspection also holds this lease. Wait before admission, then reload
+    // the current pool identity; neither socket publication nor serving is replayed.
+    let lock = lifecycle::startup_lease(&c.state_root.join("run/smolvm"), wait)?;
     let current = status(c)?;
     if current.phase != "running" || current.process_alive != Some(true) {
         return Err(super::error());
@@ -59,3 +68,89 @@ pub(crate) fn stop(c: &Candidate, owner: &state::Owner) -> Result<(), CandidateE
     super::ownership::stop(c, &path, hash)?;
     Ok(())
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::{
+        fs,
+        time::{Instant, SystemTime, UNIX_EPOCH},
+    };
+
+    struct Fixture(PathBuf);
+    impl Fixture {
+        fn new() -> Self {
+            let root = fs::canonicalize(std::env::temp_dir())
+                .unwrap()
+                .join(format!(
+                    "hkl-authority-admission-{}-{}",
+                    std::process::id(),
+                    SystemTime::now()
+                        .duration_since(UNIX_EPOCH)
+                        .unwrap()
+                        .as_nanos()
+                ));
+            state::private_directory(&root).unwrap();
+            Self(root)
+        }
+        fn candidate(&self) -> Candidate {
+            Candidate::discover(&self.0).unwrap()
+        }
+    }
+    impl Drop for Fixture {
+        fn drop(&mut self) {
+            let _ = fs::remove_dir_all(&self.0);
+        }
+    }
+
+    #[test]
+    fn authority_startup_waits_for_inspection_before_reading_pool_state() {
+        let fixture = Fixture::new();
+        let candidate = fixture.candidate();
+        let root = candidate.state_root.join("run/smolvm");
+        let held = state::Lock::acquire(&root).unwrap();
+        let worker = std::thread::spawn(move || {
+            std::thread::sleep(Duration::from_millis(150));
+            drop(held);
+        });
+        let start = Instant::now();
+        let result = serve_with_startup_wait(&candidate, None, Duration::from_secs(2));
+        worker.join().unwrap();
+        assert!(start.elapsed() >= Duration::from_millis(100));
+        // The exact production path gets past contention and rejects the unprepared
+        // pool. It must not publish an authority just because the lock became free.
+        assert!(matches!(result, Err(e) if e.code == "hostname_authority"));
+        assert_eq!(fs::read_dir(&root).unwrap().count(), 1);
+        assert!(!root.join("owner.json").exists());
+        assert!(
+            !candidate
+                .state_root
+                .join("run/certificate-admission")
+                .exists()
+        );
+        assert!(state::Lock::acquire(&root).is_ok());
+    }
+
+    #[test]
+    fn authority_startup_contention_deadline_admits_no_pool_or_socket_effect() {
+        let fixture = Fixture::new();
+        let candidate = fixture.candidate();
+        let root = candidate.state_root.join("run/smolvm");
+        let _held = state::Lock::acquire(&root).unwrap();
+        let start = Instant::now();
+        let result = serve_with_startup_wait(&candidate, None, Duration::from_millis(100));
+        assert!(start.elapsed() >= Duration::from_millis(75));
+        assert!(
+            matches!(result, Err(e) if e.code == "provider_busy" && e.message.contains("no operation was admitted"))
+        );
+        assert_eq!(fs::read_dir(&root).unwrap().count(), 1);
+        assert!(!root.join("owner.json").exists());
+        assert!(
+            !candidate
+                .state_root
+                .join("run/certificate-admission")
+                .exists()
+        );
+        assert!(matches!(state::Lock::acquire(&root), Err(e) if e.code == "provider_busy"));
+    }
+}

packages/runtime-core/src/provider/lifecycle.rs

@@ -273,12 +273,12 @@ fn operation_lease<T>(
 
 /// How long ordinary startup waits for a provider lease that another operation holds, such as
 /// the `graph inspect` behind `hack ps`.
-const STARTUP_LEASE_WAIT: Duration = Duration::from_secs(5);
+pub(super) const STARTUP_LEASE_WAIT: Duration = Duration::from_secs(5);
 
 /// Startup's provider lease. Everything `start_pool` does before taking it is read-only
 /// validation, admission sampling and RAII guards, so waiting for a briefly held lease admits
 /// nothing early, and on expiry it refuses `provider_busy` with nothing admitted.
-fn startup_lease(root: &Path, wait: Duration) -> Result<state::Lock, CandidateError> {
+pub(super) fn startup_lease(root: &Path, wait: Duration) -> Result<state::Lock, CandidateError> {
     operation_lease(root, Some(Instant::now() + wait), || Ok(())).map(|(lock, ())| lock)
 }
 

src/backends/native-https-owner-protocol.ts

@@ -272,7 +272,13 @@ export function decodeNativeHttpsOwnerFrame(bytes: Buffer): unknown {
     throw nativeHttpsOwnerRefused();
   }
   try {
-    return JSON.parse(bytes.toString("utf8"));
+    const value: unknown = JSON.parse(bytes.toString("utf8"));
+    // This private protocol is emitted by encodeNativeHttpsOwnerFrame. Require
+    // its exact encoding so duplicate keys cannot turn a refusal into an ack.
+    if (!bytes.equals(encodeNativeHttpsOwnerFrame(value))) {
+      throw nativeHttpsOwnerRefused();
+    }
+    return value;
   } catch {
     throw nativeHttpsOwnerRefused();
   }

src/backends/native-https-owner-server.ts

@@ -38,13 +38,27 @@ import {
   nativeHttpsRemoveFile,
   nativeHttpsWriteNew,
 } from "./native-https-owner-storage.ts";
+import {
+  type NativeHttpsReleaseStage,
+  nativeHttpsReleaseFailureFrame,
+  sendNativeHttpsReleaseFailure,
+} from "./native-https-release-failure.ts";
 import {
   NativeHttpsStartupError,
   recordNativeHttpsStartupFailure,
 } from "./native-https-startup-failure.ts";
 import { startNativeProjectHttps } from "./native-project-https.ts";
 import { invokeNativeRuntime } from "./native-runtime-client.ts";
 
+type ReleaseProgress = { stage: NativeHttpsReleaseStage };
+function markReleaseStage(
+  progress: ReleaseProgress | undefined,
+  stage: NativeHttpsReleaseStage
+): void {
+  if (progress) {
+    progress.stage = stage;
+  }
+}
 type Frontend = Awaited<ReturnType<typeof startNativeProjectHttps>>;
 export interface NativeHttpsOwnerServerDependencies {
   readonly start: (binding: NativeHttpsOwnerBinding) => Promise<Frontend>;
@@ -322,13 +336,19 @@ export async function serveNativeHttpsOwner(opts: {
     socket: Socket;
     identity: NativeHttpsLeaseIdentity;
     file?: NativeHttpsFileIdentity;
+    progress: ReleaseProgress;
   }) => {
+    markReleaseStage(released?.progress, "owner-validation");
     await checkPaths();
+    markReleaseStage(released?.progress, "idle-verification");
     await opts.dependencies.verifyIdle(binding);
     state = "closing";
+    markReleaseStage(released?.progress, "frontend-close");
     await frontend?.close();
+    markReleaseStage(released?.progress, "owner-retirement");
     await checkPaths();
     if (released) {
+      released.progress.stage = "release-publication";
       await nativeHttpsRecordRelease({
         version: 1,
         identity: released.identity,
@@ -343,6 +363,7 @@ export async function serveNativeHttpsOwner(opts: {
       }
       leases.delete(released.identity.leaseId);
     }
+    markReleaseStage(released?.progress, "owner-retirement");
     if (leases.size !== 0 || (await readdir(leaseRoot)).length !== 0) {
       throw nativeHttpsOwnerRefused();
     }
@@ -435,8 +456,10 @@ export async function serveNativeHttpsOwner(opts: {
   };
   const releaseUnadmittedLease = async (
     socket: Socket,
-    identity: NativeHttpsLeaseIdentity
+    identity: NativeHttpsLeaseIdentity,
+    progress: ReleaseProgress
   ) => {
+    progress.stage = "lease-validation";
     if (
       !sameNativeHttpsLease(
         nativeHttpsLeaseIdentity(configuration, identity),
@@ -445,7 +468,9 @@ export async function serveNativeHttpsOwner(opts: {
     ) {
       throw nativeHttpsOwnerRefused();
     }
+    progress.stage = "graph-verification";
     await opts.dependencies.verify(binding, identity, "release");
+    progress.stage = "lease-validation";
     try {
       await lstat(join(leaseRoot, `${identity.leaseId}.json`));
       throw nativeHttpsOwnerRefused();
@@ -455,9 +480,10 @@ export async function serveNativeHttpsOwner(opts: {
       }
     }
     if (leases.size === 0) {
-      await retire({ socket, identity });
+      await retire({ socket, identity, progress });
       return;
     }
+    progress.stage = "release-publication";
     // A persisted acquire intent may never have been delivered. Only this
     // generation's live serialized owner can certify it was never admitted.
     try {
@@ -482,26 +508,38 @@ export async function serveNativeHttpsOwner(opts: {
     );
     return;
   };
-  const handle = async (socket: Socket, request: NativeHttpsOwnerRequest) => {
+  const handle = async (
+    socket: Socket,
+    request: NativeHttpsOwnerRequest,
+    progress: ReleaseProgress
+  ) => {
     if (state !== "running" || frontendFailed) {
       throw nativeHttpsOwnerRefused();
     }
     await checkPaths();
     if (request.operation === "acquire") {
       return acquireLease(socket, request);
     }
+    progress.stage = "lease-validation";
     const entry = leases.get(request.identity.leaseId);
     if (!entry) {
-      return releaseUnadmittedLease(socket, request.identity);
+      return releaseUnadmittedLease(socket, request.identity, progress);
     }
     if (!sameNativeHttpsLease(entry.identity, request.identity)) {
       throw nativeHttpsOwnerRefused();
     }
+    progress.stage = "graph-verification";
     await opts.dependencies.verify(binding, entry.identity, "release");
     if (leases.size === 1) {
-      await retire({ socket, identity: entry.identity, file: entry.file });
+      await retire({
+        socket,
+        identity: entry.identity,
+        file: entry.file,
+        progress,
+      });
       return;
     }
+    progress.stage = "release-publication";
     await nativeHttpsRecordRelease({
       version: 1,
       identity: entry.identity,
@@ -567,8 +605,23 @@ export async function serveNativeHttpsOwner(opts: {
       bytes = Buffer.alloc(0);
       busy = true;
       socket.setTimeout(0);
-      void serialize(() => handle(socket, request))
-        .catch(() => {
+      const progress: ReleaseProgress = { stage: "owner-validation" };
+      void serialize(() => handle(socket, request, progress))
+        .catch(async (error: unknown) => {
+          if (request.operation === "release") {
+            try {
+              await sendNativeHttpsReleaseFailure(
+                socket,
+                nativeHttpsReleaseFailureFrame({
+                  identity: request.identity,
+                  stage: progress.stage,
+                  error,
+                })
+              );
+            } catch {
+              // Diagnostic delivery never grants authority or prevents the existing refusal.
+            }
+          }
           socket.destroy();
           // If retirement began, never accept another lease in a half-closed state.
           if (state === "closing") {

src/backends/native-https-owner.ts

@@ -36,6 +36,7 @@ import {
   nativeHttpsReadRetiredOwner,
   nativeHttpsWriteNew,
 } from "./native-https-owner-storage.ts";
+import { nativeHttpsReleaseReplyError } from "./native-https-release-failure.ts";
 import {
   NativeHttpsStartupError,
   readNativeHttpsStartupFailure,
@@ -377,7 +378,16 @@ export async function requestNativeHttpsOwner(
         return;
       }
       try {
-        finish(undefined, decodeNativeHttpsOwnerFrame(bytes));
+        const reply = decodeNativeHttpsOwnerFrame(bytes);
+        if (
+          isRecord(value) &&
+          value.operation === "release" &&
+          isNativeHttpsLeaseIdentity(value.identity)
+        ) {
+          finish(nativeHttpsReleaseReplyError(reply, value.identity), reply);
+          return;
+        }
+        finish(undefined, reply);
       } catch {
         finish(nativeHttpsOwnerRefused());
       }