fix: preserve exact-lease HTTPS release diagnostics

docs/guides/native-candidate.md

@@ -1401,3 +1401,12 @@ This record is diagnostic evidence only: it does not acknowledge retirement or
 permit a replacement owner. A missing record (including a helper crash before
 publication) leaves the cause unknown. Preserve the retained owner and finalization
 records for inspection; do not delete them to force another startup.
+
+When an HTTPS owner refuses an exact lease release, it now attempts a bounded
+failure response containing only the release stage and a reviewed native error
+code. The client verifies the complete lease identity and rejects extra or
+noncanonical protocol data. This response is not a release acknowledgement and
+never triggers a retry: transport loss still leaves the outcome unconfirmed.
+For example, an observed `graph-verification: provider_busy` would locate the
+refusal without exposing native stderr; it would not prove cleanup completed or
+permit another owner to start. Preserve the original records while investigating.

src/backends/native-https-owner-protocol.ts

@@ -272,7 +272,13 @@ export function decodeNativeHttpsOwnerFrame(bytes: Buffer): unknown {
     throw nativeHttpsOwnerRefused();
   }
   try {
-    return JSON.parse(bytes.toString("utf8"));
+    const value: unknown = JSON.parse(bytes.toString("utf8"));
+    // This private protocol is emitted by encodeNativeHttpsOwnerFrame. Require
+    // its exact encoding so duplicate keys cannot turn a refusal into an ack.
+    if (!bytes.equals(encodeNativeHttpsOwnerFrame(value))) {
+      throw nativeHttpsOwnerRefused();
+    }
+    return value;
   } catch {
     throw nativeHttpsOwnerRefused();
   }

src/backends/native-https-owner-server.ts

@@ -38,13 +38,27 @@ import {
   nativeHttpsRemoveFile,
   nativeHttpsWriteNew,
 } from "./native-https-owner-storage.ts";
+import {
+  type NativeHttpsReleaseStage,
+  nativeHttpsReleaseFailureFrame,
+  sendNativeHttpsReleaseFailure,
+} from "./native-https-release-failure.ts";
 import {
   NativeHttpsStartupError,
   recordNativeHttpsStartupFailure,
 } from "./native-https-startup-failure.ts";
 import { startNativeProjectHttps } from "./native-project-https.ts";
 import { invokeNativeRuntime } from "./native-runtime-client.ts";
 
+type ReleaseProgress = { stage: NativeHttpsReleaseStage };
+function markReleaseStage(
+  progress: ReleaseProgress | undefined,
+  stage: NativeHttpsReleaseStage
+): void {
+  if (progress) {
+    progress.stage = stage;
+  }
+}
 type Frontend = Awaited<ReturnType<typeof startNativeProjectHttps>>;
 export interface NativeHttpsOwnerServerDependencies {
   readonly start: (binding: NativeHttpsOwnerBinding) => Promise<Frontend>;
@@ -322,13 +336,19 @@ export async function serveNativeHttpsOwner(opts: {
     socket: Socket;
     identity: NativeHttpsLeaseIdentity;
     file?: NativeHttpsFileIdentity;
+    progress: ReleaseProgress;
   }) => {
+    markReleaseStage(released?.progress, "owner-validation");
     await checkPaths();
+    markReleaseStage(released?.progress, "idle-verification");
     await opts.dependencies.verifyIdle(binding);
     state = "closing";
+    markReleaseStage(released?.progress, "frontend-close");
     await frontend?.close();
+    markReleaseStage(released?.progress, "owner-retirement");
     await checkPaths();
     if (released) {
+      released.progress.stage = "release-publication";
       await nativeHttpsRecordRelease({
         version: 1,
         identity: released.identity,
@@ -343,6 +363,7 @@ export async function serveNativeHttpsOwner(opts: {
       }
       leases.delete(released.identity.leaseId);
     }
+    markReleaseStage(released?.progress, "owner-retirement");
     if (leases.size !== 0 || (await readdir(leaseRoot)).length !== 0) {
       throw nativeHttpsOwnerRefused();
     }
@@ -435,8 +456,10 @@ export async function serveNativeHttpsOwner(opts: {
   };
   const releaseUnadmittedLease = async (
     socket: Socket,
-    identity: NativeHttpsLeaseIdentity
+    identity: NativeHttpsLeaseIdentity,
+    progress: ReleaseProgress
   ) => {
+    progress.stage = "lease-validation";
     if (
       !sameNativeHttpsLease(
         nativeHttpsLeaseIdentity(configuration, identity),
@@ -445,7 +468,9 @@ export async function serveNativeHttpsOwner(opts: {
     ) {
       throw nativeHttpsOwnerRefused();
     }
+    progress.stage = "graph-verification";
     await opts.dependencies.verify(binding, identity, "release");
+    progress.stage = "lease-validation";
     try {
       await lstat(join(leaseRoot, `${identity.leaseId}.json`));
       throw nativeHttpsOwnerRefused();
@@ -455,9 +480,10 @@ export async function serveNativeHttpsOwner(opts: {
       }
     }
     if (leases.size === 0) {
-      await retire({ socket, identity });
+      await retire({ socket, identity, progress });
       return;
     }
+    progress.stage = "release-publication";
     // A persisted acquire intent may never have been delivered. Only this
     // generation's live serialized owner can certify it was never admitted.
     try {
@@ -482,26 +508,38 @@ export async function serveNativeHttpsOwner(opts: {
     );
     return;
   };
-  const handle = async (socket: Socket, request: NativeHttpsOwnerRequest) => {
+  const handle = async (
+    socket: Socket,
+    request: NativeHttpsOwnerRequest,
+    progress: ReleaseProgress
+  ) => {
     if (state !== "running" || frontendFailed) {
       throw nativeHttpsOwnerRefused();
     }
     await checkPaths();
     if (request.operation === "acquire") {
       return acquireLease(socket, request);
     }
+    progress.stage = "lease-validation";
     const entry = leases.get(request.identity.leaseId);
     if (!entry) {
-      return releaseUnadmittedLease(socket, request.identity);
+      return releaseUnadmittedLease(socket, request.identity, progress);
     }
     if (!sameNativeHttpsLease(entry.identity, request.identity)) {
       throw nativeHttpsOwnerRefused();
     }
+    progress.stage = "graph-verification";
     await opts.dependencies.verify(binding, entry.identity, "release");
     if (leases.size === 1) {
-      await retire({ socket, identity: entry.identity, file: entry.file });
+      await retire({
+        socket,
+        identity: entry.identity,
+        file: entry.file,
+        progress,
+      });
       return;
     }
+    progress.stage = "release-publication";
     await nativeHttpsRecordRelease({
       version: 1,
       identity: entry.identity,
@@ -567,8 +605,23 @@ export async function serveNativeHttpsOwner(opts: {
       bytes = Buffer.alloc(0);
       busy = true;
       socket.setTimeout(0);
-      void serialize(() => handle(socket, request))
-        .catch(() => {
+      const progress: ReleaseProgress = { stage: "owner-validation" };
+      void serialize(() => handle(socket, request, progress))
+        .catch(async (error: unknown) => {
+          if (request.operation === "release") {
+            try {
+              await sendNativeHttpsReleaseFailure(
+                socket,
+                nativeHttpsReleaseFailureFrame({
+                  identity: request.identity,
+                  stage: progress.stage,
+                  error,
+                })
+              );
+            } catch {
+              // Diagnostic delivery never grants authority or prevents the existing refusal.
+            }
+          }
           socket.destroy();
           // If retirement began, never accept another lease in a half-closed state.
           if (state === "closing") {

src/backends/native-https-owner.ts

@@ -36,6 +36,7 @@ import {
   nativeHttpsReadRetiredOwner,
   nativeHttpsWriteNew,
 } from "./native-https-owner-storage.ts";
+import { nativeHttpsReleaseReplyError } from "./native-https-release-failure.ts";
 import {
   NativeHttpsStartupError,
   readNativeHttpsStartupFailure,
@@ -377,7 +378,16 @@ export async function requestNativeHttpsOwner(
         return;
       }
       try {
-        finish(undefined, decodeNativeHttpsOwnerFrame(bytes));
+        const reply = decodeNativeHttpsOwnerFrame(bytes);
+        if (
+          isRecord(value) &&
+          value.operation === "release" &&
+          isNativeHttpsLeaseIdentity(value.identity)
+        ) {
+          finish(nativeHttpsReleaseReplyError(reply, value.identity), reply);
+          return;
+        }
+        finish(undefined, reply);
       } catch {
         finish(nativeHttpsOwnerRefused());
       }

src/backends/native-https-release-failure.ts

@@ -0,0 +1,127 @@
+import type { Socket } from "node:net";
+import { isRecord } from "../lib/guards.ts";
+import {
+  encodeNativeHttpsOwnerFrame,
+  isNativeHttpsLeaseIdentity,
+  type NativeHttpsLeaseIdentity,
+  nativeHttpsOwnerRefused,
+  sameNativeHttpsLease,
+} from "./native-https-owner-protocol.ts";
+import { NativeRuntimeRequestError } from "./native-runtime-client.ts";
+
+const STAGES = [
+  "owner-validation",
+  "lease-validation",
+  "graph-verification",
+  "idle-verification",
+  "frontend-close",
+  "release-publication",
+  "owner-retirement",
+] as const;
+export type NativeHttpsReleaseStage = (typeof STAGES)[number];
+const CODES = new Set([
+  "provider_busy",
+  "provider_state",
+  "foreign_state",
+  "invalid_receipt",
+  "recovery_required",
+  "host_endpoint_identity",
+  "engine_protocol",
+]);
+
+/** A failure response carries no proof of effects: successful retirement requires its normal acknowledgement. */
+export function nativeHttpsReleaseFailureFrame(opts: {
+  readonly identity: NativeHttpsLeaseIdentity;
+  readonly stage: NativeHttpsReleaseStage;
+  readonly error: unknown;
+}): Buffer {
+  if (
+    !(
+      isNativeHttpsLeaseIdentity(opts.identity) &&
+      STAGES.some((stage) => stage === opts.stage)
+    )
+  ) {
+    throw nativeHttpsOwnerRefused();
+  }
+  const nativeCode =
+    opts.error instanceof NativeRuntimeRequestError &&
+    opts.error.nativeCode &&
+    CODES.has(opts.error.nativeCode)
+      ? opts.error.nativeCode
+      : null;
+  return encodeNativeHttpsOwnerFrame({
+    version: 1,
+    ok: false,
+    operation: "release",
+    identity: opts.identity,
+    stage: opts.stage,
+    nativeCode,
+  });
+}
+
+/** Reject extra fields, arbitrary diagnostics and another attempt's response before reporting any code. */
+export function parseNativeHttpsReleaseFailure(
+  value: unknown,
+  identity: NativeHttpsLeaseIdentity
+): Error {
+  if (
+    !(
+      isRecord(value) &&
+      Object.keys(value).sort().join() ===
+        "identity,nativeCode,ok,operation,stage,version" &&
+      value.version === 1 &&
+      value.ok === false &&
+      value.operation === "release" &&
+      isNativeHttpsLeaseIdentity(value.identity) &&
+      sameNativeHttpsLease(identity, value.identity) &&
+      STAGES.some((stage) => stage === value.stage) &&
+      (value.nativeCode === null ||
+        (typeof value.nativeCode === "string" && CODES.has(value.nativeCode)))
+    )
+  ) {
+    return nativeHttpsOwnerRefused();
+  }
+  return new Error(
+    `Native HTTPS release is unconfirmed (${value.stage}${value.nativeCode ? `: ${value.nativeCode}` : ""}); ownership evidence is retained, no request was replayed. Values omitted.`
+  );
+}
+
+/** Best-effort bounded delivery, then the caller closes the failed request connection. */
+export async function sendNativeHttpsReleaseFailure(
+  socket: Socket,
+  frame: Buffer
+): Promise<void> {
+  await new Promise<void>((resolve) => {
+    const finish = () => {
+      clearTimeout(timer);
+      socket.off("close", finish);
+      socket.off("error", finish);
+      resolve();
+    };
+    const timer = setTimeout(finish, 1000);
+    socket.once("close", finish);
+    socket.once("error", finish);
+    try {
+      socket.write(frame, finish);
+    } catch {
+      finish();
+    }
+  });
+}
+
+/** A diagnostic, malformed reply, or lost response never acknowledges release. */
+export function nativeHttpsReleaseReplyError(
+  value: unknown,
+  identity: NativeHttpsLeaseIdentity
+): Error | undefined {
+  if (
+    isRecord(value) &&
+    Object.keys(value).sort().join() === "ok,released,version" &&
+    value.version === 1 &&
+    value.ok === true &&
+    value.released === identity.leaseId
+  ) {
+    return;
+  }
+  return parseNativeHttpsReleaseFailure(value, identity);
+}