fix: preserve native HTTPS startup failure diagnostics

docs/guides/native-candidate.md

@@ -1386,3 +1386,14 @@ Legacy mappings without explicit selectors and services with older launcher moun
 refuse before execution; restart with the matching candidate to adopt this path.
 Services without managed values use ordinary exec. The native live qualification
 of this fresh-delivery path remains separate from its unit and transport tests.
+
+
+A detached HTTPS helper that fails during startup may retain a generation-bound
+`startup-failure.json` beside its owner configuration. The CLI reports only the
+reviewed startup stage and an allowlisted native error code; child output, paths
+and application values are omitted. If lease cleanup also fails, the original
+acquisition diagnostic remains visible alongside the unconfirmed cleanup status.
+This record is diagnostic evidence only: it does not acknowledge retirement or
+permit a replacement owner. A missing record (including a helper crash before
+publication) leaves the cause unknown. Preserve the retained owner and finalization
+records for inspection; do not delete them to force another startup.

src/backends/native-https-owner-server.ts

@@ -38,6 +38,10 @@ import {
   nativeHttpsRemoveFile,
   nativeHttpsWriteNew,
 } from "./native-https-owner-storage.ts";
+import {
+  NativeHttpsStartupError,
+  recordNativeHttpsStartupFailure,
+} from "./native-https-startup-failure.ts";
 import { startNativeProjectHttps } from "./native-project-https.ts";
 import { invokeNativeRuntime } from "./native-runtime-client.ts";
 
@@ -580,6 +584,8 @@ export async function serveNativeHttpsOwner(opts: {
     void serialize(failOwner);
   };
   let startupTimer: ReturnType<typeof setTimeout> | undefined;
+  let startupStage: ConstructorParameters<typeof NativeHttpsStartupError>[0] =
+    "frontend-start";
   try {
     frontend = await opts.dependencies.start(binding);
     const exited = () => {
@@ -592,7 +598,9 @@ export async function serveNativeHttpsOwner(opts: {
     // Keep the published inode at once: failure cleanup then closes this listener
     // and retires only this inode, and every later observation compares against it.
     // The helper set its mode before publication, so the path is never chmodded.
+    startupStage = "control-publication";
     socketIdentity = await listenPublishedUnixSocket(server, socketPath);
+    startupStage = "control-verification";
     await opts.dependencies.afterPublish?.(socketPath);
     const published = await lstat(socketPath);
     if (
@@ -607,6 +615,7 @@ export async function serveNativeHttpsOwner(opts: {
     if (frontendFailed) {
       throw nativeHttpsOwnerRefused();
     }
+    startupStage = "endpoint-publication";
     endpointIdentity = await nativeHttpsWriteNew(join(root, "endpoint.json"), {
       version: 1,
       ownerGeneration,
@@ -632,7 +641,24 @@ export async function serveNativeHttpsOwner(opts: {
       });
     }, opts.startupGraceMs ?? 30_000);
     await completed;
-  } catch {
+  } catch (error) {
+    if (state === "starting") {
+      try {
+        const currentRoot = await lstat(root);
+        if (
+          currentRoot.dev === rootIdentity.dev &&
+          currentRoot.ino === rootIdentity.ino
+        ) {
+          await recordNativeHttpsStartupFailure({
+            configuration,
+            configurationIdentity: configurationFile.identity,
+            error: new NativeHttpsStartupError(startupStage, error),
+          });
+        }
+      } catch {
+        // Diagnostics never overwrite foreign state or change cleanup authority.
+      }
+    }
     await failOwner();
     throw nativeHttpsOwnerRefused();
   } finally {

src/backends/native-https-owner.ts

@@ -36,6 +36,10 @@ import {
   nativeHttpsReadRetiredOwner,
   nativeHttpsWriteNew,
 } from "./native-https-owner-storage.ts";
+import {
+  NativeHttpsStartupError,
+  readNativeHttpsStartupFailure,
+} from "./native-https-startup-failure.ts";
 import {
   inspectActiveNativeHttpsOwner,
   verifyActiveNativeHttpsConnection,
@@ -82,11 +86,23 @@ export interface NativeHttpsAcquireOptions {
 }
 export class NativeHttpsLeaseAcquisitionError extends Error {
   readonly identity: NativeHttpsLeaseIdentity;
-  constructor(identity: NativeHttpsLeaseIdentity) {
+  readonly startupFailure: NativeHttpsStartupError | undefined;
+  constructor(
+    identity: NativeHttpsLeaseIdentity,
+    opts?: {
+      readonly startupFailure?: unknown;
+      readonly cleanupUnconfirmed?: true;
+    }
+  ) {
+    const startupFailure =
+      opts?.startupFailure instanceof NativeHttpsStartupError
+        ? new NativeHttpsStartupError("frontend-start", opts.startupFailure)
+        : undefined;
     super(
-      "Shared native HTTPS acquisition is uncertain; use the retained exact lease identity after graph cleanup."
+      `Shared native HTTPS acquisition is uncertain; use the retained exact lease identity after graph cleanup.${startupFailure ? ` ${startupFailure.message}` : ""}${opts?.cleanupUnconfirmed ? " Native startup cleanup is unconfirmed; retained ownership evidence was kept." : ""}`
     );
     this.identity = identity;
+    this.startupFailure = startupFailure;
   }
 }
 
@@ -253,6 +269,10 @@ async function endpointFor(
   ) {
     throw nativeHttpsOwnerRefused();
   }
+  const startupFailure = await readNativeHttpsStartupFailure(configuration);
+  if (startupFailure) {
+    throw startupFailure;
+  }
   const { bytes } = await nativeHttpsReadFile(
     join(
       nativeHttpsOwnerRoot(configuration.binding.runtime.home),
@@ -418,8 +438,8 @@ export async function acquireNativeHttpsLease(
   let socket: Socket;
   try {
     socket = await connectNativeHttpsOwner(configuration);
-  } catch {
-    throw new NativeHttpsLeaseAcquisitionError(intended);
+  } catch (startupFailure) {
+    throw new NativeHttpsLeaseAcquisitionError(intended, { startupFailure });
   }
   const exited = new Promise<{ component: string; code: number }>((resolve) => {
     socket.once("close", () =>

src/backends/native-https-startup-failure.ts

@@ -0,0 +1,156 @@
+import { join } from "node:path";
+import { isRecord } from "../lib/guards.ts";
+import {
+  type NativeHttpsOwnerConfiguration,
+  nativeHttpsOwnerRefused,
+} from "./native-https-owner-protocol.ts";
+import {
+  type NativeHttpsFileIdentity,
+  nativeHttpsOwnerRoot,
+  nativeHttpsPrivateDirectory,
+  nativeHttpsReadFile,
+  nativeHttpsWriteNew,
+} from "./native-https-owner-storage.ts";
+import { NativeRuntimeRequestError } from "./native-runtime-client.ts";
+
+const STAGES = [
+  "frontend-start",
+  "owner-challenge",
+  "authority-observation",
+  "authority-start",
+  "authority-ready",
+  "caddy-start",
+  "caddy-ready",
+  "listener-verification",
+  "owner-publication",
+  "owner-verification",
+  "control-publication",
+  "control-verification",
+  "endpoint-publication",
+] as const;
+type Stage = (typeof STAGES)[number];
+const NATIVE_CODES = new Set([
+  "provider_busy",
+  "provider_state",
+  "foreign_state",
+  "invalid_receipt",
+  "recovery_required",
+  "host_endpoint_identity",
+  "engine_protocol",
+]);
+interface Diagnostic {
+  readonly stage: Stage;
+  readonly nativeCode: string | null;
+  readonly cleanupUnconfirmed: boolean;
+}
+function isDiagnostic(value: unknown): value is Diagnostic {
+  return (
+    isRecord(value) &&
+    Object.keys(value).sort().join() ===
+      "cleanupUnconfirmed,nativeCode,stage" &&
+    STAGES.some((stage) => stage === value.stage) &&
+    (value.nativeCode === null ||
+      (typeof value.nativeCode === "string" &&
+        NATIVE_CODES.has(value.nativeCode))) &&
+    typeof value.cleanupUnconfirmed === "boolean"
+  );
+}
+/** Only reviewed classifications cross the detached owner boundary; never raw errors. */
+export class NativeHttpsStartupError extends Error {
+  readonly diagnostic: Diagnostic;
+  constructor(stage: Stage, error?: unknown, cleanupUnconfirmed = false) {
+    const inherited =
+      error instanceof NativeHttpsStartupError && isDiagnostic(error.diagnostic)
+        ? error.diagnostic
+        : undefined;
+    const nativeCode =
+      error instanceof NativeRuntimeRequestError &&
+      error.nativeCode &&
+      NATIVE_CODES.has(error.nativeCode)
+        ? error.nativeCode
+        : null;
+    const diagnostic = {
+      stage,
+      nativeCode,
+      ...inherited,
+      cleanupUnconfirmed:
+        cleanupUnconfirmed || inherited?.cleanupUnconfirmed === true,
+    };
+    super(
+      `Native HTTPS startup failed (${diagnostic.stage}${diagnostic.nativeCode ? `: ${diagnostic.nativeCode}` : ""})${diagnostic.cleanupUnconfirmed ? "; child cleanup unconfirmed" : ""}; values omitted.`
+    );
+    this.diagnostic = diagnostic;
+  }
+}
+/** Diagnostic evidence only: this record never acknowledges cleanup or permits adoption. */
+export async function recordNativeHttpsStartupFailure(opts: {
+  readonly configuration: NativeHttpsOwnerConfiguration;
+  readonly configurationIdentity: NativeHttpsFileIdentity;
+  readonly error: NativeHttpsStartupError;
+}): Promise<void> {
+  const root = nativeHttpsOwnerRoot(opts.configuration.binding.runtime.home);
+  await nativeHttpsPrivateDirectory(root);
+  const current = await nativeHttpsReadFile(join(root, "configuration.json"));
+  const expected = opts.configurationIdentity;
+  if (
+    current.identity.dev !== expected.dev ||
+    current.identity.ino !== expected.ino ||
+    current.identity.sha256 !== expected.sha256 ||
+    !isDiagnostic(opts.error.diagnostic)
+  ) {
+    throw nativeHttpsOwnerRefused();
+  }
+  await nativeHttpsWriteNew(join(root, "startup-failure.json"), {
+    version: 1,
+    ownerGeneration: opts.configuration.ownerGeneration,
+    configurationSha256: expected.sha256,
+    diagnostic: opts.error.diagnostic,
+  });
+}
+export async function readNativeHttpsStartupFailure(
+  configuration: NativeHttpsOwnerConfiguration
+): Promise<NativeHttpsStartupError | undefined> {
+  const root = nativeHttpsOwnerRoot(configuration.binding.runtime.home);
+  await nativeHttpsPrivateDirectory(root);
+  let bytes: Buffer;
+  try {
+    ({ bytes } = await nativeHttpsReadFile(join(root, "startup-failure.json")));
+  } catch (error) {
+    if (isRecord(error) && error.code === "ENOENT") {
+      return;
+    }
+    throw nativeHttpsOwnerRefused();
+  }
+  const current = await nativeHttpsReadFile(join(root, "configuration.json"));
+  let value: unknown;
+  let currentConfiguration: unknown;
+  try {
+    value = JSON.parse(bytes.toString("utf8"));
+    currentConfiguration = JSON.parse(current.bytes.toString("utf8"));
+  } catch {
+    throw nativeHttpsOwnerRefused();
+  }
+  if (
+    !isRecord(value) ||
+    Object.keys(value).sort().join() !==
+      "configurationSha256,diagnostic,ownerGeneration,version" ||
+    value.version !== 1 ||
+    JSON.stringify(currentConfiguration) !== JSON.stringify(configuration) ||
+    value.ownerGeneration !== configuration.ownerGeneration ||
+    value.configurationSha256 !== current.identity.sha256 ||
+    !isDiagnostic(value.diagnostic)
+  ) {
+    throw nativeHttpsOwnerRefused();
+  }
+  const diagnostic = value.diagnostic;
+  return new NativeHttpsStartupError(
+    diagnostic.stage,
+    diagnostic.nativeCode
+      ? new NativeRuntimeRequestError({
+          message: "",
+          nativeCode: diagnostic.nativeCode,
+        })
+      : undefined,
+    diagnostic.cleanupUnconfirmed
+  );
+}

src/backends/native-project-https.ts

@@ -28,6 +28,7 @@ import { connect as tlsConnect } from "node:tls";
 import { isRecord } from "../lib/guards.ts";
 import { listenPublishedUnixSocket } from "../lib/unix-socket-publish.ts";
 import { checkNativeHttpsPort } from "./native-https-port.ts";
+import { NativeHttpsStartupError } from "./native-https-startup-failure.ts";
 import {
   invokeNativeRuntime,
   type NativeRuntimeSelection,
@@ -1374,11 +1375,14 @@ export async function startNativeProjectHttps(opts: {
       }
       await removeOwnedDirectory(lock, lockIdentity, false);
     })());
+  let startupStage: ConstructorParameters<typeof NativeHttpsStartupError>[0] =
+    "owner-challenge";
   try {
     owner = await startOwnerChallenge(
       ownerSocket,
       deps.afterOwnerSocketPublish
     );
+    startupStage = "authority-observation";
     const before = await inspect();
     if (
       !(
@@ -1394,6 +1398,7 @@ export async function startNativeProjectHttps(opts: {
     await privateDirectory(session);
     sessionIdentity = await lstat(session);
     const env: Record<string, string> = { PATH: "/usr/bin:/bin", HOME: home };
+    startupStage = "authority-start";
     authority = deps.spawn({
       argv: [
         opts.runtime.binary,
@@ -1413,6 +1418,7 @@ export async function startNativeProjectHttps(opts: {
     });
     const deadline = Date.now() + 10_000;
     const startedAuthority = authority;
+    startupStage = "authority-ready";
     ownedAuthority = await waitReady(
       deadline,
       () => dead,
@@ -1438,6 +1444,7 @@ export async function startNativeProjectHttps(opts: {
       }),
       { mode: 0o600, flag: "wx" }
     );
+    startupStage = "caddy-start";
     caddy = deps.spawn({
       argv: [
         opts.caddyBinary,
@@ -1459,6 +1466,7 @@ export async function startNativeProjectHttps(opts: {
     void caddy.exited.then(() => {
       dead = true;
     });
+    startupStage = "caddy-ready";
     const caPath = join(data, "caddy/pki/authorities/local/root.crt");
     await waitReady(
       deadline,
@@ -1476,6 +1484,7 @@ export async function startNativeProjectHttps(opts: {
         await validCa(caPath);
       }
     );
+    startupStage = "listener-verification";
     const listener = await inspectHostListener({
       runtime: opts.runtime,
       invoke: deps.invoke,
@@ -1484,6 +1493,7 @@ export async function startNativeProjectHttps(opts: {
       executable: opts.caddyBinary,
     });
     const caSha256 = caDerSha256(await validCa(caPath));
+    startupStage = "owner-publication";
     receiptIdentity = await publishOwnerReceipt({
       path: receiptPath,
       receipt: {
@@ -1504,6 +1514,7 @@ export async function startNativeProjectHttps(opts: {
         },
       },
     });
+    startupStage = "owner-verification";
     const activeOwner = await inspectActiveNativeHttpsOwner({
       runtime: opts.runtime,
       invoke: deps.invoke,
@@ -1533,7 +1544,12 @@ export async function startNativeProjectHttps(opts: {
       close,
     };
   } catch (error) {
-    await close();
-    throw error;
+    let cleanupUnconfirmed = false;
+    try {
+      await close();
+    } catch {
+      cleanupUnconfirmed = true;
+    }
+    throw new NativeHttpsStartupError(startupStage, error, cleanupUnconfirmed);
   }
 }