Retry and rate-pace TestOps uploads

packages/cli/src/commands/open.ts

@@ -10,6 +10,7 @@ import { Command, Option } from "clipanion";
 import { red } from "yoctocolors";
 
 import { findFilesByGlobs } from "./../utils/fileSystem.js";
+import { notifySignals, waitForAbort } from "./../utils/signals.js";
 import { generate } from "./commons/generate.js";
 
 export class OpenCommand extends Command {
@@ -83,12 +84,14 @@ export class OpenCommand extends Command {
       });
 
       // clean up temp report directory on ctrl-c
-      process.on("SIGINT", async () => {
+      const notifier = notifySignals(["SIGINT", "SIGTERM"]);
+
+      void waitForAbort(notifier.signal).then(async () => {
         try {
           await rm(config.output, { recursive: true });
         } catch {}
 
-        process.exit(0);
+        process.exit(notifier.info()?.code ?? 0);
       });
 
       await serve({

packages/cli/src/utils/index.ts

@@ -3,3 +3,4 @@ export * from "./terminal.js";
 export * from "./logs.js";
 export * from "./execution-context.js";
 export * from "./fileSystem.js";
+export * from "./signals.js";

packages/cli/src/utils/signals.ts

@@ -0,0 +1,92 @@
+import process from "node:process";
+
+export const SIGNAL_EXIT_CODES: Partial<Record<NodeJS.Signals, number>> = {
+  SIGINT: 130,
+  SIGTERM: 143,
+};
+
+export const GRACEFUL_SHUTDOWN_TIMEOUT_MS = 60_000;
+export const COMMAND_TERMINATION_GRACE_MS = 5_000;
+
+export type SignalInfo = {
+  signal: NodeJS.Signals;
+  code: number;
+  receivedAt: number;
+  deadline: number;
+};
+
+const exitCodeForSignal = (signal: NodeJS.Signals): number => SIGNAL_EXIT_CODES[signal] ?? 1;
+
+export type SignalNotifier = {
+  /** aborts the moment the first SIGINT/SIGTERM is received */
+  signal: AbortSignal;
+  /** the signal that triggered the abort, once received */
+  info: () => SignalInfo | undefined;
+  /** stop listening for signals */
+  dispose: () => void;
+};
+
+export const notifySignals = (
+  signals: NodeJS.Signals[] = ["SIGINT", "SIGTERM"],
+  onRepeat?: (signal: NodeJS.Signals) => void,
+): SignalNotifier => {
+  const controller = new AbortController();
+  let info: SignalInfo | undefined;
+
+  const handler = (signal: NodeJS.Signals) => {
+    if (info) {
+      onRepeat?.(signal);
+      return;
+    }
+
+    const receivedAt = Date.now();
+
+    info = {
+      signal,
+      code: exitCodeForSignal(signal),
+      receivedAt,
+      deadline: receivedAt + GRACEFUL_SHUTDOWN_TIMEOUT_MS,
+    };
+
+    controller.abort();
+  };
+
+  for (const signal of signals) {
+    process.on(signal, handler);
+  }
+
+  return {
+    signal: controller.signal,
+    info: () => info,
+    dispose: () => {
+      for (const signal of signals) {
+        process.off(signal, handler);
+      }
+    },
+  };
+};
+
+/** time left until the graceful-shutdown deadline, floored at 0; undefined if no signal received yet */
+export const gracefulShutdownRemaining = (info: SignalInfo | undefined): number | undefined => {
+  if (!info) {
+    return undefined;
+  }
+
+  return Math.max(0, info.deadline - Date.now());
+};
+
+export const boundedTerminationSignal = (
+  info: SignalInfo | undefined,
+  graceMs: number = COMMAND_TERMINATION_GRACE_MS,
+): AbortSignal => {
+  const remaining = gracefulShutdownRemaining(info);
+  const timeoutMs = remaining === undefined ? graceMs : Math.min(remaining, graceMs);
+
+  return AbortSignal.timeout(timeoutMs);
+};
+
+/** resolves once the given signal aborts */
+export const waitForAbort = (signal: AbortSignal): Promise<void> =>
+  signal.aborted
+    ? Promise.resolve()
+    : new Promise((resolve) => signal.addEventListener("abort", () => resolve(), { once: true }));

packages/plugin-testops/README.md

@@ -80,6 +80,8 @@ The plugin accepts the following options:
 | `autocloseLaunch`  | When `true` (default), the launch is closed automatically when the plugin finishes; set to `false` to keep the launch open | `boolean` | `true`          |
 | `gitFlow`          | When `true`, collect Git metadata for TestOps Git Flow on CI uploads (opt-in)                                 | `boolean` | `false`         |
 | `ancestorLimit`    | How many ancestor commits to attach to the launch for history linking in TestOps | `number`  | `100`           |
+| `uploadRateLimit`  | Caps how fast uploads are sent to TestOps within a rolling time window (requests/files/bytes per window); pass `false` to disable pacing entirely | `{ windowMs: number; maxRequestsPerWindow?: number; maxFilesPerWindow?: number; maxBytesPerWindow?: number } \| false` | `20` req/s, `1000` files/s, `1 GiB`/s |
+| `reopenClosedLaunch` | When `true`, a launch that TestOps reports as closed is reopened automatically instead of failing the upload | `boolean` | `false`         |
 
 ### Using options from environment variables