Stop retrying a failure that has already repeated N times (#25)

README.md

@@ -524,6 +524,7 @@ The top-level shape:
 | `process.cwd`      | str           | `.`                              | working dir, relative to the project root (the dir containing `.autosentry/`) |
 | `process.env`      | dict[str,str] | `{}`                             | env vars; values can interpolate `$VAR` / `${VAR}` |
 | `process.restart_policy.max_restarts` | int | `10`                 | when exceeded, monitor gives up |
+| `process.restart_policy.max_identical_failures` | int | `5` | consecutive *identical* failures before giving up, regardless of `max_restarts`. A deterministic error (bad path, missing binary) otherwise burns the whole budget one repeat at a time. A different failure resets the streak; `0` disables |
 | `process.restart_policy.cooldown_seconds` | int | `60`             | wait before restart |
 | `process.lifecycle`                   | enum          | `restart_on_failure`             | `restart_on_failure` (clean exit ends the supervisor; default since 0.8.5), `one_shot` (any exit ends it), `restart_always` (both clean and dirty exits route through the healer — pre-0.8.5 behavior) |
 | `dispatch.mode`                       | enum          | `builtin`                        | `builtin` (monitor runs the healer) or `session` (Claude Code session dispatches via `/autosentry` skill — preferred for Claude Code users; free) |

src/autosentry/config.py

@@ -49,6 +49,30 @@ class RestartPolicy(BaseModel):
     # entirely (the supervisor will keep restarting until something
     # external stops it).
     max_restarts: int = 50
+
+    # Consecutive *identical* failures before the supervisor stops retrying.
+    #
+    # ``max_restarts`` is a budget for how many times a fix may fail to
+    # stick. It cannot tell a transient apart from a deterministic error,
+    # because both just decrement it. So a config mistake -- a missing
+    # binary, a bad path, an unwritable directory -- consumes the whole
+    # budget one identical failure at a time, and on a long pipeline that
+    # is measured in hours.
+    #
+    # Reported case: a stage failed with ``pixi: command not found``
+    # (exit 127, a PATH problem that could never resolve itself) and
+    # retried 55 times across ~14 hours, each attempt re-running a
+    # completed job's 10-minute evaluation step.
+    #
+    # This counts consecutive detections carrying the same signature and
+    # gives up once they exceed the cap, regardless of remaining
+    # ``max_restarts``. Any *different* detection resets the run: a flaky
+    # error interleaved with real progress is not a repeat.
+    #
+    # Default 5: enough that a genuinely transient fault repeating a few
+    # times still heals, low enough that an unfixable one surfaces in
+    # minutes instead of overnight. ``0`` disables the guard.
+    max_identical_failures: int = 5
     backoff: Literal["fixed", "exponential"] = "exponential"
     cooldown_seconds: int = 60
 

src/autosentry/monitor.py

@@ -11,6 +11,7 @@
 from __future__ import annotations
 
 import json
+import re
 import signal
 import threading
 import time
@@ -134,6 +135,9 @@ def __init__(self, cfg: AutoSentryConfig, *, stage: StageContext | None = None)
         self._recent_attempts: dict[str, list[float]] = {}
         # When a budget burns through, remember which detectors are paused.
         self._budget_paused: set[str] = set()
+        # Consecutive-identical-failure tracking; see _note_repeat.
+        self._repeat_signature: str | None = None
+        self._repeat_count: int = 0
         # Force-Claude escalation. Flipped on when state.restarts reaches
         # the threshold; flipped off on the next kept verification. While
         # set, the next detection skips the rule healer and goes straight
@@ -678,6 +682,36 @@ def _restart_policy_fallback(self, det: Detection) -> None:
             ),
         )
 
+    @staticmethod
+    def _detection_signature(det: Detection) -> str:
+        """Identity of a failure for repeat-detection purposes.
+
+        Digits are collapsed so that messages differing only by a pid,
+        timestamp, line number or byte count still count as the same
+        failure -- "exited with code 127" at 03:11 and at 03:24 is one
+        recurring problem, not two.
+        """
+        return f"{det.detector}:{det.kind}:{re.sub(r'[0-9]+', '#', det.message)[:200]}"
+
+    def _note_repeat(self, det: Detection) -> bool:
+        """Track consecutive identical failures; True once the cap is passed.
+
+        Returns True only on the transition past the cap and on every
+        detection after it, so the caller can stop retrying. A detection
+        with a different signature resets the streak: interleaved
+        progress means the failure is not deterministic.
+        """
+        cap = self.cfg.process.restart_policy.max_identical_failures
+        if cap <= 0:
+            return False
+        sig = self._detection_signature(det)
+        if sig == self._repeat_signature:
+            self._repeat_count += 1
+        else:
+            self._repeat_signature = sig
+            self._repeat_count = 1
+        return self._repeat_count > cap
+
     def _fire_detection(self, det: Detection) -> None:
         if det.kind == "anomaly":
             log().anomaly(f"[{det.detector}] {det.message}")
@@ -701,8 +735,30 @@ def _fire_detection(self, det: Detection) -> None:
 
         # Refuse to attempt further fixes for a detector that has burned
         # through its budget. Still write the incident and notify.
+        repeat_exhausted = self._note_repeat(det)
+        if repeat_exhausted:
+            cap = self.cfg.process.restart_policy.max_identical_failures
+            log().error(
+                f"[{det.detector}] this exact failure has now repeated "
+                f"{self._repeat_count} times in a row (cap {cap}) — it is not "
+                f"transient. Recording the incident and stopping; restarting "
+                f"again would only repeat it. Fix the cause, then relaunch."
+            )
+            self._notify(
+                "exit",
+                f"repeated failure: {det.message[:100]}",
+                f"{self._repeat_count} identical failures in a row (cap {cap}); "
+                f"supervisor stopping rather than consuming the restart budget",
+            )
+            # Stop for real. Clearing `outcome` alone only declines to heal;
+            # `_restart_policy_fallback` would still restart the child on the
+            # no-action path, which is the loop this guard exists to break.
+            self._stop = True
+
         if session_dispatch:
             outcome = None
+        elif repeat_exhausted:
+            outcome = None
         elif det.detector in self._budget_paused or self._budget_exhausted(det.detector):
             log().recovery(
                 f"healer budget exhausted for {det.detector!r} — recording incident "

tests/test_repeat_failure_guard.py

@@ -0,0 +1,138 @@
+"""Consecutive-identical-failure guard.
+
+``max_restarts`` is a budget for how many times a fix may fail to stick, and
+it cannot distinguish a transient from a deterministic error because both
+merely decrement it. So a config mistake burns the entire budget one identical
+failure at a time.
+
+The reported case: a pipeline stage failed with ``pixi: command not found``
+(exit 127 — a PATH problem that could never resolve itself) and retried 55
+times over roughly 14 hours, each attempt re-running a completed job's
+10-minute evaluation step.
+
+These tests pin the three behaviours that matter:
+
+* an identical failure repeating past the cap stops the supervisor, whatever
+  ``max_restarts`` still allows;
+* a *different* failure resets the streak, so an occasional error interleaved
+  with real progress is never mistaken for a deterministic one;
+* the guard is off when the cap is 0, preserving the previous behaviour for
+  anyone who wants it.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from textwrap import dedent
+
+from autosentry.config import load_config
+from autosentry.detectors.base import Detection
+from autosentry.monitor import Monitor
+
+
+def _cfg(tmp_path: Path, cap: int) -> Path:
+    p = tmp_path / "autosentry.yaml"
+    p.write_text(
+        dedent(
+            f"""\
+            process:
+              kind: local
+              command: ["true"]
+              restart_policy:
+                max_restarts: 50
+                max_identical_failures: {cap}
+            healing:
+              claude:
+                enabled: false
+            state_path: ".autosentry/state.json"
+            incidents_dir: ".autosentry/incidents"
+            """
+        )
+    )
+    return p
+
+
+def _monitor(tmp_path: Path, cap: int) -> Monitor:
+    return Monitor(load_config(_cfg(tmp_path, cap)))
+
+
+def _det(message: str, detector: str = "exit_code") -> Detection:
+    return Detection(detector=detector, kind="error", message=message)
+
+
+def test_identical_failures_trip_the_guard_at_the_cap(tmp_path: Path) -> None:
+    m = _monitor(tmp_path, cap=3)
+    d = _det("process exited with code 127")
+    # Up to and including the cap the guard stays quiet: a fault that repeats
+    # a few times may still be a slow-healing transient.
+    assert [m._note_repeat(d) for _ in range(3)] == [False, False, False]
+    # Past it, every subsequent detection reports exhausted.
+    assert m._note_repeat(d) is True
+    assert m._note_repeat(d) is True
+
+
+def test_a_different_failure_resets_the_streak(tmp_path: Path) -> None:
+    """Interleaved progress means the failure is not deterministic.
+
+    This is the guard's whole safety property: without it, a flaky error
+    appearing occasionally among successful restarts would eventually trip
+    the cap and stop a supervisor that was recovering perfectly well.
+    """
+    m = _monitor(tmp_path, cap=2)
+    same = _det("process exited with code 127")
+    other = _det("CUDA call failed")
+    assert m._note_repeat(same) is False
+    assert m._note_repeat(same) is False
+    assert m._note_repeat(other) is False  # streak broken, counter restarts
+    assert m._note_repeat(same) is False
+    assert m._note_repeat(same) is False
+    assert m._note_repeat(same) is True  # three in a row past a cap of 2
+
+
+def test_varying_numbers_do_not_defeat_the_guard(tmp_path: Path) -> None:
+    """The signature collapses digits.
+
+    The failure this guard was written for carried a changing pid and
+    timestamp on every retry. Comparing raw message text would have treated
+    all 55 occurrences as distinct and never fired.
+    """
+    m = _monitor(tmp_path, cap=2)
+    for i in range(3):
+        got = m._note_repeat(_det(f"pid {1000 + i}: pixi: command not found"))
+    assert got is True
+
+
+def test_different_detectors_are_never_the_same_failure(tmp_path: Path) -> None:
+    m = _monitor(tmp_path, cap=1)
+    msg = "process exited with code 1"
+    # Same message text, different detector: the switch resets the streak, so
+    # the stall detection starts its own run rather than inheriting the
+    # exit_code one.
+    assert m._note_repeat(_det(msg, detector="exit_code")) is False
+    assert m._note_repeat(_det(msg, detector="stall")) is False
+    assert m._note_repeat(_det(msg, detector="stall")) is True
+
+
+def test_cap_zero_disables_the_guard(tmp_path: Path) -> None:
+    m = _monitor(tmp_path, cap=0)
+    d = _det("process exited with code 127")
+    assert [m._note_repeat(d) for _ in range(20)] == [False] * 20
+
+
+def test_default_cap_is_set(tmp_path: Path) -> None:
+    """A guard that ships off by default protects nobody who hasn't already
+    been bitten, which is exactly the population that needs it."""
+    p = tmp_path / "autosentry.yaml"
+    p.write_text(
+        dedent(
+            """\
+            process:
+              kind: local
+              command: ["true"]
+            state_path: ".autosentry/state.json"
+            incidents_dir: ".autosentry/incidents"
+            """
+        )
+    )
+    cfg = load_config(p)
+    assert cfg.process.restart_policy.max_identical_failures > 0