diff --git a/CHANGELOG.md b/CHANGELOG.md index 733ef55..3d2bc86 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,16 @@ project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added + +- **Demo GIF in the README.** Shows the heal loop end to end: a + supervised run OOMs, the `oom_halve_batch` rule restarts it with + `BATCH_SIZE` halved, training completes, and the incident lands in the + ledger. It's a real run — the stand-in `train.py` genuinely fails at + `BATCH_SIZE=8` and genuinely succeeds at `4`, so a regression in the + healing path shows up as a demo that no longer demonstrates anything. + Source and regeneration steps in `docs/demo/`. + ## [0.14.0] — 2026-07-28 ### Fixed diff --git a/README.md b/README.md index 90fcab1..94d630c 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,12 @@ Watch a command, catch the failure, fix it, leave a paper trail. [![pyrefly](https://img.shields.io/badge/typed-pyrefly-blueviolet.svg)](https://github.com/facebook/pyrefly) [![License: Apache 2.0](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](./LICENSE) +
+ +A terminal session: autosentry supervises a training run, catches a CUDA out-of-memory error in the log, matches it to the oom_halve_batch rule, restarts the process with BATCH_SIZE halved from 8 to 4, and the run completes. The process log shows the crash at batch_size=8 and the success at batch_size=4; autosentry incidents list shows the incident recorded with its detector, rule, and action. + +A real run — `train.py` actually OOMs at `BATCH_SIZE=8`. Source in docs/demo/. + `autosentry` supervises a long-running command — an ML training run, a data diff --git a/docs/demo.gif b/docs/demo.gif new file mode 100644 index 0000000..860b2bc Binary files /dev/null and b/docs/demo.gif differ diff --git a/docs/demo/README.md b/docs/demo/README.md new file mode 100644 index 0000000..f943b49 --- /dev/null +++ b/docs/demo/README.md @@ -0,0 +1,49 @@ +# README demo + +Source for `docs/demo.gif` — the GIF at the top of the main README. + +The demo is a real autosentry run, not a mock-up. `train.py` genuinely +runs out of memory at `BATCH_SIZE=8` and genuinely succeeds at `4`, so +the `oom_halve_batch` rule has an actual failure to fix. If the healing +path breaks, re-rendering the GIF fails to show a fix — which is the +point of keeping it reproducible. + +## Files + +| File | What it is | +|---|---| +| `demo.tape` | [VHS](https://github.com/charmbracelet/vhs) script — the source of truth for the recording | +| `train.py` | Stand-in training script: OOMs at `BATCH_SIZE >= 8`, completes below it | +| `autosentry.yaml` | Demo config — one `oom` pattern detector, one rule, Claude disabled | + +## Regenerating + +Needs `vhs` (which pulls in `ttyd` + `ffmpeg`) and `gifsicle`: + +```bash +brew install vhs gifsicle +``` + +From the repo root: + +```bash +make install # so `autosentry` is the local build +PATH="$(make -s venv-info)/bin:$PATH" vhs docs/demo/demo.tape +gifsicle -O3 --lossy=30 --colors 64 docs/demo.gif -o docs/demo.gif +``` + +The tape puts `PATH` in front deliberately: it records whatever +`autosentry` resolves to, so without this the GIF would silently capture +a stale globally-installed version instead of the code in this checkout. + +The recording runs out of `/tmp/autosentry-demo` (set up and torn down +inside the tape) rather than the repo, so the supervisor's absolute +`cwd=` stays short enough to read at GIF width and each take starts from +a clean state directory. + +## If you change the demo + +Keep it under ~25 seconds and re-run `gifsicle` — the committed GIF +should stay well under 1 MB so the README loads fast. Check the result +before committing; a truncated GIF renders as a broken image on GitHub +and is easy to miss locally. diff --git a/docs/demo/autosentry.yaml b/docs/demo/autosentry.yaml new file mode 100644 index 0000000..2e9af9b --- /dev/null +++ b/docs/demo/autosentry.yaml @@ -0,0 +1,41 @@ +process: + kind: local + command: ["python3", "train.py"] + cwd: "." + restart_policy: + max_restarts: 5 + cooldown_seconds: 0 + +monitor: + # Tighter than the 30s default so the demo doesn't spend its runtime + # waiting on a poll tick. + poll_interval_seconds: 1 + log_dir: ".autosentry/logs" + +detectors: + - kind: pattern + name: oom + regex: "(OutOfMemoryError|CUDA out of memory)" + - kind: exit_code + +rules: + - name: oom_halve_batch + match: { detector: oom } + action: + kind: restart_with_env + set: + BATCH_SIZE: half + notify: true + +healing: + claude: + enabled: false + +vault: + enabled: false + +notifiers: + - kind: log + +state_path: ".autosentry/state.json" +incidents_dir: ".autosentry/incidents" diff --git a/docs/demo/demo.tape b/docs/demo/demo.tape new file mode 100644 index 0000000..61d0d63 --- /dev/null +++ b/docs/demo/demo.tape @@ -0,0 +1,51 @@ +# autosentry README demo — the whole heal loop in one take. +# +# Regenerate from the repo root: +# PATH="$(make -s venv-info)/bin:$PATH" vhs docs/demo/demo.tape +# +# The recording runs out of /tmp/autosentry-demo rather than the repo so +# the supervisor's absolute `cwd=` in the log stays short and readable. +# Everything it needs is copied there in the hidden setup below, so the +# demo is a clean room every run — no leftover state from a prior take. + +Output docs/demo.gif + +Set Shell "bash" +Set FontSize 15 +Set Width 1000 +Set Height 720 +Set Padding 20 +Set Theme "Dracula" +Set TypingSpeed 40ms + +# ----- setup (not recorded) -------------------------------------------- +Hide +Type "rm -rf /tmp/autosentry-demo && mkdir -p /tmp/autosentry-demo/.autosentry" +Enter +Type "cp docs/demo/train.py /tmp/autosentry-demo/" +Enter +Type "cp docs/demo/autosentry.yaml /tmp/autosentry-demo/.autosentry/" +Enter +# BATCH_SIZE has to be a real env var, not just a config value: the +# `half` keyword resolves against the supervisor's environment. +# DEMO_STEP_DELAY paces train.py so the log is readable at GIF speed. +Type "cd /tmp/autosentry-demo && export BATCH_SIZE=8 DEMO_STEP_DELAY=0.45" +Enter +Type "clear" +Enter +Show + +# ----- beat 1: it catches the failure and fixes it ---------------------- +Type "autosentry run" +Enter +Sleep 9s + +# ----- beat 2: the fix actually worked ---------------------------------- +Type `grep -E "batch_size|OutOfMemory|complete" .autosentry/logs/process.log` +Enter +Sleep 4s + +# ----- beat 3: the paper trail ------------------------------------------ +Type "autosentry incidents list" +Enter +Sleep 5s diff --git a/docs/demo/train.py b/docs/demo/train.py new file mode 100644 index 0000000..b931b60 --- /dev/null +++ b/docs/demo/train.py @@ -0,0 +1,38 @@ +"""Stand-in training script for the autosentry demo. + +Emits realistic training log lines. At BATCH_SIZE >= 8 it runs out of +memory partway through and dies with a CUDA OOM traceback; at anything +smaller it trains to completion. That gives the demo a failure the +`oom_halve_batch` rule can actually fix, rather than a scripted fake. +""" + +import os +import sys +import time + +BATCH = int(os.environ.get("BATCH_SIZE", "8")) +STEP_DELAY = float(os.environ.get("DEMO_STEP_DELAY", "0.28")) + + +def log(msg: str) -> None: + print(msg, flush=True) + + +log(f"loading dataset shards … ok (batch_size={BATCH})") +time.sleep(STEP_DELAY) + +for step in range(1, 6): + if BATCH >= 8 and step == 4: + sys.stderr.write( + "Traceback (most recent call last):\n" + ' File "train.py", line 61, in \n' + " loss = model(batch).backward()\n" + "torch.cuda.OutOfMemoryError: CUDA out of memory. Tried to " + "allocate 2.31 GiB (GPU 0; 23.68 GiB total capacity)\n" + ) + sys.stderr.flush() + raise SystemExit(1) + log(f"step {step}/5 loss {2.91 - step * 0.17:.3f} {14 * BATCH} tok/s") + time.sleep(STEP_DELAY) + +log("training complete — checkpoint written to out/final.pt")