fix(perf): eliminate redundant terminal wake work

AGENTS.md

@@ -58,7 +58,10 @@ Inside pane-scaled render and layout loops:
 
 Prefer deterministic operation or architecture tests to wall-clock CI limits.
 Performance benchmarks are supporting evidence, not substitutes for behavioral
-coverage.
+coverage. Before a stable release, `just bench-release-smoke` must compare the
+candidate with the current stable binary under hidden and visible output. When
+the result moves materially or when validating performance work, repeat it with
+`HERDR_PERF_SAMPLE_SECONDS=60` and investigate the affected scenario.
 
 ### Runtime/client boundary guardrail
 

CHANGELOG.md

@@ -2,8 +2,6 @@
 
 ## Unreleased
 
-## [0.8.1] - 2026-08-18
-
 ### Added
 - CLI help now points coding agents to Herdr's plain-text guide, documentation index, and built-in control skill.
 - Added Qwen Code detection for idle, working, and user-confirmation states, plus optional native session restore. (#2730, #2743)
@@ -28,6 +26,7 @@
 - Experimental pane graphics now support bounded named layers, acknowledged full-RGBA primary-layer direct file frames on audited local terminals, owned BGRA fallback, exact pixel mouse input, and placement-only resize replay.
 
 ### Fixed
+- High-rate output from many hidden panes no longer floods the server loop with redundant wakeups, and terminal input-mode synchronization no longer formats pane scrollback to read one keyboard flag.
 - Chinese IME commits now reach panes on macOS when the focused application requests printable key-release events. (#2924)
 - Windows now recognizes `Ctrl+1` through `Ctrl+9` keybindings instead of decoding those key records as control characters. (#2910)
 - PowerShell panes now keep their process-reported working directory synchronized with the shell's logical location. (#2879, thanks @Pimpmuckl)

docs/next/CHANGELOG.md

@@ -2,8 +2,6 @@
 
 ## Unreleased
 
-## [0.8.1] - 2026-08-18
-
 ### Added
 - CLI help now points coding agents to Herdr's plain-text guide, documentation index, and built-in control skill.
 - Added Qwen Code detection for idle, working, and user-confirmation states, plus optional native session restore. (#2730, #2743)
@@ -28,6 +26,7 @@
 - Experimental pane graphics now support bounded named layers, acknowledged full-RGBA primary-layer direct file frames on audited local terminals, owned BGRA fallback, exact pixel mouse input, and placement-only resize replay.
 
 ### Fixed
+- High-rate output from many hidden panes no longer floods the server loop with redundant wakeups, and terminal input-mode synchronization no longer formats pane scrollback to read one keyboard flag.
 - Chinese IME commits now reach panes on macOS when the focused application requests printable key-release events. (#2924)
 - Windows now recognizes `Ctrl+1` through `Ctrl+9` keybindings instead of decoding those key records as control characters. (#2910)
 - PowerShell panes now keep their process-reported working directory synchronized with the shell's logical location. (#2879, thanks @Pimpmuckl)

justfile

@@ -73,6 +73,11 @@ build:
 bench-render-scale:
     cargo test --release --locked --bin herdr render_scale_profile -- --ignored --nocapture --test-threads=1
 
+# ~3-5 minute CPU comparison; downloads stable unless HERDR_PERF_BASELINE_BIN is set
+bench-release-smoke:
+    cargo build --release --locked
+    scripts/release_perf_smoke.sh "${CARGO_TARGET_DIR:-target}/release/herdr"
+
 # Build the website and documentation
 website-build:
     cd website && bun install --frozen-lockfile && bun run build
@@ -130,10 +135,11 @@ release-docs-check:
     just website-build
     cd website && bun run build:draft
 
-# Validate release docs and review full-render scaling before release preparation
+# Validate release docs, render scaling, and end-to-end CPU before release preparation
 pre-release-check:
     just release-docs-check
     just bench-render-scale
+    just bench-release-smoke
     @echo "release review required: investigate material render-scaling regressions before publishing."
     @echo "release review required: update skills/herdr/SKILL.md for this stable release so it matches the current CLI, IDs, agent lifecycle semantics, and safety guidance."
     @echo "release policy: do not update skills/herdr/SKILL.md between stable releases; preview builds keep the latest stable skill."

scripts/release_perf_case.sh

@@ -0,0 +1,149 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ $# -ne 8 ]]; then
+  echo "usage: $0 <binary> <variant> <scenario> <round> <seconds> <warmup> <output-root> <platform>" >&2
+  exit 2
+fi
+
+bin_dir=$(cd "$(dirname "$1")" && pwd)
+bin="$bin_dir/$(basename "$1")"
+variant=$2
+scenario=$3
+round=$4
+seconds=$5
+warmup=$6
+out_root=$(cd "$7" && pwd)
+platform=$8
+script_dir=$(cd "$(dirname "$0")" && pwd)
+producer="$script_dir/release_perf_producer.pl"
+cols=86
+rows=47
+
+case "$scenario" in
+  visible30) scenario_tag=v3; total_panes=1; writers=visible; rate=30 ;;
+  hidden50) scenario_tag=h5; total_panes=50; writers=hidden; rate=60 ;;
+  *) echo "unknown scenario: $scenario" >&2; exit 2 ;;
+esac
+case "$platform" in linux) platform_tag=l ;; macos) platform_tag=m ;; *) exit 2 ;; esac
+variant_tag=${variant:0:1}
+name="rps${platform_tag}${variant_tag}${scenario_tag}r${round}x$$"
+state="/var/tmp/herdr-release-perf-$name"
+xdg="$state/xdg"
+runtime="$state/run"
+gate="$state/start-output"
+out="$out_root/$variant/$scenario/r$round"
+mkdir -p "$xdg" "$runtime" "$out"
+
+launch_env=(env -u HERDR_BIN_PATH -u HERDR_ENV -u HERDR_SOCKET_PATH -u HERDR_CLIENT_SOCKET_PATH -u HERDR_SESSION -u HERDR_STARTUP_CWD -u HERDR_WORKSPACE_ID -u HERDR_TAB_ID -u HERDR_PANE_ID XDG_CONFIG_HOME="$xdg" XDG_RUNTIME_DIR="$runtime" HERDR_DISABLE_SOUND=1 SHELL=/bin/sh)
+control_env=(env -u HERDR_BIN_PATH -u HERDR_ENV -u HERDR_SOCKET_PATH -u HERDR_CLIENT_SOCKET_PATH -u HERDR_STARTUP_CWD -u HERDR_WORKSPACE_ID -u HERDR_TAB_ID -u HERDR_PANE_ID XDG_CONFIG_HOME="$xdg" XDG_RUNTIME_DIR="$runtime" HERDR_DISABLE_SOUND=1 SHELL=/bin/sh HERDR_SESSION="$name")
+
+cleaned=0
+cleanup() {
+  if [[ $cleaned -eq 1 ]]; then return; fi
+  cleaned=1
+  "${control_env[@]}" "$bin" session stop "$name" >/dev/null 2>&1 || true
+  for _ in $(seq 1 50); do
+    if "${control_env[@]}" "$bin" session delete "$name" >/dev/null 2>&1; then break; fi
+    sleep 0.1
+  done
+  tmux kill-session -t "$name" >/dev/null 2>&1 || true
+  rm -rf "$state"
+}
+trap cleanup EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
+
+printf -v launch 'exec '
+printf -v quoted '%q ' "${launch_env[@]}" "$bin" --session "$name"
+launch+=$quoted
+tmux new-session -d -s "$name" -x "$cols" -y "$rows" "$launch"
+
+panes_json=
+for _ in $(seq 1 150); do
+  if panes_json=$("${control_env[@]}" "$bin" pane list 2>/dev/null); then break; fi
+  sleep 0.1
+done
+[[ -n "$panes_json" ]] || { echo "session API did not become ready" >&2; exit 1; }
+root_pane=$(printf '%s\n' "$panes_json" | jq -r '.result.panes[0].pane_id')
+workspace_id=$("${control_env[@]}" "$bin" workspace list | jq -r '.result.workspaces[0].workspace_id')
+[[ -n "$root_pane" && "$root_pane" != null ]] || { echo "session did not report a root pane" >&2; exit 1; }
+[[ -n "$workspace_id" && "$workspace_id" != null ]] || { echo "session did not report a workspace" >&2; exit 1; }
+pane_file="$state/pane-ids.txt"
+printf '%s\n' "$root_pane" > "$pane_file"
+
+for ((index = 2; index <= total_panes; index++)); do
+  created=$("${control_env[@]}" "$bin" tab create --workspace "$workspace_id" --label "bench-$index" --no-focus)
+  pane_id=$(printf '%s\n' "$created" | jq -r '.result.root_pane.pane_id')
+  [[ -n "$pane_id" && "$pane_id" != null ]] || { echo "tab $index did not return a pane" >&2; exit 1; }
+  printf '%s\n' "$pane_id" >> "$pane_file"
+done
+
+index=0
+while IFS= read -r pane_id; do
+  index=$((index + 1))
+  if [[ $writers == visible && $index -eq 1 ]] || [[ $writers == hidden && $index -gt 1 ]]; then
+    "${control_env[@]}" "$bin" pane run "$pane_id" "$producer" "$rate" "$gate" "p$index" >/dev/null
+  fi
+done < "$pane_file"
+touch "$gate"
+
+socket="$xdg/herdr/sessions/$name/herdr.sock"
+server_pid=
+for _ in $(seq 1 80); do
+  server_pid=$(lsof -t "$socket" 2>/dev/null | head -n1 || true)
+  [[ -n "$server_pid" ]] && break
+  sleep 0.1
+done
+[[ -n "$server_pid" ]] || { echo "could not find server pid" >&2; exit 1; }
+client_pid=$(tmux list-panes -s -t "$name" -F '#{pane_pid}')
+[[ -n "$client_pid" ]] || { echo "could not find client pid" >&2; exit 1; }
+all_pids=("$server_pid" "$client_pid")
+
+sleep "$warmup"
+raw="$out/cpu-raw.txt"
+if [[ $platform == linux ]]; then
+  pid_csv=$(IFS=,; echo "${all_pids[*]}")
+  LC_ALL=C pidstat -h -u -p "$pid_csv" 1 "$seconds" > "$raw"
+else
+  top_args=(top -l $((seconds + 1)) -s 1 -stats pid,cpu,time -n 2)
+  for pid in "${all_pids[@]}"; do top_args+=(-pid "$pid"); done
+  LC_ALL=C "${top_args[@]}" > "$raw"
+fi
+
+mean_linux() {
+  awk -v target="$2" '
+    /^Linux/ || /^#/ || NF < 5 { next }
+    $3 == target && $(NF-2) ~ /^[0-9]+([.][0-9]+)?$/ { sum += $(NF-2); count++ }
+    END { if (!count) exit 1; printf "%.6f,%d", sum/count, count }
+  ' "$1"
+}
+mean_macos() {
+  awk -v target="$2" '
+    $1 == target && $2 ~ /^[0-9]+([.][0-9]+)?%?$/ {
+      seen++; if (seen == 1) next; value=$2; gsub(/%/, "", value); sum += value; count++ }
+    END { if (!count) exit 1; printf "%.6f,%d", sum/count, count }
+  ' "$1"
+}
+
+total=0
+for pid in "${all_pids[@]}"; do
+  if [[ $platform == linux ]]; then parsed=$(mean_linux "$raw" "$pid"); else parsed=$(mean_macos "$raw" "$pid"); fi
+  mean=${parsed%,*}
+  samples=${parsed#*,}
+  [[ $samples -eq $seconds ]] || { echo "expected $seconds samples for pid $pid, got $samples" >&2; exit 1; }
+  total=$(awk -v total="$total" -v mean="$mean" 'BEGIN { printf "%.6f", total + mean }')
+done
+
+index=0
+while IFS= read -r pane_id; do
+  index=$((index + 1))
+  if [[ $writers == visible && $index -eq 1 ]] || [[ $writers == hidden && $index -gt 1 ]]; then
+    read_file="$out/pane-$index.txt"
+    "${control_env[@]}" "$bin" pane read "$pane_id" --source visible --format text > "$read_file"
+    grep -q 'bench-output-' "$read_file" || { echo "writer pane $index produced no output" >&2; exit 1; }
+  fi
+done < "$pane_file"
+
+printf '%s\n' "$total" > "$out/total-cpu.txt"
+printf '%s,%s,%s,%s\n' "$variant" "$scenario" "$round" "$total"

scripts/release_perf_producer.pl

@@ -0,0 +1,26 @@
+#!/usr/bin/env perl
+use strict;
+use warnings;
+use Time::HiRes qw(clock_gettime sleep CLOCK_MONOTONIC);
+
+my ($rate, $gate, $label) = @ARGV;
+die "usage: $0 <rate-hz> <gate-file> <label>\n"
+    unless defined $rate && defined $gate && defined $label
+    && length $gate && length $label;
+die "rate must be a positive number\n"
+    unless $rate =~ /\A(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)\z/ && $rate > 0;
+
+sleep 0.01 until -e $gate;
+$| = 1;
+my $period = 1 / $rate;
+my $next = clock_gettime(CLOCK_MONOTONIC);
+my $sequence = 0;
+while (1) {
+    $sequence++;
+    printf "\rbench-output-%08d-%s", $sequence, $label;
+    $next += $period;
+    my $now = clock_gettime(CLOCK_MONOTONIC);
+    $next = $now + $period if $next < $now - $period;
+    my $remaining = $next - $now;
+    sleep $remaining if $remaining > 0;
+}

scripts/release_perf_smoke.sh

@@ -0,0 +1,92 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ $# -ne 1 ]]; then
+  echo "usage: $0 <candidate-binary>" >&2
+  exit 2
+fi
+
+candidate=$(cd "$(dirname "$1")" && pwd)/$(basename "$1")
+[[ -x "$candidate" ]] || { echo "candidate binary is not executable: $candidate" >&2; exit 1; }
+script_dir=$(cd "$(dirname "$0")" && pwd)
+repo_root=$(cd "$script_dir/.." && pwd)
+baseline=${HERDR_PERF_BASELINE_BIN:-}
+for command in jq lsof perl tmux; do
+  command -v "$command" >/dev/null || { echo "required command not found: $command" >&2; exit 1; }
+done
+if [[ -z "$baseline" ]]; then
+  command -v curl >/dev/null || { echo "required command not found: curl" >&2; exit 1; }
+fi
+
+case "$(uname -s)" in
+  Linux) platform=linux; command -v pidstat >/dev/null || { echo "required command not found: pidstat" >&2; exit 1; } ;;
+  Darwin) platform=macos ;;
+  *) echo "release performance smoke supports Linux and macOS" >&2; exit 1 ;;
+esac
+case "$(uname -m)" in
+  x86_64|amd64) arch=x86_64 ;;
+  arm64|aarch64) arch=aarch64 ;;
+  *) echo "unsupported architecture: $(uname -m)" >&2; exit 1 ;;
+esac
+
+root=$(mktemp -d /var/tmp/herdr-release-perf-smoke.XXXXXX)
+cleanup() { rm -rf "$root"; }
+trap cleanup EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
+mkdir -p "$root/results"
+
+if [[ -z "$baseline" ]]; then
+  baseline_version=$(jq -er '.version' "$repo_root/website/latest.json")
+  baseline="$root/herdr-baseline"
+  curl -fL --retry 3 \
+    "https://github.com/herdrdev/herdr/releases/download/v${baseline_version}/herdr-${platform}-${arch}" \
+    -o "$baseline"
+  chmod +x "$baseline"
+else
+  baseline=$(cd "$(dirname "$baseline")" && pwd)/$(basename "$baseline")
+fi
+[[ -x "$baseline" ]] || { echo "baseline binary is not executable: $baseline" >&2; exit 1; }
+
+case_script="$script_dir/release_perf_case.sh"
+seconds=${HERDR_PERF_SAMPLE_SECONDS:-10}
+warmup=${HERDR_PERF_WARMUP_SECONDS:-3}
+
+for round in 1 2; do
+  if [[ $round -eq 1 ]]; then variants="baseline candidate"; else variants="candidate baseline"; fi
+  for scenario in hidden50 visible30; do
+    for variant in $variants; do
+      if [[ $variant == baseline ]]; then binary=$baseline; else binary=$candidate; fi
+      "$case_script" "$binary" "$variant" "$scenario" "$round" "$seconds" "$warmup" "$root/results" "$platform"
+    done
+  done
+done
+
+mean_total() {
+  awk '{ sum += $1; count++ } END { if (!count) exit 1; printf "%.3f", sum/count }' \
+    "$root/results/$1/$2/r1/total-cpu.txt" \
+    "$root/results/$1/$2/r2/total-cpu.txt"
+}
+
+failed=0
+printf '\nrelease performance smoke (%s/%s, two rounds, %ss samples)\n' "$platform" "$arch" "$seconds"
+printf '%-12s %12s %12s %12s\n' scenario baseline candidate change
+for scenario in hidden50 visible30; do
+  baseline_total=$(mean_total baseline "$scenario")
+  candidate_total=$(mean_total candidate "$scenario")
+  if awk -v before="$baseline_total" -v after="$candidate_total" 'BEGIN { exit !(before <= 0 || after <= 0) }'; then
+    echo "error: $scenario measured no CPU usage; the benchmark did not exercise the binaries" >&2
+    failed=1
+  fi
+  change=$(awk -v before="$baseline_total" -v after="$candidate_total" 'BEGIN { if (before == 0) print "n/a"; else printf "%+.1f%%", (after-before)/before*100 }')
+  printf '%-12s %12s %12s %12s\n' "$scenario" "$baseline_total" "$candidate_total" "$change"
+  if awk -v before="$baseline_total" -v after="$candidate_total" 'BEGIN { exit !(after > before * 1.25 && after - before > 0.5) }'; then
+    echo "error: $scenario candidate CPU exceeds baseline by more than 25% and 0.5 CPU points" >&2
+    failed=1
+  fi
+done
+
+if [[ $failed -ne 0 ]]; then
+  exit 1
+fi
+echo "release performance smoke passed"