From cea63ef53f73754c59038500df97a20d13d845c1 Mon Sep 17 00:00:00 2001 From: LahkLeKey Date: Tue, 26 May 2026 23:58:45 -0500 Subject: [PATCH 01/36] parrallel tests --- pyproject.toml | 3 ++ scripts/dev/test.sh | 103 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+) create mode 100644 scripts/dev/test.sh diff --git a/pyproject.toml b/pyproject.toml index 49f18365f0..d9ccfd4492 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,7 @@ packages = ["src/specify_cli"] test = [ "pytest>=7.0", "pytest-cov>=4.0", + "pytest-xdist>=3.5", ] [tool.pytest.ini_options] @@ -60,6 +61,8 @@ addopts = [ "-v", "--strict-markers", "--tb=short", + "-n=auto", + "--dist=worksteal", ] [tool.coverage.run] diff --git a/scripts/dev/test.sh b/scripts/dev/test.sh new file mode 100644 index 0000000000..82add63149 --- /dev/null +++ b/scripts/dev/test.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +# Spec Kit local test wrapper — chunked FIFO dispatch over pytest-xdist. +# +# Design (matches the chunked-task / bounded-memory pattern): +# * Collect node ids once: `pytest --collect-only -q`. +# * Split the collection into fixed-size chunks (default 200 nodes). +# * Dispatch chunks sequentially as a FIFO queue; inside each chunk, +# pytest-xdist's `--dist=load` hands tests out one at a time to +# workers as they finish — natural FIFO progression with bounded +# in-flight memory. +# * Persist the cursor (next chunk index) to +# `.pytest_cache/fast-test-cursor` after every successful chunk so +# `--resume` continues exactly where a crash left off. +# * `--reset` clears the cursor; `--bench` reports wall-time only. +# +# Usage: +# scripts/dev/test.sh # full suite, chunked +# scripts/dev/test.sh --chunk-size 100 +# scripts/dev/test.sh --resume # continue from cursor +# scripts/dev/test.sh --reset # clear cursor +# scripts/dev/test.sh --bench # time only, no -v +# scripts/dev/test.sh -- tests/test_merge.py # pass-through to pytest + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +CURSOR_FILE="$REPO_ROOT/.pytest_cache/fast-test-cursor" + +CHUNK_SIZE=200 +RESUME=0 +RESET=0 +BENCH=0 +PASSTHROUGH=() + +while (( $# )); do + case "$1" in + --chunk-size) CHUNK_SIZE="$2"; shift 2 ;; + --resume) RESUME=1; shift ;; + --reset) RESET=1; shift ;; + --bench) BENCH=1; shift ;; + --) shift; PASSTHROUGH+=("$@"); break ;; + -h|--help) sed -n '2,22p' "$0"; exit 0 ;; + *) PASSTHROUGH+=("$1"); shift ;; + esac +done + +if (( RESET )); then + rm -f "$CURSOR_FILE" + echo "[fast-test] cursor cleared" + exit 0 +fi + +cd "$REPO_ROOT" +mkdir -p "$(dirname "$CURSOR_FILE")" + +# 1. Collect node ids (override addopts so collection itself is serial/quiet). +echo "[fast-test] collecting tests ..." +mapfile -t NODES < <( + uv run pytest -o addopts= --collect-only -q "${PASSTHROUGH[@]}" \ + 2>/dev/null | grep -E '::' || true +) +TOTAL="${#NODES[@]}" +if (( TOTAL == 0 )); then + echo "[fast-test] no tests collected" >&2 + exit 1 +fi + +# 2. Determine starting cursor. +START=0 +if (( RESUME )) && [[ -f "$CURSOR_FILE" ]]; then + START="$(cat "$CURSOR_FILE")" + echo "[fast-test] resuming from chunk cursor: test #$START" +fi + +CHUNKS=$(( (TOTAL - START + CHUNK_SIZE - 1) / CHUNK_SIZE )) +echo "[fast-test] $TOTAL tests · chunk=$CHUNK_SIZE · $CHUNKS chunk(s) queued · workers=auto" + +# 3. FIFO dispatch. +T_START=$(date +%s) +i="$START" +chunk_idx=0 +while (( i < TOTAL )); do + end=$(( i + CHUNK_SIZE )) + (( end > TOTAL )) && end="$TOTAL" + chunk_idx=$(( chunk_idx + 1 )) + echo "[fast-test] chunk $chunk_idx/$CHUNKS tests $((i+1))..$end" + + PYTEST_FLAGS=(-o addopts= -n auto --dist=load --tb=short) + (( BENCH )) && PYTEST_FLAGS+=(-q) || PYTEST_FLAGS+=(--no-header -q) + + if ! uv run pytest "${PYTEST_FLAGS[@]}" "${NODES[@]:i:CHUNK_SIZE}"; then + echo "[fast-test] chunk failed — cursor preserved at test #$i (use --resume to retry)" + echo "$i" > "$CURSOR_FILE" + exit 1 + fi + + i="$end" + echo "$i" > "$CURSOR_FILE" +done + +T_END=$(date +%s) +rm -f "$CURSOR_FILE" +echo "[fast-test] all $TOTAL tests passed in $((T_END - T_START))s" From 9bf891ceaff2826c0cb0c61a36f0adc8727861bb Mon Sep 17 00:00:00 2001 From: LahkLeKey Date: Thu, 28 May 2026 16:49:07 -0500 Subject: [PATCH 02/36] fix(test): keep xdist wrapper-only and harden collection --- pyproject.toml | 2 -- scripts/dev/test.sh | 22 ++++++++++++++-------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d9ccfd4492..5d2afdb4f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,8 +61,6 @@ addopts = [ "-v", "--strict-markers", "--tb=short", - "-n=auto", - "--dist=worksteal", ] [tool.coverage.run] diff --git a/scripts/dev/test.sh b/scripts/dev/test.sh index 82add63149..ed8fdf43b6 100644 --- a/scripts/dev/test.sh +++ b/scripts/dev/test.sh @@ -53,12 +53,18 @@ fi cd "$REPO_ROOT" mkdir -p "$(dirname "$CURSOR_FILE")" -# 1. Collect node ids (override addopts so collection itself is serial/quiet). +# 1. Collect node ids. echo "[fast-test] collecting tests ..." -mapfile -t NODES < <( - uv run pytest -o addopts= --collect-only -q "${PASSTHROUGH[@]}" \ - 2>/dev/null | grep -E '::' || true -) +COLLECT_ERR="$(mktemp)" +COLLECT_OUT="$(mktemp)" +if ! uv run pytest --collect-only -qq "${PASSTHROUGH[@]}" >"$COLLECT_OUT" 2>"$COLLECT_ERR"; then + echo "[fast-test] test collection failed" >&2 + [[ -s "$COLLECT_ERR" ]] && { echo "--- collection stderr ---"; cat "$COLLECT_ERR"; } >&2 + rm -f "$COLLECT_ERR" "$COLLECT_OUT" + exit 1 +fi +mapfile -t NODES < <(grep -E '::' "$COLLECT_OUT" || true) +rm -f "$COLLECT_ERR" "$COLLECT_OUT" TOTAL="${#NODES[@]}" if (( TOTAL == 0 )); then echo "[fast-test] no tests collected" >&2 @@ -69,7 +75,7 @@ fi START=0 if (( RESUME )) && [[ -f "$CURSOR_FILE" ]]; then START="$(cat "$CURSOR_FILE")" - echo "[fast-test] resuming from chunk cursor: test #$START" + echo "[fast-test] resuming from next test index: $START" fi CHUNKS=$(( (TOTAL - START + CHUNK_SIZE - 1) / CHUNK_SIZE )) @@ -85,11 +91,11 @@ while (( i < TOTAL )); do chunk_idx=$(( chunk_idx + 1 )) echo "[fast-test] chunk $chunk_idx/$CHUNKS tests $((i+1))..$end" - PYTEST_FLAGS=(-o addopts= -n auto --dist=load --tb=short) + PYTEST_FLAGS=(-n auto --dist=load) (( BENCH )) && PYTEST_FLAGS+=(-q) || PYTEST_FLAGS+=(--no-header -q) if ! uv run pytest "${PYTEST_FLAGS[@]}" "${NODES[@]:i:CHUNK_SIZE}"; then - echo "[fast-test] chunk failed — cursor preserved at test #$i (use --resume to retry)" + echo "[fast-test] chunk failed — cursor preserved at next test index $i (use --resume to retry)" echo "$i" > "$CURSOR_FILE" exit 1 fi From 9449e2405c40a43348f4d0389edb280ede892e88 Mon Sep 17 00:00:00 2001 From: LahkLeKey Date: Sat, 30 May 2026 17:28:50 -0500 Subject: [PATCH 03/36] fix(test): forward runtime passthrough and validate cursor --- scripts/dev/test.sh | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/scripts/dev/test.sh b/scripts/dev/test.sh index ed8fdf43b6..edb7e9c393 100644 --- a/scripts/dev/test.sh +++ b/scripts/dev/test.sh @@ -31,6 +31,7 @@ RESUME=0 RESET=0 BENCH=0 PASSTHROUGH=() +RUNTIME_PASSTHROUGH=() while (( $# )); do case "$1" in @@ -44,6 +45,15 @@ while (( $# )); do esac done +# Collection and execution do not share every pytest flag. Keep runtime +# passthrough aligned with user intent while stripping collect-only toggles. +for arg in "${PASSTHROUGH[@]}"; do + case "$arg" in + --collect-only|--co) ;; + *) RUNTIME_PASSTHROUGH+=("$arg") ;; + esac +done + if (( RESET )); then rm -f "$CURSOR_FILE" echo "[fast-test] cursor cleared" @@ -74,8 +84,14 @@ fi # 2. Determine starting cursor. START=0 if (( RESUME )) && [[ -f "$CURSOR_FILE" ]]; then - START="$(cat "$CURSOR_FILE")" - echo "[fast-test] resuming from next test index: $START" + RAW_START="$(tr -d '[:space:]' < "$CURSOR_FILE")" + if [[ "$RAW_START" =~ ^[0-9]+$ ]]; then + START="$RAW_START" + (( START > TOTAL )) && START="$TOTAL" + echo "[fast-test] resuming from next test index: $START" + else + echo "[fast-test] cursor file is invalid ('$RAW_START'); starting from 0" >&2 + fi fi CHUNKS=$(( (TOTAL - START + CHUNK_SIZE - 1) / CHUNK_SIZE )) @@ -94,7 +110,7 @@ while (( i < TOTAL )); do PYTEST_FLAGS=(-n auto --dist=load) (( BENCH )) && PYTEST_FLAGS+=(-q) || PYTEST_FLAGS+=(--no-header -q) - if ! uv run pytest "${PYTEST_FLAGS[@]}" "${NODES[@]:i:CHUNK_SIZE}"; then + if ! uv run pytest "${PYTEST_FLAGS[@]}" "${RUNTIME_PASSTHROUGH[@]}" "${NODES[@]:i:CHUNK_SIZE}"; then echo "[fast-test] chunk failed — cursor preserved at next test index $i (use --resume to retry)" echo "$i" > "$CURSOR_FILE" exit 1 From e317e4e887f612b6b7937c274fc7e12d2b892567 Mon Sep 17 00:00:00 2001 From: LahkLeKey Date: Tue, 2 Jun 2026 19:38:45 -0500 Subject: [PATCH 04/36] fix(test): harden chunk sizing and resume behavior --- scripts/dev/test.sh | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/scripts/dev/test.sh b/scripts/dev/test.sh index edb7e9c393..85b7ea6ef3 100644 --- a/scripts/dev/test.sh +++ b/scripts/dev/test.sh @@ -35,7 +35,14 @@ RUNTIME_PASSTHROUGH=() while (( $# )); do case "$1" in - --chunk-size) CHUNK_SIZE="$2"; shift 2 ;; + --chunk-size) + if (( $# < 2 )); then + echo "[fast-test] --chunk-size requires a value" >&2 + exit 1 + fi + CHUNK_SIZE="$2" + shift 2 + ;; --resume) RESUME=1; shift ;; --reset) RESET=1; shift ;; --bench) BENCH=1; shift ;; @@ -45,6 +52,11 @@ while (( $# )); do esac done +if ! [[ "$CHUNK_SIZE" =~ ^[1-9][0-9]*$ ]]; then + echo "[fast-test] --chunk-size must be a positive integer (got '$CHUNK_SIZE')" >&2 + exit 1 +fi + # Collection and execution do not share every pytest flag. Keep runtime # passthrough aligned with user intent while stripping collect-only toggles. for arg in "${PASSTHROUGH[@]}"; do @@ -73,7 +85,10 @@ if ! uv run pytest --collect-only -qq "${PASSTHROUGH[@]}" >"$COLLECT_OUT" 2>"$CO rm -f "$COLLECT_ERR" "$COLLECT_OUT" exit 1 fi -mapfile -t NODES < <(grep -E '::' "$COLLECT_OUT" || true) +NODES=() +while IFS= read -r node; do + NODES+=("$node") +done < <(grep -E '::' "$COLLECT_OUT" || true) rm -f "$COLLECT_ERR" "$COLLECT_OUT" TOTAL="${#NODES[@]}" if (( TOTAL == 0 )); then @@ -94,6 +109,12 @@ if (( RESUME )) && [[ -f "$CURSOR_FILE" ]]; then fi fi +if (( START >= TOTAL )); then + echo "[fast-test] nothing to resume (cursor at end: $START/$TOTAL)" + rm -f "$CURSOR_FILE" + exit 0 +fi + CHUNKS=$(( (TOTAL - START + CHUNK_SIZE - 1) / CHUNK_SIZE )) echo "[fast-test] $TOTAL tests · chunk=$CHUNK_SIZE · $CHUNKS chunk(s) queued · workers=auto" From e72b5b4e6a7f5f288623a43042297eccc0045dc6 Mon Sep 17 00:00:00 2001 From: LahkLeKey Date: Wed, 3 Jun 2026 17:45:16 -0500 Subject: [PATCH 05/36] fix(test): use -q instead of -qq for pytest collect-only to preserve node ids --- scripts/dev/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/dev/test.sh b/scripts/dev/test.sh index 85b7ea6ef3..4bab370eda 100644 --- a/scripts/dev/test.sh +++ b/scripts/dev/test.sh @@ -79,7 +79,7 @@ mkdir -p "$(dirname "$CURSOR_FILE")" echo "[fast-test] collecting tests ..." COLLECT_ERR="$(mktemp)" COLLECT_OUT="$(mktemp)" -if ! uv run pytest --collect-only -qq "${PASSTHROUGH[@]}" >"$COLLECT_OUT" 2>"$COLLECT_ERR"; then +if ! uv run pytest --collect-only -q "${PASSTHROUGH[@]}" >"$COLLECT_OUT" 2>"$COLLECT_ERR"; then echo "[fast-test] test collection failed" >&2 [[ -s "$COLLECT_ERR" ]] && { echo "--- collection stderr ---"; cat "$COLLECT_ERR"; } >&2 rm -f "$COLLECT_ERR" "$COLLECT_OUT" From 2caac75db1120f9457ebefae9f67cd4cf154b357 Mon Sep 17 00:00:00 2001 From: LahkLeKey Date: Thu, 4 Jun 2026 17:52:27 -0500 Subject: [PATCH 06/36] (copilot-commit-suggestion) Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- scripts/dev/test.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/dev/test.sh b/scripts/dev/test.sh index 4bab370eda..06a67af5ef 100644 --- a/scripts/dev/test.sh +++ b/scripts/dev/test.sh @@ -8,10 +8,10 @@ # pytest-xdist's `--dist=load` hands tests out one at a time to # workers as they finish — natural FIFO progression with bounded # in-flight memory. -# * Persist the cursor (next chunk index) to +# * Persist the cursor (next test index / chunk boundary) to # `.pytest_cache/fast-test-cursor` after every successful chunk so -# `--resume` continues exactly where a crash left off. -# * `--reset` clears the cursor; `--bench` reports wall-time only. +# `--resume` continues from the last completed chunk. +# * `--reset` clears the cursor; `--bench` reduces pytest output (progress still prints). # # Usage: # scripts/dev/test.sh # full suite, chunked From 2227434c7bbeb405bfb289f2d5eed67f1848f850 Mon Sep 17 00:00:00 2001 From: LahkLeKey Date: Thu, 4 Jun 2026 18:04:21 -0500 Subject: [PATCH 07/36] chore: harden fast-test cursor handling --- scripts/dev/test.sh | 40 ++++++++++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/scripts/dev/test.sh b/scripts/dev/test.sh index 06a67af5ef..4c085907e0 100644 --- a/scripts/dev/test.sh +++ b/scripts/dev/test.sh @@ -32,6 +32,11 @@ RESET=0 BENCH=0 PASSTHROUGH=() RUNTIME_PASSTHROUGH=() +LOCK_DIR="$REPO_ROOT/.pytest_cache/fast-test.lock" +LOCK_HELD=0 +CURSOR_TMP="$CURSOR_FILE.tmp" +COLLECT_ERR="" +COLLECT_OUT="" while (( $# )); do case "$1" in @@ -66,15 +71,38 @@ for arg in "${PASSTHROUGH[@]}"; do esac done +cleanup() { + [[ -n "$COLLECT_ERR" && -f "$COLLECT_ERR" ]] && rm -f "$COLLECT_ERR" + [[ -n "$COLLECT_OUT" && -f "$COLLECT_OUT" ]] && rm -f "$COLLECT_OUT" + [[ -n "$CURSOR_TMP" && -f "$CURSOR_TMP" ]] && rm -f "$CURSOR_TMP" + if (( LOCK_HELD )); then + rm -f "$LOCK_DIR/pid" + rmdir "$LOCK_DIR" 2>/dev/null || rm -rf "$LOCK_DIR" + fi +} + +trap cleanup EXIT + +write_cursor() { + printf '%s\n' "$1" > "$CURSOR_TMP" + mv "$CURSOR_TMP" "$CURSOR_FILE" +} +cd "$REPO_ROOT" +mkdir -p "$(dirname "$CURSOR_FILE")" + +if ! mkdir "$LOCK_DIR" 2>/dev/null; then + echo "[fast-test] another run is active (lock: $LOCK_DIR)" >&2 + exit 1 +fi +LOCK_HELD=1 +printf '%s\n' "$$" > "$LOCK_DIR/pid" + if (( RESET )); then - rm -f "$CURSOR_FILE" + rm -f "$CURSOR_FILE" "$CURSOR_TMP" echo "[fast-test] cursor cleared" exit 0 fi -cd "$REPO_ROOT" -mkdir -p "$(dirname "$CURSOR_FILE")" - # 1. Collect node ids. echo "[fast-test] collecting tests ..." COLLECT_ERR="$(mktemp)" @@ -133,12 +161,12 @@ while (( i < TOTAL )); do if ! uv run pytest "${PYTEST_FLAGS[@]}" "${RUNTIME_PASSTHROUGH[@]}" "${NODES[@]:i:CHUNK_SIZE}"; then echo "[fast-test] chunk failed — cursor preserved at next test index $i (use --resume to retry)" - echo "$i" > "$CURSOR_FILE" + write_cursor "$i" exit 1 fi i="$end" - echo "$i" > "$CURSOR_FILE" + write_cursor "$i" done T_END=$(date +%s) From dcd1475d8bc9c141562828d30f5228c7da1902cb Mon Sep 17 00:00:00 2001 From: LahkLeKey Date: Thu, 4 Jun 2026 18:07:36 -0500 Subject: [PATCH 08/36] chore: harden fast-test portability --- scripts/dev/test.sh | 57 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 50 insertions(+), 7 deletions(-) diff --git a/scripts/dev/test.sh b/scripts/dev/test.sh index 4c085907e0..a169730b24 100644 --- a/scripts/dev/test.sh +++ b/scripts/dev/test.sh @@ -32,6 +32,7 @@ RESET=0 BENCH=0 PASSTHROUGH=() RUNTIME_PASSTHROUGH=() +PYTEST_CMD=() LOCK_DIR="$REPO_ROOT/.pytest_cache/fast-test.lock" LOCK_HELD=0 CURSOR_TMP="$CURSOR_FILE.tmp" @@ -62,11 +63,32 @@ if ! [[ "$CHUNK_SIZE" =~ ^[1-9][0-9]*$ ]]; then exit 1 fi +mktemp_file() { + local tmp + if tmp="$(mktemp -t fast-test.XXXXXX 2>/dev/null)"; then + printf '%s\n' "$tmp" + return 0 + fi + if tmp="$(mktemp "${TMPDIR:-/tmp}/fast-test.XXXXXX" 2>/dev/null)"; then + printf '%s\n' "$tmp" + return 0 + fi + mktemp +} + # Collection and execution do not share every pytest flag. Keep runtime -# passthrough aligned with user intent while stripping collect-only toggles. +# passthrough aligned with user intent while stripping collect-only toggles +# and xdist-specific options that this script owns. +skip_next=0 for arg in "${PASSTHROUGH[@]}"; do + if (( skip_next )); then + skip_next=0 + continue + fi case "$arg" in --collect-only|--co) ;; + -n|--numprocesses|--dist) skip_next=1 ;; + -n*|--numprocesses=*|--dist=*) ;; *) RUNTIME_PASSTHROUGH+=("$arg") ;; esac done @@ -90,6 +112,18 @@ write_cursor() { cd "$REPO_ROOT" mkdir -p "$(dirname "$CURSOR_FILE")" +if [[ -d "$LOCK_DIR" ]]; then + if [[ -f "$LOCK_DIR/pid" ]]; then + PID_CONTENTS="$(tr -d '[:space:]' < "$LOCK_DIR/pid" 2>/dev/null || true)" + if [[ "$PID_CONTENTS" =~ ^[0-9]+$ ]] && kill -0 "$PID_CONTENTS" 2>/dev/null; then + echo "[fast-test] another run is active (lock: $LOCK_DIR)" >&2 + exit 1 + fi + echo "[fast-test] removing stale lock (pid: ${PID_CONTENTS:-unknown})" >&2 + fi + rm -f "$LOCK_DIR/pid" + rmdir "$LOCK_DIR" 2>/dev/null || rm -rf "$LOCK_DIR" +fi if ! mkdir "$LOCK_DIR" 2>/dev/null; then echo "[fast-test] another run is active (lock: $LOCK_DIR)" >&2 exit 1 @@ -104,10 +138,19 @@ if (( RESET )); then fi # 1. Collect node ids. +if command -v uv >/dev/null 2>&1; then + PYTEST_CMD=(uv run pytest) +elif command -v python >/dev/null 2>&1; then + PYTEST_CMD=(python -m pytest) +else + echo "[fast-test] pytest runner not found (install uv or ensure python is on PATH)" >&2 + exit 1 +fi + echo "[fast-test] collecting tests ..." -COLLECT_ERR="$(mktemp)" -COLLECT_OUT="$(mktemp)" -if ! uv run pytest --collect-only -q "${PASSTHROUGH[@]}" >"$COLLECT_OUT" 2>"$COLLECT_ERR"; then +COLLECT_ERR="$(mktemp_file)" +COLLECT_OUT="$(mktemp_file)" +if ! "${PYTEST_CMD[@]}" --collect-only -q "${PASSTHROUGH[@]}" >"$COLLECT_OUT" 2>"$COLLECT_ERR"; then echo "[fast-test] test collection failed" >&2 [[ -s "$COLLECT_ERR" ]] && { echo "--- collection stderr ---"; cat "$COLLECT_ERR"; } >&2 rm -f "$COLLECT_ERR" "$COLLECT_OUT" @@ -115,8 +158,8 @@ if ! uv run pytest --collect-only -q "${PASSTHROUGH[@]}" >"$COLLECT_OUT" 2>"$COL fi NODES=() while IFS= read -r node; do - NODES+=("$node") -done < <(grep -E '::' "$COLLECT_OUT" || true) + [[ -n "$node" ]] && NODES+=("$node") +done < "$COLLECT_OUT" rm -f "$COLLECT_ERR" "$COLLECT_OUT" TOTAL="${#NODES[@]}" if (( TOTAL == 0 )); then @@ -159,7 +202,7 @@ while (( i < TOTAL )); do PYTEST_FLAGS=(-n auto --dist=load) (( BENCH )) && PYTEST_FLAGS+=(-q) || PYTEST_FLAGS+=(--no-header -q) - if ! uv run pytest "${PYTEST_FLAGS[@]}" "${RUNTIME_PASSTHROUGH[@]}" "${NODES[@]:i:CHUNK_SIZE}"; then + if ! "${PYTEST_CMD[@]}" "${PYTEST_FLAGS[@]}" "${RUNTIME_PASSTHROUGH[@]}" "${NODES[@]:i:CHUNK_SIZE}"; then echo "[fast-test] chunk failed — cursor preserved at next test index $i (use --resume to retry)" write_cursor "$i" exit 1 From 533e7a79ced7ecf0b6a95178d31c6ad48435aa89 Mon Sep 17 00:00:00 2001 From: LahkLeKey Date: Thu, 4 Jun 2026 18:10:43 -0500 Subject: [PATCH 09/36] chore: harden fast-test collection --- scripts/dev/test.sh | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/scripts/dev/test.sh b/scripts/dev/test.sh index a169730b24..3ded1e21d6 100644 --- a/scripts/dev/test.sh +++ b/scripts/dev/test.sh @@ -18,7 +18,7 @@ # scripts/dev/test.sh --chunk-size 100 # scripts/dev/test.sh --resume # continue from cursor # scripts/dev/test.sh --reset # clear cursor -# scripts/dev/test.sh --bench # time only, no -v +# scripts/dev/test.sh --bench # quieter pytest output # scripts/dev/test.sh -- tests/test_merge.py # pass-through to pytest set -euo pipefail @@ -32,6 +32,7 @@ RESET=0 BENCH=0 PASSTHROUGH=() RUNTIME_PASSTHROUGH=() +COLLECT_PASSTHROUGH=() PYTEST_CMD=() LOCK_DIR="$REPO_ROOT/.pytest_cache/fast-test.lock" LOCK_HELD=0 @@ -73,7 +74,12 @@ mktemp_file() { printf '%s\n' "$tmp" return 0 fi - mktemp + if tmp="$(mktemp "${TMPDIR:-/tmp}/fast-test.XXXXXX" 2>/dev/null)"; then + printf '%s\n' "$tmp" + return 0 + fi + echo "[fast-test] mktemp failed; set TMPDIR or install a compatible mktemp" >&2 + return 1 } # Collection and execution do not share every pytest flag. Keep runtime @@ -89,7 +95,10 @@ for arg in "${PASSTHROUGH[@]}"; do --collect-only|--co) ;; -n|--numprocesses|--dist) skip_next=1 ;; -n*|--numprocesses=*|--dist=*) ;; - *) RUNTIME_PASSTHROUGH+=("$arg") ;; + *) + RUNTIME_PASSTHROUGH+=("$arg") + COLLECT_PASSTHROUGH+=("$arg") + ;; esac done @@ -150,7 +159,7 @@ fi echo "[fast-test] collecting tests ..." COLLECT_ERR="$(mktemp_file)" COLLECT_OUT="$(mktemp_file)" -if ! "${PYTEST_CMD[@]}" --collect-only -q "${PASSTHROUGH[@]}" >"$COLLECT_OUT" 2>"$COLLECT_ERR"; then +if ! "${PYTEST_CMD[@]}" --collect-only -q "${COLLECT_PASSTHROUGH[@]}" >"$COLLECT_OUT" 2>"$COLLECT_ERR"; then echo "[fast-test] test collection failed" >&2 [[ -s "$COLLECT_ERR" ]] && { echo "--- collection stderr ---"; cat "$COLLECT_ERR"; } >&2 rm -f "$COLLECT_ERR" "$COLLECT_OUT" From 91e873af4685dfb853d0f208bec41834d9f15cef Mon Sep 17 00:00:00 2001 From: LahkLeKey Date: Thu, 4 Jun 2026 18:12:32 -0500 Subject: [PATCH 10/36] chore: tune fast-test output --- scripts/dev/test.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/dev/test.sh b/scripts/dev/test.sh index 3ded1e21d6..671fce3967 100644 --- a/scripts/dev/test.sh +++ b/scripts/dev/test.sh @@ -74,10 +74,6 @@ mktemp_file() { printf '%s\n' "$tmp" return 0 fi - if tmp="$(mktemp "${TMPDIR:-/tmp}/fast-test.XXXXXX" 2>/dev/null)"; then - printf '%s\n' "$tmp" - return 0 - fi echo "[fast-test] mktemp failed; set TMPDIR or install a compatible mktemp" >&2 return 1 } @@ -209,7 +205,11 @@ while (( i < TOTAL )); do echo "[fast-test] chunk $chunk_idx/$CHUNKS tests $((i+1))..$end" PYTEST_FLAGS=(-n auto --dist=load) - (( BENCH )) && PYTEST_FLAGS+=(-q) || PYTEST_FLAGS+=(--no-header -q) + if (( BENCH )); then + PYTEST_FLAGS+=(--no-header -q) + else + PYTEST_FLAGS+=(--no-header) + fi if ! "${PYTEST_CMD[@]}" "${PYTEST_FLAGS[@]}" "${RUNTIME_PASSTHROUGH[@]}" "${NODES[@]:i:CHUNK_SIZE}"; then echo "[fast-test] chunk failed — cursor preserved at next test index $i (use --resume to retry)" From e55ca411e9dfd096a7b8cb6a879217979666cd4e Mon Sep 17 00:00:00 2001 From: LahkLeKey Date: Thu, 4 Jun 2026 18:18:38 -0500 Subject: [PATCH 11/36] chore: harden fast-test locking --- scripts/dev/test.sh | 120 ++++++++++++++++++++++++++++++++------------ 1 file changed, 88 insertions(+), 32 deletions(-) diff --git a/scripts/dev/test.sh b/scripts/dev/test.sh index 671fce3967..33f68ef68e 100644 --- a/scripts/dev/test.sh +++ b/scripts/dev/test.sh @@ -34,11 +34,17 @@ PASSTHROUGH=() RUNTIME_PASSTHROUGH=() COLLECT_PASSTHROUGH=() PYTEST_CMD=() -LOCK_DIR="$REPO_ROOT/.pytest_cache/fast-test.lock" +LOCK_FILE="$REPO_ROOT/.pytest_cache/fast-test.lock" +LOCK_DIR="$REPO_ROOT/.pytest_cache/fast-test.lockdir" +LOCK_MODE="" LOCK_HELD=0 +LOCK_NAME="$(basename "$0")" CURSOR_TMP="$CURSOR_FILE.tmp" COLLECT_ERR="" COLLECT_OUT="" +COLLECT_FILTERED="" +CHUNK_NODES=() +XDIST_IGNORED=0 while (( $# )); do case "$1" in @@ -81,28 +87,43 @@ mktemp_file() { # Collection and execution do not share every pytest flag. Keep runtime # passthrough aligned with user intent while stripping collect-only toggles # and xdist-specific options that this script owns. -skip_next=0 -for arg in "${PASSTHROUGH[@]}"; do - if (( skip_next )); then - skip_next=0 - continue - fi +idx=0 +while (( idx < ${#PASSTHROUGH[@]} )); do + arg="${PASSTHROUGH[$idx]}" case "$arg" in --collect-only|--co) ;; - -n|--numprocesses|--dist) skip_next=1 ;; - -n*|--numprocesses=*|--dist=*) ;; + -n|--numprocesses|--dist) + next="${PASSTHROUGH[$((idx + 1))]:-}" + if [[ -z "$next" || "$next" == -* ]]; then + echo "[fast-test] $arg requires a value, but xdist flags are managed by this script" >&2 + exit 1 + fi + XDIST_IGNORED=1 + idx=$((idx + 1)) + ;; + -n*|--numprocesses=*|--dist=*) XDIST_IGNORED=1 ;; *) RUNTIME_PASSTHROUGH+=("$arg") COLLECT_PASSTHROUGH+=("$arg") ;; esac + idx=$((idx + 1)) done +if (( XDIST_IGNORED )); then + echo "[fast-test] ignoring xdist flags (-n/--numprocesses/--dist); this script manages them" >&2 +fi + cleanup() { [[ -n "$COLLECT_ERR" && -f "$COLLECT_ERR" ]] && rm -f "$COLLECT_ERR" [[ -n "$COLLECT_OUT" && -f "$COLLECT_OUT" ]] && rm -f "$COLLECT_OUT" + [[ -n "$COLLECT_FILTERED" && -f "$COLLECT_FILTERED" ]] && rm -f "$COLLECT_FILTERED" [[ -n "$CURSOR_TMP" && -f "$CURSOR_TMP" ]] && rm -f "$CURSOR_TMP" - if (( LOCK_HELD )); then + if [[ "$LOCK_MODE" == "flock" ]]; then + flock -u 9 2>/dev/null || true + exec 9>&- || true + rm -f "$LOCK_FILE" + elif (( LOCK_HELD )); then rm -f "$LOCK_DIR/pid" rmdir "$LOCK_DIR" 2>/dev/null || rm -rf "$LOCK_DIR" fi @@ -114,27 +135,59 @@ write_cursor() { printf '%s\n' "$1" > "$CURSOR_TMP" mv "$CURSOR_TMP" "$CURSOR_FILE" } + +load_chunk_nodes() { + local start_index="$1" + local end_index="$2" + local start_line=$(( start_index + 1 )) + local end_line="$end_index" + + CHUNK_NODES=() + while IFS= read -r node; do + [[ -n "$node" ]] && CHUNK_NODES+=("$node") + done < <(awk -v start="$start_line" -v end="$end_line" 'NR < start {next} NR > end {exit} {print}' "$COLLECT_OUT") +} + cd "$REPO_ROOT" mkdir -p "$(dirname "$CURSOR_FILE")" -if [[ -d "$LOCK_DIR" ]]; then - if [[ -f "$LOCK_DIR/pid" ]]; then - PID_CONTENTS="$(tr -d '[:space:]' < "$LOCK_DIR/pid" 2>/dev/null || true)" - if [[ "$PID_CONTENTS" =~ ^[0-9]+$ ]] && kill -0 "$PID_CONTENTS" 2>/dev/null; then - echo "[fast-test] another run is active (lock: $LOCK_DIR)" >&2 - exit 1 +if command -v flock >/dev/null 2>&1; then + exec 9>"$LOCK_FILE" || { echo "[fast-test] unable to open lock file: $LOCK_FILE" >&2; exit 1; } + if ! flock -n 9; then + echo "[fast-test] another run is active (lock: $LOCK_FILE)" >&2 + exit 1 + fi + LOCK_MODE="flock" + LOCK_HELD=1 +else + if [[ -d "$LOCK_DIR" ]]; then + if [[ -f "$LOCK_DIR/pid" ]]; then + PID_CONTENTS="$(tr -d '[:space:]' < "$LOCK_DIR/pid" 2>/dev/null || true)" + if [[ "$PID_CONTENTS" =~ ^[0-9]+$ ]] && kill -0 "$PID_CONTENTS" 2>/dev/null; then + if command -v ps >/dev/null 2>&1; then + CMDLINE="$(ps -p "$PID_CONTENTS" -o args= 2>/dev/null || true)" + if [[ -z "$CMDLINE" || "$CMDLINE" == *"$LOCK_NAME"* ]]; then + echo "[fast-test] another run is active (lock: $LOCK_DIR)" >&2 + exit 1 + fi + else + echo "[fast-test] another run is active (lock: $LOCK_DIR)" >&2 + exit 1 + fi + fi + echo "[fast-test] removing stale lock (pid: ${PID_CONTENTS:-unknown})" >&2 fi - echo "[fast-test] removing stale lock (pid: ${PID_CONTENTS:-unknown})" >&2 + rm -f "$LOCK_DIR/pid" + rmdir "$LOCK_DIR" 2>/dev/null || rm -rf "$LOCK_DIR" fi - rm -f "$LOCK_DIR/pid" - rmdir "$LOCK_DIR" 2>/dev/null || rm -rf "$LOCK_DIR" -fi -if ! mkdir "$LOCK_DIR" 2>/dev/null; then - echo "[fast-test] another run is active (lock: $LOCK_DIR)" >&2 - exit 1 + if ! mkdir "$LOCK_DIR" 2>/dev/null; then + echo "[fast-test] another run is active (lock: $LOCK_DIR)" >&2 + exit 1 + fi + LOCK_MODE="dir" + LOCK_HELD=1 + printf '%s\n' "$$" > "$LOCK_DIR/pid" fi -LOCK_HELD=1 -printf '%s\n' "$$" > "$LOCK_DIR/pid" if (( RESET )); then rm -f "$CURSOR_FILE" "$CURSOR_TMP" @@ -145,6 +198,8 @@ fi # 1. Collect node ids. if command -v uv >/dev/null 2>&1; then PYTEST_CMD=(uv run pytest) +elif command -v python3 >/dev/null 2>&1; then + PYTEST_CMD=(python3 -m pytest) elif command -v python >/dev/null 2>&1; then PYTEST_CMD=(python -m pytest) else @@ -161,12 +216,11 @@ if ! "${PYTEST_CMD[@]}" --collect-only -q "${COLLECT_PASSTHROUGH[@]}" >"$COLLECT rm -f "$COLLECT_ERR" "$COLLECT_OUT" exit 1 fi -NODES=() -while IFS= read -r node; do - [[ -n "$node" ]] && NODES+=("$node") -done < "$COLLECT_OUT" -rm -f "$COLLECT_ERR" "$COLLECT_OUT" -TOTAL="${#NODES[@]}" +rm -f "$COLLECT_ERR" +COLLECT_FILTERED="$(mktemp_file)" +awk 'NF' "$COLLECT_OUT" > "$COLLECT_FILTERED" +mv "$COLLECT_FILTERED" "$COLLECT_OUT" +TOTAL="$(wc -l < "$COLLECT_OUT" | tr -d '[:space:]')" if (( TOTAL == 0 )); then echo "[fast-test] no tests collected" >&2 exit 1 @@ -211,7 +265,8 @@ while (( i < TOTAL )); do PYTEST_FLAGS+=(--no-header) fi - if ! "${PYTEST_CMD[@]}" "${PYTEST_FLAGS[@]}" "${RUNTIME_PASSTHROUGH[@]}" "${NODES[@]:i:CHUNK_SIZE}"; then + load_chunk_nodes "$i" "$end" + if ! "${PYTEST_CMD[@]}" "${PYTEST_FLAGS[@]}" "${RUNTIME_PASSTHROUGH[@]}" "${CHUNK_NODES[@]}"; then echo "[fast-test] chunk failed — cursor preserved at next test index $i (use --resume to retry)" write_cursor "$i" exit 1 @@ -222,5 +277,6 @@ while (( i < TOTAL )); do done T_END=$(date +%s) +rm -f "$COLLECT_OUT" rm -f "$CURSOR_FILE" echo "[fast-test] all $TOTAL tests passed in $((T_END - T_START))s" From 4a73eefebd20753b0fccf2c9cae7b97c2ab3368e Mon Sep 17 00:00:00 2001 From: LahkLeKey Date: Thu, 4 Jun 2026 18:22:59 -0500 Subject: [PATCH 12/36] chore: stream fast-test chunks --- scripts/dev/test.sh | 51 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 39 insertions(+), 12 deletions(-) diff --git a/scripts/dev/test.sh b/scripts/dev/test.sh index 33f68ef68e..e6fd48fb98 100644 --- a/scripts/dev/test.sh +++ b/scripts/dev/test.sh @@ -60,7 +60,7 @@ while (( $# )); do --reset) RESET=1; shift ;; --bench) BENCH=1; shift ;; --) shift; PASSTHROUGH+=("$@"); break ;; - -h|--help) sed -n '2,22p' "$0"; exit 0 ;; + -h|--help) print_help; exit 0 ;; *) PASSTHROUGH+=("$1"); shift ;; esac done @@ -84,6 +84,17 @@ mktemp_file() { return 1 } +print_help() { + awk ' + /^# Usage:/ {printing=1} + printing { + if ($0 !~ /^#/) {exit} + sub(/^# ?/, "", $0) + print + } + ' "$0" +} + # Collection and execution do not share every pytest flag. Keep runtime # passthrough aligned with user intent while stripping collect-only toggles # and xdist-specific options that this script owns. @@ -115,6 +126,7 @@ if (( XDIST_IGNORED )); then fi cleanup() { + exec 3<&- 2>/dev/null || true [[ -n "$COLLECT_ERR" && -f "$COLLECT_ERR" ]] && rm -f "$COLLECT_ERR" [[ -n "$COLLECT_OUT" && -f "$COLLECT_OUT" ]] && rm -f "$COLLECT_OUT" [[ -n "$COLLECT_FILTERED" && -f "$COLLECT_FILTERED" ]] && rm -f "$COLLECT_FILTERED" @@ -136,16 +148,16 @@ write_cursor() { mv "$CURSOR_TMP" "$CURSOR_FILE" } -load_chunk_nodes() { - local start_index="$1" - local end_index="$2" - local start_line=$(( start_index + 1 )) - local end_line="$end_index" - +read_chunk() { + local count=0 CHUNK_NODES=() - while IFS= read -r node; do - [[ -n "$node" ]] && CHUNK_NODES+=("$node") - done < <(awk -v start="$start_line" -v end="$end_line" 'NR < start {next} NR > end {exit} {print}' "$COLLECT_OUT") + while (( count < CHUNK_SIZE )); do + if ! IFS= read -r node <&3; then + break + fi + CHUNK_NODES+=("$node") + count=$((count + 1)) + done } cd "$REPO_ROOT" @@ -248,6 +260,15 @@ fi CHUNKS=$(( (TOTAL - START + CHUNK_SIZE - 1) / CHUNK_SIZE )) echo "[fast-test] $TOTAL tests · chunk=$CHUNK_SIZE · $CHUNKS chunk(s) queued · workers=auto" +exec 3< "$COLLECT_OUT" +skip_count=0 +while (( skip_count < START )); do + if ! IFS= read -r _ <&3; then + break + fi + skip_count=$((skip_count + 1)) +done + # 3. FIFO dispatch. T_START=$(date +%s) i="$START" @@ -256,6 +277,12 @@ while (( i < TOTAL )); do end=$(( i + CHUNK_SIZE )) (( end > TOTAL )) && end="$TOTAL" chunk_idx=$(( chunk_idx + 1 )) + read_chunk + if (( ${#CHUNK_NODES[@]} == 0 )); then + echo "[fast-test] no tests read for chunk; stopping" >&2 + break + fi + end=$(( i + ${#CHUNK_NODES[@]} )) echo "[fast-test] chunk $chunk_idx/$CHUNKS tests $((i+1))..$end" PYTEST_FLAGS=(-n auto --dist=load) @@ -265,9 +292,8 @@ while (( i < TOTAL )); do PYTEST_FLAGS+=(--no-header) fi - load_chunk_nodes "$i" "$end" if ! "${PYTEST_CMD[@]}" "${PYTEST_FLAGS[@]}" "${RUNTIME_PASSTHROUGH[@]}" "${CHUNK_NODES[@]}"; then - echo "[fast-test] chunk failed — cursor preserved at next test index $i (use --resume to retry)" + echo "[fast-test] chunk failed — cursor preserved at next test index $i (use --resume to retry)" >&2 write_cursor "$i" exit 1 fi @@ -277,6 +303,7 @@ while (( i < TOTAL )); do done T_END=$(date +%s) +exec 3<&- 2>/dev/null || true rm -f "$COLLECT_OUT" rm -f "$CURSOR_FILE" echo "[fast-test] all $TOTAL tests passed in $((T_END - T_START))s" From aeb476ec5a20d160c8a5c9915e2928b1a45860b8 Mon Sep 17 00:00:00 2001 From: LahkLeKey Date: Thu, 4 Jun 2026 18:26:49 -0500 Subject: [PATCH 13/36] chore: harden fast-test fd handling --- scripts/dev/test.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/dev/test.sh b/scripts/dev/test.sh index e6fd48fb98..16a57fa33a 100644 --- a/scripts/dev/test.sh +++ b/scripts/dev/test.sh @@ -126,7 +126,7 @@ if (( XDIST_IGNORED )); then fi cleanup() { - exec 3<&- 2>/dev/null || true + { exec 3<&-; } 2>/dev/null || true [[ -n "$COLLECT_ERR" && -f "$COLLECT_ERR" ]] && rm -f "$COLLECT_ERR" [[ -n "$COLLECT_OUT" && -f "$COLLECT_OUT" ]] && rm -f "$COLLECT_OUT" [[ -n "$COLLECT_FILTERED" && -f "$COLLECT_FILTERED" ]] && rm -f "$COLLECT_FILTERED" @@ -280,7 +280,8 @@ while (( i < TOTAL )); do read_chunk if (( ${#CHUNK_NODES[@]} == 0 )); then echo "[fast-test] no tests read for chunk; stopping" >&2 - break + write_cursor "$i" + exit 1 fi end=$(( i + ${#CHUNK_NODES[@]} )) echo "[fast-test] chunk $chunk_idx/$CHUNKS tests $((i+1))..$end" @@ -303,7 +304,7 @@ while (( i < TOTAL )); do done T_END=$(date +%s) -exec 3<&- 2>/dev/null || true +{ exec 3<&-; } 2>/dev/null || true rm -f "$COLLECT_OUT" rm -f "$CURSOR_FILE" echo "[fast-test] all $TOTAL tests passed in $((T_END - T_START))s" From 500c843a9fb5f3950e35f0720398ec43e11fe168 Mon Sep 17 00:00:00 2001 From: LahkLeKey Date: Thu, 4 Jun 2026 18:28:43 -0500 Subject: [PATCH 14/36] chore: fix fast-test help and lock --- scripts/dev/test.sh | 34 +++++++++++++--------------------- 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/scripts/dev/test.sh b/scripts/dev/test.sh index 16a57fa33a..8073f73902 100644 --- a/scripts/dev/test.sh +++ b/scripts/dev/test.sh @@ -46,6 +46,17 @@ COLLECT_FILTERED="" CHUNK_NODES=() XDIST_IGNORED=0 +print_help() { + awk ' + /^# Usage:/ {printing=1} + printing { + if ($0 !~ /^#/) {exit} + sub(/^# ?/, "", $0) + print + } + ' "$0" +} + while (( $# )); do case "$1" in --chunk-size) @@ -84,17 +95,6 @@ mktemp_file() { return 1 } -print_help() { - awk ' - /^# Usage:/ {printing=1} - printing { - if ($0 !~ /^#/) {exit} - sub(/^# ?/, "", $0) - print - } - ' "$0" -} - # Collection and execution do not share every pytest flag. Keep runtime # passthrough aligned with user intent while stripping collect-only toggles # and xdist-specific options that this script owns. @@ -176,16 +176,8 @@ else if [[ -f "$LOCK_DIR/pid" ]]; then PID_CONTENTS="$(tr -d '[:space:]' < "$LOCK_DIR/pid" 2>/dev/null || true)" if [[ "$PID_CONTENTS" =~ ^[0-9]+$ ]] && kill -0 "$PID_CONTENTS" 2>/dev/null; then - if command -v ps >/dev/null 2>&1; then - CMDLINE="$(ps -p "$PID_CONTENTS" -o args= 2>/dev/null || true)" - if [[ -z "$CMDLINE" || "$CMDLINE" == *"$LOCK_NAME"* ]]; then - echo "[fast-test] another run is active (lock: $LOCK_DIR)" >&2 - exit 1 - fi - else - echo "[fast-test] another run is active (lock: $LOCK_DIR)" >&2 - exit 1 - fi + echo "[fast-test] another run is active (lock: $LOCK_DIR)" >&2 + exit 1 fi echo "[fast-test] removing stale lock (pid: ${PID_CONTENTS:-unknown})" >&2 fi From f711540aad948dae788904b6e89d6bb594b6da4a Mon Sep 17 00:00:00 2001 From: LahkLeKey Date: Thu, 4 Jun 2026 18:30:41 -0500 Subject: [PATCH 15/36] chore: tighten fast-test locks --- scripts/dev/test.sh | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/scripts/dev/test.sh b/scripts/dev/test.sh index 8073f73902..a0ae061209 100644 --- a/scripts/dev/test.sh +++ b/scripts/dev/test.sh @@ -134,10 +134,14 @@ cleanup() { if [[ "$LOCK_MODE" == "flock" ]]; then flock -u 9 2>/dev/null || true exec 9>&- || true - rm -f "$LOCK_FILE" - elif (( LOCK_HELD )); then - rm -f "$LOCK_DIR/pid" - rmdir "$LOCK_DIR" 2>/dev/null || rm -rf "$LOCK_DIR" + elif [[ "$LOCK_MODE" == "dir" ]] && (( LOCK_HELD )); then + if [[ -f "$LOCK_DIR/pid" ]]; then + PID_CONTENTS="$(tr -d '[:space:]' < "$LOCK_DIR/pid" 2>/dev/null || true)" + if [[ "$PID_CONTENTS" == "$$" ]]; then + rm -f "$LOCK_DIR/pid" + rmdir "$LOCK_DIR" 2>/dev/null || true + fi + fi fi } @@ -211,6 +215,12 @@ else exit 1 fi +if ! "${PYTEST_CMD[@]}" --help 2>/dev/null | grep -Eq '(-n[[:space:]]|--numprocesses)'; then + echo "[fast-test] pytest-xdist is required for this wrapper (missing -n/--numprocesses)." >&2 + echo "[fast-test] install test extras, e.g. 'uv pip install -e .[test]'" >&2 + exit 1 +fi + echo "[fast-test] collecting tests ..." COLLECT_ERR="$(mktemp_file)" COLLECT_OUT="$(mktemp_file)" From f5e0a017ccdc19553492a1111dba00bf7d26ef5f Mon Sep 17 00:00:00 2001 From: LahkLeKey Date: Thu, 4 Jun 2026 18:34:02 -0500 Subject: [PATCH 16/36] chore: clarify fast-test resume --- scripts/dev/test.sh | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/scripts/dev/test.sh b/scripts/dev/test.sh index a0ae061209..2a68fbefe4 100644 --- a/scripts/dev/test.sh +++ b/scripts/dev/test.sh @@ -11,6 +11,8 @@ # * Persist the cursor (next test index / chunk boundary) to # `.pytest_cache/fast-test-cursor` after every successful chunk so # `--resume` continues from the last completed chunk. +# * Resume is tied to the current collection order; changing selection +# or order can skip or re-run tests. # * `--reset` clears the cursor; `--bench` reduces pytest output (progress still prints). # # Usage: @@ -20,6 +22,10 @@ # scripts/dev/test.sh --reset # clear cursor # scripts/dev/test.sh --bench # quieter pytest output # scripts/dev/test.sh -- tests/test_merge.py # pass-through to pytest +# +# Notes: +# --resume assumes the same collection order; changing selection/order +# can skip or re-run tests. set -euo pipefail @@ -45,6 +51,7 @@ COLLECT_OUT="" COLLECT_FILTERED="" CHUNK_NODES=() XDIST_IGNORED=0 +COLLECT_ONLY_REQUESTED=0 print_help() { awk ' @@ -81,6 +88,10 @@ if ! [[ "$CHUNK_SIZE" =~ ^[1-9][0-9]*$ ]]; then exit 1 fi +if (( CHUNK_SIZE > 1000 )); then + echo "[fast-test] warning: large --chunk-size may exceed OS argument limits" >&2 +fi + mktemp_file() { local tmp if tmp="$(mktemp -t fast-test.XXXXXX 2>/dev/null)"; then @@ -102,7 +113,7 @@ idx=0 while (( idx < ${#PASSTHROUGH[@]} )); do arg="${PASSTHROUGH[$idx]}" case "$arg" in - --collect-only|--co) ;; + --collect-only|--co) COLLECT_ONLY_REQUESTED=1 ;; -n|--numprocesses|--dist) next="${PASSTHROUGH[$((idx + 1))]:-}" if [[ -z "$next" || "$next" == -* ]]; then @@ -121,6 +132,11 @@ while (( idx < ${#PASSTHROUGH[@]} )); do idx=$((idx + 1)) done +if (( COLLECT_ONLY_REQUESTED )); then + echo "[fast-test] --collect-only is not supported; run pytest --collect-only directly" >&2 + exit 1 +fi + if (( XDIST_IGNORED )); then echo "[fast-test] ignoring xdist flags (-n/--numprocesses/--dist); this script manages them" >&2 fi From ba75dab89556723b06229800df4dc67c04c9732b Mon Sep 17 00:00:00 2001 From: LahkLeKey Date: Thu, 4 Jun 2026 18:37:22 -0500 Subject: [PATCH 17/36] chore: route fast-test logs to stderr --- scripts/dev/test.sh | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/scripts/dev/test.sh b/scripts/dev/test.sh index 2a68fbefe4..7d3057c6b4 100644 --- a/scripts/dev/test.sh +++ b/scripts/dev/test.sh @@ -53,6 +53,10 @@ CHUNK_NODES=() XDIST_IGNORED=0 COLLECT_ONLY_REQUESTED=0 +log() { + echo "$*" >&2 +} + print_help() { awk ' /^# Usage:/ {printing=1} @@ -89,7 +93,7 @@ if ! [[ "$CHUNK_SIZE" =~ ^[1-9][0-9]*$ ]]; then fi if (( CHUNK_SIZE > 1000 )); then - echo "[fast-test] warning: large --chunk-size may exceed OS argument limits" >&2 + log "[fast-test] warning: large --chunk-size may exceed OS argument limits" fi mktemp_file() { @@ -138,7 +142,7 @@ if (( COLLECT_ONLY_REQUESTED )); then fi if (( XDIST_IGNORED )); then - echo "[fast-test] ignoring xdist flags (-n/--numprocesses/--dist); this script manages them" >&2 + log "[fast-test] ignoring xdist flags (-n/--numprocesses/--dist); this script manages them" fi cleanup() { @@ -170,6 +174,7 @@ write_cursor() { read_chunk() { local count=0 + local node CHUNK_NODES=() while (( count < CHUNK_SIZE )); do if ! IFS= read -r node <&3; then @@ -215,7 +220,7 @@ fi if (( RESET )); then rm -f "$CURSOR_FILE" "$CURSOR_TMP" - echo "[fast-test] cursor cleared" + log "[fast-test] cursor cleared" exit 0 fi @@ -237,7 +242,7 @@ if ! "${PYTEST_CMD[@]}" --help 2>/dev/null | grep -Eq '(-n[[:space:]]|--numproce exit 1 fi -echo "[fast-test] collecting tests ..." +log "[fast-test] collecting tests ..." COLLECT_ERR="$(mktemp_file)" COLLECT_OUT="$(mktemp_file)" if ! "${PYTEST_CMD[@]}" --collect-only -q "${COLLECT_PASSTHROUGH[@]}" >"$COLLECT_OUT" 2>"$COLLECT_ERR"; then @@ -263,20 +268,21 @@ if (( RESUME )) && [[ -f "$CURSOR_FILE" ]]; then if [[ "$RAW_START" =~ ^[0-9]+$ ]]; then START="$RAW_START" (( START > TOTAL )) && START="$TOTAL" - echo "[fast-test] resuming from next test index: $START" + log "[fast-test] resuming from next test index: $START" else echo "[fast-test] cursor file is invalid ('$RAW_START'); starting from 0" >&2 + rm -f "$CURSOR_FILE" fi fi if (( START >= TOTAL )); then - echo "[fast-test] nothing to resume (cursor at end: $START/$TOTAL)" + log "[fast-test] nothing to resume (cursor at end: $START/$TOTAL)" rm -f "$CURSOR_FILE" exit 0 fi CHUNKS=$(( (TOTAL - START + CHUNK_SIZE - 1) / CHUNK_SIZE )) -echo "[fast-test] $TOTAL tests · chunk=$CHUNK_SIZE · $CHUNKS chunk(s) queued · workers=auto" +log "[fast-test] $TOTAL tests · chunk=$CHUNK_SIZE · $CHUNKS chunk(s) queued · workers=auto" exec 3< "$COLLECT_OUT" skip_count=0 @@ -302,7 +308,7 @@ while (( i < TOTAL )); do exit 1 fi end=$(( i + ${#CHUNK_NODES[@]} )) - echo "[fast-test] chunk $chunk_idx/$CHUNKS tests $((i+1))..$end" + log "[fast-test] chunk $chunk_idx/$CHUNKS tests $((i+1))..$end" PYTEST_FLAGS=(-n auto --dist=load) if (( BENCH )); then @@ -325,4 +331,4 @@ T_END=$(date +%s) { exec 3<&-; } 2>/dev/null || true rm -f "$COLLECT_OUT" rm -f "$CURSOR_FILE" -echo "[fast-test] all $TOTAL tests passed in $((T_END - T_START))s" +log "[fast-test] all $TOTAL tests passed in $((T_END - T_START))s" From ec063444351a4d5197be8afb046baaf7a915cf58 Mon Sep 17 00:00:00 2001 From: LahkLeKey Date: Thu, 4 Jun 2026 18:45:13 -0500 Subject: [PATCH 18/36] chore: harden fast-test argv sizing --- scripts/dev/test.sh | 61 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 55 insertions(+), 6 deletions(-) diff --git a/scripts/dev/test.sh b/scripts/dev/test.sh index 7d3057c6b4..47059a5d86 100644 --- a/scripts/dev/test.sh +++ b/scripts/dev/test.sh @@ -52,6 +52,7 @@ COLLECT_FILTERED="" CHUNK_NODES=() XDIST_IGNORED=0 COLLECT_ONLY_REQUESTED=0 +FD3_OPEN=0 log() { echo "$*" >&2 @@ -92,10 +93,6 @@ if ! [[ "$CHUNK_SIZE" =~ ^[1-9][0-9]*$ ]]; then exit 1 fi -if (( CHUNK_SIZE > 1000 )); then - log "[fast-test] warning: large --chunk-size may exceed OS argument limits" -fi - mktemp_file() { local tmp if tmp="$(mktemp -t fast-test.XXXXXX 2>/dev/null)"; then @@ -110,6 +107,15 @@ mktemp_file() { return 1 } +arg_bytes() { + local total=0 + local arg + for arg in "$@"; do + total=$(( total + ${#arg} + 1 )) + done + printf '%s\n' "$total" +} + # Collection and execution do not share every pytest flag. Keep runtime # passthrough aligned with user intent while stripping collect-only toggles # and xdist-specific options that this script owns. @@ -146,7 +152,10 @@ if (( XDIST_IGNORED )); then fi cleanup() { - { exec 3<&-; } 2>/dev/null || true + if (( FD3_OPEN )); then + { exec 3<&-; } 2>/dev/null || true + FD3_OPEN=0 + fi [[ -n "$COLLECT_ERR" && -f "$COLLECT_ERR" ]] && rm -f "$COLLECT_ERR" [[ -n "$COLLECT_OUT" && -f "$COLLECT_OUT" ]] && rm -f "$COLLECT_OUT" [[ -n "$COLLECT_FILTERED" && -f "$COLLECT_FILTERED" ]] && rm -f "$COLLECT_FILTERED" @@ -197,6 +206,10 @@ if command -v flock >/dev/null 2>&1; then LOCK_MODE="flock" LOCK_HELD=1 else + if [[ -L "$LOCK_DIR" ]]; then + echo "[fast-test] lock path is a symlink; refusing to use $LOCK_DIR" >&2 + exit 1 + fi if [[ -d "$LOCK_DIR" ]]; then if [[ -f "$LOCK_DIR/pid" ]]; then PID_CONTENTS="$(tr -d '[:space:]' < "$LOCK_DIR/pid" 2>/dev/null || true)" @@ -261,6 +274,38 @@ if (( TOTAL == 0 )); then exit 1 fi +ARG_MAX="" +if command -v getconf >/dev/null 2>&1; then + ARG_MAX="$(getconf ARG_MAX 2>/dev/null || true)" +fi +if [[ "$ARG_MAX" =~ ^[0-9]+$ ]]; then + MAX_NODE_LEN="$(awk 'length > max { max = length } END { print max + 0 }' "$COLLECT_OUT")" + BASE_PYTEST_ARGS=(-n auto --dist=load --no-header) + if (( BENCH )); then + BASE_PYTEST_ARGS+=(-q) + fi + BASE_ARGS_SIZE="$(arg_bytes "${PYTEST_CMD[@]}" "${BASE_PYTEST_ARGS[@]}" "${RUNTIME_PASSTHROUGH[@]}")" + SAFETY_MARGIN=2048 + AVAILABLE=$(( ARG_MAX - BASE_ARGS_SIZE - SAFETY_MARGIN )) + (( AVAILABLE < 0 )) && AVAILABLE=0 + PER_NODE=$(( MAX_NODE_LEN + 1 )) + if (( PER_NODE > 0 )); then + MAX_SAFE=$(( AVAILABLE / PER_NODE )) + if (( MAX_SAFE <= 0 )); then + echo "[fast-test] chunk size too large for ARG_MAX; reduce --chunk-size" >&2 + exit 1 + fi + if (( CHUNK_SIZE > MAX_SAFE )); then + log "[fast-test] reducing --chunk-size from $CHUNK_SIZE to $MAX_SAFE to avoid ARG_MAX" + CHUNK_SIZE="$MAX_SAFE" + fi + fi +else + if (( CHUNK_SIZE > 1000 )); then + log "[fast-test] warning: large --chunk-size may exceed OS argument limits" + fi +fi + # 2. Determine starting cursor. START=0 if (( RESUME )) && [[ -f "$CURSOR_FILE" ]]; then @@ -285,6 +330,7 @@ CHUNKS=$(( (TOTAL - START + CHUNK_SIZE - 1) / CHUNK_SIZE )) log "[fast-test] $TOTAL tests · chunk=$CHUNK_SIZE · $CHUNKS chunk(s) queued · workers=auto" exec 3< "$COLLECT_OUT" +FD3_OPEN=1 skip_count=0 while (( skip_count < START )); do if ! IFS= read -r _ <&3; then @@ -328,7 +374,10 @@ while (( i < TOTAL )); do done T_END=$(date +%s) -{ exec 3<&-; } 2>/dev/null || true +if (( FD3_OPEN )); then + { exec 3<&-; } 2>/dev/null || true + FD3_OPEN=0 +fi rm -f "$COLLECT_OUT" rm -f "$CURSOR_FILE" log "[fast-test] all $TOTAL tests passed in $((T_END - T_START))s" From abb8cb5fe02e761526f3096ed99ffb39251642e5 Mon Sep 17 00:00:00 2001 From: LahkLeKey Date: Thu, 4 Jun 2026 18:53:18 -0500 Subject: [PATCH 19/36] chore: align test wrapper naming --- scripts/dev/test.sh | 100 ++++++++++++++++++++++++++------------------ 1 file changed, 59 insertions(+), 41 deletions(-) diff --git a/scripts/dev/test.sh b/scripts/dev/test.sh index 47059a5d86..b5e72116e4 100644 --- a/scripts/dev/test.sh +++ b/scripts/dev/test.sh @@ -9,7 +9,7 @@ # workers as they finish — natural FIFO progression with bounded # in-flight memory. # * Persist the cursor (next test index / chunk boundary) to -# `.pytest_cache/fast-test-cursor` after every successful chunk so +# `.pytest_cache/