From 811190aa490b1edf004134aba756795f1890d9c4 Mon Sep 17 00:00:00 2001 From: Aaron McHale Date: Sun, 8 Nov 2026 18:46:04 +0000 Subject: [PATCH 1/3] Allow test runner to accept paths to specific test files --- .../tests-writing-test-functions/SKILL.md | 10 ++--- docs/Tests.md | 14 +++--- tests/run-tests.sh | 25 ++++++----- tests/test-cli-0.sh | 45 +++++++++++-------- tests/test-cli-helper-functions.sh | 8 ++-- 5 files changed, 61 insertions(+), 41 deletions(-) diff --git a/.agents/skills/tests-writing-test-functions/SKILL.md b/.agents/skills/tests-writing-test-functions/SKILL.md index 0cc9768..6f624b7 100644 --- a/.agents/skills/tests-writing-test-functions/SKILL.md +++ b/.agents/skills/tests-writing-test-functions/SKILL.md @@ -15,30 +15,30 @@ Instructions: - Use `return 0` for success and `return 1` for failure. - Write code procedurally inside test functions and always return early when errors occur; do not use subshells or pipelines that would prevent returning the correct exit code. - Do not use `exit` inside test functions. -- Use the `$TEST_DIR` variable when referencing paths in tests. +- Use the `$TESTROOT` variable when referencing paths in tests. - Ensure cleanup is performed in `teardown` function when necessary. Example test file: ```Bash # Example setup function setup() { - echo 'echo "Test command executed"' > "$TEST_DIR/../commands/test-cmd.sh" + cd "$TESTROOT/.." + echo 'echo "Test command executed"' > "./commands/test-cmd.sh" } # Example test function test_cli_output_for_test_cmd() { echo "Testing cli.sh output for our test command" - cd .. if [[ "$(./cli.sh)" != *"test-cmd"* ]]; then echo "FAIL: Test command test-cmd not found in cli.sh output." return 1 fi - cd "$TEST_DIR" || { echo "Failed to cd back to tests directory"; return 1; } } # Example teardown function teardown() { - rm "$TEST_DIR/../commands/test-cmd.sh" || true + cd "$TESTROOT/.." + rm "./commands/test-cmd.sh" || true } ``` diff --git a/docs/Tests.md b/docs/Tests.md index 97cfb61..b010429 100644 --- a/docs/Tests.md +++ b/docs/Tests.md @@ -8,13 +8,15 @@ To run all tests, run `./tests/run-tests.sh`, this is known as the test runner. The test runner finds all shell scripts in `./tests` which start with `test-` and end in `.sh`. Each script is sourced and all functions starting `test_` are run. +You can also pass one or more files into the test runner as arguments, and it will run only those specific test files. + Each test file is sourced in a sub-shell, this means that variables and functions defined in test files should not leak out of the test file. ## Structure of a test file A test file may contian: -1. A `setup` function which is responsible for setting up the environment, this could include creating files and folders that are needed for the tests to run. If `setup` exists, the test runner will run it before running any tests, this means test functions do not need to call `setup`. +1. A `setup` function which is responsible for setting up the environment, this could include creating files and folders that are needed for the tests to run. If `setup` exists, the test runner will run it before running any tests, this means test functions do not need to call `setup`. If tests need to be run from a specific directory, the `setup` function can use `cd`. 2. One or more test functions, each test function must start with `test_`. 3. A `teardown` function which is responsible for cleaning up the environment, if `setup` or any test functions create files, directories or make any other changes, `teardown` must undo those and restore the environment. If `teardown` exists, the test runner will run it after all tests complete, or after any tests fail, this means test functions do not need to call `teardown`. Teardown functions should try to always true. @@ -22,19 +24,20 @@ A test file may contian: Tests functions should be written as follows: * Test functions must be named starting with `test_`, they will be run automatically by the test runner. -* Use the `setup` function for setting up the environment, doing things like creating files and directories used by tests. +* Use the `setup` function for setting up the environment, doing things like creating files and directories used by tests and changing the current directory. * Use the `teardown` function for restoring the environment, deleting anything created in the `setup` function. * Test functions must never call `setup` and `teardown`, the test runner will call these. * Test functions should return `0` if the test is successful, or `1` if the test fails. * Test functions must never use `exit`, they should always use `return` instead, this ensures control is properly handed back to the test runner. Tests should `return 1` for fail, and `return 0` for success. -* The variable `$TEST_DIR` contains the absolute path to the `tests` directory, this is defined by the test runner. +* The variable `$TESTROOT` contains the absolute path to the `tests` directory, this is defined by the test runner. ## Examples ```Bash # Example setup function setup() { - echo 'echo "Test command executed"' > "$TEST_DIR/../commands/test-cmd.sh" + cd "$TESTROOT/.." + echo 'echo "Test command executed"' > "./commands/test-cmd.sh" } # Example test function @@ -50,6 +53,7 @@ test_cli_output_for_test_cmd() { # Example teardown function teardown() { - rm "$TEST_DIR/../commands/test-cmd.sh" || true + cd "$TESTROOT/.." + rm "./commands/test-cmd.sh" || true } ``` diff --git a/tests/run-tests.sh b/tests/run-tests.sh index cf6c366..f5fd1d5 100755 --- a/tests/run-tests.sh +++ b/tests/run-tests.sh @@ -3,22 +3,20 @@ set -u # Run all test_ functions in test-*.sh scripts -# Cd to the directory of this script, so it can be run from anywhere +# Get the absolute path to the tests directory. realpath="$(realpath "$0")" || { echo "Failed to get realpath"; exit 1; } -cd "${realpath%/*}" || { echo "Failed to cd to script directory"; exit 1; } - -# Save the current directory if tests need a absolute path reference -# shellcheck disable=SC2034 -TEST_DIR="$PWD" +TESTROOT="${realpath%/*}" # Run a test file. # Uses a subshell so that each test file is isolated. # Usage: run_tests "test-file.sh" run_tests() {( - local test_file="$1" - echo -e "Running tests in $test_file...\n" + local test_path="$1" + local test_file="${1##*/}" + echo -e "Running tests in $test_file..." + echo -e "Path: $test_path\n" # shellcheck disable=SC1090 - source "$test_file" + source "$test_path" # As we're running in a subshell, we can use trap to run teardown and print message on exit. # shellcheck disable=SC2329 @@ -56,7 +54,14 @@ run_tests() {( done < <(declare -F) )} -TEST_FILES=(test-*.sh) +# If test files are provided, run only those specific tests. +# Otherwise, run all test-*.sh files in the tests directory. +if [[ $# -gt 0 ]]; then + TEST_FILES=("$@") +else + TEST_FILES=("$TESTROOT"/test-*.sh) +fi + for test_file in "${TEST_FILES[@]}"; do # Confirm that test_file is a file and is not empty before trying to run it if [[ -f "$test_file" && -s "$test_file" ]]; then diff --git a/tests/test-cli-0.sh b/tests/test-cli-0.sh index a88e3d9..e264eaf 100644 --- a/tests/test-cli-0.sh +++ b/tests/test-cli-0.sh @@ -1,47 +1,57 @@ #!/usr/bin/env bash teardown() { - rm "$TEST_DIR/../commands/test-cmd.sh" || true - rm "$TEST_DIR/../commands/test-cmd.help.txt" || true - rm "$TEST_DIR/cli-symlink.sh" || true + cd "$TESTROOT/.." + rm "commands/test-cmd.sh" || true + rm "commands/test-cmd.help.txt" || true + rm "tests/cli-symlink.sh" || true } setup() { - echo 'echo "Test command executed"' > "$TEST_DIR/../commands/test-cmd.sh" - echo "this is a test command" > "$TEST_DIR/../commands/test-cmd.help.txt" - echo "this line should not show on the command list" >> "$TEST_DIR/../commands/test-cmd.help.txt" - ln -s ../cli.sh cli-symlink.sh || true + TMPDIR="$(mktemp -d -t web-router-cli-wrapper-tests )" || { + echo "FAIL: could not generate temp dir." && + return 1 + } + # Most tests here expect to be in the project root. + cd "$TESTROOT/.." + echo 'echo "Test command executed"' > "commands/test-cmd.sh" + echo "this is a test command" > "commands/test-cmd.help.txt" + echo "this line should not show on the command list" >> "commands/test-cmd.help.txt" + ln -s cli.sh tests/cli-symlink.sh } test_cli_output_for_test_cmd() { echo "Testing cli.sh output for our test command" - cd .. if [[ "$(./cli.sh)" != *"test-cmd"* ]]; then echo "FAIL: Test command test-cmd not found in cli.sh output." return 1 fi - cd "$TEST_DIR" || { echo "Failed to cd back to tests directory"; return 1; } } test_cli_from_tests_dir() { echo "Testing running cli.sh from tests directory, to proove that it can be run from anywhere" + # This test needs to be in a different directory from the project root + cd "$TESTROOT" if [[ "$(./../cli.sh)" != *"test-cmd"* ]]; then echo "FAIL: Test command test-cmd not found when running from tests dir." return 1 fi + cd .. } test_cli_symlink_from_tests_dir() { echo "Testing cli.sh as a symlink from tests directory, to prove that it can be run as a symlink from anywhere" + cd "$TESTROOT" if [[ "$(./cli-symlink.sh)" != *"test-cmd"* ]]; then echo "FAIL: Test command test-cmd not found when running cli.sh as a symlink from tests dir" return 1 fi + cd .. } test_cli_list_output() { echo "Testing list sub-command to check output" - if [[ "$(./../cli.sh list)" != *"test-cmd"* ]]; then + if [[ "$(./cli.sh list)" != *"test-cmd"* ]]; then echo "FAIL: Test command test-cmd not found in cli.sh list output" return 1 fi @@ -50,13 +60,12 @@ test_cli_list_output() { test_sourcing_cli_is_silent_and_uses_current_directory() { echo "Testing sourcing cli.sh is silent and sets CLIROOT to the current directory" local output expected_cli_root - expected_cli_root="$(cd "$TEST_DIR/.." && pwd)" || return 1 + expected_cli_root="$PWD" output=$(bash -c ' - cd "$1" || exit 1 - source "$2" + source "$1" command_exists "test-cmd" || exit 1 printf "%s" "$CLIROOT" - ' bash "$expected_cli_root" "$expected_cli_root/cli.sh" 2>&1) || { + ' bash "$expected_cli_root/cli.sh" 2>&1) || { echo "FAIL: Sourcing cli.sh did not expose its helpers." return 1 } @@ -68,7 +77,7 @@ test_sourcing_cli_is_silent_and_uses_current_directory() { test_list_output_for_test_cmd_help() { echo "Testing help text file file for test command" - if [[ "$(./../cli.sh list)" != *"this is a test command"* ]]; then + if [[ "$(./cli.sh list)" != *"this is a test command"* ]]; then echo "FAIL: content not found in cli.sh list output" return 1 fi @@ -76,7 +85,7 @@ test_list_output_for_test_cmd_help() { test_help_output_for_test_cmd() { echo "Testing help output for test command" - if [[ "$(./../cli.sh help test-cmd)" != *"this is a test command"* ]]; then + if [[ "$(./cli.sh help test-cmd)" != *"this is a test command"* ]]; then echo "FAIL: content not found in cli.sh help output" return 1 fi @@ -84,7 +93,7 @@ test_help_output_for_test_cmd() { test_list_only_shows_first_line_of_help_file() { echo "Test that a second line in the help file does not show on command list" - if [[ "$(./../cli.sh list)" = *"this line should not show on the command list"* ]]; then + if [[ "$(./cli.sh list)" = *"this line should not show on the command list"* ]]; then echo "FAIL: content not found in cli.sh list output" return 1 fi @@ -92,7 +101,7 @@ test_list_only_shows_first_line_of_help_file() { test_help_shows_all_lines_of_help_file() { echo "Test that second line of help file shows when running help test-cmd" - if [[ "$(./../cli.sh help test-cmd)" != *"this line should not show on the command list"* ]]; then + if [[ "$(./cli.sh help test-cmd)" != *"this line should not show on the command list"* ]]; then echo "FAIL: content not found in cli.sh help output" return 1 fi diff --git a/tests/test-cli-helper-functions.sh b/tests/test-cli-helper-functions.sh index bab593b..91c304e 100644 --- a/tests/test-cli-helper-functions.sh +++ b/tests/test-cli-helper-functions.sh @@ -3,15 +3,16 @@ # Tests for warning, error, and success output formatting setup() { + cd "$TESTROOT/.." # Use Bash builtin printf to write the command script printf '%s\n' '#!/usr/bin/env bash' \ 'command_exists "formatting-test" && echo "formatting-test exists"' \ 'warning This is a warning' \ 'error This is an error' \ - 'success This is a success' > "$TEST_DIR/../commands/formatting-test.sh" + 'success This is a success' > "./commands/formatting-test.sh" # Run the command and store output in a variable for tests - TEST_FORMATTING_OUTPUT=$("$TEST_DIR/../cli.sh" formatting-test 2>&1) + TEST_FORMATTING_OUTPUT=$(./cli.sh formatting-test 2>&1) } test_command_exists() { @@ -63,5 +64,6 @@ test_success_output() { } teardown() { - rm -f "$TEST_DIR/../commands/formatting-test.sh" || true + cd "$TESTROOT/.." + rm -f "commands/formatting-test.sh" || true } From fef9aa53847f2cc899ca8f3b0951bbc9d0679b6e Mon Sep 17 00:00:00 2001 From: Aaron McHale Date: Sun, 15 Nov 2026 13:24:25 +0000 Subject: [PATCH 2/3] Fix tests and linting --- tests/test-cli-0.sh | 22 +++++++++++----------- tests/test-cli-helper-functions.sh | 4 ++-- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/test-cli-0.sh b/tests/test-cli-0.sh index e264eaf..9a9d243 100644 --- a/tests/test-cli-0.sh +++ b/tests/test-cli-0.sh @@ -1,19 +1,19 @@ #!/usr/bin/env bash teardown() { - cd "$TESTROOT/.." + cd "$TESTROOT/.." || return 1 rm "commands/test-cmd.sh" || true rm "commands/test-cmd.help.txt" || true rm "tests/cli-symlink.sh" || true } setup() { - TMPDIR="$(mktemp -d -t web-router-cli-wrapper-tests )" || { + TMPDIR="$(mktemp -d -t web-router-cli-wrapper-tests.XXXXXX )" || { echo "FAIL: could not generate temp dir." && return 1 } # Most tests here expect to be in the project root. - cd "$TESTROOT/.." + cd "$TESTROOT/.." || return 1 echo 'echo "Test command executed"' > "commands/test-cmd.sh" echo "this is a test command" > "commands/test-cmd.help.txt" echo "this line should not show on the command list" >> "commands/test-cmd.help.txt" @@ -28,26 +28,26 @@ test_cli_output_for_test_cmd() { fi } -test_cli_from_tests_dir() { +test_cli_from_tests_dir() {( echo "Testing running cli.sh from tests directory, to proove that it can be run from anywhere" # This test needs to be in a different directory from the project root - cd "$TESTROOT" + # Test is run in a sub-shell, so we don't need to cd back. + cd "$TESTROOT" || return 1 if [[ "$(./../cli.sh)" != *"test-cmd"* ]]; then echo "FAIL: Test command test-cmd not found when running from tests dir." return 1 fi - cd .. -} +)} -test_cli_symlink_from_tests_dir() { +test_cli_symlink_from_tests_dir() {( echo "Testing cli.sh as a symlink from tests directory, to prove that it can be run as a symlink from anywhere" - cd "$TESTROOT" + # Test is run in a sub-shell, so we don't need to cd back. + cd "$TESTROOT" || return 1 if [[ "$(./cli-symlink.sh)" != *"test-cmd"* ]]; then echo "FAIL: Test command test-cmd not found when running cli.sh as a symlink from tests dir" return 1 fi - cd .. -} +)} test_cli_list_output() { echo "Testing list sub-command to check output" diff --git a/tests/test-cli-helper-functions.sh b/tests/test-cli-helper-functions.sh index 91c304e..2aefa95 100644 --- a/tests/test-cli-helper-functions.sh +++ b/tests/test-cli-helper-functions.sh @@ -3,7 +3,7 @@ # Tests for warning, error, and success output formatting setup() { - cd "$TESTROOT/.." + cd "$TESTROOT/.." || return 1 # Use Bash builtin printf to write the command script printf '%s\n' '#!/usr/bin/env bash' \ 'command_exists "formatting-test" && echo "formatting-test exists"' \ @@ -64,6 +64,6 @@ test_success_output() { } teardown() { - cd "$TESTROOT/.." + cd "$TESTROOT/.." || return 1 rm -f "commands/formatting-test.sh" || true } From 6af424b301f52678186ce5b73ef7935db5c197ad Mon Sep 17 00:00:00 2001 From: Aaron McHale Date: Sun, 15 Nov 2026 13:32:55 +0000 Subject: [PATCH 3/3] Fix tests --- tests/test-cli-0.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/test-cli-0.sh b/tests/test-cli-0.sh index 9a9d243..f759eb8 100644 --- a/tests/test-cli-0.sh +++ b/tests/test-cli-0.sh @@ -4,7 +4,6 @@ teardown() { cd "$TESTROOT/.." || return 1 rm "commands/test-cmd.sh" || true rm "commands/test-cmd.help.txt" || true - rm "tests/cli-symlink.sh" || true } setup() { @@ -17,7 +16,7 @@ setup() { echo 'echo "Test command executed"' > "commands/test-cmd.sh" echo "this is a test command" > "commands/test-cmd.help.txt" echo "this line should not show on the command list" >> "commands/test-cmd.help.txt" - ln -s cli.sh tests/cli-symlink.sh + ln -s "$PWD/cli.sh" "$TMPDIR/cli-symlink.sh" || return 1 } test_cli_output_for_test_cmd() { @@ -40,9 +39,9 @@ test_cli_from_tests_dir() {( )} test_cli_symlink_from_tests_dir() {( - echo "Testing cli.sh as a symlink from tests directory, to prove that it can be run as a symlink from anywhere" + echo "Testing cli.sh as a symlink from $TMPDIR, to prove that it can be run as a symlink from anywhere" # Test is run in a sub-shell, so we don't need to cd back. - cd "$TESTROOT" || return 1 + cd "$TMPDIR" || return 1 if [[ "$(./cli-symlink.sh)" != *"test-cmd"* ]]; then echo "FAIL: Test command test-cmd not found when running cli.sh as a symlink from tests dir" return 1