diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6f99f7fa0..2e1e6e112 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -34,8 +34,8 @@ jobs: - uses: actions/checkout@v6 - run: npx pyright --stats - pytest-cram: - name: pytest-cram (${{ matrix.conda-args }} ${{ matrix.pip-args }}) + pytest-prysk: + name: pytest-prysk (${{ matrix.conda-args }} ${{ matrix.pip-args }}) runs-on: ubuntu-latest strategy: matrix: @@ -105,21 +105,24 @@ jobs: run: | if [[ -n "${COVERAGE_FILE:-}" ]]; then echo "Running pytest with coverage enabled" - pytest --cov=augur + pytest --cov=augur --ignore=tests/functional -n auto else echo "Running pytest without coverage" - pytest --no-cov + pytest --no-cov --ignore=tests/functional -n auto fi - - name: Run cram tests + # Run prysk tests separately to try to even spread out workers among long + # running functional tests. + - name: Run prysk tests run: | if [[ -n "${COVERAGE_FILE:-}" ]]; then - echo "Running cram tests with coverage enabled" + echo "Running prysk tests with coverage enabled" export AUGUR="coverage run -a ${{ github.workspace }}/bin/augur" + pytest --no-cov -n auto tests/functional/ else - echo "Running cram tests without coverage" + echo "Running prysk tests without coverage" export AUGUR="${{ github.workspace }}/bin/augur" + pytest --no-cov -n auto tests/functional/ fi - scripts/run-cram-parallel.py - name: Upload coverage if: env.COVERAGE_FILE uses: actions/upload-artifact@v7 @@ -342,7 +345,7 @@ jobs: codecov: if: github.repository == 'nextstrain/augur' - needs: [pytest-cram] + needs: [pytest-prysk] runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 @@ -408,7 +411,7 @@ jobs: release: # Only run when called by the release workflow on the default branch if: github.workflow_ref == format('{0}/.github/workflows/release.yaml@refs/heads/{1}', github.repository, github.event.repository.default_branch) - needs: [pytest-cram] + needs: [pytest-prysk] runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 diff --git a/.cramrc b/.pryskrc similarity index 78% rename from .cramrc rename to .pryskrc index 153d20f5f..82fac79f2 100644 --- a/.cramrc +++ b/.pryskrc @@ -1,3 +1,3 @@ -[cram] +[prysk] shell = /bin/bash indent = 2 diff --git a/docs/contribute/DEV_DOCS.md b/docs/contribute/DEV_DOCS.md index 1b1502768..bfcafd018 100644 --- a/docs/contribute/DEV_DOCS.md +++ b/docs/contribute/DEV_DOCS.md @@ -61,7 +61,7 @@ They can be helpful for testing a real-world example and determining if a regres #### 3. Functional tests -Augur's command line interface is tested by functional tests implemented with the [Cram framework](https://bitheap.org/cram/). +Augur's command line interface is tested by functional tests implemented with the [Prysk framework](https://www.prysk.net/). These tests complement existing unit tests of individual augur Python functions by running augur commands in the shell and confirming that these commands: 1. execute without any errors @@ -69,11 +69,11 @@ These tests complement existing unit tests of individual augur Python functions These tests can reveal bugs resulting from untested internal functions or untested combinations fo internal functions. -Over time, we have changed the way we design and organize Augur's Cram tests. You might find older practices in existing tests that haven't been updated yet, but these are the latest guidelines that we've discovered to be helpful. +Over time, we have changed the way we design and organize Augur's Prysk tests. You might find older practices in existing tests that haven't been updated yet, but these are the latest guidelines that we've discovered to be helpful. -1. Keep cram files modular. This makes it easier to see which command is failing. +1. Keep prysk files modular. This makes it easier to see which command is failing. 2. Create files in the initial working directory (e.g. `./file.txt` or simply `file.txt`), as it is a temporary working directory unique to the test. Note that the name of the `$TMP` directory is misleading - although it is temporary, it is shared across all tests so you'll have to explicitly remove files at the end of each test to avoid affecting other tests. The initial directory of each test is a unique directory within `$TMP`. -3. Each directory containing cram tests should have a setup script named `_setup.sh`. Keep all shared setup commands in this file. +3. Each directory containing prysk tests should have a setup script named `_setup.sh`. Keep all shared setup commands in this file. ##### Comparing outputs of augur commands @@ -105,6 +105,12 @@ Next, run all augur tests with the following command from the root, top-level of ./run_tests.sh ``` +For faster execution by runnning tests in parallel, add the `-n` argument. +Using the `-n auto` will use all available cores: + +```bash +./run_test.sh -n auto +``` For rapid execution of a subset of unit tests (as during test-driven development), the `-k` argument will disable code coverage and functional tests and pass directly to pytest to limit the tests that are run. For example, the following command only runs unit tests related to augur mask. @@ -112,27 +118,27 @@ For example, the following command only runs unit tests related to augur mask. ./run_tests.sh -k test_mask ``` -You can run specific integration test(s) with `cram` directly or via our parallel-wrapper which will use all -available CPUs by default. For instance to run `tests/functional/clades.t` these will both work: +You can run specific integration test(s) with the `-k` argument as well, via pytest, or prysk directly. ```bash -cram tests/functional/clades.t -./scripts/run-cram-parallel.py tests/functional/clades.t +./run_tests.sh -k tests/functional/clades/ +pytest tests/functional/clades/ +prysk tests/functional/clades/ ``` To run all tests in parallel simply run ```bash -./scripts/run-cram-parallel.py +./run_tests.sh -k tests/functional -n auto ``` -To run cram tests locally and capture test coverage data, you can use this invocation: +To run prysk tests locally and capture test coverage data, you can use this invocation: ```bash -AUGUR="coverage run --data-file="$PWD/.coverage" $PWD/bin/augur" cram +AUGUR="coverage run --data-file="$PWD/.coverage" $PWD/bin/augur" prysk ``` -You can provide one or more cram test file names to get coverage for just those tests, or omit file names to run the entire cram test suite. +You can provide one or more prysk test file names to get coverage for just those tests, or omit file names to run the entire prysk test suite. Troubleshooting tip: As tests run on the development code in the augur repository, your environment should not have an existing augur installation that could cause a conflict in pytest. diff --git a/pytest.ini b/pytest.ini index 5845e427e..1a8180ef9 100644 --- a/pytest.ini +++ b/pytest.ini @@ -30,6 +30,11 @@ addopts = # output short traceback --tb=short + # prysk settings since .pryskrc is not used with the pytest plugin + # See + --prysk-shell=/bin/bash + --prysk-indent=2 + testpaths = augur/ tests/ diff --git a/run_tests.sh b/run_tests.sh index 9dd2850af..a5206f806 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,7 +1,6 @@ #!/bin/sh # Default to running all tests without coverage. -partial_test=0 run_coverage=0 # Check if user explicitly requests coverage @@ -9,12 +8,6 @@ case "$@" in *--cov*) run_coverage=1 ;; esac -# If user requests a subset of tests for pytest, note this preference to avoid -# running other tests. -case "$@" in - *-k*) partial_test=1 ;; -esac - if [ "$run_coverage" = 1 ]; then # Remove --cov from args since pytest will handle coverage via --cov=augur echo "Running tests with coverage enabled, remove --cov to disable" @@ -37,15 +30,7 @@ else coverage_arg='--no-cov' fi -echo "Running unit tests and doctests with pytest" +echo "Running unit tests, doctests,and functional tests with pytest" python3 -m pytest $coverage_arg $filtered_args -# Only run functional tests if we are not running a subset of tests for pytest. -if [ "$partial_test" = 0 ]; then - echo "Running functional tests with cram in parallel via our run-cram-parallel.py runner" - ./scripts/run-cram-parallel.py -else - echo "Skipping functional tests when running a subset of unit tests" -fi - exit $? diff --git a/scripts/run-cram-parallel.py b/scripts/run-cram-parallel.py deleted file mode 100755 index f6f999adc..000000000 --- a/scripts/run-cram-parallel.py +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env python3 -"""Run cram tests in parallel using a worker pool.""" -import argparse -import os -import subprocess -import sys -import time -from concurrent.futures import ProcessPoolExecutor, as_completed -from pathlib import Path - -# These tests were identified as being particularly slow -# We can re-check these over time as we speed up individual tests. -# We run these tests first to improve parallel efficiency. -SLOW_TESTS = [ - "tests/functional/merge/cram/merge-metadata.t", - "tests/functional/tree/cram/iqtree-more-threads.t", - "tests/functional/subsample/cram/proximal-subsampling.t", - "tests/functional/measurements_export.t", - "tests/functional/curate/cram/metadata-input.t", - "tests/functional/export_v2/cram/metadata-columns.t", - "tests/functional/curate/cram/titlecase.t", - "tests/functional/tree/cram/iqtree-override-args.t", - "tests/functional/subsample/cram/proximal-subsampling-errors.t", - "tests/functional/merge/cram/merge-metadata-and-sequences.t", -] - - -def run_test(test_file, cram_args): - start = time.monotonic() - result = subprocess.run( - ["cram", *cram_args, str(test_file)], - capture_output=True, - ) - elapsed = time.monotonic() - start - return test_file, result.returncode, elapsed, result.stdout, result.stderr - - -def main(): - parser = argparse.ArgumentParser( - description=__doc__, - usage="%(prog)s [OPTIONS] [TESTS] [-- CRAM_ARGS...]", - ) - parser.add_argument( - "-j", "--jobs", type=int, default=os.cpu_count(), - help="number of parallel workers (default: all available (%(default)s))", - ) - parser.add_argument( - "tests", nargs="*", default=["tests/"], - help="files or directories to find .t files in (default: tests/)", - ) - - argv = sys.argv[1:] - if "--" in argv: - split = argv.index("--") - args = parser.parse_args(argv[:split]) - cram_args = argv[split + 1:] - else: - args = parser.parse_args(argv) - cram_args = [] - - all_tests = [] - for path in map(Path, args.tests): - if path.is_file(): - all_tests.append(path) - elif path.is_dir(): - all_tests.extend(path.rglob("*.t")) - else: - parser.error(f"not a file or directory: {path}") - all_tests = sorted(set(all_tests)) - if not all_tests: - parser.error(f"no .t files found in {' '.join(args.tests)}") - - slow_set = [Path(p) for p in SLOW_TESTS] - slow_tests = [t for t in slow_set if t in all_tests] - rest_tests = [t for t in all_tests if t not in slow_set] - test_files = slow_tests + rest_tests - - cram_cmd = " ".join(["cram", *cram_args]) - print(f"Running {len(test_files)} tests with {args.jobs} workers") - print(f" ({len(slow_tests)} slow tests scheduled first)") - print(f"cram invocation: {cram_cmd} \n") - - results = [] - passed = failed = 0 - wall_start = time.monotonic() - - with ProcessPoolExecutor(max_workers=args.jobs) as pool: - futures = { - pool.submit(run_test, t, cram_args): t for t in test_files - } - for future in as_completed(futures): - test_file, rc, elapsed, stdout, stderr = future.result() - results.append((elapsed, rc, test_file)) - status = "PASS" if rc == 0 else "FAIL" - if rc == 0: - passed += 1 - else: - failed += 1 - print(f" {status} {elapsed:6.1f}s {test_file}") - if rc != 0: - if stdout: - print(stdout.decode(errors="replace")) - if stderr: - print(stderr.decode(errors="replace")) - - wall_elapsed = time.monotonic() - wall_start - total_cpu = sum(e for e, _, _ in results) - - print(f"\n{'='*60}") - print(f"Passed: {passed} Failed: {failed} Total: {len(results)}") - print(f"Wall time: {wall_elapsed:.1f}s") - print(f"Total CPU: {total_cpu:.1f}s") - print(f"Speedup: {total_cpu / wall_elapsed:.1f}x") - - failures = [(e, rc, t) for e, rc, t in results if rc != 0] - if failures: - print(f"\n{'='*60}") - print(f"Failing tests ({len(failures)}):\n") - for elapsed, rc, test_file in sorted(failures, key=lambda x: x[2]): - print(f" {elapsed:6.1f}s exit={rc} {test_file}") - - print(f"\n{'='*60}") - print("Slowest tests:\n") - results.sort(reverse=True) - for elapsed, rc, test_file in results[:20]: - status = "PASS" if rc == 0 else "FAIL" - print(f" {elapsed:6.1f}s {status} {test_file}") - - sys.exit(1 if failures else 0) - - -if __name__ == "__main__": - main() diff --git a/setup.py b/setup.py index 9fb52dadb..7a90be809 100644 --- a/setup.py +++ b/setup.py @@ -75,17 +75,18 @@ ], extras_require = { 'dev': [ - "cram >=0.7", "deepdiff >=4.3.2, <8.0.0", "flake8 >=7.0.0, <8", "freezegun >=0.3.15", "mypy >=1.18.1", "nextstrain-sphinx-theme >=2022.5", "pandas-stubs >=1.4.0, <3", + "prysk[pytest-plugin]", "pylint >=1.7.6", "pytest >=5.4.1", "pytest-cov >=2.8.1", "pytest-mock >= 2.0.0", + "pytest-xdist", "recommonmark >=0.5.0", "Sphinx >=2.0.1", "sphinx-autobuild >=2021.3.14", diff --git a/tests/functional/subsample/cram/include-file.t b/tests/functional/subsample/cram/include-file.t index 6c4745d57..b14f3065b 100644 --- a/tests/functional/subsample/cram/include-file.t +++ b/tests/functional/subsample/cram/include-file.t @@ -137,7 +137,7 @@ Error when file not found in any search path ERROR: File 'nonexistent.txt' not resolvable from any of the following paths: config - .*/include-file.t (re) + .* (re) [2] --search-paths overrides AUGUR_SEARCH_PATHS