tests: Cram to Prysk

.github/workflows/ci.yaml

@@ -34,8 +34,8 @@ jobs:
     - uses: actions/checkout@v6
     - run: npx pyright --stats
 
-  pytest-cram:
-    name: pytest-cram (${{ matrix.conda-args }} ${{ matrix.pip-args }})
+  pytest-prysk:
+    name: pytest-prysk (${{ matrix.conda-args }} ${{ matrix.pip-args }})
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -105,21 +105,24 @@ jobs:
       run: |
         if [[ -n "${COVERAGE_FILE:-}" ]]; then
           echo "Running pytest with coverage enabled"
-          pytest --cov=augur
+          pytest --cov=augur --ignore=tests/functional -n auto
         else
           echo "Running pytest without coverage"
-          pytest --no-cov
+          pytest --no-cov --ignore=tests/functional -n auto
         fi
-    - name: Run cram tests
+    # Run prysk tests separately to try to even spread out workers among long
+    # running functional tests.
+    - name: Run prysk tests
       run: |
         if [[ -n "${COVERAGE_FILE:-}" ]]; then
-          echo "Running cram tests with coverage enabled"
+          echo "Running prysk tests with coverage enabled"
           export AUGUR="coverage run -a ${{ github.workspace }}/bin/augur"
+          pytest --no-cov -n auto tests/functional/
         else
-          echo "Running cram tests without coverage"
+          echo "Running prysk tests without coverage"
           export AUGUR="${{ github.workspace }}/bin/augur"
+          pytest --no-cov -n auto tests/functional/
         fi
-        scripts/run-cram-parallel.py
     - name: Upload coverage
       if: env.COVERAGE_FILE
       uses: actions/upload-artifact@v7
@@ -342,7 +345,7 @@ jobs:
 
   codecov:
     if: github.repository == 'nextstrain/augur'
-    needs: [pytest-cram]
+    needs: [pytest-prysk]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v6
@@ -408,7 +411,7 @@ jobs:
   release:
     # Only run when called by the release workflow on the default branch
     if: github.workflow_ref == format('{0}/.github/workflows/release.yaml@refs/heads/{1}', github.repository, github.event.repository.default_branch)
-    needs: [pytest-cram]
+    needs: [pytest-prysk]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v6

.cramrc → .pryskrc

@@ -1,3 +1,3 @@
-[cram]
+[prysk]
 shell = /bin/bash
 indent = 2

docs/contribute/DEV_DOCS.md

@@ -61,19 +61,19 @@ They can be helpful for testing a real-world example and determining if a regres
 
 #### 3. Functional tests
 
-Augur's command line interface is tested by functional tests implemented with the [Cram framework](https://bitheap.org/cram/).
+Augur's command line interface is tested by functional tests implemented with the [Prysk framework](https://www.prysk.net/).
 These tests complement existing unit tests of individual augur Python functions by running augur commands in the shell and confirming that these commands:
 
 1. execute without any errors
 2. produce exactly the expected outputs for the given inputs
 
 These tests can reveal bugs resulting from untested internal functions or untested combinations fo internal functions.
 
-Over time, we have changed the way we design and organize Augur's Cram tests. You might find older practices in existing tests that haven't been updated yet, but these are the latest guidelines that we've discovered to be helpful.
+Over time, we have changed the way we design and organize Augur's Prysk tests. You might find older practices in existing tests that haven't been updated yet, but these are the latest guidelines that we've discovered to be helpful.
 
-1. Keep cram files modular. This makes it easier to see which command is failing.
+1. Keep prysk files modular. This makes it easier to see which command is failing.
 2. Create files in the initial working directory (e.g. `./file.txt` or simply `file.txt`), as it is a temporary working directory unique to the test. Note that the name of the `$TMP` directory is misleading - although it is temporary, it is shared across all tests so you'll have to explicitly remove files at the end of each test to avoid affecting other tests. The initial directory of each test is a unique directory within `$TMP`.
-3. Each directory containing cram tests should have a setup script named `_setup.sh`. Keep all shared setup commands in this file.
+3. Each directory containing prysk tests should have a setup script named `_setup.sh`. Keep all shared setup commands in this file.
 
 ##### Comparing outputs of augur commands
 
@@ -105,34 +105,40 @@ Next, run all augur tests with the following command from the root, top-level of
 ./run_tests.sh
 ```
 
+For faster execution by runnning tests in parallel, add the `-n` argument.
+Using the `-n auto` will use all available cores:
+
+```bash
+./run_test.sh -n auto
+```
 For rapid execution of a subset of unit tests (as during test-driven development), the `-k` argument will disable code coverage and functional tests and pass directly to pytest to limit the tests that are run.
 For example, the following command only runs unit tests related to augur mask.
 
 ```bash
 ./run_tests.sh -k test_mask
 ```
 
-You can run specific integration test(s) with `cram` directly or via our parallel-wrapper which will use all
-available CPUs by default. For instance to run `tests/functional/clades.t` these will both work:
+You can run specific integration test(s) with the `-k` argument as well, via pytest, or prysk directly.
 
 ```bash
-cram tests/functional/clades.t
-./scripts/run-cram-parallel.py tests/functional/clades.t
+./run_tests.sh -k tests/functional/clades/
+pytest tests/functional/clades/
+prysk tests/functional/clades/
 ```
 
 To run all tests in parallel simply run
 
 ```bash
-./scripts/run-cram-parallel.py
+./run_tests.sh -k tests/functional -n auto
 ```
 
-To run cram tests locally and capture test coverage data, you can use this invocation:
+To run prysk tests locally and capture test coverage data, you can use this invocation:
 
 ```bash
-AUGUR="coverage run --data-file="$PWD/.coverage" $PWD/bin/augur" cram
+AUGUR="coverage run --data-file="$PWD/.coverage" $PWD/bin/augur" prysk
 ```
 
-You can provide one or more cram test file names to get coverage for just those tests, or omit file names to run the entire cram test suite.
+You can provide one or more prysk test file names to get coverage for just those tests, or omit file names to run the entire prysk test suite.
 
 Troubleshooting tip: As tests run on the development code in the augur repository, your environment should not have an existing augur installation that could cause a conflict in pytest.
 

pytest.ini

@@ -30,6 +30,11 @@ addopts =
     # output short traceback
     --tb=short
 
+    # prysk settings since .pryskrc is not used with the pytest plugin
+    # See <https://github.com/prysk/pytest-prysk/issues/15>
+    --prysk-shell=/bin/bash
+    --prysk-indent=2
+
 testpaths =
     augur/
     tests/

run_tests.sh

@@ -1,20 +1,13 @@
 #!/bin/sh
 
 # Default to running all tests without coverage.
-partial_test=0
 run_coverage=0
 
 # Check if user explicitly requests coverage
 case "$@" in
     *--cov*) run_coverage=1 ;;
 esac
 
-# If user requests a subset of tests for pytest, note this preference to avoid
-# running other tests.
-case "$@" in
-    *-k*) partial_test=1 ;;
-esac
-
 if [ "$run_coverage" = 1 ]; then
     # Remove --cov from args since pytest will handle coverage via --cov=augur
     echo "Running tests with coverage enabled, remove --cov to disable"
@@ -37,15 +30,7 @@ else
     coverage_arg='--no-cov'
 fi
 
-echo "Running unit tests and doctests with pytest"
+echo "Running unit tests, doctests,and functional tests with pytest"
 python3 -m pytest $coverage_arg $filtered_args
 
-# Only run functional tests if we are not running a subset of tests for pytest.
-if [ "$partial_test" = 0 ]; then
-    echo "Running functional tests with cram in parallel via our run-cram-parallel.py runner"
-    ./scripts/run-cram-parallel.py
-else
-    echo "Skipping functional tests when running a subset of unit tests"
-fi
-
 exit $?

setup.py

@@ -75,17 +75,18 @@
     ],
     extras_require = {
         'dev': [
-            "cram >=0.7",
             "deepdiff >=4.3.2, <8.0.0",
             "flake8 >=7.0.0, <8",
             "freezegun >=0.3.15",
             "mypy >=1.18.1",
             "nextstrain-sphinx-theme >=2022.5",
             "pandas-stubs >=1.4.0, <3",
+            "prysk[pytest-plugin]",
             "pylint >=1.7.6",
             "pytest >=5.4.1",
             "pytest-cov >=2.8.1",
             "pytest-mock >= 2.0.0",
+            "pytest-xdist",
             "recommonmark >=0.5.0",
             "Sphinx >=2.0.1",
             "sphinx-autobuild >=2021.3.14",

tests/functional/subsample/cram/include-file.t

@@ -137,7 +137,7 @@ Error when file not found in any search path
   ERROR: File 'nonexistent.txt' not resolvable from any of the following paths:
 
     config
-    .*/include-file.t (re)
+    .* (re)
   [2]
 
 --search-paths overrides AUGUR_SEARCH_PATHS