Skip to content

Provide option to use EESSI modules instead of init script #313

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
casparvl wants to merge 14 commits into
base: main
Choose a base branch
from
Open

Provide option to use EESSI modules instead of init script #313

Changes from 3 commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
d46efdd
Make run_reframe.sh loop over versions defined in EESSI_VERSIONS (int…
Jan 19, 2026
35a8b06
Add 2025.06 to default list of EESSI_VERSION
Jan 19, 2026
e937e4e
Improve behaviour in case EESSI module is NOT used, or in case local …
Jan 19, 2026
ed54378
Apply suggestion from @casparvl
casparvl Apr 9, 2026
7270887
Apply suggestion from @casparvl
casparvl Apr 9, 2026
47e28b3
Remove loop. Still WIP. But need to figure out if we should specify t…
Apr 30, 2026
7813dcc
Merge remote-tracking branch 'refs/remotes/origin/make_run_reframe_lo…
Apr 30, 2026
d13df26
Make sure we support EESSI and local programming environments, and pa…
Apr 30, 2026
89b1ace
Remove stray 'REFRAME_URL='
Jun 17, 2026
c1e3ded
No longer needed now that we've internalized the hpctestlib into EESS…
Jun 17, 2026
8fea8eb
Merge branch 'support_multiple_eessi_environments' into make_run_refr…
Jun 17, 2026
2bd4564
Remove more things related to cloning the reframe repo for the hpctes…
Jun 17, 2026
281d1ac
Merge branch 'main' into make_run_reframe_loop_eessi_versions
Jun 22, 2026
7b25663
Corrected path, it was missing 'versions' in the prefix
Jul 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
188 changes: 96 additions & 92 deletions CI/run_reframe.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,21 +37,11 @@ fi
source "${CI_CONFIG}"

# Set default configuration, but let anything set by CI_CONFIG take priority
if [ -z "${REFRAME_ARGS}" ]; then
REFRAME_ARGS="--tag CI --tag 1_node"
fi
if [ -z "${REFRAME_VERSION}"]; then
REFRAME_VERSION=4.8.1
fi
if [ -z "${REFRAME_URL}" ]; then
REFRAME_URL='https://github.com/reframe-hpc/reframe.git'
fi
if [ -z "${REFRAME_BRANCH}" ]; then
REFRAME_BRANCH="v${REFRAME_VERSION}"
fi
if [ -z "${EESSI_TESTSUITE_URL}" ]; then
EESSI_TESTSUITE_URL='https://github.com/EESSI/test-suite.git'
fi
REFRAME_ARGS="${REFRAME_ARGS:---tag CI --tag 1_node}"
REFRAME_VERSION="${REFRAME_VERSION:-4.8.1}"
REFRAME_URL="${REFRAME_URL:-REFRAME_URL=https://github.com/reframe-hpc/reframe.git}"
REFRAME_BRANCH="${REFRAME_BRANCH:-v${REFRAME_VERSION}}"
EESSI_TESTSUITE_URL="${EESSI_TESTSUITE_URL:-https://github.com/EESSI/test-suite.git}"
if [ -z "${EESSI_TESTSUITE_BRANCH}" ]; then
git clone -n --depth=1 --filter=tree:0 ${EESSI_TESTSUITE_URL} "${TEMPDIR}/test-suite-version-checkout"
cd "${TEMPDIR}/test-suite-version-checkout"
Expand All @@ -62,44 +52,24 @@ if [ -z "${EESSI_TESTSUITE_BRANCH}" ]; then
EESSI_TESTSUITE_BRANCH="${LATEST_VERSION}"
cd ${TEMPDIR}
fi
if [ -z "${EESSI_CONFIGS_TESTSUITE_URL}" ]; then
EESSI_CONFIGS_TESTSUITE_URL="${EESSI_TESTSUITE_URL}"
fi
if [ -z "${EESSI_CONFIGS_TESTSUITE_BRANCH}" ]; then
EESSI_CONFIGS_TESTSUITE_BRANCH="${EESSI_TESTSUITE_BRANCH}"
fi
if [ -z "${USE_EESSI_SOFTWARE_STACK}" ] || [ "$USE_EESSI_SOFTWARE_STACK" == "True" ]; then
export USE_EESSI_SOFTWARE_STACK=True
if [ -z "${EESSI_CVMFS_REPO}" ]; then
export EESSI_CVMFS_REPO=/cvmfs/software.eessi.io
fi
if [ -z "${EESSI_VERSION}" ]; then
export EESSI_VERSION=2023.06
fi
fi
if [ -z "${RFM_CONFIG_FILES}" ]; then
export RFM_CONFIG_FILES="${TEMPDIR}/configs/config/${EESSI_CI_SYSTEM_NAME}.py"
fi
if [ -z "${RFM_CHECK_SEARCH_PATH}" ]; then
export RFM_CHECK_SEARCH_PATH="${TEMPDIR}/test-suite/eessi/testsuite/tests/"
fi
if [ -z "${RFM_CHECK_SEARCH_RECURSIVE}" ]; then
export RFM_CHECK_SEARCH_RECURSIVE=1
fi
if [ -z "${RFM_PREFIX}" ]; then
export RFM_PREFIX="${HOME}/reframe_CI_runs"
fi
if [ -z "${REFRAME_TIMEOUT}" ]; then
# 10 minutes short of 1 day, since typically the test suite will be run daily.
# This will prevent multiple ReFrame runs from piling up and exceeding the quota on our Magic Castle clusters
export REFRAME_TIMEOUT=1430m
fi
if [ -z "${UNSET_MODULEPATH}" ]; then
export UNSET_MODULEPATH=True
fi
if [ -z "${SET_LOCAL_MODULE_ENV}" ]; then
export SET_LOCAL_MODULE_ENV=False
fi
EESSI_CONFIGS_TESTSUITE_URL="${EESSI_CONFIGS_TESTSUITE_URL:-${EESSI_TESTSUITE_URL}}"
EESSI_CONFIGS_TESTSUITE_BRANCH="${EESSI_CONFIGS_TESTSUITE_BRANCH:-${EESSI_TESTSUITE_BRANCH}}"
export USE_EESSI_SOFTWARE_STACK="${USE_EESSI_SOFTWARE_STACK:-True}"
if [ "$USE_EESSI_SOFTWARE_STACK" == "True" ]; then
export EESSI_CVMFS_REPO="${EESSI_CVMFS_REPO:-/cvmfs/software.eessi.io}"
export EESSI_VERSION="${EESSI_VERSION:-2023.06,2025.06}"
fi
export RFM_CONFIG_FILES="${RFM_CONFIG_FILES:-${TEMPDIR}/configs/config/${EESSI_CI_SYSTEM_NAME}.py}"
export RFM_CHECK_SEARCH_PATH="${RFM_CHECK_SEARCH_PATH:-${TEMPDIR}/test-suite/eessi/testsuite/tests/}"
export RFM_CHECK_SEARCH_RECURSIVE="${RFM_CHECK_SEARCH_RECURSIVE:-1}"
export RFM_PREFIX="${RFM_PREFIX:-${HOME}/reframe_CI_runs}"
# 10 minutes short of 1 day, since typically the test suite will be run daily.
# This will prevent multiple ReFrame runs from piling up and exceeding the quota on our Magic Castle clusters
export REFRAME_TIMEOUT="${REFRAME_TIMEOUT:-1430m}"
export UNSET_MODULEPATH="${UNSET_MODULEPATH:-True}"
export SET_LOCAL_MODULE_ENV="${SET_LOCAL_MODULE_ENV:-False}"
# If false, the script under versions/<eessi_version>/init/bash to initialize the environment will be sourced instead
export USE_EESSI_MODULE="${USE_EESSI_MODULE:-True}"

# Create virtualenv for ReFrame using system python
python3 -m venv "${TEMPDIR}"/reframe_venv
Expand Down Expand Up @@ -149,48 +119,82 @@ if [ "$SET_LOCAL_MODULE_ENV" == "True" ]; then
module load "${LOCAL_MODULES}"
fi

# Start the EESSI environment
# Add correct path to MODULEPATH (only once, so outside the loop below)
if [ "$USE_EESSI_SOFTWARE_STACK" == "True" ]; then
eessi_init_path="${EESSI_CVMFS_REPO}"/versions/"${EESSI_VERSION}"/init/bash
source "${eessi_init_path}"
if [ "$USE_EESSI_MODULE" == "True" ]; then
if command -v module &>/dev/null; then
# Module command available
module use "${EESSI_CVMFS_REPO}/init/modules/"
fi
fi
Comment thread
casparvl marked this conversation as resolved.
fi

# Needed in order to make sure the reframe from our TEMPDIR is first on the PATH,
# prior to the one shipped with the 2021.12 compat layer
# Probably no longer needed with newer compat layer that doesn't include ReFrame
deactivate
source "${TEMPDIR}"/reframe_venv/bin/activate

# Print ReFrame config
echo "Starting CI run with the follwing settings:"
echo ""
echo "TEMPDIR: ${TEMPDIR}"
echo "PYTHONPATH: ${PYTHONPATH}"
echo "EESSI test suite URL: ${EESSI_TESTSUITE_URL}"
echo "EESSI test suite version: ${EESSI_TESTSUITE_BRANCH}"
echo "EESSI test suite URL for configs: ${EESSI_CONFIGS_TESTSUITE_URL}"
echo "EESSI test suite version for configs: ${EESSI_CONFIGS_TESTSUITE_BRANCH}"
echo "HPCtestlib from ReFrame URL: ${REFRAME_URL}"
echo "HPCtestlib from ReFrame branch: ${REFRAME_BRANCH}"
echo "ReFrame executable: $(which reframe)"
echo "ReFrame version: $(reframe --version)"
echo "ReFrame config file: ${RFM_CONFIG_FILES}"
echo "ReFrame check search path: ${RFM_CHECK_SEARCH_PATH}"
echo "ReFrame check search recursive: ${RFM_CHECK_SEARCH_RECURSIVE}"
echo "ReFrame prefix: ${RFM_PREFIX}"
echo "ReFrame args: ${REFRAME_ARGS}"
echo "Using EESSI: ${USE_EESSI_SOFTWARE_STACK}"
echo "Using local software stack ${SET_LOCAL_MODULE_ENV}"
echo "MODULEPATH: ${MODULEPATH}"
echo ""

# List tests
echo "Listing tests:"
reframe ${REFRAME_ARGS} --list

# Run
echo "Run tests:"
timeout -v --preserve-status -s SIGTERM ${REFRAME_TIMEOUT} reframe ${REFRAME_ARGS} --run --setvar EESSI_CONFIGS_URL=${EESSI_CONFIGS_TESTSUITE_URL} --setvar EESSI_CONFIGS_BRANCH=${EESSI_CONFIGS_TESTSUITE_BRANCH}
# EESSI_VERSION can consist of multiple, comma-separated entries. Loop over those:
IFS=',' read -r -a eessi_version_array <<< "$EESSI_VERSION"

# Don't loop over EESSI_VERSIONS if we a) don't use the EESSI software stack or b) don't load it from the module
# In this case, simply only retain the first element of the eessi_version_array to avoid looping
if (( ${#eessi_version_array[@]} > 1 )) && \
[[ $USE_EESSI_SOFTWARE_STACK == "True" && $USE_EESSI_MODULE != "True" ]]; then
eessi_version_array=("${eessi_version_array[0]}")
fi
Comment on lines +144 to +152

@laraPPr laraPPr Jul 9, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah it this stuff that seems to cause issues

@casparvl casparvl Jul 9, 2026

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmmm, weird. It's supposed to define the ReFrame programming environments (like EESSI-2023.06, EESSI-2025.06) and take into account that you may want to run the test also on a local programming environment (say local-2025 or whatever name you use in the reframe config file for your local programming env). But for you, they all seem to be empty? Not sure why that would happen. If I recall, REFRAME_LOCAL_PROGRAMMING_ENVS may be empty, but we account for that. And REFRAME_EESSSI_PROGRAMMING_ENVS should default to EESSI-2023.06,EESSI-2025.06. So unless you'd explicitely set that env var (which I'm assuming you don't), I don't see why those would not be in there...

Can you add a few echo statements to see the values of REFRAME_LOCAL_PROGRAMMING_ENVS and REFRAME_EESSI_PROGRAMMING_ENVS, and maybe REFRAME_PROGRAMMING_ENVS as it goes through that code section?


for EESSI_VERSION in "${eessi_version_array[@]}"; do
# Start the EESSI environment
if [ "$USE_EESSI_SOFTWARE_STACK" == "True" ]; then
if [ "$USE_EESSI_MODULE" == "True" ]; then
if command -v module &>/dev/null; then
# Module command available
module load "EESSI/${EESSI_VERSION}"
else
source "${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION}/init/lmod/bash"
Comment thread
casparvl marked this conversation as resolved.
Outdated
fi
else
eessi_init_path="${EESSI_CVMFS_REPO}"/versions/"${EESSI_VERSION}"/init/bash
source "${eessi_init_path}"
fi
fi

# Needed in order to make sure the reframe from our TEMPDIR is first on the PATH,
# prior to the one shipped with the 2021.12 compat layer
# Probably no longer needed with newer compat layer that doesn't include ReFrame
deactivate
source "${TEMPDIR}"/reframe_venv/bin/activate

# Print ReFrame config
echo "Starting CI run with the follwing settings:"
echo ""
echo "TEMPDIR: ${TEMPDIR}"
echo "PYTHONPATH: ${PYTHONPATH}"
echo "EESSI test suite URL: ${EESSI_TESTSUITE_URL}"
echo "EESSI test suite version: ${EESSI_TESTSUITE_BRANCH}"
echo "EESSI test suite URL for configs: ${EESSI_CONFIGS_TESTSUITE_URL}"
echo "EESSI test suite version for configs: ${EESSI_CONFIGS_TESTSUITE_BRANCH}"
echo "HPCtestlib from ReFrame URL: ${REFRAME_URL}"
echo "HPCtestlib from ReFrame branch: ${REFRAME_BRANCH}"
echo "ReFrame executable: $(which reframe)"
echo "ReFrame version: $(reframe --version)"
echo "ReFrame config file: ${RFM_CONFIG_FILES}"
echo "ReFrame check search path: ${RFM_CHECK_SEARCH_PATH}"
echo "ReFrame check search recursive: ${RFM_CHECK_SEARCH_RECURSIVE}"
echo "ReFrame prefix: ${RFM_PREFIX}"
echo "ReFrame args: ${REFRAME_ARGS}"
echo "Using EESSI: ${USE_EESSI_SOFTWARE_STACK}"
echo "Using the EESSI module: ${USE_EESSI_MODULE}"
echo "Using EESSI version: ${EESSI_VERSION}"
echo "Using local software stack ${SET_LOCAL_MODULE_ENV}"
echo "MODULEPATH: ${MODULEPATH}"
echo ""

# List tests
echo "Listing tests:"
reframe ${REFRAME_ARGS} --list

# Run
echo "Run tests:"
timeout -v --preserve-status -s SIGTERM ${REFRAME_TIMEOUT} reframe ${REFRAME_ARGS} --run --setvar EESSI_CONFIGS_URL=${EESSI_CONFIGS_TESTSUITE_URL} --setvar EESSI_CONFIGS_BRANCH=${EESSI_CONFIGS_TESTSUITE_BRANCH}

done

# Cleanup
rm -rf "${TEMPDIR}"