diff --git a/docs/source/contributing/architecture.rst b/docs/source/contributing/architecture.rst index 09417af002..07fabff3e3 100644 --- a/docs/source/contributing/architecture.rst +++ b/docs/source/contributing/architecture.rst @@ -185,7 +185,6 @@ src/CSET/cset_workflow ├── site # Site-specific cylc configuration. │   └── localhost.cylc ├── flow.cylc # Workflow definition detailing how tasks are run. - ├── install_restricted_files.sh # Script for installing site-specific files. ├── README.md ├── rose-suite.conf # User configuration of workflow and diagnostics. └── rose-suite.conf.example # Blank user configuration to be copied. diff --git a/docs/source/getting-started/run_full_cylc_workflow.rst b/docs/source/getting-started/run_full_cylc_workflow.rst index 4c83b4606f..9da6dadfd7 100644 --- a/docs/source/getting-started/run_full_cylc_workflow.rst +++ b/docs/source/getting-started/run_full_cylc_workflow.rst @@ -71,13 +71,19 @@ With the newly created conda environment activated, run the ``cset extract-workflow`` command to unpack the workflow from inside the CSET package into a directory of your choosing. A sensible choice is ``~/cylc-src``, which is the default location where cylc will search for workflows. +If you are at a Momentum® Partnership site with restricted site specific files +you should also include the ``--restricted`` flag to install them. .. code-block:: bash # Create the cylc-src directory if it doesn't exist. mkdir -p ~/cylc-src + # Extract the workflow from CSET into the chosen directory. cset extract-workflow ~/cylc-src + # Alternatively install the restricted site-specific files at Momentum sites. + # cset extract-workflow --restricted ~/cylc-src + # Change into the freshly unpacked workflow directory. cd ~/cylc-src/cset-workflow-vX.Y.Z @@ -86,18 +92,19 @@ Your directory should look like this: .. code-block:: bash $ ls - app conda-environment includes lib opt rose-suite.conf.example - bin flow.cylc install_restricted_files.sh meta README.md site + app conda-environment includes meta README.md site + bin flow.cylc lib opt rose-suite.conf.example If you are at a site with specific CSET integration, such as the Met Office or -Momentum Partnership, you will want to install the site specific configuration +Momentum Partnership, and did not use the ``--restricted`` option to ``cset +extract-workflow`` you will want to install the site specific configuration files that specify where cylc will run the tasks. This is done by running the -``install_restricted_files.sh`` script. For other users, you can skip this step -and use the ``localhost`` site instead. +``cset install-restricted-files`` command. For other users, you can skip this +step and use the ``localhost`` site instead. .. code-block:: bash - ./install_restricted_files.sh + cset install-restricted-files /path/to/workflow You have now installed the CSET workflow and are ready to use it. diff --git a/docs/source/reference/cli.rst b/docs/source/reference/cli.rst index abaac2605d..5341f9b305 100644 --- a/docs/source/reference/cli.rst +++ b/docs/source/reference/cli.rst @@ -98,10 +98,40 @@ usage see :doc:`/getting-started/run_full_cylc_workflow`. .. code-block:: text - usage: cset extract-workflow [-h] location + usage: cset extract-workflow [-h] [--restricted] + [--restricted-url RESTRICTED_URL] + location positional arguments: - location directory to save workflow into + location directory to save workflow into options: - -h, --help show this help message and exit + -h, --help show this help message and exit + --restricted install restricted site-specific files during + extraction + --restricted-url RESTRICTED_URL + Alternative Git URL to fetch the restricted files + from. If omitted, defaults to trying to clone first + from 'localmirrors:', then from GitHub via SSH and + HTTPS. + +cset install-restricted-files +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Download and install restricted site-specific files into the CSET workflow. + +.. code-block:: text + + usage: cset install-restricted-files [-h] [--restricted-url RESTRICTED_URL] + location + + positional arguments: + location directory containing workflow + + options: + -h, --help show this help message and exit + --restricted-url RESTRICTED_URL + Alternative Git URL to fetch the restricted files + from. If omitted, defaults to trying to clone first + from 'localmirrors:', then from GitHub via SSH and + HTTPS. diff --git a/docs/source/usage/add-site.rst b/docs/source/usage/add-site.rst index 4adaf03ff4..f33272d2d0 100644 --- a/docs/source/usage/add-site.rst +++ b/docs/source/usage/add-site.rst @@ -129,8 +129,8 @@ add your site file. If you would prefer to keep your site-specific configuration non-public, and are a Momentum Partnership member, we have a designated `CSET site-specific config repository`_ that contains these configurations for various Momentum Partners. -It is this repository that is installed via the ``install_restricted_files.sh`` -script. Even when your file remains restricted like this you should still +It is this repository that is installed via ``cset install-restricted-files``. +Even when your file remains restricted like this you should still contribute your rose metadata changes to the `main CSET GitHub repository`_ so your site shows up as an option to users. diff --git a/src/CSET/__init__.py b/src/CSET/__init__.py index 5fd32713cd..42af397b96 100644 --- a/src/CSET/__init__.py +++ b/src/CSET/__init__.py @@ -1,4 +1,4 @@ -# © Crown copyright, Met Office (2022-2025) and CSET contributors. +# © Crown copyright, Met Office (2022-2026) and CSET contributors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -169,8 +169,40 @@ def setup_argument_parser() -> argparse.ArgumentParser: parser_extract_workflow.add_argument( "location", type=Path, help="directory to save workflow into" ) + parser_extract_workflow.add_argument( + "--restricted", + action="store_true", + help="install restricted site-specific files during extraction", + ) + parser_extract_workflow.add_argument( + "--restricted-url", + type=str, + help=( + "Alternative Git URL to fetch the restricted files from. " + "If omitted, defaults to trying to clone first from " + "'localmirrors:', then from GitHub via SSH and HTTPS." + ), + ) parser_extract_workflow.set_defaults(func=_extract_workflow_command) + parser_install_restricted_files = subparsers.add_parser( + "install-restricted-files", + help="download and install restricted site-specific files for the CSET cylc workflow", + ) + parser_install_restricted_files.add_argument( + "location", type=Path, help="directory containing workflow" + ) + parser_install_restricted_files.add_argument( + "--restricted-url", + type=str, + help=( + "Alternative Git URL to fetch the restricted files from. " + "If omitted, defaults to trying to clone first from " + "'localmirrors:', then from GitHub via SSH and HTTPS." + ), + ) + parser_install_restricted_files.set_defaults(func=_install_restricted_files_command) + return parser @@ -256,6 +288,14 @@ def _cookbook_command(args, unparsed_args): def _extract_workflow_command(args, unparsed_args): - from CSET.extract_workflow import install_workflow + from CSET.extract_workflow import install_restricted_files, install_workflow + + workflow_dir = install_workflow(args.location) + if args.restricted: + install_restricted_files(workflow_dir, args.restricted_url) + + +def _install_restricted_files_command(args, unparsed_args): + from CSET.extract_workflow import install_restricted_files - install_workflow(args.location) + install_restricted_files(args.location, args.restricted_url) diff --git a/src/CSET/cset_workflow/install_restricted_files.sh b/src/CSET/cset_workflow/install_restricted_files.sh deleted file mode 100755 index 2c0fea8db1..0000000000 --- a/src/CSET/cset_workflow/install_restricted_files.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail -IFS="$(printf '\n\t')" - -show_help() -{ - cat << EOF -Usage: ${0} [-b branch-name] [-R repository] - - * Repository defaults to https://github.com/MetOffice/CSET-restricted-files - * Branch defaults to main - * Most people should omit both, and use the default values - -This script downloads and install Momentum® Partnership restricted files and -code for use in CSET. Make sure you have git authentication setup for the -repository at https://github.com/MetOffice/CSET-restricted-files -EOF - exit "$1" -} - -# Set defaults. These are left unchanged if the option isn't specified. -branch="main" -httpurl="https://github.com/MetOffice/CSET-restricted-files" - -# Parse options. -while getopts "hb:R:" opt -do - case "$opt" in - b) branch="$OPTARG" - ;; - R) httpurl="$OPTARG" - ;; - h) show_help 0 - ;; - ?) show_help 1 - esac -done - -# Basic check that we are in the right folder. -if [[ ! -f flow.cylc ]] -then - echo "You must be in the cset-workflow directory when running this script." - exit 1 -fi - -# Convert GitHub HTTPS url into SSH one. -# GitLab prepends `ssh.` to its SSH URLs, so is not currently supported. -sshurl="$(echo "$httpurl" | sed -e 's/https:\/\//git@/' -e 's/\//:/')" - -# Make a temporary directory into which to clone the repository. -tempdir="$(mktemp -d)" - -# We don't need history, so shallow git clone for speed. -# First try with SSH cloning. -echo "Cloning ${branch} from ${sshurl}" -if ! git clone --branch "${branch}" --depth 1 "${sshurl}" "${tempdir}" -then - echo "Cannot clone via SSH, falling back to HTTPS." - echo "Cloning ${branch} from ${httpurl}" - if ! git clone --branch "${branch}" --depth 1 "${httpurl}" "${tempdir}" - then - echo "Problem cloning git repository." - echo "Check you have set up access for ${httpurl}" - exit 1 - fi -fi - -# Copy most files from there into workflow directory, clobering existing ones. -# Don't copy some files, like README.md or top-level hidden files. -rm "${tempdir}/README.md" -# Ignores top level hidden files. -cp -rv "${tempdir}"/* "$PWD" -# Clean up. Must force here to remove .git folder. -rm -rf "${tempdir}" - -# Report success. -cat << EOF - ___________ ____________ - / ____/ ___// ____/_ __/ - / / \__ \/ __/ / / -/ /___ ___/ / /___ / / -\____//____/_____/ /_/ - -Restricted files installed! -EOF diff --git a/src/CSET/extract_workflow.py b/src/CSET/extract_workflow.py index e862c3f1d8..c6f967c687 100644 --- a/src/CSET/extract_workflow.py +++ b/src/CSET/extract_workflow.py @@ -1,4 +1,4 @@ -# © Crown copyright, Met Office (2022-2025) and CSET contributors. +# © Crown copyright, Met Office (2022-2026) and CSET contributors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,8 +18,11 @@ import importlib.resources import logging import os +import re import shutil import stat +import subprocess +import tempfile from pathlib import Path import CSET.cset_workflow @@ -51,7 +54,7 @@ def make_script_executable(p: Path): p.chmod(mode) -def install_workflow(location: Path): +def install_workflow(location: Path) -> Path: """Install the workflow's files and link the conda environment. Arguments @@ -89,3 +92,119 @@ def install_workflow(location: Path): logger.warning("CONDA_PREFIX not defined. Skipping linking environment.") print(f"Workflow written to {workflow_dir}") + return workflow_dir + + +def list_refs(url: str) -> list[str]: + """List release refs for the repository. + + This serves both as an access check and to find an appropriate ref. + """ + # Disable interactively asking for authentication. + env = os.environ.copy() + env["GIT_TERMINAL_PROMPT"] = "false" + + cmd = ("git", "ls-remote", "--quiet", url, "releases/**", "v*") + logger.debug("Running %s", " ".join(cmd)) + try: + p = subprocess.run(cmd, env=env, check=True, capture_output=True, text=True) + except subprocess.CalledProcessError as err: + raise ValueError(f"Cannot access Git repository at {url}") from err + + return [line.split(maxsplit=1)[-1] for line in p.stdout.splitlines()] + + +def clone_ref(ref: str, url: str, location: str): + """Clone the specified ref.""" + # Disable interactively asking for authentication. + env = os.environ.copy() + env["GIT_TERMINAL_PROMPT"] = "false" + + cmd = ("git", "clone", "--quiet", "--depth", "1", "--branch", ref, url, location) + logger.debug("Running %s", " ".join(cmd)) + try: + subprocess.run(cmd, env=env, check=True, capture_output=True) + except subprocess.CalledProcessError as err: + raise ValueError(f"Cannot access Git repository at {url}") from err + + +def install_restricted_files(workflow_dir: Path, alternative_url: str | None = None): + """Install restricted site-specific restricted files into CSET workflow. + + Parameters + ---------- + workflow_dir: str + The workflow directory into which the restricted files will be copied. + alternative_url: str, optional + Alternative Git URL to fetch the restricted files from. If omitted, + defaults to trying to clone first from 'localmirrors:', then from GitHub + via SSH and HTTPS. + + Notes + ----- + Requires Git to be installed to function. + """ + # Basic check workflow_dir is correct. + if not (workflow_dir / "flow.cylc").is_file(): + raise ValueError("workflow_dir should be a CSET workflow directory.") + + # Determine target tag/branch from version. + version_tag = f"v{importlib.metadata.version('CSET')}" + logger.debug("Running for CSET %s", version_tag) + m = re.match(r"v\d+\.\d+", version_tag) + if m is None: + logger.error("Unknown CSET version: %s", version_tag) + m = [version_tag] + release_branch = f"releases/{m[0]}" + + # Default URLs to try in order, or use alternative if provided. + urls = ( + ( + "localmirrors:MetOffice/CSET-restricted-files.git", + "git@github.com:MetOffice/CSET-restricted-files.git", + "https://github.com/MetOffice/CSET-restricted-files.git", + ) + if alternative_url is None + else (alternative_url,) + ) + + # Find first working URL and list its refs. + for url in urls: + try: + refs = list_refs(url) + break + except ValueError: + continue + else: + raise ValueError( + "Could not read from restricted repository. Have you got access?" + ) + + # Use most specific ref, falling back to main. + logger.debug("Release refs: %s", refs) + if version_tag in refs: + ref = version_tag + elif release_branch in refs: + ref = release_branch + else: + ref = "main" + logger.info("Fetching restricted files from ref %s of %s", ref, url) + + # Checkout git repository to temporary location. + with tempfile.TemporaryDirectory() as tempdir: + # Clone restrict file repository. + logger.debug("Cloning to %s", tempdir) + clone_ref(ref, url, tempdir) + + # Delete unwanted top-level README. + os.unlink(f"{tempdir}/README.md") + + # Copy remaining files, skipping hidden files. + shutil.copytree( + tempdir, + workflow_dir, + ignore=shutil.ignore_patterns(".*"), + symlinks=True, + dirs_exist_ok=True, + ) + logger.info("Installation complete.") diff --git a/tests/workflow_utils/test_install_workflow.py b/tests/workflow_utils/test_install_workflow.py index 0ee6b3b1cf..5b1304ad68 100644 --- a/tests/workflow_utils/test_install_workflow.py +++ b/tests/workflow_utils/test_install_workflow.py @@ -125,8 +125,6 @@ def test_install_workflow(monkeypatch, tmp_path): assert wd.is_dir() # Regular files are coped. assert (wd / "flow.cylc").is_file() - # Scripts are made executable. - assert (wd / "install_restricted_files.sh").stat().st_mode & stat.S_IXUSR # Conda environment is linked. assert (wd / "conda-environment").is_symlink() assert (wd / "conda-environment").readlink() == conda_env