Enable switching off CDDS extract

CMEW/app/configure_for/rose-app.conf

@@ -1,4 +1,4 @@
-# (C) Crown Copyright 2024-2025, Met Office.
+# (C) Crown Copyright 2022-2025, Met Office.
 # The LICENSE.md file contains full licensing details.
 
 [command]

CMEW/app/configure_standardise/bin/create_request_file.py

@@ -17,9 +17,16 @@ def create_request():
     configparser.ConfigParser()
         CDDS request configuration.
     """
+    extract_flag = os.environ.get("EXTRACT", "true").lower() == "true"
+    extract_data_path = os.environ.get("EXTRACT_DATA_PATH", "").strip()
     end_year = int(os.environ["START_YEAR"]) + int(
         os.environ["NUMBER_OF_YEARS"]
     )
+    if not extract_flag and not extract_data_path:
+        raise ValueError(
+            "EXTRACT=False but EXTRACT_DATA_PATH is empty. "
+            "Provide a full path to previously extracted model output."
+        )
     request = configparser.ConfigParser()
     request["metadata"] = {
         "base_date": "1850-01-01T00:00:00",
@@ -47,6 +54,8 @@ def create_request():
         "root_data_dir": os.environ["ROOT_DATA_DIR"],
         "workflow_basename": "CMEW",
     }
+    if not extract_flag:
+        request["common"]["root_data_dir"] = extract_data_path
     request["data"] = {
         "end_date": f"{end_year}-01-01T00:00:00",
         "mass_data_class": "crum",
@@ -65,6 +74,9 @@ def create_request():
         "skip_archive": "True",
         "cylc_args": "--no-detach -v",
     }
+    if not extract_flag:
+        request["conversion"]["skip_extract"] = "True"
+
     return request
 
 

CMEW/app/configure_standardise/bin/test_create_request_file.py

@@ -1,12 +1,17 @@
 # (C) Crown Copyright 2024-2025, Met Office.
 # The LICENSE.md file contains full licensing details.
+
 import os
+import pytest
 
 from create_request_file import create_request
 
 
-def test_create_request(monkeypatch):
-    # In the order defined in 'create_request_file.py'.
+def _set_base_env(monkeypatch):
+    """
+    Set the base environment variables needed by create_request().
+    In the order defined in 'create_request_file.py'.
+    """
     monkeypatch.setenv("START_YEAR", "1993")
     monkeypatch.setenv("NUMBER_OF_YEARS", "1")
     monkeypatch.setenv("CALENDAR", "360_day")
@@ -17,10 +22,24 @@ def test_create_request(monkeypatch):
     monkeypatch.setenv("SUITE_ID", "u-az513")
     monkeypatch.setenv("VARIABLES_PATH", "/path/to/variables.txt")
 
+
+def _clear_extract_env(monkeypatch):
+    """Ensure EXTRACT/EXTRACT_DATA_PATH do not leak from the suite env."""
+    monkeypatch.delenv("EXTRACT", raising=False)
+    monkeypatch.delenv("EXTRACT_DATA_PATH", raising=False)
+
+
+def test_create_request_default_extract(monkeypatch):
+    """EXTRACT default (True) - no skip_extract, root_data_dir unchanged."""
+    _clear_extract_env(monkeypatch)
+    _set_base_env(monkeypatch)
+    # Do not set EXTRACT or EXTRACT_DATA_PATH → EXTRACT defaults to True.
+
     config = create_request()
     actual = {
         section: dict(config.items(section)) for section in config.sections()
     }
+
     expected = {
         "metadata": {
             "branch_method": "no parent",
@@ -65,6 +84,43 @@ def test_create_request(monkeypatch):
             "mip_convert_plugin": "UKESM1",
             "skip_archive": "True",
             "cylc_args": "--no-detach -v",
+            # NOTE: no 'skip_extract' key when EXTRACT defaults to True
         },
     }
+
     assert actual == expected
+
+
+def test_create_request_extract_false_with_path(monkeypatch):
+    """EXTRACT=False + EXTRACT_DATA_PATH set →
+    skip_extract + override root_data_dir."""
+    _clear_extract_env(monkeypatch)
+    _set_base_env(monkeypatch)
+    monkeypatch.setenv("EXTRACT", "false")
+    monkeypatch.setenv("EXTRACT_DATA_PATH", "/pre/extracted/data")
+
+    config = create_request()
+    actual = {
+        section: dict(config.items(section)) for section in config.sections()
+    }
+
+    # Ensure skip_extract is set in [conversion]
+    assert actual["conversion"]["skip_extract"] == "True"
+
+    # Ensure root_data_dir is taken from EXTRACT_DATA_PATH, not ROOT_DATA_DIR
+    assert actual["common"]["root_data_dir"] == "/pre/extracted/data"
+
+    # Optional extra checks to ensure other keys unaffected:
+    assert actual["common"]["root_proc_dir"] == "/path/to/proc/dir/"
+    assert actual["conversion"]["mip_convert_plugin"] == "UKESM1"
+
+
+def test_create_request_extract_false_without_path_raises(monkeypatch):
+    """EXTRACT=False with no EXTRACT_DATA_PATH → fail with ValueError."""
+    _clear_extract_env(monkeypatch)
+    _set_base_env(monkeypatch)
+    monkeypatch.setenv("EXTRACT", "false")
+    # EXTRACT_DATA_PATH intentionally not set (or could set to empty)
+
+    with pytest.raises(ValueError, match="EXTRACT=False"):
+        create_request()

CMEW/app/configure_standardise/rose-app.conf

@@ -1,4 +1,4 @@
-# (C) Crown Copyright 2024-2025, Met Office.
+# (C) Crown Copyright 2022-2025, Met Office.
 # The LICENSE.md file contains full licensing details.
 
 [command]

CMEW/app/housekeeping/rose-app.conf

@@ -1,4 +1,4 @@
-# (C) Crown Copyright 2024-2025, Met Office.
+# (C) Crown Copyright 2022-2025, Met Office.
 # The LICENSE.md file contains full licensing details.
 
 meta=rose_prune

CMEW/app/standardise_model_data/rose-app.conf

@@ -1,4 +1,4 @@
-# (C) Crown Copyright 2024-2025, Met Office.
+# (C) Crown Copyright 2022-2025, Met Office.
 # The LICENSE.md file contains full licensing details.
 
 [command]

CMEW/app/unittest/rose-app.conf

@@ -1,4 +1,4 @@
-# (C) Crown Copyright 2024-2025, Met Office.
+# (C) Crown Copyright 2022-2025, Met Office.
 # The LICENSE.md file contains full licensing details.
 
 [command]

CMEW/flow.cylc

@@ -21,7 +21,8 @@
             install_env_file => configure_recipe & configure_for<recipe>
             configure_for<recipe> => configure_standardise<recipe>
             configure_recipe => run_recipe<recipe>
-            configure_standardise<recipe> => standardise_model_data => run_recipe<recipe> & housekeeping
+            configure_standardise<recipe> => standardise_model_data
+            standardise_model_data => run_recipe<recipe> & housekeeping
     {%- if TEST %}
             run_recipe<recipe> => compare<recipe>
     {%- endif %}
@@ -35,6 +36,8 @@
         platform = localhost
         [[[environment]]]
             ESMVALTOOL_MODULE_NAME = {{ ESMVALTOOL_MODULE_NAME }}
+            EXTRACT = {{ EXTRACT }}
+            EXTRACT_DATA_PATH = {{ EXTRACT_DATA_PATH }}
             USER_CONFIG_DIR = ${CYLC_WORKFLOW_SHARE_DIR}/etc
             USER_CONFIG_PATH = ${USER_CONFIG_DIR}/config-user.yml
             OUTPUT_DIR = ${CYLC_WORKFLOW_SHARE_DIR}/cycle/${CYLC_TASK_CYCLE_POINT}
@@ -56,6 +59,13 @@
             # Workaround for bug in CDDS: ROOT_SOFTWARE_DIR: unbound variable.
             ROOT_SOFTWARE_DIR = ${CDDS_SOFTWARE_DIR}
             CDDS_VERSION = {{ CDDS_VERSION }}
+            {%- if EXTRACT in (False, "false", "False") %}
+            # When EXTRACT is False, reuse pre-extracted data from EXTRACT_DATA_PATH.
+            # Align ROOT_DATA_DIR (used by restructure_for_cmip6) with the path
+            # that create_request_file.py writes into [common].root_data_dir.
+            ROOT_DATA_DIR = "{{ EXTRACT_DATA_PATH }}"
+            {%- endif %}
+
 
     [[install_env_file]]
         [[[environment]]]

CMEW/rose-suite.conf

@@ -14,6 +14,8 @@ DRS_OBS="default"
 DRS_OBS4MIPS="default"
 DRS_OBS6="default"
 ESMVALTOOL_MODULE_NAME=""
+EXTRACT=true
+EXTRACT_DATA_PATH=""
 MAX_PARALLEL_TASKS=4
 MODEL_ID="UKESM1-0-LL"
 NUMBER_OF_YEARS=1