From 2be1e0cf76c932f4764f1a8faa8c2ecbbe779e37 Mon Sep 17 00:00:00 2001 From: Sylviabohnenstengel Date: Wed, 24 Jun 2026 17:47:15 +0100 Subject: [PATCH 1/9] Add RMSE scores based vertical pressure profile recipe and include plotting of vertical profiles without sequence_coordinate for 1D cubes. --- .../includes/metplus_point_stat.cylc | 2 +- src/CSET/cset_workflow/site/nci-gadi.cylc | 2 +- src/CSET/operators/plot.py | 54 ++++++++++++------- .../generic_level_rmse_scores_profile.yaml | 36 +++++++++++++ 4 files changed, 74 insertions(+), 20 deletions(-) create mode 100644 src/CSET/recipes/verification/generic_level_rmse_scores_profile.yaml diff --git a/src/CSET/cset_workflow/includes/metplus_point_stat.cylc b/src/CSET/cset_workflow/includes/metplus_point_stat.cylc index 894c04cf0..9343880c1 100644 --- a/src/CSET/cset_workflow/includes/metplus_point_stat.cylc +++ b/src/CSET/cset_workflow/includes/metplus_point_stat.cylc @@ -103,7 +103,7 @@ # Runs VerPy metloader utility to create the VerPy databases inherit = VERPY_METLOADER [[[environment]]] - VERPY_DIR = {{VERPY_DIR}} + VERPY_DIR = {{VERPY_DIR|default("")}} VER_METHOD = area STAT_TYPE = cnt STREAM = point_stat diff --git a/src/CSET/cset_workflow/site/nci-gadi.cylc b/src/CSET/cset_workflow/site/nci-gadi.cylc index 543ef4ea5..3e83a92f0 100644 --- a/src/CSET/cset_workflow/site/nci-gadi.cylc +++ b/src/CSET/cset_workflow/site/nci-gadi.cylc @@ -23,7 +23,7 @@ """ [[[ environment ]]] PROJECT = {{ PROJECT }} - PYTHONPATH = "{{VERPY_DIR}}" + PYTHONPATH = "{{VERPY_DIR|default("")}}" {% if RUN_METPLUS_GRID_STAT|default(False) or RUN_METPLUS_POINT_STAT|default(False) %} [[METPLUS]] diff --git a/src/CSET/operators/plot.py b/src/CSET/operators/plot.py index 5ec8d7315..e8a5a7315 100644 --- a/src/CSET/operators/plot.py +++ b/src/CSET/operators/plot.py @@ -2181,28 +2181,46 @@ def plot_vertical_line_series( vmin = min(x_levels) vmax = max(x_levels) - # Matching the slices (matching by seq coord point; it may happen that - # evaluated models do not cover the same seq coord range, hence matching - # necessary) - cube_iterables = _find_matched_slices(cubes, sequence_coordinate) + # Check if the cube has a sequence coordinate (e.g. time). If not, plot + # a single profile directly without iterating over a sequence. + has_sequence_coord = all(cube.coords(sequence_coordinate) for cube in cubes) - # Create a plot for each value of the sequence coordinate. - # Allowing for multiple cubes in a CubeList to be plotted in the same plot for - # similar sequence values. Passing a CubeList into the internal plotting function - # for similar values of the sequence coordinate. cube_slice can be an iris.cube.Cube - # or an iris.cube.CubeList. plot_index = [] - nplot = np.size(cubes[0].coord(sequence_coordinate).points) - for cubes_slice in cube_iterables: - # Format the coordinate value in a unit appropriate way. - seq_coord = cubes_slice[0].coord(sequence_coordinate) - plot_title, plot_filename = _set_title_and_filename( - seq_coord, nplot, recipe_title, filename - ) + if has_sequence_coord: + # Matching the slices (matching by seq coord point; it may happen that + # evaluated models do not cover the same seq coord range, hence matching + # necessary) + cube_iterables = _find_matched_slices(cubes, sequence_coordinate) + nplot = np.size(cubes[0].coord(sequence_coordinate).points) + for cubes_slice in cube_iterables: + # Format the coordinate value in a unit appropriate way. + seq_coord = cubes_slice[0].coord(sequence_coordinate) + plot_title, plot_filename = _set_title_and_filename( + seq_coord, nplot, recipe_title, filename + ) + + # Do the actual plotting. + _plot_and_save_vertical_line_series( + cubes_slice, + coords, + "realization", + plot_filename, + series_coordinate, + title=plot_title, + vmin=vmin, + vmax=vmax, + ) + plot_index.append(plot_filename) + else: + # 1D case: no sequence coordinate, plot a single profile. + plot_title = recipe_title + if filename: + plot_filename = filename + else: + plot_filename = f"{slugify(plot_title)}.png" - # Do the actual plotting. _plot_and_save_vertical_line_series( - cubes_slice, + cubes, coords, "realization", plot_filename, diff --git a/src/CSET/recipes/verification/generic_level_rmse_scores_profile.yaml b/src/CSET/recipes/verification/generic_level_rmse_scores_profile.yaml new file mode 100644 index 000000000..f5b55db49 --- /dev/null +++ b/src/CSET/recipes/verification/generic_level_rmse_scores_profile.yaml @@ -0,0 +1,36 @@ +category: Scores +title: "$VARNAME\nRMSE vertical profile between $OTHER_MODEL and $BASE_MODEL" +description: | + Extracts and plots the Root Mean Square Error (RMSE) of $VARNAME + as a vertical profile on pressure levels. The RMSE is calculated + pairwise between $BASE_MODEL and $OTHER_MODEL at each grid point + and level, then aggregated over all time and horizontal grid points + to produce one RMSE value per pressure level. + + A larger RMSE implies a greater error than a smaller RMSE. An + RMSE of zero implies the two fields match exactly. + +steps: + - operator: read.read_cubes + file_paths: $INPUT_PATHS + model_names: [$BASE_MODEL, $OTHER_MODEL] + constraint: + operator: constraints.combine_constraints + variable_constraint: + operator: constraints.generate_var_constraint + varname: $VARNAME + level_constraint: + operator: constraints.generate_level_constraint + coordinate: pressure + levels: "*" + subarea_type: $SUBAREA_TYPE + subarea_extent: $SUBAREA_EXTENT + + - operator: scoreswrappers.scores_rmse + preserved_coordinates: ["pressure"] + + - operator: plot.plot_vertical_line_series + series_coordinate: pressure + + - operator: write.write_cube_to_nc + overwrite: True From 8004edb8ce554714be0e7e8cbc9aad17b564652c Mon Sep 17 00:00:00 2001 From: Sylviabohnenstengel Date: Wed, 24 Jun 2026 18:20:37 +0100 Subject: [PATCH 2/9] add functionality to calculate RMSE for each vertical level for a spatial field and preserve time as sequence coordinate to slide over. This required an addition to scoreswrappers.py that maps the xarray time dimensions with iris auxilliary time dimension requirements. The scoreswrapper was developed with help of Github copilot. --- src/CSET/operators/scoreswrappers.py | 60 +++++++++++++++++-- .../generic_level_rmse_scores_profile.yaml | 8 ++- 2 files changed, 61 insertions(+), 7 deletions(-) diff --git a/src/CSET/operators/scoreswrappers.py b/src/CSET/operators/scoreswrappers.py index 6ca1ee9bf..3fac176ff 100644 --- a/src/CSET/operators/scoreswrappers.py +++ b/src/CSET/operators/scoreswrappers.py @@ -140,6 +140,53 @@ def _sort_cubes_for_verification(cubes: CubeList): return base, other +def _resolve_preserve_dims( + cube: Cube, + data_array: xr.DataArray, + preserved_coordinates: list[str] | str | None, +) -> list[str] | None: + """Resolve preserve coordinates to xarray dimension names. + + The ``scores`` package expects preserve dimensions to match xarray + dimension names. In Iris data, commonly used coordinates such as ``time`` + may be auxiliary coordinates attached to a differently named dimension + (e.g. ``dim0``). This helper maps coordinate names to their underlying + dimension names. + """ + if preserved_coordinates is None: + return None + + coord_names = ( + [preserved_coordinates] + if isinstance(preserved_coordinates, str) + else preserved_coordinates + ) + preserve_dims: list[str] = [] + + for coord_name in coord_names: + # Already an xarray dimension name. + if coord_name in data_array.dims: + if coord_name not in preserve_dims: + preserve_dims.append(coord_name) + continue + + # Otherwise, map coordinate name to dimension index/indices. + try: + dim_indices = cube.coord_dims(coord_name) + except iris.exceptions.CoordinateNotFoundError: + # Keep original name so scores raises a clear error for unknown keys. + if coord_name not in preserve_dims: + preserve_dims.append(coord_name) + continue + + for dim_index in dim_indices: + dim_name = data_array.dims[dim_index] + if dim_name not in preserve_dims: + preserve_dims.append(dim_name) + + return preserve_dims + + def scores_rmse(cubes: CubeList, preserved_coordinates: list[str] | str | None = None): r"""Calculate the Root Mean Square Error (RMSE) using scores. @@ -154,7 +201,7 @@ def scores_rmse(cubes: CubeList, preserved_coordinates: list[str] | str | None = A CubeList containing exactly two cubes: a base and an "other" model, this can be an analysis and the model. preserved_coordinates: list[str] | str | None, default is None. - The coordinates that you wish to preserve in the calculaiton of the + The coordinates (or xarray dimension names) that you wish to preserve in the calculaiton of the RMSE. For example if you want a map of each time you can preserve ["time","grid_latitude", "grid_longitude"] or if you want a time series you can preserve ["time"], if you want to collapse to a single value @@ -181,13 +228,18 @@ def scores_rmse(cubes: CubeList, preserved_coordinates: list[str] | str | None = forecasts, predictions or models (2.5.0)". Zenodo. doi: 10.5281/zenodo.18638494 """ base, other = _sort_cubes_for_verification(cubes) + + other_xr = xr.DataArray.from_iris(other) + base_xr = xr.DataArray.from_iris(base) + preserve_dims = _resolve_preserve_dims(other, other_xr, preserved_coordinates) + # Scores operators on xarray data arrays, so we transform the iris cube into an array, # apply scores, and then transform it back. RMSE = xr.DataArray.to_iris( scores.continuous.rmse( - xr.DataArray.from_iris(other), - xr.DataArray.from_iris(base), - preserve_dims=preserved_coordinates, + other_xr, + base_xr, + preserve_dims=preserve_dims, ) ) RMSE.rename(f"RMSE_of_{base.name()}") diff --git a/src/CSET/recipes/verification/generic_level_rmse_scores_profile.yaml b/src/CSET/recipes/verification/generic_level_rmse_scores_profile.yaml index f5b55db49..df745e6f9 100644 --- a/src/CSET/recipes/verification/generic_level_rmse_scores_profile.yaml +++ b/src/CSET/recipes/verification/generic_level_rmse_scores_profile.yaml @@ -4,8 +4,9 @@ description: | Extracts and plots the Root Mean Square Error (RMSE) of $VARNAME as a vertical profile on pressure levels. The RMSE is calculated pairwise between $BASE_MODEL and $OTHER_MODEL at each grid point - and level, then aggregated over all time and horizontal grid points - to produce one RMSE value per pressure level. + over time on each vertical level, then aggregated over all time and horizontal grid points + to produce one RMSE value per pressure level. The RMSE is calculated in + a single operation for each level that includes the time dimension. A larger RMSE implies a greater error than a smaller RMSE. An RMSE of zero implies the two fields match exactly. @@ -27,10 +28,11 @@ steps: subarea_extent: $SUBAREA_EXTENT - operator: scoreswrappers.scores_rmse - preserved_coordinates: ["pressure"] + preserved_coordinates: ["time","pressure"] - operator: plot.plot_vertical_line_series series_coordinate: pressure + sequence_coordinate: time - operator: write.write_cube_to_nc overwrite: True From 603588cda6adde5adf07ba84485454b715e5f22f Mon Sep 17 00:00:00 2001 From: Sylviabohnenstengel Date: Thu, 25 Jun 2026 12:21:03 +0100 Subject: [PATCH 3/9] include aggregation period into title of vertical profile line plot for rmse_scores_profile when aggregating over an entire case study's time coordinate resulting in a single rmse profile. --- src/CSET/operators/plot.py | 30 ++++++++++++++++++- src/CSET/operators/scoreswrappers.py | 29 ++++++++++++++++++ .../generic_level_rmse_scores_profile.yaml | 5 ++-- 3 files changed, 60 insertions(+), 4 deletions(-) diff --git a/src/CSET/operators/plot.py b/src/CSET/operators/plot.py index e8a5a7315..a544add3d 100644 --- a/src/CSET/operators/plot.py +++ b/src/CSET/operators/plot.py @@ -2183,7 +2183,17 @@ def plot_vertical_line_series( # Check if the cube has a sequence coordinate (e.g. time). If not, plot # a single profile directly without iterating over a sequence. - has_sequence_coord = all(cube.coords(sequence_coordinate) for cube in cubes) + sequence_coords = [ + cube.coord(sequence_coordinate) + for cube in cubes + if cube.coords(sequence_coordinate) + ] + has_sequence_coord = len(sequence_coords) == len(cubes) and all( + np.size(coord.points) > 1 for coord in sequence_coords + ) + has_scalar_sequence_coord = len(sequence_coords) == len(cubes) and all( + np.size(coord.points) == 1 for coord in sequence_coords + ) plot_index = [] if has_sequence_coord: @@ -2211,6 +2221,24 @@ def plot_vertical_line_series( vmax=vmax, ) plot_index.append(plot_filename) + elif has_scalar_sequence_coord: + # Scalar sequence coordinate (typically aggregated time bounds): + # make one plot and include sequence period in title/filename. + plot_title, plot_filename = _set_title_and_filename( + sequence_coords[0], 1, recipe_title, filename + ) + + _plot_and_save_vertical_line_series( + cubes, + coords, + "realization", + plot_filename, + series_coordinate, + title=plot_title, + vmin=vmin, + vmax=vmax, + ) + plot_index.append(plot_filename) else: # 1D case: no sequence coordinate, plot a single profile. plot_title = recipe_title diff --git a/src/CSET/operators/scoreswrappers.py b/src/CSET/operators/scoreswrappers.py index 3fac176ff..a85b2babe 100644 --- a/src/CSET/operators/scoreswrappers.py +++ b/src/CSET/operators/scoreswrappers.py @@ -242,5 +242,34 @@ def scores_rmse(cubes: CubeList, preserved_coordinates: list[str] | str | None = preserve_dims=preserve_dims, ) ) + + # If time is aggregated out, attach a scalar time coordinate with bounds + # so plotting can display the aggregated period in the title. + try: + if not RMSE.coords("time"): + base_time = base.coord("time") + time_vals = ( + base_time.bounds.flatten() + if base_time.has_bounds() + else base_time.points + ) + t_start = float(time_vals[0]) + t_end = float(time_vals[-1]) + t_mid = 0.5 * (t_start + t_end) + + RMSE.add_aux_coord( + iris.coords.AuxCoord( + t_mid, + standard_name=base_time.standard_name, + long_name=base_time.long_name, + var_name=base_time.var_name, + units=base_time.units, + bounds=np.array([t_start, t_end]), + attributes=base_time.attributes.copy(), + ) + ) + except iris.exceptions.CoordinateNotFoundError: + pass + RMSE.rename(f"RMSE_of_{base.name()}") return RMSE diff --git a/src/CSET/recipes/verification/generic_level_rmse_scores_profile.yaml b/src/CSET/recipes/verification/generic_level_rmse_scores_profile.yaml index df745e6f9..f027fa776 100644 --- a/src/CSET/recipes/verification/generic_level_rmse_scores_profile.yaml +++ b/src/CSET/recipes/verification/generic_level_rmse_scores_profile.yaml @@ -4,9 +4,8 @@ description: | Extracts and plots the Root Mean Square Error (RMSE) of $VARNAME as a vertical profile on pressure levels. The RMSE is calculated pairwise between $BASE_MODEL and $OTHER_MODEL at each grid point - over time on each vertical level, then aggregated over all time and horizontal grid points - to produce one RMSE value per pressure level. The RMSE is calculated in - a single operation for each level that includes the time dimension. + and vertical level, then aggregated over all time and horizontal grid points + to produce one RMSE value per pressure level. A larger RMSE implies a greater error than a smaller RMSE. An RMSE of zero implies the two fields match exactly. From 6a1bc6e673154ec10d0e954bd1b6cdbf09aa67d2 Mon Sep 17 00:00:00 2001 From: Sylviabohnenstengel Date: Thu, 25 Jun 2026 15:10:54 +0100 Subject: [PATCH 4/9] wiring the recipe into the cylc workflow --- .../meta/verification/rose-meta.conf | 7 +++++++ src/CSET/cset_workflow/rose-suite.conf.example | 1 + src/CSET/loaders/verification.py | 17 +++++++++++++++++ .../generic_level_rmse_scores_profile.yaml | 2 +- 4 files changed, 26 insertions(+), 1 deletion(-) diff --git a/src/CSET/cset_workflow/meta/verification/rose-meta.conf b/src/CSET/cset_workflow/meta/verification/rose-meta.conf index 53c5cbfb5..d39839af3 100644 --- a/src/CSET/cset_workflow/meta/verification/rose-meta.conf +++ b/src/CSET/cset_workflow/meta/verification/rose-meta.conf @@ -73,3 +73,10 @@ description=Create a timeseries of the RMSE for the specified surface fields. type=python_boolean compulsory=true sort-key=scores2 + +[template variables=SCORES_RMSE_VERTICAL_PROFILES] +ns=Verification/Scores +description=Create vertical profile RMSE plots for the specified level fields. +type=python_boolean +compulsory=true +sort-key=scores3 diff --git a/src/CSET/cset_workflow/rose-suite.conf.example b/src/CSET/cset_workflow/rose-suite.conf.example index 984415ba6..8a3d91631 100644 --- a/src/CSET/cset_workflow/rose-suite.conf.example +++ b/src/CSET/cset_workflow/rose-suite.conf.example @@ -127,6 +127,7 @@ RAIN_PRESENCE_SPATIAL_DIFFERENCE=False RAIN_PRESENCE_SPATIAL_PLOT=False SCORES_RMSE_SPATIAL=False SCORES_RMSE_TIMESERIES=False +SCORES_RMSE_VERTICAL_PROFILES=True SCREEN_LEVEL_TEMPERATURE_PROBABILITIES=False !!SCREEN_LEVEL_TEMPERATURE_SPATIAL_PROBABILITY_WITHOUT_CONTROL_MEMBER=False SELECT_SUBAREA=False diff --git a/src/CSET/loaders/verification.py b/src/CSET/loaders/verification.py index 88d4cc3f4..e0d608cbf 100644 --- a/src/CSET/loaders/verification.py +++ b/src/CSET/loaders/verification.py @@ -63,3 +63,20 @@ def load(conf: Config): model_ids=[base_model["id"], model["id"]], aggregation=False, ) + if conf.SCORES_RMSE_VERTICAL_PROFILES: + base_model = models[0] + for model, field in itertools.product(models[1:], conf.PRESSURE_LEVEL_FIELDS): + yield RawRecipe( + recipe="generic_level_rmse_scores_profile.yaml", + variables={ + "VARNAME": field, + "BASE_MODEL": base_model["name"], + "OTHER_MODEL": model["name"], + "SUBAREA_TYPE": conf.SUBAREA_TYPE if conf.SELECT_SUBAREA else None, + "SUBAREA_EXTENT": conf.SUBAREA_EXTENT + if conf.SELECT_SUBAREA + else None, + }, + model_ids=[base_model["id"], model["id"]], + aggregation=False, + ) diff --git a/src/CSET/recipes/verification/generic_level_rmse_scores_profile.yaml b/src/CSET/recipes/verification/generic_level_rmse_scores_profile.yaml index f027fa776..fcc7f67eb 100644 --- a/src/CSET/recipes/verification/generic_level_rmse_scores_profile.yaml +++ b/src/CSET/recipes/verification/generic_level_rmse_scores_profile.yaml @@ -27,7 +27,7 @@ steps: subarea_extent: $SUBAREA_EXTENT - operator: scoreswrappers.scores_rmse - preserved_coordinates: ["time","pressure"] + preserved_coordinates: ["pressure"] - operator: plot.plot_vertical_line_series series_coordinate: pressure From f52bde4158c89b5b738044060ddcd9d9eb3dac22 Mon Sep 17 00:00:00 2001 From: Sylviabohnenstengel Date: Thu, 25 Jun 2026 16:44:07 +0100 Subject: [PATCH 5/9] functionality to select rmse profile for aggregated across all time steps in a case study and in additional rmse profile for evevry single time step. --- .../meta/verification/rose-meta.conf | 15 +++++++++++++++ src/CSET/cset_workflow/rose-suite.conf.example | 1 + src/CSET/loaders/verification.py | 11 ++++++++++- .../generic_level_rmse_scores_profile.yaml | 11 +++++++---- 4 files changed, 33 insertions(+), 5 deletions(-) diff --git a/src/CSET/cset_workflow/meta/verification/rose-meta.conf b/src/CSET/cset_workflow/meta/verification/rose-meta.conf index d39839af3..49e97fa10 100644 --- a/src/CSET/cset_workflow/meta/verification/rose-meta.conf +++ b/src/CSET/cset_workflow/meta/verification/rose-meta.conf @@ -79,4 +79,19 @@ ns=Verification/Scores description=Create vertical profile RMSE plots for the specified level fields. type=python_boolean compulsory=true +trigger=template variables=SCORES_RMSE_VERTICAL_PROFILES_TIMESERIES: True; sort-key=scores3 + +[template variables=SCORES_RMSE_VERTICAL_PROFILES_TIMESERIES] +ns=Verification/Scores +description=Produce one RMSE vertical profile per time step rather than a single + profile aggregated over the whole time coordinate. +help=When True, the RMSE is calculated at each grid point and level for every + time step, collapsing only the spatial (horizontal) dimensions. This produces + a sequence of vertical RMSE profile plots, one per time step. + + When False (default), the RMSE is collapsed over both horizontal space and + time, yielding a single vertical RMSE profile for the whole period. +type=python_boolean +compulsory=true +sort-key=scores4 diff --git a/src/CSET/cset_workflow/rose-suite.conf.example b/src/CSET/cset_workflow/rose-suite.conf.example index 8a3d91631..ccb8df193 100644 --- a/src/CSET/cset_workflow/rose-suite.conf.example +++ b/src/CSET/cset_workflow/rose-suite.conf.example @@ -128,6 +128,7 @@ RAIN_PRESENCE_SPATIAL_PLOT=False SCORES_RMSE_SPATIAL=False SCORES_RMSE_TIMESERIES=False SCORES_RMSE_VERTICAL_PROFILES=True +SCORES_RMSE_VERTICAL_PROFILES_TIMESERIES=False SCREEN_LEVEL_TEMPERATURE_PROBABILITIES=False !!SCREEN_LEVEL_TEMPERATURE_SPATIAL_PROBABILITY_WITHOUT_CONTROL_MEMBER=False SELECT_SUBAREA=False diff --git a/src/CSET/loaders/verification.py b/src/CSET/loaders/verification.py index e0d608cbf..d171a6f55 100644 --- a/src/CSET/loaders/verification.py +++ b/src/CSET/loaders/verification.py @@ -65,13 +65,22 @@ def load(conf: Config): ) if conf.SCORES_RMSE_VERTICAL_PROFILES: base_model = models[0] - for model, field in itertools.product(models[1:], conf.PRESSURE_LEVEL_FIELDS): + # List of aggregation modes to generate recipes for + agg_modes = [("pressure", ["pressure"])] + if conf.SCORES_RMSE_VERTICAL_PROFILES_TIMESERIES: + agg_modes.append(("timeseries", ["time", "pressure"])) + + for (mode_name, preserved_coords), (model, field) in itertools.product( + agg_modes, itertools.product(models[1:], conf.PRESSURE_LEVEL_FIELDS) + ): yield RawRecipe( recipe="generic_level_rmse_scores_profile.yaml", variables={ "VARNAME": field, "BASE_MODEL": base_model["name"], "OTHER_MODEL": model["name"], + "PRESERVED_COORDS": preserved_coords, + "AGGREGATION_MODE": mode_name, "SUBAREA_TYPE": conf.SUBAREA_TYPE if conf.SELECT_SUBAREA else None, "SUBAREA_EXTENT": conf.SUBAREA_EXTENT if conf.SELECT_SUBAREA diff --git a/src/CSET/recipes/verification/generic_level_rmse_scores_profile.yaml b/src/CSET/recipes/verification/generic_level_rmse_scores_profile.yaml index fcc7f67eb..6c3cf3fe4 100644 --- a/src/CSET/recipes/verification/generic_level_rmse_scores_profile.yaml +++ b/src/CSET/recipes/verification/generic_level_rmse_scores_profile.yaml @@ -1,11 +1,14 @@ category: Scores -title: "$VARNAME\nRMSE vertical profile between $OTHER_MODEL and $BASE_MODEL" +title: "$VARNAME\nRMSE vertical profile ($AGGREGATION_MODE) between $OTHER_MODEL and $BASE_MODEL" description: | Extracts and plots the Root Mean Square Error (RMSE) of $VARNAME as a vertical profile on pressure levels. The RMSE is calculated pairwise between $BASE_MODEL and $OTHER_MODEL at each grid point - and vertical level, then aggregated over all time and horizontal grid points - to produce one RMSE value per pressure level. + and vertical level, then aggregated over horizontal grid points. + + Aggregation mode: + - pressure: Aggregates over the entire time period, producing one RMSE profile + - timeseries: Preserves time, producing one RMSE profile per time step A larger RMSE implies a greater error than a smaller RMSE. An RMSE of zero implies the two fields match exactly. @@ -27,7 +30,7 @@ steps: subarea_extent: $SUBAREA_EXTENT - operator: scoreswrappers.scores_rmse - preserved_coordinates: ["pressure"] + preserved_coordinates: $PRESERVED_COORDS - operator: plot.plot_vertical_line_series series_coordinate: pressure From 7081246ed3e85785985222442dbaaca09c1f0b7a Mon Sep 17 00:00:00 2001 From: Sylviabohnenstengel Date: Mon, 29 Jun 2026 18:13:27 +0100 Subject: [PATCH 6/9] removing dependency between vertical profile RMSE for evevry timestep (sequence) and vertical profile RMSE over whole case. In addition identifying base_model at start of loader. --- .../meta/verification/rose-meta.conf | 12 +++---- src/CSET/loaders/verification.py | 35 ++++++++++++------- 2 files changed, 29 insertions(+), 18 deletions(-) diff --git a/src/CSET/cset_workflow/meta/verification/rose-meta.conf b/src/CSET/cset_workflow/meta/verification/rose-meta.conf index 49e97fa10..865f5522c 100644 --- a/src/CSET/cset_workflow/meta/verification/rose-meta.conf +++ b/src/CSET/cset_workflow/meta/verification/rose-meta.conf @@ -77,21 +77,21 @@ sort-key=scores2 [template variables=SCORES_RMSE_VERTICAL_PROFILES] ns=Verification/Scores description=Create vertical profile RMSE plots for the specified level fields. + RMSE profiles are calculated at each grid point for all timesteps within a case study at each pressure level. type=python_boolean compulsory=true -trigger=template variables=SCORES_RMSE_VERTICAL_PROFILES_TIMESERIES: True; sort-key=scores3 [template variables=SCORES_RMSE_VERTICAL_PROFILES_TIMESERIES] ns=Verification/Scores -description=Produce one RMSE vertical profile per time step rather than a single - profile aggregated over the whole time coordinate. +description=Also produce one RMSE vertical profile per time step. help=When True, the RMSE is calculated at each grid point and level for every time step, collapsing only the spatial (horizontal) dimensions. This produces - a sequence of vertical RMSE profile plots, one per time step. + a sequence of vertical RMSE profile plots, one per time step, in addition to + the profile that is aggregated across the whole case study. - When False (default), the RMSE is collapsed over both horizontal space and - time, yielding a single vertical RMSE profile for the whole period. + When False (default), only the aggregated vertical RMSE profile is produced. type=python_boolean compulsory=true +#trigger=template variables=SCORES_RMSE_VERTICAL_PROFILES: True; sort-key=scores4 diff --git a/src/CSET/loaders/verification.py b/src/CSET/loaders/verification.py index d171a6f55..b53f55176 100644 --- a/src/CSET/loaders/verification.py +++ b/src/CSET/loaders/verification.py @@ -24,9 +24,9 @@ def load(conf: Config): # Load a list of model detail dictionaries. models = get_models(conf.asdict()) # Models are listed in order, so model 1 is the first element. + base_model = models[0] if conf.SCORES_RMSE_SPATIAL: - base_model = models[0] for model, field, method in itertools.product( models[1:], conf.SURFACE_FIELDS, conf.SPATIAL_SURFACE_FIELD_METHOD ): @@ -47,7 +47,6 @@ def load(conf: Config): ) if conf.SCORES_RMSE_TIMESERIES: - base_model = models[0] for model, field in itertools.product(models[1:], conf.SURFACE_FIELDS): yield RawRecipe( recipe="timeseries_surface_rmse_scores.yaml", @@ -63,24 +62,36 @@ def load(conf: Config): model_ids=[base_model["id"], model["id"]], aggregation=False, ) + if conf.SCORES_RMSE_VERTICAL_PROFILES: - base_model = models[0] - # List of aggregation modes to generate recipes for - agg_modes = [("pressure", ["pressure"])] - if conf.SCORES_RMSE_VERTICAL_PROFILES_TIMESERIES: - agg_modes.append(("timeseries", ["time", "pressure"])) + for model, field in itertools.product(models[1:], conf.PRESSURE_LEVEL_FIELDS): + yield RawRecipe( + recipe="generic_level_rmse_scores_profile.yaml", + variables={ + "VARNAME": field, + "BASE_MODEL": base_model["name"], + "OTHER_MODEL": model["name"], + "PRESERVED_COORDS": ["pressure"], + "AGGREGATION_MODE": "pressure", + "SUBAREA_TYPE": conf.SUBAREA_TYPE if conf.SELECT_SUBAREA else None, + "SUBAREA_EXTENT": conf.SUBAREA_EXTENT + if conf.SELECT_SUBAREA + else None, + }, + model_ids=[base_model["id"], model["id"]], + aggregation=False, + ) - for (mode_name, preserved_coords), (model, field) in itertools.product( - agg_modes, itertools.product(models[1:], conf.PRESSURE_LEVEL_FIELDS) - ): + if conf.SCORES_RMSE_VERTICAL_PROFILES_TIMESERIES: + for model, field in itertools.product(models[1:], conf.PRESSURE_LEVEL_FIELDS): yield RawRecipe( recipe="generic_level_rmse_scores_profile.yaml", variables={ "VARNAME": field, "BASE_MODEL": base_model["name"], "OTHER_MODEL": model["name"], - "PRESERVED_COORDS": preserved_coords, - "AGGREGATION_MODE": mode_name, + "PRESERVED_COORDS": ["time", "pressure"], + "AGGREGATION_MODE": "timeseries", "SUBAREA_TYPE": conf.SUBAREA_TYPE if conf.SELECT_SUBAREA else None, "SUBAREA_EXTENT": conf.SUBAREA_EXTENT if conf.SELECT_SUBAREA From 537809eaad3df4d09353f024c1a1a5e2eaf10a9e Mon Sep 17 00:00:00 2001 From: Sylviabohnenstengel Date: Mon, 6 Jul 2026 17:25:34 +0100 Subject: [PATCH 7/9] coorect name for vertical profile at each timepoint to SEQUENCE. --- src/CSET/cset_workflow/rose-suite.conf.example | 2 +- src/CSET/loaders/verification.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/CSET/cset_workflow/rose-suite.conf.example b/src/CSET/cset_workflow/rose-suite.conf.example index ccb8df193..511bb56a4 100644 --- a/src/CSET/cset_workflow/rose-suite.conf.example +++ b/src/CSET/cset_workflow/rose-suite.conf.example @@ -128,7 +128,7 @@ RAIN_PRESENCE_SPATIAL_PLOT=False SCORES_RMSE_SPATIAL=False SCORES_RMSE_TIMESERIES=False SCORES_RMSE_VERTICAL_PROFILES=True -SCORES_RMSE_VERTICAL_PROFILES_TIMESERIES=False +SCORES_RMSE_VERTICAL_PROFILES_SEQUENCE=False SCREEN_LEVEL_TEMPERATURE_PROBABILITIES=False !!SCREEN_LEVEL_TEMPERATURE_SPATIAL_PROBABILITY_WITHOUT_CONTROL_MEMBER=False SELECT_SUBAREA=False diff --git a/src/CSET/loaders/verification.py b/src/CSET/loaders/verification.py index b53f55176..021931648 100644 --- a/src/CSET/loaders/verification.py +++ b/src/CSET/loaders/verification.py @@ -82,7 +82,7 @@ def load(conf: Config): aggregation=False, ) - if conf.SCORES_RMSE_VERTICAL_PROFILES_TIMESERIES: + if conf.SCORES_RMSE_VERTICAL_PROFILES_SEQUENCE: for model, field in itertools.product(models[1:], conf.PRESSURE_LEVEL_FIELDS): yield RawRecipe( recipe="generic_level_rmse_scores_profile.yaml", From 723649518538b5cb1b04d6bfe1166eeeb0c94825 Mon Sep 17 00:00:00 2001 From: Sylviabohnenstengel Date: Mon, 6 Jul 2026 17:42:58 +0100 Subject: [PATCH 8/9] correctly name rmse profile to sequence for profiles at each timestep. --- src/CSET/cset_workflow/meta/verification/rose-meta.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CSET/cset_workflow/meta/verification/rose-meta.conf b/src/CSET/cset_workflow/meta/verification/rose-meta.conf index 865f5522c..a108475f6 100644 --- a/src/CSET/cset_workflow/meta/verification/rose-meta.conf +++ b/src/CSET/cset_workflow/meta/verification/rose-meta.conf @@ -82,7 +82,7 @@ type=python_boolean compulsory=true sort-key=scores3 -[template variables=SCORES_RMSE_VERTICAL_PROFILES_TIMESERIES] +[template variables=SCORES_RMSE_VERTICAL_PROFILES_SEQUENCE] ns=Verification/Scores description=Also produce one RMSE vertical profile per time step. help=When True, the RMSE is calculated at each grid point and level for every From 1a8d04e9dae5f07bb8576b49e687466d1e4d853d Mon Sep 17 00:00:00 2001 From: Sylviabohnenstengel Date: Mon, 6 Jul 2026 18:15:08 +0100 Subject: [PATCH 9/9] preserve enembles for vertical profile RMSE plots. --- src/CSET/loaders/verification.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/CSET/loaders/verification.py b/src/CSET/loaders/verification.py index 021931648..ba5568919 100644 --- a/src/CSET/loaders/verification.py +++ b/src/CSET/loaders/verification.py @@ -71,7 +71,7 @@ def load(conf: Config): "VARNAME": field, "BASE_MODEL": base_model["name"], "OTHER_MODEL": model["name"], - "PRESERVED_COORDS": ["pressure"], + "PRESERVED_COORDS": ["pressure", "realization"], "AGGREGATION_MODE": "pressure", "SUBAREA_TYPE": conf.SUBAREA_TYPE if conf.SELECT_SUBAREA else None, "SUBAREA_EXTENT": conf.SUBAREA_EXTENT @@ -90,7 +90,7 @@ def load(conf: Config): "VARNAME": field, "BASE_MODEL": base_model["name"], "OTHER_MODEL": model["name"], - "PRESERVED_COORDS": ["time", "pressure"], + "PRESERVED_COORDS": ["time", "pressure", "realization"], "AGGREGATION_MODE": "timeseries", "SUBAREA_TYPE": conf.SUBAREA_TYPE if conf.SELECT_SUBAREA else None, "SUBAREA_EXTENT": conf.SUBAREA_EXTENT