From 3e7c4fa5bac3c5ac472b38687ddb6fe845348470 Mon Sep 17 00:00:00 2001 From: Scott Wales Date: Thu, 18 Jun 2026 09:44:41 +1000 Subject: [PATCH 1/5] Update NCI site to allow changing module and CSET path --- .../opt/rose-suite-nci-gadi.conf | 14 +++++++++++ src/CSET/cset_workflow/site/nci-gadi.cylc | 25 ++++++++++++------- 2 files changed, 30 insertions(+), 9 deletions(-) create mode 100644 src/CSET/cset_workflow/opt/rose-suite-nci-gadi.conf diff --git a/src/CSET/cset_workflow/opt/rose-suite-nci-gadi.conf b/src/CSET/cset_workflow/opt/rose-suite-nci-gadi.conf new file mode 100644 index 000000000..30495ae90 --- /dev/null +++ b/src/CSET/cset_workflow/opt/rose-suite-nci-gadi.conf @@ -0,0 +1,14 @@ +[template variables] +SITE="nci-gadi" + +# Storage flags to add +NCI_STORAGE=["gdata/dp9"] + +# Where to store output website (view from ARE virtual desktop) +WEB_DIR="~/public_html/$CYLC_WORKFLOW_NAME" + +# Default module +CSET_ENV_USE_MODULES = True +CSET_ENV_SEPARATE_MET = False +MODULES_PURGE = "" +MODULES_LIST = "/g/data/access/ngm/modules/cset/26.4.0" diff --git a/src/CSET/cset_workflow/site/nci-gadi.cylc b/src/CSET/cset_workflow/site/nci-gadi.cylc index 543ef4ea5..d2171247e 100644 --- a/src/CSET/cset_workflow/site/nci-gadi.cylc +++ b/src/CSET/cset_workflow/site/nci-gadi.cylc @@ -2,10 +2,14 @@ # Site configuration for NCI Gadi # +# Using the opt file 'nci-gadi' is recommended to ensure the CSET module is +# loaded correctly +# # Important rose-suite.conf settings: # SITE = "nci-gadi" # Enable the site # PROJECT # NCI project code # VERPY_DIR # Path to Verpy source code +# NCI_STORAGE # List of NCI storage locations to use # METPLUS_OBS_SYSTEM # Where to source obs from # # Can be a Bureau forecast system e.g. # # 'access_c3_dn' or 'custom' @@ -14,16 +18,19 @@ # # Leave METPLUS_FCST_DIR and METPLUS_OBS_DIR unset +{% set STORAGE = [ + 'scratch/'~PROJECT, + 'gdata/'~PROJECT, + 'gdata/access', + 'gdata/hr22', +] + (NCI_STORAGE | default([])) %} + [runtime] [[root]] platform = gadi_background - init-script = """ - module use /g/data/access/ngm/modules - module load cset/26.4.0 - """ [[[ environment ]]] PROJECT = {{ PROJECT }} - PYTHONPATH = "{{VERPY_DIR}}" + PYTHONPATH = {{VERPY_DIR}}{% if CSET_DIR is defined %}:{{CSET_DIR}}{% endif %} {% if RUN_METPLUS_GRID_STAT|default(False) or RUN_METPLUS_POINT_STAT|default(False) %} [[METPLUS]] @@ -34,14 +41,14 @@ -q = normal -l ncpus = 1 -l mem = 4gb - -l storage = scratch/{{PROJECT}}+gdata/{{PROJECT}}+gdata/access+gdata/dp9+gdata/hr22+gdata/ig2+scratch/dx2 + -l storage = {{ STORAGE | join('+') }}+gdata/ig2 -W umask = 0022 [[[ environment ]]] ANALYSIS_LENGTH = {{ANALYSIS_LENGTH}} [[metplus_prep_obs]] [[[ environment ]]] - OBS_TIMES = R{{ANALYSIS_LENGTH | duration_as('h') | int }}/$CYLC_TASK_CYCLE_POINT/PT1H + OBS_TIMES = R{{(ANALYSIS_LENGTH | duration_as('h') | int) + 1}}/$CYLC_TASK_CYCLE_POINT/PT1H {% if METPLUS_OBS_SYSTEM != "custom" %} # Forecast system to pull obs from (e.g. 'access_c3_dn') METPLUS_OBS_SYSTEM = {{METPLUS_OBS_SYSTEM}} @@ -60,7 +67,7 @@ -q = normal -l ncpus = 48 -l mem = 180gb - -l storage = scratch/{{PROJECT}}+gdata/{{PROJECT}}+gdata/access+gdata/dp9+gdata/hr22+gdata/xp65+scratch/dx2 + -l storage = {{ STORAGE | join('+') }} -W umask = 0022 [[[ environment ]]] BUNCH_POOL_SIZE = $PBS_NCPUS @@ -73,7 +80,7 @@ -q = normal -l ncpus = 48 -l mem = 180gb - -l storage = scratch/{{PROJECT}}+gdata/{{PROJECT}}+gdata/access+gdata/dp9+gdata/hr22+gdata/xp65+scratch/dx2 + -l storage = {{ STORAGE | join('+') }} -W umask = 0022 [[[ environment ]]] BUNCH_POOL_SIZE = $PBS_NCPUS From fb566809a48b2ea6355dab1ef539674bef9ca7f1 Mon Sep 17 00:00:00 2001 From: Scott Wales Date: Thu, 18 Jun 2026 09:46:38 +1000 Subject: [PATCH 2/5] Remove time shift now that times are correctly processed --- .../app/metplus_point_stat/file/nci/pointstat.conf | 3 --- .../app/metplus_point_stat/file/nci/surface.conf | 4 ++-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/CSET/cset_workflow/app/metplus_point_stat/file/nci/pointstat.conf b/src/CSET/cset_workflow/app/metplus_point_stat/file/nci/pointstat.conf index 5d7b1880d..52bf7c95b 100644 --- a/src/CSET/cset_workflow/app/metplus_point_stat/file/nci/pointstat.conf +++ b/src/CSET/cset_workflow/app/metplus_point_stat/file/nci/pointstat.conf @@ -38,9 +38,6 @@ INIT_END = {ENV[TASK_START_TIME]} INIT_INCREMENT = 1H LEAD_SEQ = begin_end_incr(0,{ENV[FORECAST_LENGTH]},1) -# Number of seconds to shift times in the fcst file (try half the time increment) -FCST_SHIFT = 1800 - ### # File I/O # https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#directory-and-filename-template-info diff --git a/src/CSET/cset_workflow/app/metplus_point_stat/file/nci/surface.conf b/src/CSET/cset_workflow/app/metplus_point_stat/file/nci/surface.conf index d61168298..e1c42c5b4 100644 --- a/src/CSET/cset_workflow/app/metplus_point_stat/file/nci/surface.conf +++ b/src/CSET/cset_workflow/app/metplus_point_stat/file/nci/surface.conf @@ -17,7 +17,7 @@ POINT_STAT_FCST_FILE_TYPE = NETCDF_NCCF POINT_STAT_MESSAGE_TYPE = "ADPSFC" FCST_POINT_STAT_VAR1_NAME = air_temperature -FCST_POINT_STAT_VAR1_LEVELS = "({valid?fmt=%Y%m%d_%H%M%S?shift={FCST_SHIFT}},*,*)" +FCST_POINT_STAT_VAR1_LEVELS = "({valid?fmt=%Y%m%d_%H%M%S},*,*)" FCST_POINT_STAT_VAR1_THRESH = <=273, >273 OBS_POINT_STAT_VAR1_NAME = t2m @@ -25,7 +25,7 @@ OBS_POINT_STAT_VAR1_LEVELS = Z0 OBS_POINT_STAT_VAR1_THRESH = <=273, >273 FCST_POINT_STAT_VAR2_NAME = relative_humidity -FCST_POINT_STAT_VAR2_LEVELS = "({valid?fmt=%Y%m%d_%H%M%S?shift={FCST_SHIFT}},*,*)" +FCST_POINT_STAT_VAR2_LEVELS = "({valid?fmt=%Y%m%d_%H%M%S},*,*)" FCST_POINT_STAT_VAR2_THRESH = <60, >95 OBS_POINT_STAT_VAR2_NAME = rh2m From 8545fe17f1f2200fa3b20e4796619690aa4eccab Mon Sep 17 00:00:00 2001 From: Scott Wales Date: Thu, 18 Jun 2026 09:47:50 +1000 Subject: [PATCH 3/5] Change ODB2 processing to output one ASCII file per valid time --- .../metplus_ascii2nc/file/nci/ascii2nc.conf | 10 +------- .../app/metplus_prep_obs/bin/odb2/odb2.py | 25 +++++++++++++++---- .../app/metplus_prep_obs/bin/prepBureauNCI.py | 13 +++------- .../app/metplus_prep_obs/bin/prepODB2.py | 16 ++++-------- .../opt/rose-app-nci-gadi.conf | 2 +- .../metplus_prep_obs/opt/rose-app-odb2.conf | 2 +- 6 files changed, 31 insertions(+), 37 deletions(-) diff --git a/src/CSET/cset_workflow/app/metplus_ascii2nc/file/nci/ascii2nc.conf b/src/CSET/cset_workflow/app/metplus_ascii2nc/file/nci/ascii2nc.conf index f0f5dddf4..ee77d8c68 100644 --- a/src/CSET/cset_workflow/app/metplus_ascii2nc/file/nci/ascii2nc.conf +++ b/src/CSET/cset_workflow/app/metplus_ascii2nc/file/nci/ascii2nc.conf @@ -43,16 +43,8 @@ LEAD_SEQ = # https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#directory-and-filename-template-info ### ASCII2NC_INPUT_DIR = {INPUT_BASE} -ASCII2NC_INPUT_TEMPLATE = *.ascii +ASCII2NC_INPUT_TEMPLATE = obs.{valid?fmt=%Y%m%dT%H%MZ}.ascii ASCII2NC_INPUT_FORMAT = met_point ASCII2NC_OUTPUT_DIR = {ENV[CYLC_WORKFLOW_SHARE_DIR]}/obs_nc ASCII2NC_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%dT%H}.nc - -### -# ASCII2NC Settings -# https://metplus.readthedocs.io/en/latest/Users_Guide/wrappers.html#ascii2nc -### - -ASCII2NC_WINDOW_BEGIN = 0 -ASCII2NC_WINDOW_END = 0 diff --git a/src/CSET/cset_workflow/app/metplus_prep_obs/bin/odb2/odb2.py b/src/CSET/cset_workflow/app/metplus_prep_obs/bin/odb2/odb2.py index ad7f45110..2f7110f93 100644 --- a/src/CSET/cset_workflow/app/metplus_prep_obs/bin/odb2/odb2.py +++ b/src/CSET/cset_workflow/app/metplus_prep_obs/bin/odb2/odb2.py @@ -11,6 +11,8 @@ from glob import glob from pathlib import Path from typing import Iterable, TextIO +import sys +from contextlib import nullcontext import metomi.isodatetime.parsers import numpy @@ -340,13 +342,26 @@ def read_odb(self, valid_time: TimePoint) -> Iterable[DataFrame]: """Read ODB2 data.""" raise NotImplementedError - def odb2ascii(self, output: TextIO, valid_times: Iterable[TimePoint]): - """Write all the observations to a MET ASCII file.""" + def odb2ascii(self, output_pattern: str, valid_times: Iterable[TimePoint]): + """ + Write all the observations to a MET ASCII file. + + If output_pattern contains a strftime-style pattern then the valid time + will be used to replace the pattern. + """ for t in valid_times: + output = t.strftime(output_pattern) + + if output == '-': + out_context = nullcontext(sys.stdout) + else: + out_context = open(output, "wt") + log.info("Processing %s", t) - for obs in self.read_odb(t): - ascii = odb2ascii_dataframe(obs) - write_ascii(ascii, output) + with out_context as f: + for obs in self.read_odb(t): + ascii = odb2ascii_dataframe(obs) + write_ascii(ascii, f) class PrepODB2Pattern(PrepODB2): diff --git a/src/CSET/cset_workflow/app/metplus_prep_obs/bin/prepBureauNCI.py b/src/CSET/cset_workflow/app/metplus_prep_obs/bin/prepBureauNCI.py index 2c3e1a191..90e993814 100755 --- a/src/CSET/cset_workflow/app/metplus_prep_obs/bin/prepBureauNCI.py +++ b/src/CSET/cset_workflow/app/metplus_prep_obs/bin/prepBureauNCI.py @@ -6,17 +6,16 @@ Valid times can be either ISO timepoints or recurrences, and are used to replace any strftime patterns. Data is sourced from the mirror in the ig2 project, not all times are available. - ./prepODB2.py \ + ./prepBureauNCI.py \ --system access-c3-dn \ --valid-time 20010101T0000Z \ --valid-time R4/20010102T0000Z/PT6H \ - --output obs.ascii + --output obs.%Y%m%dT%H%MZ.ascii """ import argparse import logging import sys -from contextlib import nullcontext from odb2 import valid_times_iterator from odb2.bom import BOM_SYSTEMS, PrepBomNci @@ -56,13 +55,7 @@ def main(argv: list[str]): logging.basicConfig(level=logging.INFO) sys.tracebacklimit = 0 - if args.output == "-": - out_context = nullcontext(sys.stdout) - else: - out_context = open(args.output, "wt") - - with out_context as output: - PrepBomNci(args.system).odb2ascii(output, valid_times_iterator(args.valid_time)) + PrepBomNci(args.system).odb2ascii(args.output, valid_times_iterator(args.valid_time)) if __name__ == "__main__": diff --git a/src/CSET/cset_workflow/app/metplus_prep_obs/bin/prepODB2.py b/src/CSET/cset_workflow/app/metplus_prep_obs/bin/prepODB2.py index c021a261d..97a5103bd 100755 --- a/src/CSET/cset_workflow/app/metplus_prep_obs/bin/prepODB2.py +++ b/src/CSET/cset_workflow/app/metplus_prep_obs/bin/prepODB2.py @@ -9,7 +9,7 @@ ./prepODB2.py /path/to/%Y/%m/%Y%m%dT%H%MZ/*.odb \ --valid-time 20010101T0000Z \ --valid-time R4/20010102T0000Z/PT6H \ - --output obs.ascii + --output obs.%Y%m%dT%H%MZ.ascii """ import argparse @@ -71,16 +71,10 @@ def main(argv: list[str]): # Valid time unset, hopefully the pattern isn't using times args.valid_time = ["00010101T0000Z"] - if args.output == "-": - out_context = nullcontext(sys.stdout) - else: - out_context = open(args.output, "wt") - - with out_context as output: - for pattern in args.file: - PrepODB2Pattern(pattern).odb2ascii( - output, valid_times_iterator(args.valid_time) - ) + for pattern in args.file: + PrepODB2Pattern(pattern).odb2ascii( + args.output, valid_times_iterator(args.valid_time) + ) if __name__ == "__main__": diff --git a/src/CSET/cset_workflow/app/metplus_prep_obs/opt/rose-app-nci-gadi.conf b/src/CSET/cset_workflow/app/metplus_prep_obs/opt/rose-app-nci-gadi.conf index 61375dd49..1fa93ef26 100644 --- a/src/CSET/cset_workflow/app/metplus_prep_obs/opt/rose-app-nci-gadi.conf +++ b/src/CSET/cset_workflow/app/metplus_prep_obs/opt/rose-app-nci-gadi.conf @@ -1,4 +1,4 @@ [command] # Read in files from ig2 default = mkdir -p "$METPLUS_OBS_DIR" - = app_env_wrapper prepBureauNCI.py --system "$METPLUS_OBS_SYSTEM" --valid-time "$OBS_TIMES" --output "$METPLUS_OBS_DIR/${CYLC_TASK_CYCLE_POINT}.ascii" + = app_env_wrapper prepBureauNCI.py --system "$METPLUS_OBS_SYSTEM" --valid-time "$OBS_TIMES" --output "$METPLUS_OBS_DIR/obs.%Y%m%dT%H%MZ.ascii" diff --git a/src/CSET/cset_workflow/app/metplus_prep_obs/opt/rose-app-odb2.conf b/src/CSET/cset_workflow/app/metplus_prep_obs/opt/rose-app-odb2.conf index a47fea382..7ad4f9f67 100644 --- a/src/CSET/cset_workflow/app/metplus_prep_obs/opt/rose-app-odb2.conf +++ b/src/CSET/cset_workflow/app/metplus_prep_obs/opt/rose-app-odb2.conf @@ -1,4 +1,4 @@ [command] # Read in files from ODB2 files by providing a strftime pattern default = mkdir -p "$METPLUS_OBS_DIR" - = app_env_wrapper prepODB2.py --valid-time "$OBS_TIMES" --output "$METPLUS_OBS_DIR/${CYLC_TASK_CYCLE_POINT}.ascii" $CUSTOM_ODB2_PATTERN + = app_env_wrapper prepODB2.py --valid-time "$OBS_TIMES" --output "$METPLUS_OBS_DIR/obs.%Y%m%dT%H%MZ.ascii" $CUSTOM_ODB2_PATTERN From 6c8a693133294c66695d278dacb181e040d9c70d Mon Sep 17 00:00:00 2001 From: Scott Wales Date: Thu, 18 Jun 2026 09:48:25 +1000 Subject: [PATCH 4/5] Lint --- .../cset_workflow/app/metplus_prep_obs/bin/odb2/odb2.py | 8 ++++---- .../app/metplus_prep_obs/bin/prepBureauNCI.py | 4 +++- .../cset_workflow/app/metplus_prep_obs/bin/prepODB2.py | 1 - 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/CSET/cset_workflow/app/metplus_prep_obs/bin/odb2/odb2.py b/src/CSET/cset_workflow/app/metplus_prep_obs/bin/odb2/odb2.py index 2f7110f93..c4446a589 100644 --- a/src/CSET/cset_workflow/app/metplus_prep_obs/bin/odb2/odb2.py +++ b/src/CSET/cset_workflow/app/metplus_prep_obs/bin/odb2/odb2.py @@ -7,12 +7,12 @@ import functools import json import logging +import sys from abc import ABC, abstractmethod +from contextlib import nullcontext from glob import glob from pathlib import Path from typing import Iterable, TextIO -import sys -from contextlib import nullcontext import metomi.isodatetime.parsers import numpy @@ -345,14 +345,14 @@ def read_odb(self, valid_time: TimePoint) -> Iterable[DataFrame]: def odb2ascii(self, output_pattern: str, valid_times: Iterable[TimePoint]): """ Write all the observations to a MET ASCII file. - + If output_pattern contains a strftime-style pattern then the valid time will be used to replace the pattern. """ for t in valid_times: output = t.strftime(output_pattern) - if output == '-': + if output == "-": out_context = nullcontext(sys.stdout) else: out_context = open(output, "wt") diff --git a/src/CSET/cset_workflow/app/metplus_prep_obs/bin/prepBureauNCI.py b/src/CSET/cset_workflow/app/metplus_prep_obs/bin/prepBureauNCI.py index 90e993814..477b855c8 100755 --- a/src/CSET/cset_workflow/app/metplus_prep_obs/bin/prepBureauNCI.py +++ b/src/CSET/cset_workflow/app/metplus_prep_obs/bin/prepBureauNCI.py @@ -55,7 +55,9 @@ def main(argv: list[str]): logging.basicConfig(level=logging.INFO) sys.tracebacklimit = 0 - PrepBomNci(args.system).odb2ascii(args.output, valid_times_iterator(args.valid_time)) + PrepBomNci(args.system).odb2ascii( + args.output, valid_times_iterator(args.valid_time) + ) if __name__ == "__main__": diff --git a/src/CSET/cset_workflow/app/metplus_prep_obs/bin/prepODB2.py b/src/CSET/cset_workflow/app/metplus_prep_obs/bin/prepODB2.py index 97a5103bd..500ed02d7 100755 --- a/src/CSET/cset_workflow/app/metplus_prep_obs/bin/prepODB2.py +++ b/src/CSET/cset_workflow/app/metplus_prep_obs/bin/prepODB2.py @@ -15,7 +15,6 @@ import argparse import logging import sys -from contextlib import nullcontext from odb2.odb2 import PrepODB2Pattern, valid_times_iterator From b5ee78b12773a9de184196ef45ec6777d6256b62 Mon Sep 17 00:00:00 2001 From: Scott Wales Date: Thu, 18 Jun 2026 10:51:32 +1000 Subject: [PATCH 5/5] Add more comments to the opt file --- src/CSET/cset_workflow/opt/rose-suite-nci-gadi.conf | 11 +++++++++++ src/CSET/cset_workflow/site/nci-gadi.cylc | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/CSET/cset_workflow/opt/rose-suite-nci-gadi.conf b/src/CSET/cset_workflow/opt/rose-suite-nci-gadi.conf index 30495ae90..30461f232 100644 --- a/src/CSET/cset_workflow/opt/rose-suite-nci-gadi.conf +++ b/src/CSET/cset_workflow/opt/rose-suite-nci-gadi.conf @@ -12,3 +12,14 @@ CSET_ENV_USE_MODULES = True CSET_ENV_SEPARATE_MET = False MODULES_PURGE = "" MODULES_LIST = "/g/data/access/ngm/modules/cset/26.4.0" + +# # Paths to local checkouts of repositories +# VERPY_DIR = "/path/to/verpy" +# CSET_DIR = "/path/to/cset" + +# # Metplus information +# # OBS_SYSTEM can be access_g{3,4}, access_c{3,4}_{ad,bn,dn,nq,ph,sy,vt} or custom +# METPLUS_OBS_SYSTEM = "access_g4" +# +# # If OBS_SYSTEM is custom, what path should be read +# CUSTOM_ODB_PATTERN = "/path/to/files/%Y%m%dT%H%MZ.odb2" diff --git a/src/CSET/cset_workflow/site/nci-gadi.cylc b/src/CSET/cset_workflow/site/nci-gadi.cylc index d2171247e..44f7904b6 100644 --- a/src/CSET/cset_workflow/site/nci-gadi.cylc +++ b/src/CSET/cset_workflow/site/nci-gadi.cylc @@ -30,7 +30,7 @@ platform = gadi_background [[[ environment ]]] PROJECT = {{ PROJECT }} - PYTHONPATH = {{VERPY_DIR}}{% if CSET_DIR is defined %}:{{CSET_DIR}}{% endif %} + PYTHONPATH = {{VERPY_DIR}}{% if CSET_DIR is defined %}:{{CSET_DIR}}/src{% endif %} {% if RUN_METPLUS_GRID_STAT|default(False) or RUN_METPLUS_POINT_STAT|default(False) %} [[METPLUS]]