From 25a959e1f8d6489c1cc73588ce93bb4f888fa47f Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Tue, 14 May 2024 14:57:31 +0000 Subject: [PATCH 01/89] Rename exglobal_stage script ahead of rewrite - Change the extension of the exglobal_stage ex-script from "sh" to "py". Refs #2475 --- scripts/{exglobal_stage_ic.sh => exglobal_stage_ic.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/{exglobal_stage_ic.sh => exglobal_stage_ic.py} (100%) diff --git a/scripts/exglobal_stage_ic.sh b/scripts/exglobal_stage_ic.py similarity index 100% rename from scripts/exglobal_stage_ic.sh rename to scripts/exglobal_stage_ic.py From 9bf02ee7c3c1644fbf8a5f87f2658c112773e4f1 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Tue, 14 May 2024 14:58:35 +0000 Subject: [PATCH 02/89] Update for pythonization of stage script - Update script extension for ex-script from "sh" to "py". - Pull COM* variable declares up from ex-script. Refs #2475 --- jobs/JGLOBAL_STAGE_IC | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/jobs/JGLOBAL_STAGE_IC b/jobs/JGLOBAL_STAGE_IC index c460e91c9e..4837eaae41 100755 --- a/jobs/JGLOBAL_STAGE_IC +++ b/jobs/JGLOBAL_STAGE_IC @@ -9,8 +9,31 @@ rCDUMP=${CDUMP} [[ ${CDUMP} = "gfs" ]] && export rCDUMP="gdas" export rCDUMP +# Locally scoped variables and functions +# shellcheck disable=SC2153 +GDATE=$(date --utc -d "${PDY} ${cyc} - ${assim_freq} hours" +%Y%m%d%H) +gPDY="${GDATE:0:8}" +gcyc="${GDATE:8:2}" + +# Declare COMs +if [[ ${EXP_WARM_START:-".false."} = ".true." ]]; then + RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL + RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl COM_MED_RESTART_PREV:COM_MED_RESTART_TMPL +else + YMD=${PDY} HH=${cyc} declare_from_tmpl COM_ATMOS_INPUT +fi +if [[ "${DO_OCN:-}" = "YES" ]]; then + RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl COM_OCEAN_RESTART_PREV:COM_OCEAN_RESTART_TMPL +fi +if [[ "${DO_ICE:-}" = "YES" ]]; then + RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl COM_ICE_RESTART_PREV:COM_ICE_RESTART_TMPL +fi +if [[ "${DO_WAVE:-}" = "YES" ]]; then + YMD=${PDY} HH=${cyc} declare_from_tmpl COM_WAVE_RESTART +fi + # Execute the Script -"${SCRgfs}/exglobal_stage_ic.sh" +"${SCRgfs}/exglobal_stage_ic.py" ########################################## # Remove the Temporary working directory From 8d2203d9fb9e4df89cad5f697efede61d93c3272 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Tue, 14 May 2024 15:00:39 +0000 Subject: [PATCH 03/89] Add PYTHONPATH to stage_ic.sh Refs #2475 --- jobs/rocoto/stage_ic.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/jobs/rocoto/stage_ic.sh b/jobs/rocoto/stage_ic.sh index 5e7b3395d2..5b79bb9e7d 100755 --- a/jobs/rocoto/stage_ic.sh +++ b/jobs/rocoto/stage_ic.sh @@ -7,6 +7,11 @@ source "${HOMEgfs}/ush/preamble.sh" status=$? [[ "${status}" -ne 0 ]] && exit "${status}" +############################################################### +# setup python path for workflow utilities and tasks +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${HOMEgfs}/ush/python" +export PYTHONPATH + export job="stage_ic" export jobid="${job}.$$" From ac9fe808912da6072a8929036a54ba2bed1d374c Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Tue, 14 May 2024 15:02:42 +0000 Subject: [PATCH 04/89] Remove IC staging from setup Remove the functions and calls to set up symlinks to ICs in ROTDIR. Refs #2475 --- workflow/setup_expt.py | 228 +---------------------------------------- 1 file changed, 1 insertion(+), 227 deletions(-) diff --git a/workflow/setup_expt.py b/workflow/setup_expt.py index 9602b66b60..224f182343 100755 --- a/workflow/setup_expt.py +++ b/workflow/setup_expt.py @@ -29,231 +29,6 @@ def makedirs_if_missing(dirname): os.makedirs(dirname) -def fill_ROTDIR(host, inputs): - """ - Method to populate the ROTDIR for supported modes. - INPUTS: - host: host object from class Host - inputs: user inputs to setup_expt.py - """ - - fill_modes = { - 'cycled': fill_ROTDIR_cycled, - 'forecast-only': fill_ROTDIR_forecasts - } - - try: - fill_modes[inputs.mode](host, inputs) - except KeyError: - raise NotImplementedError(f'{inputs.mode} is not a supported mode.\n' + - 'Currently supported modes are:\n' + - f'{" | ".join(fill_modes.keys())}') - - return - - -def fill_ROTDIR_cycled(host, inputs): - """ - Implementation of 'fill_ROTDIR' for cycled mode - """ - - rotdir = os.path.join(inputs.comroot, inputs.pslot) - - do_ocean = do_ice = do_med = False - - if 'S2S' in inputs.app: - do_ocean = do_ice = do_med = True - - if inputs.icsdir is None: - warnings.warn("User did not provide '--icsdir' to stage initial conditions") - return - - rdatestr = datetime_to_YMDH(inputs.idate - to_timedelta('T06H')) - idatestr = datetime_to_YMDH(inputs.idate) - - # Test if we are using the new COM structure or the old flat one for ICs - if inputs.start in ['warm']: - pathstr = os.path.join(inputs.icsdir, f'{inputs.cdump}.{rdatestr[:8]}', - rdatestr[8:], 'model_data', 'atmos') - else: - pathstr = os.path.join(inputs.icsdir, f'{inputs.cdump}.{idatestr[:8]}', - idatestr[8:], 'model_data', 'atmos') - - if os.path.isdir(pathstr): - flat_structure = False - else: - flat_structure = True - - # Destination always uses the new COM structure - # These should match the templates defined in config.com - if inputs.start in ['warm']: - dst_atm_dir = os.path.join('model_data', 'atmos', 'restart') - dst_med_dir = os.path.join('model_data', 'med', 'restart') - else: - dst_atm_dir = os.path.join('model_data', 'atmos', 'input') - dst_med_dir = '' # no mediator files for a "cold start" - do_med = False - dst_ocn_rst_dir = os.path.join('model_data', 'ocean', 'restart') - dst_ocn_anl_dir = os.path.join('analysis', 'ocean') - dst_ice_rst_dir = os.path.join('model_data', 'ice', 'restart') - dst_atm_anl_dir = os.path.join('analysis', 'atmos') - - if flat_structure: - # ICs are in the old flat COM structure - if inputs.start in ['warm']: # This is warm start experiment - src_atm_dir = os.path.join('atmos', 'RESTART') - src_med_dir = os.path.join('med', 'RESTART') - elif inputs.start in ['cold']: # This is a cold start experiment - src_atm_dir = os.path.join('atmos', 'INPUT') - src_med_dir = '' # no mediator files for a "cold start" - do_med = False - # ocean and ice have the same filenames for warm and cold - src_ocn_rst_dir = os.path.join('ocean', 'RESTART') - src_ocn_anl_dir = 'ocean' - src_ice_rst_dir = os.path.join('ice', 'RESTART') - src_atm_anl_dir = 'atmos' - else: - src_atm_dir = dst_atm_dir - src_med_dir = dst_med_dir - src_ocn_rst_dir = dst_ocn_rst_dir - src_ocn_anl_dir = dst_ocn_anl_dir - src_ice_rst_dir = dst_ice_rst_dir - src_atm_anl_dir = dst_atm_anl_dir - - def link_files_from_src_to_dst(src_dir, dst_dir): - files = os.listdir(src_dir) - for fname in files: - os.symlink(os.path.join(src_dir, fname), - os.path.join(dst_dir, fname)) - return - - # Link ensemble member initial conditions - if inputs.nens > 0: - previous_cycle_dir = f'enkf{inputs.cdump}.{rdatestr[:8]}/{rdatestr[8:]}' - current_cycle_dir = f'enkf{inputs.cdump}.{idatestr[:8]}/{idatestr[8:]}' - - for ii in range(1, inputs.nens + 1): - memdir = f'mem{ii:03d}' - # Link atmospheric files - if inputs.start in ['warm']: - dst_dir = os.path.join(rotdir, previous_cycle_dir, memdir, dst_atm_dir) - src_dir = os.path.join(inputs.icsdir, previous_cycle_dir, memdir, src_atm_dir) - elif inputs.start in ['cold']: - dst_dir = os.path.join(rotdir, current_cycle_dir, memdir, dst_atm_dir) - src_dir = os.path.join(inputs.icsdir, current_cycle_dir, memdir, src_atm_dir) - makedirs_if_missing(dst_dir) - link_files_from_src_to_dst(src_dir, dst_dir) - - # Link ocean files - if do_ocean: - dst_dir = os.path.join(rotdir, previous_cycle_dir, memdir, dst_ocn_rst_dir) - src_dir = os.path.join(inputs.icsdir, previous_cycle_dir, memdir, src_ocn_rst_dir) - makedirs_if_missing(dst_dir) - link_files_from_src_to_dst(src_dir, dst_dir) - - # First 1/2 cycle needs a MOM6 increment - incfile = f'enkf{inputs.cdump}.t{idatestr[8:]}z.ocninc.nc' - src_file = os.path.join(inputs.icsdir, current_cycle_dir, memdir, src_ocn_anl_dir, incfile) - dst_file = os.path.join(rotdir, current_cycle_dir, memdir, dst_ocn_anl_dir, incfile) - makedirs_if_missing(os.path.join(rotdir, current_cycle_dir, memdir, dst_ocn_anl_dir)) - os.symlink(src_file, dst_file) - - # Link ice files - if do_ice: - dst_dir = os.path.join(rotdir, previous_cycle_dir, memdir, dst_ice_rst_dir) - src_dir = os.path.join(inputs.icsdir, previous_cycle_dir, memdir, src_ice_rst_dir) - makedirs_if_missing(dst_dir) - link_files_from_src_to_dst(src_dir, dst_dir) - - # Link mediator files - if do_med: - dst_dir = os.path.join(rotdir, previous_cycle_dir, memdir, dst_med_dir) - src_dir = os.path.join(inputs.icsdir, previous_cycle_dir, memdir, src_med_dir) - makedirs_if_missing(dst_dir) - link_files_from_src_to_dst(src_dir, dst_dir) - - # Link deterministic initial conditions - previous_cycle_dir = f'{inputs.cdump}.{rdatestr[:8]}/{rdatestr[8:]}' - current_cycle_dir = f'{inputs.cdump}.{idatestr[:8]}/{idatestr[8:]}' - - # Link atmospheric files - if inputs.start in ['warm']: - dst_dir = os.path.join(rotdir, previous_cycle_dir, dst_atm_dir) - src_dir = os.path.join(inputs.icsdir, previous_cycle_dir, src_atm_dir) - elif inputs.start in ['cold']: - dst_dir = os.path.join(rotdir, current_cycle_dir, dst_atm_dir) - src_dir = os.path.join(inputs.icsdir, current_cycle_dir, src_atm_dir) - - makedirs_if_missing(dst_dir) - link_files_from_src_to_dst(src_dir, dst_dir) - - # Link ocean files - if do_ocean: - dst_dir = os.path.join(rotdir, previous_cycle_dir, dst_ocn_rst_dir) - src_dir = os.path.join(inputs.icsdir, previous_cycle_dir, src_ocn_rst_dir) - makedirs_if_missing(dst_dir) - link_files_from_src_to_dst(src_dir, dst_dir) - - # First 1/2 cycle needs a MOM6 increment - incfile = f'{inputs.cdump}.t{idatestr[8:]}z.ocninc.nc' - src_file = os.path.join(inputs.icsdir, current_cycle_dir, src_ocn_anl_dir, incfile) - dst_file = os.path.join(rotdir, current_cycle_dir, dst_ocn_anl_dir, incfile) - makedirs_if_missing(os.path.join(rotdir, current_cycle_dir, dst_ocn_anl_dir)) - os.symlink(src_file, dst_file) - - # Link ice files - if do_ice: - dst_dir = os.path.join(rotdir, previous_cycle_dir, dst_ice_rst_dir) - src_dir = os.path.join(inputs.icsdir, previous_cycle_dir, src_ice_rst_dir) - makedirs_if_missing(dst_dir) - link_files_from_src_to_dst(src_dir, dst_dir) - - # Link mediator files - if do_med: - dst_dir = os.path.join(rotdir, previous_cycle_dir, dst_med_dir) - src_dir = os.path.join(inputs.icsdir, previous_cycle_dir, src_med_dir) - makedirs_if_missing(dst_dir) - link_files_from_src_to_dst(src_dir, dst_dir) - - # Link bias correction and radiance diagnostics files - src_dir = os.path.join(inputs.icsdir, current_cycle_dir, src_atm_anl_dir) - dst_dir = os.path.join(rotdir, current_cycle_dir, dst_atm_anl_dir) - makedirs_if_missing(dst_dir) - for ftype in ['abias', 'abias_pc', 'abias_air', 'radstat']: - fname = f'{inputs.cdump}.t{idatestr[8:]}z.{ftype}' - src_file = os.path.join(src_dir, fname) - if os.path.exists(src_file): - os.symlink(src_file, os.path.join(dst_dir, fname)) - # First 1/2 cycle also needs a atmos increment if doing warm start - if inputs.start in ['warm']: - for ftype in ['atmi003.nc', 'atminc.nc', 'atmi009.nc']: - fname = f'{inputs.cdump}.t{idatestr[8:]}z.{ftype}' - src_file = os.path.join(src_dir, fname) - if os.path.exists(src_file): - os.symlink(src_file, os.path.join(dst_dir, fname)) - if inputs.nens > 0: - current_cycle_dir = f'enkf{inputs.cdump}.{idatestr[:8]}/{idatestr[8:]}' - for ii in range(1, inputs.nens + 1): - memdir = f'mem{ii:03d}' - src_dir = os.path.join(inputs.icsdir, current_cycle_dir, memdir, src_atm_anl_dir) - dst_dir = os.path.join(rotdir, current_cycle_dir, memdir, dst_atm_anl_dir) - makedirs_if_missing(dst_dir) - for ftype in ['ratmi003.nc', 'ratminc.nc', 'ratmi009.nc']: - fname = f'enkf{inputs.cdump}.t{idatestr[8:]}z.{ftype}' - src_file = os.path.join(src_dir, fname) - if os.path.exists(src_file): - os.symlink(src_file, os.path.join(dst_dir, fname)) - - return - - -def fill_ROTDIR_forecasts(host, inputs): - """ - Implementation of 'fill_ROTDIR' for forecast-only mode - """ - print('forecast-only mode treats ICs differently and cannot be staged here') - - def fill_EXPDIR(inputs): """ Method to copy config files from workflow to experiment directory @@ -464,7 +239,7 @@ def _gefs_args(parser): description = """ Setup files and directories to start a GFS parallel.\n Create EXPDIR, copy config files.\n - Create ROTDIR experiment directory structure, + Create ROTDIR experiment directory, """ parser = ArgumentParser(description=description, @@ -576,7 +351,6 @@ def main(*argv): if create_rotdir: makedirs_if_missing(rotdir) - fill_ROTDIR(host, user_inputs) if create_expdir: makedirs_if_missing(expdir) From 04d75e375932ecb9e6641d88961e6a97dd49535f Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Tue, 14 May 2024 15:04:57 +0000 Subject: [PATCH 05/89] Add initial changes to pythonize and yamlize staging - Add initial new yaml files for staging information - Add new stage.py to python tasks. - Add first draft pythonization of stage ex-script. Much more work is still to be done. Refs #2475 --- parm/stage/da.yaml.j2 | 6 + parm/stage/fv3.yaml.j2 | 36 ++++ parm/stage/fv3_nest.yaml.j2 | 41 +++++ parm/stage/ice.yaml.j2 | 5 + parm/stage/master_forecast_only.yaml.j2 | 39 ++++ parm/stage/mediator.yaml.j2 | 7 + parm/stage/ocean.yaml.j2 | 20 ++ parm/stage/wave.yaml.j2 | 9 + scripts/exglobal_stage_ic.py | 234 ++++++------------------ ush/python/pygfs/task/stage.py | 137 ++++++++++++++ 10 files changed, 352 insertions(+), 182 deletions(-) create mode 100644 parm/stage/da.yaml.j2 create mode 100644 parm/stage/fv3.yaml.j2 create mode 100644 parm/stage/fv3_nest.yaml.j2 create mode 100644 parm/stage/ice.yaml.j2 create mode 100644 parm/stage/master_forecast_only.yaml.j2 create mode 100644 parm/stage/mediator.yaml.j2 create mode 100644 parm/stage/ocean.yaml.j2 create mode 100644 parm/stage/wave.yaml.j2 create mode 100644 ush/python/pygfs/task/stage.py diff --git a/parm/stage/da.yaml.j2 b/parm/stage/da.yaml.j2 new file mode 100644 index 0000000000..b5aede5a7a --- /dev/null +++ b/parm/stage/da.yaml.j2 @@ -0,0 +1,6 @@ +#for ftype in ['abias', 'abias_pc', 'abias_air', 'radstat']: +da: + name: "DA" + source: "{{ BASE_CPLIC }}/{{ CPL_ICEIC }}/{{ cycle_YMD }}{{ cycle_HH }}/{{ MEMDIR }}/ice" + required: + - "{{ COM_ICE_RESTART_PREV | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.cice_model.res.nc" diff --git a/parm/stage/fv3.yaml.j2 b/parm/stage/fv3.yaml.j2 new file mode 100644 index 0000000000..df2f59960a --- /dev/null +++ b/parm/stage/fv3.yaml.j2 @@ -0,0 +1,36 @@ +# # First 1/2 cycle also needs a atmos increment if doing warm start +# if inputs.start in ['warm']: +# for ftype in ['atmi003.nc', 'atminc.nc', 'atmi009.nc']: +# if inputs.nens > 0: +#for ftype in ['ratmi003.nc', 'ratminc.nc', 'ratmi009.nc'] +fv3: + name: "FV3" + source: "{{ BASE_CPLIC }}/{{ CPL_ATMIC }}/{{ cycle_YMD }}{{ cycle_HH }}/{{ MEMDIR }}/atmos" + required: + #--------------- + # WARM START + #--------------- + {% if EXP_WARM_START %} + {% set head = "{{ COM_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000." %} + {% for ftype in coupler.res fv_core.res.nc %} + - "{{ head }}{{ ftype }}" + {% endfor %} + {% for ftype in ca_data fv_core.res fv_srf_wnd.res fv_tracer.res phy_data sfc_data %} + {% for ntile in $(seq 1 ntiles) %} + - "{{ head }}{{ ftype }}.tile{{ ntile }}.nc" + {% endfor %} # ntile + {% endfor %} # ftype + #--------------- # end warm start + {% else %} + #--------------- + # COLD START + #--------------- + {% set head = "{{ COM_ATMOS_INPUT | relpath(ROTDIR) }}" %} + - "{{ head }}/gfs_ctrl.nc" + {% for ftype in gfs_data sfc_data %} + {% for ntile in $(seq 1 ntiles) %} + - "{{ head }}/{{ ftype }}.tile{{ ntile }}.nc" + {% endfor %} # ntile + {% endfor %} # ftype + #--------------- # end cold start + {% endif %} diff --git a/parm/stage/fv3_nest.yaml.j2 b/parm/stage/fv3_nest.yaml.j2 new file mode 100644 index 0000000000..09c7c5afe4 --- /dev/null +++ b/parm/stage/fv3_nest.yaml.j2 @@ -0,0 +1,41 @@ +#WARM +# for ftype in ca_data fv_core.res fv_srf_wnd.res fv_tracer.res phy_data sfc_data; do +# src="${BASE_CPLIC}/${CPL_ATMIC:-}/${PDY}${cyc}/${MEMDIR}/atmos/${PDY}.${cyc}0000.${ftype}.tile${tt}.nc" +# if (( tt > 6 )) ; then +# tgt="${COM_ATMOS_RESTART_PREV}/${PDY}.${cyc}0000.${ftype}.nest0$((tt-5)).tile${tt}.nc" +# fi +#COLD +# for ftype in gfs_data sfc_data; do +# for ((tt = 1; tt <= ntiles; tt++)); do +# src="${BASE_CPLIC}/${CPL_ATMIC:-}/${PDY}${cyc}/${MEMDIR}/atmos/${ftype}.tile${tt}.nc" +# tgt="${COM_ATMOS_INPUT}/${ftype}.tile${tt}.nc" +# if (( ntiles > 6 )); then +# ${NLN} "${COM_ATMOS_INPUT}/${ftype}.tile7.nc" "${COM_ATMOS_INPUT}/${ftype}.nest02.tile7.nc" +fv3: + name: "FV3" + source: "{{ BASE_CPLIC }}/{{ CPL_ATMIC }}/{{ cycle_YMD }}{{ cycle_HH }}/{{ MEMDIR }}/atmos" + {% set ntile = 7 %} + required: + #--------------- + # WARM START + #--------------- + {% if EXP_WARM_START %} + {% set head = "{{ COM_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000." %} + {% set ntile = 7 %} + {% for ftype in ca_data fv_core.res fv_srf_wnd.res fv_tracer.res phy_data sfc_data %} + #TODO: figure out how to copy one name to the other in here + - "{{ head }}{{ ftype }}.nest0{{ ntile-5 }}.tile{{ ntile }}.nc" + {% endfor %} # ftype + #--------------- # end warm start + {% else %} + #--------------- + # COLD START + #--------------- + {% set head = "{{ COM_ATMOS_INPUT | relpath(ROTDIR) }}" %} + - "{{ head }}/gfs_ctrl.nc" + {% for ftype in gfs_data sfc_data %} + #TODO: figure out how to copy one name to the other in here + - "{{ head }}/{{ ftype }}.tile{{ ntile }}.nc" + {% endfor %} # ftype + #--------------- # end cold start + {% endif %} diff --git a/parm/stage/ice.yaml.j2 b/parm/stage/ice.yaml.j2 new file mode 100644 index 0000000000..3c645a75a1 --- /dev/null +++ b/parm/stage/ice.yaml.j2 @@ -0,0 +1,5 @@ +ice: + name: "ICE" + source: "{{ BASE_CPLIC }}/{{ CPL_ICEIC }}/{{ cycle_YMD }}{{ cycle_HH }}/{{ MEMDIR }}/ice" + required: + - "{{ COM_ICE_RESTART_PREV | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.cice_model.res.nc" diff --git a/parm/stage/master_forecast_only.yaml.j2 b/parm/stage/master_forecast_only.yaml.j2 new file mode 100644 index 0000000000..59d8488887 --- /dev/null +++ b/parm/stage/master_forecast_only.yaml.j2 @@ -0,0 +1,39 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMD = current_cycle | to_YMD %} +{% set cycle_YMDH = current_cycle | to_YMDH %} +datasets: +{% if EXP_WARM_START %} +{% filter indent(width=4) %} +{% include "fv3.yaml.j2" %} +{% endfilter %} +{% if DO_NEST %} +{% filter indent(width=4) %} +{% include "fv3_nest.yaml.j2" %} +{% endfilter %} +{% endif %} +{% endif %} + +{% if DO_WAVE %} +{% if EXP_WARM_START %} +{% filter indent(width=4) %} +{% include "wave.yaml.j2" %} +{% endfilter %} +{% endif %} +{% endif %} + +{% if DO_OCEAN %} +{% if EXP_WARM_START %} +{% filter indent(width=4) %} +{% include "ocean.yaml.j2" %} +{% include "mediator.yaml.j2" %} +{% endfilter %} +{% endif %} +{% endif %} + +{% if DO_ICE %} +{% if EXP_WARM_START %} +{% filter indent(width=4) %} +{% include "ice.yaml.j2" %} +{% endfilter %} +{% endif %} +{% endif %} diff --git a/parm/stage/mediator.yaml.j2 b/parm/stage/mediator.yaml.j2 new file mode 100644 index 0000000000..5c9777dfb8 --- /dev/null +++ b/parm/stage/mediator.yaml.j2 @@ -0,0 +1,7 @@ +#src="${BASE_CPLIC}/${CPL_MEDIC:-}/${PDY}${cyc}/${MEMDIR}/med/${PDY}.${cyc}0000.ufs.cpld.cpl.r.nc" +# tgt="${COM_MED_RESTART_PREV}/${PDY}.${cyc}0000.ufs.cpld.cpl.r.nc" +mediator: + name: "MEDIATOR" + source: "{{ BASE_CPLIC }}/{{ CPL_MEDIC }}/{{ cycle_YMD }}{{ cycle_HH }}/{{ MEMDIR }}/med" + required: + - "{{ COM_MED_RESTART_PREV | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.ufs.cpld.cpl.r.nc" diff --git a/parm/stage/ocean.yaml.j2 b/parm/stage/ocean.yaml.j2 new file mode 100644 index 0000000000..788c9c3dfd --- /dev/null +++ b/parm/stage/ocean.yaml.j2 @@ -0,0 +1,20 @@ +ocean: + name: "OCEAN" + source: "{{ BASE_CPLIC }}/{{ CPL_OCNIC }}/{{ cycle_YMD }}{{ cycle_HH }}/{{ MEMDIR }}/ocean" + required: + # Ocean initial conditions + #------------------------- + - "{{ COM_OCEAN_RESTART_PREV | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.MOM.res.nc" + # Resolution based initial conditions + #------------------------------------ + #NOTE: {% if OCNRES == 500 or OCNRES == 100 %} # Nothing more to do for these resolutions + {% if OCNRES == 025 %} + {% for nn in $(seq 1 3) %} + - "{{ COM_OCEAN_RESTART_PREV | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.MOM.res_{{ nn }}.nc" + {% endfor %} + {% endif %} + optional: + # Ocean Perturbation Files + #------------------------- + {% if MEMDIR > 0 and USE_OCN_PERTURB_FILES %} + - "{{ COM_OCEAN_RESTART_PREV | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.mom6_increment.nc" diff --git a/parm/stage/wave.yaml.j2 b/parm/stage/wave.yaml.j2 new file mode 100644 index 0000000000..2ae173002a --- /dev/null +++ b/parm/stage/wave.yaml.j2 @@ -0,0 +1,9 @@ +#TODO: these should be placed in $RUN.$gPDY/$gcyc) +wave: + #TODO: check if BASE_CPLIC/CPL_WAVIC exists? + name: "WAVE" + source: "{{ BASE_CPLIC }}/{{ CPL_WAVIC }}/{{ cycle_YMD }}{{ cycle_HH }}/{{ MEMDIR }}/wave" + required: + {% for grdID in waveGRD %} + - "{{ COM_WAVE_RESTART | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.restart.{{ grdID }}" + {% endfor %} diff --git a/scripts/exglobal_stage_ic.py b/scripts/exglobal_stage_ic.py index d941fa10b4..4424948185 100755 --- a/scripts/exglobal_stage_ic.py +++ b/scripts/exglobal_stage_ic.py @@ -1,182 +1,52 @@ -#!/usr/bin/env bash - -source "${USHgfs}/preamble.sh" - -# Locally scoped variables and functions -# shellcheck disable=SC2153 -GDATE=$(date --utc -d "${PDY} ${cyc} - ${assim_freq} hours" +%Y%m%d%H) -gPDY="${GDATE:0:8}" -gcyc="${GDATE:8:2}" - -MEMDIR_ARRAY=() -if [[ "${RUN:-}" = "gefs" ]]; then - # Populate the member_dirs array based on the value of NMEM_ENS - for ((ii = 0; ii <= "${NMEM_ENS:-0}"; ii++)); do - MEMDIR_ARRAY+=("mem$(printf "%03d" "${ii}")") - done -else - MEMDIR_ARRAY+=("") -fi - -# Initialize return code -err=0 - -error_message() { - echo "FATAL ERROR: Unable to copy ${1} to ${2} (Error code ${3})" -} - -############################################################### -for MEMDIR in "${MEMDIR_ARRAY[@]}"; do - - # Stage atmosphere initial conditions to ROTDIR - if [[ ${EXP_WARM_START:-".false."} = ".true." ]]; then - # Stage the FV3 restarts to ROTDIR (warm start) - RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL - [[ ! -d "${COM_ATMOS_RESTART_PREV}" ]] && mkdir -p "${COM_ATMOS_RESTART_PREV}" - for ftype in coupler.res fv_core.res.nc; do - src="${BASE_CPLIC}/${CPL_ATMIC:-}/${PDY}${cyc}/${MEMDIR}/atmos/${PDY}.${cyc}0000.${ftype}" - tgt="${COM_ATMOS_RESTART_PREV}/${PDY}.${cyc}0000.${ftype}" - ${NCP} "${src}" "${tgt}" - rc=$? - ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}" - err=$((err + rc)) - done - for ftype in ca_data fv_core.res fv_srf_wnd.res fv_tracer.res phy_data sfc_data; do - for ((tt = 1; tt <= ntiles; tt++)); do - src="${BASE_CPLIC}/${CPL_ATMIC:-}/${PDY}${cyc}/${MEMDIR}/atmos/${PDY}.${cyc}0000.${ftype}.tile${tt}.nc" - if (( tt > 6 )) ; then - tgt="${COM_ATMOS_RESTART_PREV}/${PDY}.${cyc}0000.${ftype}.nest0$((tt-5)).tile${tt}.nc" - else - tgt="${COM_ATMOS_RESTART_PREV}/${PDY}.${cyc}0000.${ftype}.tile${tt}.nc" - fi - ${NCP} "${src}" "${tgt}" - rc=$? - ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}" - err=$((err + rc)) - done - done - else - # Stage the FV3 cold-start initial conditions to ROTDIR - YMD=${PDY} HH=${cyc} declare_from_tmpl COM_ATMOS_INPUT - [[ ! -d "${COM_ATMOS_INPUT}" ]] && mkdir -p "${COM_ATMOS_INPUT}" - src="${BASE_CPLIC}/${CPL_ATMIC:-}/${PDY}${cyc}/${MEMDIR}/atmos/gfs_ctrl.nc" - tgt="${COM_ATMOS_INPUT}/gfs_ctrl.nc" - ${NCP} "${src}" "${tgt}" - rc=$? - ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}" - err=$((err + rc)) - for ftype in gfs_data sfc_data; do - for ((tt = 1; tt <= ntiles; tt++)); do - src="${BASE_CPLIC}/${CPL_ATMIC:-}/${PDY}${cyc}/${MEMDIR}/atmos/${ftype}.tile${tt}.nc" - tgt="${COM_ATMOS_INPUT}/${ftype}.tile${tt}.nc" - ${NCP} "${src}" "${tgt}" - rc=$? - ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}" - err=$((err + rc)) - done - if (( ntiles > 6 )); then - ${NLN} "${COM_ATMOS_INPUT}/${ftype}.tile7.nc" "${COM_ATMOS_INPUT}/${ftype}.nest02.tile7.nc" - fi - done - fi - - # Stage ocean initial conditions to ROTDIR (warm start) - if [[ "${DO_OCN:-}" = "YES" ]]; then - RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl COM_OCEAN_RESTART_PREV:COM_OCEAN_RESTART_TMPL - [[ ! -d "${COM_OCEAN_RESTART_PREV}" ]] && mkdir -p "${COM_OCEAN_RESTART_PREV}" - src="${BASE_CPLIC}/${CPL_OCNIC:-}/${PDY}${cyc}/${MEMDIR}/ocean/${PDY}.${cyc}0000.MOM.res.nc" - tgt="${COM_OCEAN_RESTART_PREV}/${PDY}.${cyc}0000.MOM.res.nc" - ${NCP} "${src}" "${tgt}" - rc=$? - ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}" - err=$((err + rc)) - case "${OCNRES}" in - "500" | "100") - # Nothing more to do for these resolutions - ;; - "025" ) - for nn in $(seq 1 3); do - src="${BASE_CPLIC}/${CPL_OCNIC:-}/${PDY}${cyc}/${MEMDIR}/ocean/${PDY}.${cyc}0000.MOM.res_${nn}.nc" - tgt="${COM_OCEAN_RESTART_PREV}/${PDY}.${cyc}0000.MOM.res_${nn}.nc" - ${NCP} "${src}" "${tgt}" - rc=$? - ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}" - err=$((err + rc)) - done - ;; - *) - echo "FATAL ERROR: Unsupported ocean resolution ${OCNRES}" - rc=1 - err=$((err + rc)) - ;; - esac - - # Ocean Perturbation Files - # Extra zero on MEMDIR ensure we have a number even if the string is empty - if (( 0${MEMDIR:3} > 0 )) && [[ "${USE_OCN_PERTURB_FILES:-false}" == "true" ]]; then - src="${BASE_CPLIC}/${CPL_OCNIC:-}/${PDY}${cyc}/${MEMDIR}/ocean/${PDY}.${cyc}0000.mom6_increment.nc" - tgt="${COM_OCEAN_RESTART_PREV}/${PDY}.${cyc}0000.mom6_increment.nc" - ${NCP} "${src}" "${tgt}" - rc=${?} - ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}" - err=$((err + rc)) - fi - - # TODO: Do mediator restarts exists in a ATMW configuration? - # TODO: No mediator is presumably involved in an ATMA configuration - if [[ ${EXP_WARM_START:-".false."} = ".true." ]]; then - # Stage the mediator restarts to ROTDIR (warm start/restart the coupled model) - RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl COM_MED_RESTART_PREV:COM_MED_RESTART_TMPL - [[ ! -d "${COM_MED_RESTART_PREV}" ]] && mkdir -p "${COM_MED_RESTART_PREV}" - src="${BASE_CPLIC}/${CPL_MEDIC:-}/${PDY}${cyc}/${MEMDIR}/med/${PDY}.${cyc}0000.ufs.cpld.cpl.r.nc" - tgt="${COM_MED_RESTART_PREV}/${PDY}.${cyc}0000.ufs.cpld.cpl.r.nc" - if [[ -f "${src}" ]]; then - ${NCP} "${src}" "${tgt}" - rc=$? - ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}" - err=$((err + rc)) - else - echo "WARNING: No mediator restarts available with warm_start=${EXP_WARM_START}" - fi - fi - - fi - - # Stage ice initial conditions to ROTDIR (warm start) - if [[ "${DO_ICE:-}" = "YES" ]]; then - RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl COM_ICE_RESTART_PREV:COM_ICE_RESTART_TMPL - [[ ! -d "${COM_ICE_RESTART_PREV}" ]] && mkdir -p "${COM_ICE_RESTART_PREV}" - src="${BASE_CPLIC}/${CPL_ICEIC:-}/${PDY}${cyc}/${MEMDIR}/ice/${PDY}.${cyc}0000.cice_model.res.nc" - tgt="${COM_ICE_RESTART_PREV}/${PDY}.${cyc}0000.cice_model.res.nc" - ${NCP} "${src}" "${tgt}" - rc=$? - ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}" - err=$((err + rc)) - fi - - # Stage the WW3 initial conditions to ROTDIR (warm start; TODO: these should be placed in $RUN.$gPDY/$gcyc) - if [[ "${DO_WAVE:-}" = "YES" ]]; then - YMD=${PDY} HH=${cyc} declare_from_tmpl COM_WAVE_RESTART - [[ ! -d "${COM_WAVE_RESTART}" ]] && mkdir -p "${COM_WAVE_RESTART}" - for grdID in ${waveGRD}; do # TODO: check if this is a bash array; if so adjust - src="${BASE_CPLIC}/${CPL_WAVIC:-}/${PDY}${cyc}/${MEMDIR}/wave/${PDY}.${cyc}0000.restart.${grdID}" - tgt="${COM_WAVE_RESTART}/${PDY}.${cyc}0000.restart.${grdID}" - ${NCP} "${src}" "${tgt}" - rc=$? - ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}" - err=$((err + rc)) - done - fi - -done # for MEMDIR in "${MEMDIR_ARRAY[@]}"; do - -############################################################### -# Check for errors and exit if any of the above failed -if [[ "${err}" -ne 0 ]]; then - echo "FATAL ERROR: Unable to copy ICs from ${BASE_CPLIC} to ${ROTDIR}; ABORT!" - exit "${err}" -fi - -############################################################## -# Exit cleanly -exit "${err}" +#!/usr/bin/env python3 + +import os + +from pygfs.task.stage import Stage +from wxflow import AttrDict, Logger, cast_strdict_as_dtypedict, chdir, logit + +# Initialize root logger +logger = Logger(level=os.environ.get("LOGGING_LEVEL", "DEBUG"), colored_log=True) + +@logit(logger) +def main(): + + config = cast_strdict_as_dtypedict(os.environ) + + # Instantiate the Stage object + stage = Stage(config) + + #Pull out all the configuration keys needed to run stage job + keys = ['RUN','MODE','EXP_WARM_START','CDUMP','rCDUMP', + 'ntiles', + 'BASE_CPLIC','waveGRD','OCNRES','USE_OCN_PERTURB_FILES', + 'CPL_ATMIC','CPL_ICEIC','CPL_MEDIC','CPL_OCNIC','CPL_WAVIC'] + + stage_dict = AttrDict() + for key in keys: + stage_dict[key] = stage.task_config[key] + + # Also import all COM* directory and template variables + for key in stage.task_config.keys(): + if key.startswith("COM"): + stage_dict[key] = stage.task_config[key] + + #TEST PRINT + #for key in stage_dict: + # print(f'{key} = {stage_dict[key]}') + + cwd = os.getcwd() + + os.chdir(config.ROTDIR) + + # Determine which ICs to stage + stage_set = stage.determine(stage_dict) + + # Stage ICs + # TODO - create and invoke copies + stage.execute(stage_set) + + os.chdir(cwd) + +if __name__ == '__main__': + main() diff --git a/ush/python/pygfs/task/stage.py b/ush/python/pygfs/task/stage.py new file mode 100644 index 0000000000..db26a3a160 --- /dev/null +++ b/ush/python/pygfs/task/stage.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 + +import glob +import os +import shutil +from datetime import timedelta +from logging import getLogger +from typing import Any, Dict, List + +from wxflow import (AttrDict, + FileHandler, + Hsi, + Htar, + Task, + cast_strdict_as_dtypedict, + chgrp, + get_gid, + logit, + mkdir_p, + parse_j2yaml, + rm_p, + strftime, + to_YMD, + to_YMDH, + Template, + TemplateConstants) + +logger = getLogger(__name__.split('.')[-1]) + +class Stage(Task): + """Task to stage initial conditions + """ + + @logit(logger, name="Stage") + def __init__(self, config: Dict[str, Any]) -> None: + """Constructor for the Stage task + The constructor is responsible for collecting necessary yamls based on + the runtime options and RUN. + + Parameters + ---------- + config : Dict[str, Any] + Incoming configuration for the task from the environment + + Returns + ------- + None + """ + super().__init__(config) + + rotdir = self.config.ROTDIR + os.sep + + # Find all absolute paths in the environment and get their relative paths from ${ROTDIR} + path_dict = self._gen_relative_paths(rotdir) + + self.task_config = AttrDict(**self.config, **self.runtime_config, **path_dict) + + @logit(logger) + def _gen_relative_paths(self, root_path: str) -> Dict: + """Generate a dict of paths in self.config relative to root_path + + Parameters + ---------- + root_path : str + Path to base all relative paths off of + + Return + ------ + rel_path_dict : Dict + Dictionary of paths relative to root_path. Members will be named + based on the dict names in self.config. For COM paths, the names will + follow COM_ --> _dir. For all other directories, the + names will follow --> _dir. + """ + + rel_path_dict = {} + for key, value in self.config.items(): + if isinstance(value, str): + if root_path in value: + rel_path = value.replace(root_path, "") + rel_key = (key[4:] if key.startswith("COM_") else key).lower() + "_dir" + rel_path_dict[rel_key] = rel_path + + return rel_path_dict + + @logit(logger) + def determine(self, stage_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str, Any]]): + """Determine which initial condition files need to be placed in ROTDIR. + + Parameters + ---------- + stage_dict : Dict[str, Any] + Task specific keys, e.g. runtime options (DO_AERO, DO_ICE, etc) + + Return + ------ + stage_set : Dict[str, Any] + Set of FileHandler instructions to copy files to the ROTDIR + """ + + stage_parm = os.path.join(stage_dict.PARMgfs, "stage") + + # Add the glob.glob function for capturing log filenames + # TODO remove this kludge once log filenames are explicit + stage_dict['glob'] = glob.glob + + # Add the os.path.exists function to the dict for yaml parsing + stage_dict['path_exists'] = os.path.exists + + if not os.path.isdir(stage_dict.ROTDIR): + raise FileNotFoundError(f"FATAL ERROR: The ROTDIR ({stage_dict.ROTDIR}) does not exist!") + + if stage_dict.RUN == "gfs" or stage_dict.RUN == "gdas": + if stage_dict.MODE == "cycled": + master_yaml = "master_cycled.yaml.j2" + elif stage_dict.MODE == "forecast-only": + master_yaml = "master_forecast_only.yaml.j2" + elif stage_dict.RUN == "gefs": + raise NotImplementedError("FATAL ERROR: Staging is not yet set up for GEFS runs") + elif stage_dict.RUN == "sfs": + raise NotImplementedError("FATAL ERROR: Staging is not yet set up for SFS runs") + else: + raise ValueError(f"FATAL ERROR: Staging is not enabled for {stage_dict.RUN} runs") + + parsed_sets = parse_j2yaml(os.path.join(stage_parm, master_yaml), stage_dict) + + stage_sets = [] + + for dataset in parsed_sets.datasets.values(): + + dataset["fileset"] = Stage._create_fileset(dataset) + + stage_sets.append(dataset) + + return stage_sets + +#TODO - create def for staging From f6548e3963e3cea84da03bfb7b35e38bd54b185f Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Tue, 14 May 2024 17:29:46 +0000 Subject: [PATCH 06/89] Remove warm start check for ingesting fv3.yaml.j2 Refs #2475 --- parm/stage/master_forecast_only.yaml.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/stage/master_forecast_only.yaml.j2 b/parm/stage/master_forecast_only.yaml.j2 index 59d8488887..febbe75dc2 100644 --- a/parm/stage/master_forecast_only.yaml.j2 +++ b/parm/stage/master_forecast_only.yaml.j2 @@ -2,16 +2,16 @@ {% set cycle_YMD = current_cycle | to_YMD %} {% set cycle_YMDH = current_cycle | to_YMDH %} datasets: -{% if EXP_WARM_START %} + {% filter indent(width=4) %} {% include "fv3.yaml.j2" %} {% endfilter %} + {% if DO_NEST %} {% filter indent(width=4) %} {% include "fv3_nest.yaml.j2" %} {% endfilter %} {% endif %} -{% endif %} {% if DO_WAVE %} {% if EXP_WARM_START %} From 11df1afe0262f1631841b96b3efb47b98cd90fee Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Wed, 22 May 2024 14:03:51 +0000 Subject: [PATCH 07/89] Split fv3 yaml into warm and cold Also fix to set target and remove source Refs #2475 --- parm/stage/fv3.yaml.j2 | 36 ------------------------------------ parm/stage/fv3_cold.yaml.j2 | 15 +++++++++++++++ parm/stage/fv3_warm.yaml.j2 | 19 +++++++++++++++++++ 3 files changed, 34 insertions(+), 36 deletions(-) delete mode 100644 parm/stage/fv3.yaml.j2 create mode 100644 parm/stage/fv3_cold.yaml.j2 create mode 100644 parm/stage/fv3_warm.yaml.j2 diff --git a/parm/stage/fv3.yaml.j2 b/parm/stage/fv3.yaml.j2 deleted file mode 100644 index df2f59960a..0000000000 --- a/parm/stage/fv3.yaml.j2 +++ /dev/null @@ -1,36 +0,0 @@ -# # First 1/2 cycle also needs a atmos increment if doing warm start -# if inputs.start in ['warm']: -# for ftype in ['atmi003.nc', 'atminc.nc', 'atmi009.nc']: -# if inputs.nens > 0: -#for ftype in ['ratmi003.nc', 'ratminc.nc', 'ratmi009.nc'] -fv3: - name: "FV3" - source: "{{ BASE_CPLIC }}/{{ CPL_ATMIC }}/{{ cycle_YMD }}{{ cycle_HH }}/{{ MEMDIR }}/atmos" - required: - #--------------- - # WARM START - #--------------- - {% if EXP_WARM_START %} - {% set head = "{{ COM_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000." %} - {% for ftype in coupler.res fv_core.res.nc %} - - "{{ head }}{{ ftype }}" - {% endfor %} - {% for ftype in ca_data fv_core.res fv_srf_wnd.res fv_tracer.res phy_data sfc_data %} - {% for ntile in $(seq 1 ntiles) %} - - "{{ head }}{{ ftype }}.tile{{ ntile }}.nc" - {% endfor %} # ntile - {% endfor %} # ftype - #--------------- # end warm start - {% else %} - #--------------- - # COLD START - #--------------- - {% set head = "{{ COM_ATMOS_INPUT | relpath(ROTDIR) }}" %} - - "{{ head }}/gfs_ctrl.nc" - {% for ftype in gfs_data sfc_data %} - {% for ntile in $(seq 1 ntiles) %} - - "{{ head }}/{{ ftype }}.tile{{ ntile }}.nc" - {% endfor %} # ntile - {% endfor %} # ftype - #--------------- # end cold start - {% endif %} diff --git a/parm/stage/fv3_cold.yaml.j2 b/parm/stage/fv3_cold.yaml.j2 new file mode 100644 index 0000000000..550b9bda06 --- /dev/null +++ b/parm/stage/fv3_cold.yaml.j2 @@ -0,0 +1,15 @@ +# # First 1/2 cycle also needs a atmos increment if doing warm start +# if inputs.start in ['warm']: +# for ftype in ['atmi003.nc', 'atminc.nc', 'atmi009.nc']: +# if inputs.nens > 0: +#for ftype in ['ratmi003.nc', 'ratminc.nc', 'ratmi009.nc'] +fv3_cold: + name: "FV3_COLD" + target: "{{ COM_ATMOS_INPUT }}" + required: + - "{{ BASE_CPLIC }}/{{ CPL_ATMIC }}/{{ cycle_YMD }}{{ cycle_HH }}/atmos/gfs_ctrl.nc" + {% for ftype in ["gfs_data", "sfc_data"] %} + {% for ntile in range(1, ntiles + 1) %} + - "{{ BASE_CPLIC }}/{{ CPL_ATMIC }}/{{ cycle_YMD }}{{ cycle_HH }}/atmos/{{ ftype }}.tile{{ ntile }}.nc" + {% endfor %} # ntile + {% endfor %} # ftype diff --git a/parm/stage/fv3_warm.yaml.j2 b/parm/stage/fv3_warm.yaml.j2 new file mode 100644 index 0000000000..e40aa41ba3 --- /dev/null +++ b/parm/stage/fv3_warm.yaml.j2 @@ -0,0 +1,19 @@ +# # First 1/2 cycle also needs a atmos increment if doing warm start +# if inputs.start in ['warm']: +# for ftype in ['atmi003.nc', 'atminc.nc', 'atmi009.nc']: +# if inputs.nens > 0: +#for ftype in ['ratmi003.nc', 'ratminc.nc', 'ratmi009.nc'] +fv3_warm: + name: "FV3_WARM" + target: "{{ COM_ATMOS_RESTART_PREV }}" + required: +#TODO: Set MEMDIR as empty if not set? + {% set head = "{{ BASE_CPLIC }}/{{ CPL_ATMIC }}/{{ cycle_YMD }}{{ cycle_HH }}/{{ MEMDIR }}/atmos/{{ cycle_YMD }}.{{ cycle_HH }}0000." %} + {% for ftype in ["coupler.res", "fv_core.res.nc"] %} + - "{{ head }}{{ ftype }}" + {% endfor %} + {% for ftype in ["ca_data", "fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data sfc_data"] %} + {% for ntile in range(1, ntiles) %} + - "{{ head }}{{ ftype }}.tile{{ ntile }}.nc" + {% endfor %} # ntile + {% endfor %} # ftype From 606c34a6b5d0c5b693fe28597f53353fdf97f1ac Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Wed, 22 May 2024 14:05:23 +0000 Subject: [PATCH 08/89] Fix other yamls to work Add target, remove source, and update file info Refs #2475 --- parm/stage/fv3_nest.yaml.j2 | 4 ++-- parm/stage/ice.yaml.j2 | 4 ++-- parm/stage/master_forecast_only.yaml.j2 | 12 +++++++++--- parm/stage/mediator.yaml.j2 | 6 ++---- parm/stage/ocean.yaml.j2 | 12 ++++++++---- parm/stage/wave.yaml.j2 | 4 ++-- 6 files changed, 25 insertions(+), 17 deletions(-) diff --git a/parm/stage/fv3_nest.yaml.j2 b/parm/stage/fv3_nest.yaml.j2 index 09c7c5afe4..fffad0c245 100644 --- a/parm/stage/fv3_nest.yaml.j2 +++ b/parm/stage/fv3_nest.yaml.j2 @@ -13,14 +13,14 @@ # ${NLN} "${COM_ATMOS_INPUT}/${ftype}.tile7.nc" "${COM_ATMOS_INPUT}/${ftype}.nest02.tile7.nc" fv3: name: "FV3" - source: "{{ BASE_CPLIC }}/{{ CPL_ATMIC }}/{{ cycle_YMD }}{{ cycle_HH }}/{{ MEMDIR }}/atmos" + target: "{{ COM_ATMOS_RESTART_PREV }}" {% set ntile = 7 %} required: #--------------- # WARM START #--------------- {% if EXP_WARM_START %} - {% set head = "{{ COM_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000." %} + {% set head = "{{ BASE_CPLIC }}/{{ CPL_ATMIC }}/{{ cycle_YMD }}{{ cycle_HH }}/{{ MEMDIR }}/atmos/{{ cycle_YMD }}.{{ cycle_HH }}0000." %} {% set ntile = 7 %} {% for ftype in ca_data fv_core.res fv_srf_wnd.res fv_tracer.res phy_data sfc_data %} #TODO: figure out how to copy one name to the other in here diff --git a/parm/stage/ice.yaml.j2 b/parm/stage/ice.yaml.j2 index 3c645a75a1..68f0795a24 100644 --- a/parm/stage/ice.yaml.j2 +++ b/parm/stage/ice.yaml.j2 @@ -1,5 +1,5 @@ ice: name: "ICE" - source: "{{ BASE_CPLIC }}/{{ CPL_ICEIC }}/{{ cycle_YMD }}{{ cycle_HH }}/{{ MEMDIR }}/ice" + target: "{{ COM_ICE_RESTART_PREV }}" required: - - "{{ COM_ICE_RESTART_PREV | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.cice_model.res.nc" + - "{{ BASE_CPLIC }}/{{ CPL_ICEIC }}/{{ cycle_YMD }}{{ cycle_HH }}/{{ MEMDIR }}/ice/{{ cycle_YMD }}.{{ cycle_HH }}0000.cice_model.res.nc" diff --git a/parm/stage/master_forecast_only.yaml.j2 b/parm/stage/master_forecast_only.yaml.j2 index febbe75dc2..1ad7d30898 100644 --- a/parm/stage/master_forecast_only.yaml.j2 +++ b/parm/stage/master_forecast_only.yaml.j2 @@ -3,9 +3,15 @@ {% set cycle_YMDH = current_cycle | to_YMDH %} datasets: +{% if EXP_WARM_START %} +{% filter indent(width=4) %} +{% include "fv3_warm.yaml.j2" %} +{% endfilter %} +{% else %} {% filter indent(width=4) %} -{% include "fv3.yaml.j2" %} +{% include "fv3_cold.yaml.j2" %} {% endfilter %} +{% endif %} {% if DO_NEST %} {% filter indent(width=4) %} @@ -21,11 +27,11 @@ datasets: {% endif %} {% endif %} -{% if DO_OCEAN %} +{% if DO_OCN %} {% if EXP_WARM_START %} {% filter indent(width=4) %} {% include "ocean.yaml.j2" %} -{% include "mediator.yaml.j2" %} +#TODO:#{% include "mediator.yaml.j2" %} {% endfilter %} {% endif %} {% endif %} diff --git a/parm/stage/mediator.yaml.j2 b/parm/stage/mediator.yaml.j2 index 5c9777dfb8..37cbc906cc 100644 --- a/parm/stage/mediator.yaml.j2 +++ b/parm/stage/mediator.yaml.j2 @@ -1,7 +1,5 @@ -#src="${BASE_CPLIC}/${CPL_MEDIC:-}/${PDY}${cyc}/${MEMDIR}/med/${PDY}.${cyc}0000.ufs.cpld.cpl.r.nc" -# tgt="${COM_MED_RESTART_PREV}/${PDY}.${cyc}0000.ufs.cpld.cpl.r.nc" mediator: name: "MEDIATOR" - source: "{{ BASE_CPLIC }}/{{ CPL_MEDIC }}/{{ cycle_YMD }}{{ cycle_HH }}/{{ MEMDIR }}/med" + target: "{{ COM_MED_RESTART_PREV }}" required: - - "{{ COM_MED_RESTART_PREV | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.ufs.cpld.cpl.r.nc" + - "{{ BASE_CPLIC }}/{{ CPL_MEDIC }}/{{ cycle_YMD }}{{ cycle_HH }}/{{ MEMDIR }}/med/{{ cycle_YMD }}.{{ cycle_HH }}0000.ufs.cpld.cpl.r.nc" diff --git a/parm/stage/ocean.yaml.j2 b/parm/stage/ocean.yaml.j2 index 788c9c3dfd..f38f23823e 100644 --- a/parm/stage/ocean.yaml.j2 +++ b/parm/stage/ocean.yaml.j2 @@ -1,20 +1,24 @@ ocean: name: "OCEAN" - source: "{{ BASE_CPLIC }}/{{ CPL_OCNIC }}/{{ cycle_YMD }}{{ cycle_HH }}/{{ MEMDIR }}/ocean" + target: "{{ COM_OCEAN_RESTART_PREV }}" required: + {% set head = "{{ BASE_CPLIC }}/{{ CPL_OCNIC }}/{{ cycle_YMD }}{{ cycle_HH }}/{{ MEMDIR }}/ocean" %} + #------------------------- # Ocean initial conditions #------------------------- - - "{{ COM_OCEAN_RESTART_PREV | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.MOM.res.nc" + - "{{ head }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.MOM.res.nc" + #------------------------------------ # Resolution based initial conditions #------------------------------------ #NOTE: {% if OCNRES == 500 or OCNRES == 100 %} # Nothing more to do for these resolutions {% if OCNRES == 025 %} {% for nn in $(seq 1 3) %} - - "{{ COM_OCEAN_RESTART_PREV | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.MOM.res_{{ nn }}.nc" + - "{{ head }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.MOM.res_{{ nn }}.nc" {% endfor %} {% endif %} optional: + #------------------------- # Ocean Perturbation Files #------------------------- {% if MEMDIR > 0 and USE_OCN_PERTURB_FILES %} - - "{{ COM_OCEAN_RESTART_PREV | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.mom6_increment.nc" + - "{{ head }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.mom6_increment.nc" diff --git a/parm/stage/wave.yaml.j2 b/parm/stage/wave.yaml.j2 index 2ae173002a..8197c7f739 100644 --- a/parm/stage/wave.yaml.j2 +++ b/parm/stage/wave.yaml.j2 @@ -2,8 +2,8 @@ wave: #TODO: check if BASE_CPLIC/CPL_WAVIC exists? name: "WAVE" - source: "{{ BASE_CPLIC }}/{{ CPL_WAVIC }}/{{ cycle_YMD }}{{ cycle_HH }}/{{ MEMDIR }}/wave" + target: "{{ COM_WAVE_RESTART }}" required: {% for grdID in waveGRD %} - - "{{ COM_WAVE_RESTART | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.restart.{{ grdID }}" + - "{{ BASE_CPLIC }}/{{ CPL_WAVIC }}/{{ cycle_YMD }}{{ cycle_HH }}/{{ MEMDIR }}/wave/{{ cycle_YMD }}.{{ cycle_HH }}0000.restart.{{ grdID }}" {% endfor %} From 0176d2e180f9a79aebd67d8b41f1a9a46ea4e5db Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Wed, 22 May 2024 19:22:18 +0000 Subject: [PATCH 09/89] Add COM variables exports in JGLOBAL_STAGE_IC Refs #2475 --- jobs/JGLOBAL_STAGE_IC | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/jobs/JGLOBAL_STAGE_IC b/jobs/JGLOBAL_STAGE_IC index 4837eaae41..24aafc6b77 100755 --- a/jobs/JGLOBAL_STAGE_IC +++ b/jobs/JGLOBAL_STAGE_IC @@ -19,17 +19,22 @@ gcyc="${GDATE:8:2}" if [[ ${EXP_WARM_START:-".false."} = ".true." ]]; then RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl COM_MED_RESTART_PREV:COM_MED_RESTART_TMPL + export COM_ATMOS_RESTART_PREV COM_MED_RESTART_PREV else YMD=${PDY} HH=${cyc} declare_from_tmpl COM_ATMOS_INPUT + export COM_ATMOS_INPUT fi if [[ "${DO_OCN:-}" = "YES" ]]; then RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl COM_OCEAN_RESTART_PREV:COM_OCEAN_RESTART_TMPL + export COM_OCEAN_RESTART_PREV fi if [[ "${DO_ICE:-}" = "YES" ]]; then RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl COM_ICE_RESTART_PREV:COM_ICE_RESTART_TMPL + export COM_ICE_RESTART_PREV fi if [[ "${DO_WAVE:-}" = "YES" ]]; then YMD=${PDY} HH=${cyc} declare_from_tmpl COM_WAVE_RESTART + export COM_WAVE_RESTART fi # Execute the Script From 4861ed1fb49c8a63a376bd612729175d4a4d0b16 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Wed, 22 May 2024 19:24:21 +0000 Subject: [PATCH 10/89] Update fv3_cold yaml Update to use mkdir and copy instead of target and required Refs #2475 --- parm/stage/fv3_cold.yaml.j2 | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/parm/stage/fv3_cold.yaml.j2 b/parm/stage/fv3_cold.yaml.j2 index 550b9bda06..402f61e098 100644 --- a/parm/stage/fv3_cold.yaml.j2 +++ b/parm/stage/fv3_cold.yaml.j2 @@ -3,13 +3,16 @@ # for ftype in ['atmi003.nc', 'atminc.nc', 'atmi009.nc']: # if inputs.nens > 0: #for ftype in ['ratmi003.nc', 'ratminc.nc', 'ratmi009.nc'] +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMD = current_cycle | to_YMD %} +{% set cycle_YMDH = current_cycle | to_YMDH %} fv3_cold: - name: "FV3_COLD" - target: "{{ COM_ATMOS_INPUT }}" - required: - - "{{ BASE_CPLIC }}/{{ CPL_ATMIC }}/{{ cycle_YMD }}{{ cycle_HH }}/atmos/gfs_ctrl.nc" + mkdir: + - "{{ COM_ATMOS_INPUT }}" + copy: + - ["{{ BASE_CPLIC }}/{{ CPL_ATMIC }}/{{ cycle_YMD }}{{ cycle_HH }}/atmos/gfs_ctrl.nc", "{{ COM_ATMOS_INPUT }}/gfs_ctrl.nc"] {% for ftype in ["gfs_data", "sfc_data"] %} {% for ntile in range(1, ntiles + 1) %} - - "{{ BASE_CPLIC }}/{{ CPL_ATMIC }}/{{ cycle_YMD }}{{ cycle_HH }}/atmos/{{ ftype }}.tile{{ ntile }}.nc" + - ["{{ BASE_CPLIC }}/{{ CPL_ATMIC }}/{{ cycle_YMD }}{{ cycle_HH }}/atmos/{{ ftype }}.tile{{ ntile }}.nc", "{{ COM_ATMOS_INPUT }}/{{ ftype }}.tile{{ ntile }}.nc"] {% endfor %} # ntile {% endfor %} # ftype From b7a8c82a6f2c38030895d651b70e3192f7be1679 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Wed, 22 May 2024 19:25:30 +0000 Subject: [PATCH 11/89] Update stage python scripts Refs #2475 --- scripts/exglobal_stage_ic.py | 23 +++++--- ush/python/pygfs/task/stage.py | 105 +++++++++++++++++++++++---------- 2 files changed, 89 insertions(+), 39 deletions(-) diff --git a/scripts/exglobal_stage_ic.py b/scripts/exglobal_stage_ic.py index 4424948185..da738493fd 100755 --- a/scripts/exglobal_stage_ic.py +++ b/scripts/exglobal_stage_ic.py @@ -17,10 +17,13 @@ def main(): stage = Stage(config) #Pull out all the configuration keys needed to run stage job - keys = ['RUN','MODE','EXP_WARM_START','CDUMP','rCDUMP', + keys = ['RUN','MODE','current_cycle','EXP_WARM_START','CDUMP','rCDUMP', + 'ROTDIR','PARMgfs', 'ntiles', - 'BASE_CPLIC','waveGRD','OCNRES','USE_OCN_PERTURB_FILES', - 'CPL_ATMIC','CPL_ICEIC','CPL_MEDIC','CPL_OCNIC','CPL_WAVIC'] + 'BASE_CPLIC','waveGRD','OCNRES', + #TODO: GEFS only#'USE_OCN_PERTURB_FILES', + #TODO: Need this#'CPL_MEDIC', + 'CPL_ATMIC','CPL_ICEIC','CPL_OCNIC','CPL_WAVIC'] stage_dict = AttrDict() for key in keys: @@ -32,19 +35,23 @@ def main(): stage_dict[key] = stage.task_config[key] #TEST PRINT - #for key in stage_dict: - # print(f'{key} = {stage_dict[key]}') + for key in stage_dict: + print(f'{key} = {stage_dict[key]}') cwd = os.getcwd() os.chdir(config.ROTDIR) # Determine which ICs to stage - stage_set = stage.determine(stage_dict) + #stage_sets = stage.determine_stage(stage_dict) + stage_set = stage.determine_stage(stage_dict) # Stage ICs - # TODO - create and invoke copies - stage.execute(stage_set) + #for stage_set in stage_sets: + # print(f'set = {stage_set}') + # stage.execute_stage(stage_set) + print(f'set = {stage_set}') + stage.execute_stage(stage_set) os.chdir(cwd) diff --git a/ush/python/pygfs/task/stage.py b/ush/python/pygfs/task/stage.py index db26a3a160..bfcc57e6ce 100644 --- a/ush/python/pygfs/task/stage.py +++ b/ush/python/pygfs/task/stage.py @@ -7,23 +7,9 @@ from logging import getLogger from typing import Any, Dict, List -from wxflow import (AttrDict, - FileHandler, - Hsi, - Htar, - Task, - cast_strdict_as_dtypedict, - chgrp, - get_gid, - logit, - mkdir_p, - parse_j2yaml, - rm_p, - strftime, - to_YMD, - to_YMDH, - Template, - TemplateConstants) +from wxflow import (AttrDict, FileHandler, Hsi, Htar, Task, cast_strdict_as_dtypedict, + chgrp, get_gid, logit, mkdir_p, parse_j2yaml, rm_p, strftime, + to_YMD, to_YMDH, Template, TemplateConstants) logger = getLogger(__name__.split('.')[-1]) @@ -83,14 +69,54 @@ def _gen_relative_paths(self, root_path: str) -> Dict: return rel_path_dict + @staticmethod @logit(logger) - def determine(self, stage_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str, Any]]): + def _create_fileset(stage_set: Dict[str, Any]) -> List: + """ + Collect the list of all available files from the parsed yaml dict. + Globs are expanded and if required files are missing, an error is + raised. + + TODO: expand all globs in the jinja yaml files instead of expanding + them here and issue errors here if globbing patterns (*, ?, []) + are found. + + Parameters + ---------- + stage_set: Dict + Contains full paths for required and optional files to be staged. + """ + + fileset = [] + if "required" in stage_set: + if stage_set.required is not None: + for item in stage_set.required: + glob_set = glob.glob(item) + if len(glob_set) == 0: + raise FileNotFoundError(f"FATAL ERROR: Required file, directory, or glob {item} not found!") + for entry in glob_set: + fileset.append(entry) + + if "optional" in stage_set: + if stage_set.optional is not None: + for item in stage_set.optional: + glob_set = glob.glob(item) + if len(glob_set) == 0: + logger.warning(f"WARNING: optional file/glob {item} not found!") + else: + for entry in glob_set: + fileset.append(entry) + + return fileset + + @logit(logger) + def determine_stage(self, stage_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str, Any]]): """Determine which initial condition files need to be placed in ROTDIR. Parameters ---------- stage_dict : Dict[str, Any] - Task specific keys, e.g. runtime options (DO_AERO, DO_ICE, etc) + Task specific keys, e.g. runtime options (DO_WAVE, DO_ICE, etc) Return ------ @@ -100,10 +126,6 @@ def determine(self, stage_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[st stage_parm = os.path.join(stage_dict.PARMgfs, "stage") - # Add the glob.glob function for capturing log filenames - # TODO remove this kludge once log filenames are explicit - stage_dict['glob'] = glob.glob - # Add the os.path.exists function to the dict for yaml parsing stage_dict['path_exists'] = os.path.exists @@ -114,7 +136,8 @@ def determine(self, stage_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[st if stage_dict.MODE == "cycled": master_yaml = "master_cycled.yaml.j2" elif stage_dict.MODE == "forecast-only": - master_yaml = "master_forecast_only.yaml.j2" + #master_yaml = "master_forecast_only.yaml.j2" + master_yaml = "fv3_cold.yaml.j2" elif stage_dict.RUN == "gefs": raise NotImplementedError("FATAL ERROR: Staging is not yet set up for GEFS runs") elif stage_dict.RUN == "sfs": @@ -122,16 +145,36 @@ def determine(self, stage_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[st else: raise ValueError(f"FATAL ERROR: Staging is not enabled for {stage_dict.RUN} runs") - parsed_sets = parse_j2yaml(os.path.join(stage_parm, master_yaml), stage_dict) + #parsed_sets = parse_j2yaml(os.path.join(stage_parm, master_yaml), stage_dict) + stage_set = parse_j2yaml(os.path.join(stage_parm, master_yaml), stage_dict) + #print(f'parsed_sets = {parsed_sets}') + + #stage_sets = [] - stage_sets = [] + #for dataset in parsed_sets.datasets.values(): - for dataset in parsed_sets.datasets.values(): + # dataset["fileset"] = Stage._create_fileset(dataset) - dataset["fileset"] = Stage._create_fileset(dataset) + # stage_sets.append(dataset) - stage_sets.append(dataset) + #return stage_sets + return stage_set - return stage_sets + @logit(logger) + def execute_stage(self, stage_set: Dict[str, Any]) -> None: + """Perform local staging of initial condition files. + + Parameters + ---------- + stage_set : Dict[str, Any] + FileHandler instructions to populate ROTDIR with + + Return + ------ + None + """ -#TODO - create def for staging + # Copy files to ROTDIR + for key in stage_set.keys(): + # print(f'key = {key}') + FileHandler(stage_set[key]).sync() From 5898ee209ca4cc8ee80721c55f3b87abb4435305 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Wed, 22 May 2024 19:32:28 +0000 Subject: [PATCH 12/89] Remove cycle_HH and cycle_YMD from fv3_cold yaml Refs #2475 --- parm/stage/fv3_cold.yaml.j2 | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/parm/stage/fv3_cold.yaml.j2 b/parm/stage/fv3_cold.yaml.j2 index 402f61e098..0f90b263bc 100644 --- a/parm/stage/fv3_cold.yaml.j2 +++ b/parm/stage/fv3_cold.yaml.j2 @@ -3,16 +3,14 @@ # for ftype in ['atmi003.nc', 'atminc.nc', 'atmi009.nc']: # if inputs.nens > 0: #for ftype in ['ratmi003.nc', 'ratminc.nc', 'ratmi009.nc'] -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMD = current_cycle | to_YMD %} {% set cycle_YMDH = current_cycle | to_YMDH %} fv3_cold: mkdir: - "{{ COM_ATMOS_INPUT }}" copy: - - ["{{ BASE_CPLIC }}/{{ CPL_ATMIC }}/{{ cycle_YMD }}{{ cycle_HH }}/atmos/gfs_ctrl.nc", "{{ COM_ATMOS_INPUT }}/gfs_ctrl.nc"] + - ["{{ BASE_CPLIC }}/{{ CPL_ATMIC }}/{{ cycle_YMDH }}/atmos/gfs_ctrl.nc", "{{ COM_ATMOS_INPUT }}/gfs_ctrl.nc"] {% for ftype in ["gfs_data", "sfc_data"] %} {% for ntile in range(1, ntiles + 1) %} - - ["{{ BASE_CPLIC }}/{{ CPL_ATMIC }}/{{ cycle_YMD }}{{ cycle_HH }}/atmos/{{ ftype }}.tile{{ ntile }}.nc", "{{ COM_ATMOS_INPUT }}/{{ ftype }}.tile{{ ntile }}.nc"] + - ["{{ BASE_CPLIC }}/{{ CPL_ATMIC }}/{{ cycle_YMDH }}/atmos/{{ ftype }}.tile{{ ntile }}.nc", "{{ COM_ATMOS_INPUT }}/{{ ftype }}.tile{{ ntile }}.nc"] {% endfor %} # ntile {% endfor %} # ftype From 5e8f92f603f8b1d2478e96b180bc75f406232113 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Thu, 23 May 2024 14:37:26 +0000 Subject: [PATCH 13/89] Remove unneeded funcions from stage.py Refs #2475 --- ush/python/pygfs/task/stage.py | 73 +--------------------------------- 1 file changed, 1 insertion(+), 72 deletions(-) diff --git a/ush/python/pygfs/task/stage.py b/ush/python/pygfs/task/stage.py index bfcc57e6ce..459c944319 100644 --- a/ush/python/pygfs/task/stage.py +++ b/ush/python/pygfs/task/stage.py @@ -36,78 +36,7 @@ def __init__(self, config: Dict[str, Any]) -> None: rotdir = self.config.ROTDIR + os.sep - # Find all absolute paths in the environment and get their relative paths from ${ROTDIR} - path_dict = self._gen_relative_paths(rotdir) - - self.task_config = AttrDict(**self.config, **self.runtime_config, **path_dict) - - @logit(logger) - def _gen_relative_paths(self, root_path: str) -> Dict: - """Generate a dict of paths in self.config relative to root_path - - Parameters - ---------- - root_path : str - Path to base all relative paths off of - - Return - ------ - rel_path_dict : Dict - Dictionary of paths relative to root_path. Members will be named - based on the dict names in self.config. For COM paths, the names will - follow COM_ --> _dir. For all other directories, the - names will follow --> _dir. - """ - - rel_path_dict = {} - for key, value in self.config.items(): - if isinstance(value, str): - if root_path in value: - rel_path = value.replace(root_path, "") - rel_key = (key[4:] if key.startswith("COM_") else key).lower() + "_dir" - rel_path_dict[rel_key] = rel_path - - return rel_path_dict - - @staticmethod - @logit(logger) - def _create_fileset(stage_set: Dict[str, Any]) -> List: - """ - Collect the list of all available files from the parsed yaml dict. - Globs are expanded and if required files are missing, an error is - raised. - - TODO: expand all globs in the jinja yaml files instead of expanding - them here and issue errors here if globbing patterns (*, ?, []) - are found. - - Parameters - ---------- - stage_set: Dict - Contains full paths for required and optional files to be staged. - """ - - fileset = [] - if "required" in stage_set: - if stage_set.required is not None: - for item in stage_set.required: - glob_set = glob.glob(item) - if len(glob_set) == 0: - raise FileNotFoundError(f"FATAL ERROR: Required file, directory, or glob {item} not found!") - for entry in glob_set: - fileset.append(entry) - - if "optional" in stage_set: - if stage_set.optional is not None: - for item in stage_set.optional: - glob_set = glob.glob(item) - if len(glob_set) == 0: - logger.warning(f"WARNING: optional file/glob {item} not found!") - else: - for entry in glob_set: - fileset.append(entry) - - return fileset + self.task_config = AttrDict(**self.config, **self.runtime_config) @logit(logger) def determine_stage(self, stage_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str, Any]]): From 1cf0063311ba2b5108bc5fa64720c459e10d3146 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Thu, 23 May 2024 16:02:34 +0000 Subject: [PATCH 14/89] Refactor stage yamls Refs #2475 --- parm/stage/fv3_cold.yaml.j2 | 9 ++------- parm/stage/fv3_nest.yaml.j2 | 13 ++++++++----- parm/stage/fv3_warm.yaml.j2 | 21 +++++++++------------ parm/stage/ice.yaml.j2 | 11 +++++++---- parm/stage/master_forecast_only.yaml.j2 | 9 --------- parm/stage/mediator.yaml.j2 | 11 +++++++---- parm/stage/ocean.yaml.j2 | 18 ++++++++++-------- parm/stage/wave.yaml.j2 | 13 +++++++------ 8 files changed, 50 insertions(+), 55 deletions(-) diff --git a/parm/stage/fv3_cold.yaml.j2 b/parm/stage/fv3_cold.yaml.j2 index 0f90b263bc..70472dd8ca 100644 --- a/parm/stage/fv3_cold.yaml.j2 +++ b/parm/stage/fv3_cold.yaml.j2 @@ -1,16 +1,11 @@ -# # First 1/2 cycle also needs a atmos increment if doing warm start -# if inputs.start in ['warm']: -# for ftype in ['atmi003.nc', 'atminc.nc', 'atmi009.nc']: -# if inputs.nens > 0: -#for ftype in ['ratmi003.nc', 'ratminc.nc', 'ratmi009.nc'] {% set cycle_YMDH = current_cycle | to_YMDH %} fv3_cold: mkdir: - "{{ COM_ATMOS_INPUT }}" copy: - - ["{{ BASE_CPLIC }}/{{ CPL_ATMIC }}/{{ cycle_YMDH }}/atmos/gfs_ctrl.nc", "{{ COM_ATMOS_INPUT }}/gfs_ctrl.nc"] + - ["{{ BASE_CPLIC }}/{{ CPL_ATMIC }}/{{ cycle_YMDH }}/{{ MEMDIR }}/atmos/gfs_ctrl.nc", "{{ COM_ATMOS_INPUT }}/gfs_ctrl.nc"] {% for ftype in ["gfs_data", "sfc_data"] %} {% for ntile in range(1, ntiles + 1) %} - - ["{{ BASE_CPLIC }}/{{ CPL_ATMIC }}/{{ cycle_YMDH }}/atmos/{{ ftype }}.tile{{ ntile }}.nc", "{{ COM_ATMOS_INPUT }}/{{ ftype }}.tile{{ ntile }}.nc"] + - ["{{ BASE_CPLIC }}/{{ CPL_ATMIC }}/{{ cycle_YMDH }}/{{ MEMDIR }}/atmos/{{ ftype }}.tile{{ ntile }}.nc", "{{ COM_ATMOS_INPUT }}/{{ ftype }}.tile{{ ntile }}.nc"] {% endfor %} # ntile {% endfor %} # ftype diff --git a/parm/stage/fv3_nest.yaml.j2 b/parm/stage/fv3_nest.yaml.j2 index fffad0c245..bc33eb729a 100644 --- a/parm/stage/fv3_nest.yaml.j2 +++ b/parm/stage/fv3_nest.yaml.j2 @@ -11,11 +11,14 @@ # tgt="${COM_ATMOS_INPUT}/${ftype}.tile${tt}.nc" # if (( ntiles > 6 )); then # ${NLN} "${COM_ATMOS_INPUT}/${ftype}.tile7.nc" "${COM_ATMOS_INPUT}/${ftype}.nest02.tile7.nc" -fv3: - name: "FV3" - target: "{{ COM_ATMOS_RESTART_PREV }}" - {% set ntile = 7 %} - required: +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMD = current_cycle | to_YMD %} +{% set cycle_YMDH = current_cycle | to_YMDH %} +{% set ntile = 7 %} +fv3_nest: + mkdir: + - "{{ COM_ATMOS_RESTART_PREV }}" + copy: #--------------- # WARM START #--------------- diff --git a/parm/stage/fv3_warm.yaml.j2 b/parm/stage/fv3_warm.yaml.j2 index e40aa41ba3..efd2a3a4d8 100644 --- a/parm/stage/fv3_warm.yaml.j2 +++ b/parm/stage/fv3_warm.yaml.j2 @@ -1,19 +1,16 @@ -# # First 1/2 cycle also needs a atmos increment if doing warm start -# if inputs.start in ['warm']: -# for ftype in ['atmi003.nc', 'atminc.nc', 'atmi009.nc']: -# if inputs.nens > 0: -#for ftype in ['ratmi003.nc', 'ratminc.nc', 'ratmi009.nc'] +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMD = current_cycle | to_YMD %} +{% set cycle_YMDH = current_cycle | to_YMDH %} fv3_warm: - name: "FV3_WARM" - target: "{{ COM_ATMOS_RESTART_PREV }}" - required: -#TODO: Set MEMDIR as empty if not set? - {% set head = "{{ BASE_CPLIC }}/{{ CPL_ATMIC }}/{{ cycle_YMD }}{{ cycle_HH }}/{{ MEMDIR }}/atmos/{{ cycle_YMD }}.{{ cycle_HH }}0000." %} + mkdir: + - "{{ COM_ATMOS_RESTART_PREV }}" + copy: + {% set head = "{{ BASE_CPLIC }}/{{ CPL_ATMIC }}/{{ cycle_YMDH }}/{{ MEMDIR }}/atmos/{{ cycle_YMD }}.{{ cycle_HH }}0000." %} {% for ftype in ["coupler.res", "fv_core.res.nc"] %} - - "{{ head }}{{ ftype }}" + - ["{{ head }}{{ ftype }}", "{{ COM_ATMOS_RESTART_PREV }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.{{ ftype }}"] {% endfor %} {% for ftype in ["ca_data", "fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data sfc_data"] %} {% for ntile in range(1, ntiles) %} - - "{{ head }}{{ ftype }}.tile{{ ntile }}.nc" + - ["{{ head }}{{ ftype }}.tile{{ ntile }}.nc", "{{ COM_ATMOS_RESTART_PREV }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.{{ ftype }}.tile{{ ntile }}.nc"] {% endfor %} # ntile {% endfor %} # ftype diff --git a/parm/stage/ice.yaml.j2 b/parm/stage/ice.yaml.j2 index 68f0795a24..6bd3e189ad 100644 --- a/parm/stage/ice.yaml.j2 +++ b/parm/stage/ice.yaml.j2 @@ -1,5 +1,8 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMD = current_cycle | to_YMD %} +{% set cycle_YMDH = current_cycle | to_YMDH %} ice: - name: "ICE" - target: "{{ COM_ICE_RESTART_PREV }}" - required: - - "{{ BASE_CPLIC }}/{{ CPL_ICEIC }}/{{ cycle_YMD }}{{ cycle_HH }}/{{ MEMDIR }}/ice/{{ cycle_YMD }}.{{ cycle_HH }}0000.cice_model.res.nc" + mkdir: + - "{{ COM_ICE_RESTART_PREV }}" + copy: + - ["{{ BASE_CPLIC }}/{{ CPL_ICEIC }}/{{ cycle_YMDH }}/{{ MEMDIR }}/ice/{{ cycle_YMD }}.{{ cycle_HH }}0000.cice_model.res.nc", "{{ COM_ICE_RESTART_PREV }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.cice_model.res.nc"] diff --git a/parm/stage/master_forecast_only.yaml.j2 b/parm/stage/master_forecast_only.yaml.j2 index 1ad7d30898..b3e34b9aab 100644 --- a/parm/stage/master_forecast_only.yaml.j2 +++ b/parm/stage/master_forecast_only.yaml.j2 @@ -1,6 +1,3 @@ -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMD = current_cycle | to_YMD %} -{% set cycle_YMDH = current_cycle | to_YMDH %} datasets: {% if EXP_WARM_START %} @@ -20,26 +17,20 @@ datasets: {% endif %} {% if DO_WAVE %} -{% if EXP_WARM_START %} {% filter indent(width=4) %} {% include "wave.yaml.j2" %} {% endfilter %} {% endif %} -{% endif %} {% if DO_OCN %} -{% if EXP_WARM_START %} {% filter indent(width=4) %} {% include "ocean.yaml.j2" %} #TODO:#{% include "mediator.yaml.j2" %} {% endfilter %} {% endif %} -{% endif %} {% if DO_ICE %} -{% if EXP_WARM_START %} {% filter indent(width=4) %} {% include "ice.yaml.j2" %} {% endfilter %} {% endif %} -{% endif %} diff --git a/parm/stage/mediator.yaml.j2 b/parm/stage/mediator.yaml.j2 index 37cbc906cc..5f1e4d20ba 100644 --- a/parm/stage/mediator.yaml.j2 +++ b/parm/stage/mediator.yaml.j2 @@ -1,5 +1,8 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMD = current_cycle | to_YMD %} +{% set cycle_YMDH = current_cycle | to_YMDH %} mediator: - name: "MEDIATOR" - target: "{{ COM_MED_RESTART_PREV }}" - required: - - "{{ BASE_CPLIC }}/{{ CPL_MEDIC }}/{{ cycle_YMD }}{{ cycle_HH }}/{{ MEMDIR }}/med/{{ cycle_YMD }}.{{ cycle_HH }}0000.ufs.cpld.cpl.r.nc" + mkdir: + - "{{ COM_MED_RESTART_PREV }}" + copy: + - ["{{ BASE_CPLIC }}/{{ CPL_MEDIC }}/{{ cycle_YMDH }}/{{ MEMDIR }}/med/{{ cycle_YMD }}.{{ cycle_HH }}0000.ufs.cpld.cpl.r.nc", "{{ COM_MED_RESTART_PREV }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.ufs.cpld.cpl.r.nc"] diff --git a/parm/stage/ocean.yaml.j2 b/parm/stage/ocean.yaml.j2 index f38f23823e..a073bc5242 100644 --- a/parm/stage/ocean.yaml.j2 +++ b/parm/stage/ocean.yaml.j2 @@ -1,24 +1,26 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMD = current_cycle | to_YMD %} +{% set cycle_YMDH = current_cycle | to_YMDH %} ocean: - name: "OCEAN" - target: "{{ COM_OCEAN_RESTART_PREV }}" - required: - {% set head = "{{ BASE_CPLIC }}/{{ CPL_OCNIC }}/{{ cycle_YMD }}{{ cycle_HH }}/{{ MEMDIR }}/ocean" %} + mkdir: + - "{{ COM_OCEAN_RESTART_PREV }}" + copy: + {% set head = "{{ BASE_CPLIC }}/{{ CPL_OCNIC }}/{{ cycle_YMDH }}/{{ MEMDIR }}/ocean" %} #------------------------- # Ocean initial conditions #------------------------- - - "{{ head }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.MOM.res.nc" + - ["{{ head }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.MOM.res.nc", "{{ COM_OCEAN_RESTART_PREV }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.MOM.res.nc"] #------------------------------------ # Resolution based initial conditions #------------------------------------ #NOTE: {% if OCNRES == 500 or OCNRES == 100 %} # Nothing more to do for these resolutions {% if OCNRES == 025 %} {% for nn in $(seq 1 3) %} - - "{{ head }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.MOM.res_{{ nn }}.nc" + - ["{{ head }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.MOM.res_{{ nn }}.nc", "{{ COM_OCEAN_RESTART_PREV }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.MOM.res_{{ nn }}.nc"] {% endfor %} {% endif %} - optional: #------------------------- # Ocean Perturbation Files #------------------------- {% if MEMDIR > 0 and USE_OCN_PERTURB_FILES %} - - "{{ head }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.mom6_increment.nc" + - ["{{ head }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.mom6_increment.nc", "{{ COM_OCEAN_RESTART_PREV }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.mom6_increment.nc"] diff --git a/parm/stage/wave.yaml.j2 b/parm/stage/wave.yaml.j2 index 8197c7f739..4cfc927a85 100644 --- a/parm/stage/wave.yaml.j2 +++ b/parm/stage/wave.yaml.j2 @@ -1,9 +1,10 @@ -#TODO: these should be placed in $RUN.$gPDY/$gcyc) +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMD = current_cycle | to_YMD %} +{% set cycle_YMDH = current_cycle | to_YMDH %} wave: - #TODO: check if BASE_CPLIC/CPL_WAVIC exists? - name: "WAVE" - target: "{{ COM_WAVE_RESTART }}" - required: + mkdir: + - "{{ COM_WAVE_RESTART }}" + copy: {% for grdID in waveGRD %} - - "{{ BASE_CPLIC }}/{{ CPL_WAVIC }}/{{ cycle_YMD }}{{ cycle_HH }}/{{ MEMDIR }}/wave/{{ cycle_YMD }}.{{ cycle_HH }}0000.restart.{{ grdID }}" + - ["{{ BASE_CPLIC }}/{{ CPL_WAVIC }}/{{ cycle_YMDH }}/{{ MEMDIR }}/wave/{{ cycle_YMD }}.{{ cycle_HH }}0000.restart.{{ grdID }}", "{{ COM_WAVE_RESTART }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.restart.{{ grdID }}"] {% endfor %} From 9a397875bbe68cb63fb7b012fa47dc7c8addbf4d Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Thu, 23 May 2024 16:04:13 +0000 Subject: [PATCH 15/89] Add MEMDIR array to JGLOBAL_STAGE_IC Refs #2475 --- jobs/JGLOBAL_STAGE_IC | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/jobs/JGLOBAL_STAGE_IC b/jobs/JGLOBAL_STAGE_IC index 24aafc6b77..d8657e9f5d 100755 --- a/jobs/JGLOBAL_STAGE_IC +++ b/jobs/JGLOBAL_STAGE_IC @@ -37,8 +37,38 @@ if [[ "${DO_WAVE:-}" = "YES" ]]; then export COM_WAVE_RESTART fi -# Execute the Script -"${SCRgfs}/exglobal_stage_ic.py" +# Define MEMDIR_ARRAY +MEMDIR_ARRAY=() +if [[ "${RUN:-}" = "gefs" ]]; then + # Populate the member_dirs array based on the value of NMEM_ENS + for ((ii = 0; ii <= "${NMEM_ENS:-0}"; ii++)); do + MEMDIR_ARRAY+=("mem$(printf "%03d" "${ii}")") + done +else + MEMDIR_ARRAY+=("") +fi + +# Initialize return code +err=0 + +error_message() { + echo "FATAL ERROR: Unable to copy ${1} to ${2} (Error code ${3})" +} + +############################################################### +for MEMDIR in "${MEMDIR_ARRAY[@]}"; do + + # Execute the Script + "${SCRgfs}/exglobal_stage_ic.py" + +done # for MEMDIR in "${MEMDIR_ARRAY[@]}"; do + +############################################################### +# Check for errors and exit if any of the above failed +if [[ "${err}" -ne 0 ]]; then + echo "FATAL ERROR: Unable to copy ICs from ${BASE_CPLIC} to ${ROTDIR}; ABORT!" + exit "${err}" +fi ########################################## # Remove the Temporary working directory From cd362fd53ec6fed40b909f6248bd8e93df538b98 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Thu, 23 May 2024 16:05:09 +0000 Subject: [PATCH 16/89] Remove stage_ic job dependencies from setup Refs #2475 --- workflow/rocoto/gefs_tasks.py | 45 ------------------------------- workflow/rocoto/gfs_tasks.py | 51 ----------------------------------- 2 files changed, 96 deletions(-) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index 86be494549..31f70e4ce0 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -12,55 +12,10 @@ def stage_ic(self): cpl_ic = self._configs['stage_ic'] - deps = [] - - # Atm ICs - if self.app_config.do_atm: - prefix = f"{cpl_ic['BASE_CPLIC']}/{cpl_ic['CPL_ATMIC']}/@Y@m@d@H/mem000/atmos" - for file in ['gfs_ctrl.nc'] + \ - [f'{datatype}_data.tile{tile}.nc' - for datatype in ['gfs', 'sfc'] - for tile in range(1, self.n_tiles + 1)]: - data = f"{prefix}/{file}" - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - - # Ocean ICs - if self.app_config.do_ocean: - ocn_res = f"{self._base.get('OCNRES', '025'):03d}" - prefix = f"{cpl_ic['BASE_CPLIC']}/{cpl_ic['CPL_OCNIC']}/@Y@m@d@H/mem000/ocean" - data = f"{prefix}/@Y@m@d.@H0000.MOM.res.nc" - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - if ocn_res in ['025']: - # 0.25 degree ocean model also has these additional restarts - for res in [f'res_{res_index}' for res_index in range(1, 4)]: - data = f"{prefix}/@Y@m@d.@H0000.MOM.{res}.nc" - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - - # Ice ICs - if self.app_config.do_ice: - prefix = f"{cpl_ic['BASE_CPLIC']}/{cpl_ic['CPL_ICEIC']}/@Y@m@d@H/mem000/ice" - data = f"{prefix}/@Y@m@d.@H0000.cice_model.res.nc" - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - - # Wave ICs - if self.app_config.do_wave: - prefix = f"{cpl_ic['BASE_CPLIC']}/{cpl_ic['CPL_WAVIC']}/@Y@m@d@H/mem000/wave" - for wave_grid in self._configs['waveinit']['waveGRD'].split(): - data = f"{prefix}/@Y@m@d.@H0000.restart.{wave_grid}" - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - resources = self.get_resource('stage_ic') task_name = f'stage_ic' task_dict = {'task_name': task_name, 'resources': resources, - 'dependency': dependencies, 'envars': self.envars, 'cycledef': 'gefs', 'command': f'{self.HOMEgfs}/jobs/rocoto/stage_ic.sh', diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index d61060a596..ed2fe3b371 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -20,61 +20,10 @@ def stage_ic(self): cpl_ic = self._configs['stage_ic'] - deps = [] - - # Atm ICs - if self.app_config.do_atm: - prefix = f"{cpl_ic['BASE_CPLIC']}/{cpl_ic['CPL_ATMIC']}/@Y@m@d@H/atmos" - for file in ['gfs_ctrl.nc'] + \ - [f'{datatype}_data.tile{tile}.nc' - for datatype in ['gfs', 'sfc'] - for tile in range(1, self.n_tiles + 1)]: - data = f"{prefix}/{file}" - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - else: # data-atmosphere - # TODO - need more information about how these forcings are stored - prefix = f"{cpl_ic['BASE_CPLIC']}/{cpl_ic['CPL_DATM']}/@Y@m@d@H" - data = f"{prefix}/gefs.@Y@m.nc" - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - - # Ocean ICs - if self.app_config.do_ocean: - ocn_res = f"{self._base.get('OCNRES', '025'):03d}" - prefix = f"{cpl_ic['BASE_CPLIC']}/{cpl_ic['CPL_OCNIC']}/@Y@m@d@H/ocean" - data = f"{prefix}/@Y@m@d.@H0000.MOM.res.nc" - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - if ocn_res in ['025']: - # 0.25 degree ocean model also has these additional restarts - for res in [f'res_{res_index}' for res_index in range(1, 4)]: - data = f"{prefix}/@Y@m@d.@H0000.MOM.{res}.nc" - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - - # Ice ICs - if self.app_config.do_ice: - prefix = f"{cpl_ic['BASE_CPLIC']}/{cpl_ic['CPL_ICEIC']}/@Y@m@d@H/ice" - data = f"{prefix}/@Y@m@d.@H0000.cice_model.res.nc" - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - - # Wave ICs - if self.app_config.do_wave: - prefix = f"{cpl_ic['BASE_CPLIC']}/{cpl_ic['CPL_WAVIC']}/@Y@m@d@H/wave" - for wave_grid in self._configs['waveinit']['waveGRD'].split(): - data = f"{prefix}/@Y@m@d.@H0000.restart.{wave_grid}" - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - resources = self.get_resource('stage_ic') task_name = f'{self.cdump}stage_ic' task_dict = {'task_name': task_name, 'resources': resources, - 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.cdump, 'command': f'{self.HOMEgfs}/jobs/rocoto/stage_ic.sh', From 859d31d70a151471dc74d2d06f7978e0ff2b9cac Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Thu, 23 May 2024 19:29:41 +0000 Subject: [PATCH 17/89] Set MEMDIR as empty if not set Refs #2475 --- jobs/JGLOBAL_STAGE_IC | 1 + 1 file changed, 1 insertion(+) diff --git a/jobs/JGLOBAL_STAGE_IC b/jobs/JGLOBAL_STAGE_IC index d8657e9f5d..61e437e278 100755 --- a/jobs/JGLOBAL_STAGE_IC +++ b/jobs/JGLOBAL_STAGE_IC @@ -58,6 +58,7 @@ error_message() { ############################################################### for MEMDIR in "${MEMDIR_ARRAY[@]}"; do + export MEMDIR=${MEMDIR:-} # Execute the Script "${SCRgfs}/exglobal_stage_ic.py" From a805fa817390579fdf182bbd31cadfd676614aa4 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Thu, 23 May 2024 19:31:25 +0000 Subject: [PATCH 18/89] Add keys_gefs in exglobal_stage_ic.py If RUN=gefs add keys_gefs to keys. Refs #2475 --- scripts/exglobal_stage_ic.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/scripts/exglobal_stage_ic.py b/scripts/exglobal_stage_ic.py index da738493fd..a382579599 100755 --- a/scripts/exglobal_stage_ic.py +++ b/scripts/exglobal_stage_ic.py @@ -17,14 +17,18 @@ def main(): stage = Stage(config) #Pull out all the configuration keys needed to run stage job - keys = ['RUN','MODE','current_cycle','EXP_WARM_START','CDUMP','rCDUMP', - 'ROTDIR','PARMgfs', - 'ntiles', + keys = ['RUN','MODE','DO_WAVE','DO_OCN','DO_ICE','DO_NEST', + 'current_cycle','EXP_WARM_START','CDUMP','rCDUMP', + 'ROTDIR','PARMgfs','ntiles','MEMDIR', 'BASE_CPLIC','waveGRD','OCNRES', - #TODO: GEFS only#'USE_OCN_PERTURB_FILES', - #TODO: Need this#'CPL_MEDIC', + #TODO: Need this for mediator#'CPL_MEDIC', 'CPL_ATMIC','CPL_ICEIC','CPL_OCNIC','CPL_WAVIC'] + keys_gefs = ['USE_OCN_PERTURB_FILES'] + + if stage.task_config['RUN'] == "gefs": + keys.extend(keys_gefs) + stage_dict = AttrDict() for key in keys: stage_dict[key] = stage.task_config[key] From 719701b50939cb31cbbf5d519b248287f3522d4d Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Thu, 30 May 2024 13:35:55 +0000 Subject: [PATCH 19/89] Add USE_OCN_PERTURB_FILES to gfs config.base Set to .false. by default; needed for staging job Refs #2475 --- parm/config/gfs/config.base | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index f1e25750ef..d81a6a152b 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -437,6 +437,10 @@ export INCVARS_EFOLD="5" export netcdf_diag=".true." export binary_diag=".false." +# Initialize ocean ensemble members with perturbations +# if true, only occurs for members greater than zero +export USE_OCN_PERTURB_FILES=".false." + # Verification options export DO_METP="NO" # Run METPLUS jobs - set METPLUS settings in config.metp; not supported with spack-stack export DO_FIT2OBS="YES" # Run fit to observations package From cf50f5499d4691790904743568e7ef5583badc75 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Thu, 30 May 2024 13:40:34 +0000 Subject: [PATCH 20/89] Stage job yaml updates - remove master yaml, no longer using - update fv3_cold, ice, ocean, and wave yamls Refs #2475 --- parm/stage/fv3_cold.yaml.j2 | 4 +-- parm/stage/ice.yaml.j2 | 2 +- parm/stage/master_forecast_only.yaml.j2 | 36 ------------------------- parm/stage/ocean.yaml.j2 | 15 +++++------ parm/stage/wave.yaml.j2 | 4 +-- 5 files changed, 11 insertions(+), 50 deletions(-) delete mode 100644 parm/stage/master_forecast_only.yaml.j2 diff --git a/parm/stage/fv3_cold.yaml.j2 b/parm/stage/fv3_cold.yaml.j2 index 70472dd8ca..f4d80131ae 100644 --- a/parm/stage/fv3_cold.yaml.j2 +++ b/parm/stage/fv3_cold.yaml.j2 @@ -3,9 +3,9 @@ fv3_cold: mkdir: - "{{ COM_ATMOS_INPUT }}" copy: - - ["{{ BASE_CPLIC }}/{{ CPL_ATMIC }}/{{ cycle_YMDH }}/{{ MEMDIR }}/atmos/gfs_ctrl.nc", "{{ COM_ATMOS_INPUT }}/gfs_ctrl.nc"] + - ["{{ BASE_CPLIC }}/{{ CPL_ATMIC }}/{{ cycle_YMDH }}/{{ MEMDIR }}/atmos/gfs_ctrl.nc", "{{ COM_ATMOS_INPUT }}"] {% for ftype in ["gfs_data", "sfc_data"] %} {% for ntile in range(1, ntiles + 1) %} - - ["{{ BASE_CPLIC }}/{{ CPL_ATMIC }}/{{ cycle_YMDH }}/{{ MEMDIR }}/atmos/{{ ftype }}.tile{{ ntile }}.nc", "{{ COM_ATMOS_INPUT }}/{{ ftype }}.tile{{ ntile }}.nc"] + - ["{{ BASE_CPLIC }}/{{ CPL_ATMIC }}/{{ cycle_YMDH }}/{{ MEMDIR }}/atmos/{{ ftype }}.tile{{ ntile }}.nc", "{{ COM_ATMOS_INPUT }}"] {% endfor %} # ntile {% endfor %} # ftype diff --git a/parm/stage/ice.yaml.j2 b/parm/stage/ice.yaml.j2 index 6bd3e189ad..d5a47951b9 100644 --- a/parm/stage/ice.yaml.j2 +++ b/parm/stage/ice.yaml.j2 @@ -5,4 +5,4 @@ ice: mkdir: - "{{ COM_ICE_RESTART_PREV }}" copy: - - ["{{ BASE_CPLIC }}/{{ CPL_ICEIC }}/{{ cycle_YMDH }}/{{ MEMDIR }}/ice/{{ cycle_YMD }}.{{ cycle_HH }}0000.cice_model.res.nc", "{{ COM_ICE_RESTART_PREV }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.cice_model.res.nc"] + - ["{{ BASE_CPLIC }}/{{ CPL_ICEIC }}/{{ cycle_YMDH }}/{{ MEMDIR }}/ice/{{ cycle_YMD }}.{{ cycle_HH }}0000.cice_model.res.nc", "{{ COM_ICE_RESTART_PREV }}"] diff --git a/parm/stage/master_forecast_only.yaml.j2 b/parm/stage/master_forecast_only.yaml.j2 deleted file mode 100644 index b3e34b9aab..0000000000 --- a/parm/stage/master_forecast_only.yaml.j2 +++ /dev/null @@ -1,36 +0,0 @@ -datasets: - -{% if EXP_WARM_START %} -{% filter indent(width=4) %} -{% include "fv3_warm.yaml.j2" %} -{% endfilter %} -{% else %} -{% filter indent(width=4) %} -{% include "fv3_cold.yaml.j2" %} -{% endfilter %} -{% endif %} - -{% if DO_NEST %} -{% filter indent(width=4) %} -{% include "fv3_nest.yaml.j2" %} -{% endfilter %} -{% endif %} - -{% if DO_WAVE %} -{% filter indent(width=4) %} -{% include "wave.yaml.j2" %} -{% endfilter %} -{% endif %} - -{% if DO_OCN %} -{% filter indent(width=4) %} -{% include "ocean.yaml.j2" %} -#TODO:#{% include "mediator.yaml.j2" %} -{% endfilter %} -{% endif %} - -{% if DO_ICE %} -{% filter indent(width=4) %} -{% include "ice.yaml.j2" %} -{% endfilter %} -{% endif %} diff --git a/parm/stage/ocean.yaml.j2 b/parm/stage/ocean.yaml.j2 index a073bc5242..a7fee221ba 100644 --- a/parm/stage/ocean.yaml.j2 +++ b/parm/stage/ocean.yaml.j2 @@ -5,22 +5,21 @@ ocean: mkdir: - "{{ COM_OCEAN_RESTART_PREV }}" copy: - {% set head = "{{ BASE_CPLIC }}/{{ CPL_OCNIC }}/{{ cycle_YMDH }}/{{ MEMDIR }}/ocean" %} #------------------------- # Ocean initial conditions #------------------------- - - ["{{ head }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.MOM.res.nc", "{{ COM_OCEAN_RESTART_PREV }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.MOM.res.nc"] + - ["{{ BASE_CPLIC }}/{{ CPL_OCNIC }}/{{ cycle_YMDH }}/{{ MEMDIR }}/ocean/{{ cycle_YMD }}.{{ cycle_HH }}0000.MOM.res.nc", "{{ COM_OCEAN_RESTART_PREV }}"] #------------------------------------ # Resolution based initial conditions #------------------------------------ - #NOTE: {% if OCNRES == 500 or OCNRES == 100 %} # Nothing more to do for these resolutions - {% if OCNRES == 025 %} - {% for nn in $(seq 1 3) %} - - ["{{ head }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.MOM.res_{{ nn }}.nc", "{{ COM_OCEAN_RESTART_PREV }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.MOM.res_{{ nn }}.nc"] + {% if OCNRES == "025" %} + {% for nn in range(1, 3) %} + - ["{{ BASE_CPLIC }}/{{ CPL_OCNIC }}/{{ cycle_YMDH }}/{{ MEMDIR }}/ocean/{{ cycle_YMD }}.{{ cycle_HH }}0000.MOM.res_{{ nn }}.nc", "{{ COM_OCEAN_RESTART_PREV }}"] {% endfor %} {% endif %} #------------------------- # Ocean Perturbation Files #------------------------- - {% if MEMDIR > 0 and USE_OCN_PERTURB_FILES %} - - ["{{ head }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.mom6_increment.nc", "{{ COM_OCEAN_RESTART_PREV }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.mom6_increment.nc"] + {% if MEMDIR and USE_OCN_PERTURB_FILES %} + - ["{{ BASE_CPLIC }}/{{ CPL_OCNIC }}/{{ cycle_YMDH }}/{{ MEMDIR }}/ocean/{{ cycle_YMD }}.{{ cycle_HH }}0000.mom6_increment.nc", "{{ COM_OCEAN_RESTART_PREV }}"] + {% endif %} diff --git a/parm/stage/wave.yaml.j2 b/parm/stage/wave.yaml.j2 index 4cfc927a85..db6f6502aa 100644 --- a/parm/stage/wave.yaml.j2 +++ b/parm/stage/wave.yaml.j2 @@ -5,6 +5,4 @@ wave: mkdir: - "{{ COM_WAVE_RESTART }}" copy: - {% for grdID in waveGRD %} - - ["{{ BASE_CPLIC }}/{{ CPL_WAVIC }}/{{ cycle_YMDH }}/{{ MEMDIR }}/wave/{{ cycle_YMD }}.{{ cycle_HH }}0000.restart.{{ grdID }}", "{{ COM_WAVE_RESTART }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.restart.{{ grdID }}"] - {% endfor %} + - ["{{ BASE_CPLIC }}/{{ CPL_WAVIC }}/{{ cycle_YMDH }}/{{ MEMDIR }}/wave/{{ cycle_YMD }}.{{ cycle_HH }}0000.restart.{{ waveGRD }}", "{{ COM_WAVE_RESTART }}"] From 3e754c1003615f07322e549d0c555c18486abbec Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Thu, 30 May 2024 19:33:10 +0000 Subject: [PATCH 21/89] Updates to exglobal_stage_ic.py - Add keys for GEFS - Cleanup Refs #2475 --- scripts/exglobal_stage_ic.py | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/scripts/exglobal_stage_ic.py b/scripts/exglobal_stage_ic.py index a382579599..aad20252ea 100755 --- a/scripts/exglobal_stage_ic.py +++ b/scripts/exglobal_stage_ic.py @@ -20,15 +20,10 @@ def main(): keys = ['RUN','MODE','DO_WAVE','DO_OCN','DO_ICE','DO_NEST', 'current_cycle','EXP_WARM_START','CDUMP','rCDUMP', 'ROTDIR','PARMgfs','ntiles','MEMDIR', - 'BASE_CPLIC','waveGRD','OCNRES', + 'BASE_CPLIC','waveGRD','OCNRES','USE_OCN_PERTURB_FILES', #TODO: Need this for mediator#'CPL_MEDIC', 'CPL_ATMIC','CPL_ICEIC','CPL_OCNIC','CPL_WAVIC'] - keys_gefs = ['USE_OCN_PERTURB_FILES'] - - if stage.task_config['RUN'] == "gefs": - keys.extend(keys_gefs) - stage_dict = AttrDict() for key in keys: stage_dict[key] = stage.task_config[key] @@ -42,22 +37,14 @@ def main(): for key in stage_dict: print(f'{key} = {stage_dict[key]}') - cwd = os.getcwd() - - os.chdir(config.ROTDIR) + # Add the os.path.exists function to the dict for yaml parsing + stage_dict['path_exists'] = os.path.exists # Determine which ICs to stage - #stage_sets = stage.determine_stage(stage_dict) - stage_set = stage.determine_stage(stage_dict) + stage_sets = stage.determine_stage(stage_dict) # Stage ICs - #for stage_set in stage_sets: - # print(f'set = {stage_set}') - # stage.execute_stage(stage_set) - print(f'set = {stage_set}') - stage.execute_stage(stage_set) - - os.chdir(cwd) + stage.execute_stage(stage_dict,stage_sets) if __name__ == '__main__': main() From b62590640edc3584f3679b7c876f0da6ad8c7d34 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Thu, 30 May 2024 19:34:55 +0000 Subject: [PATCH 22/89] Updates to stage.py - General cleanup - Rework determine_stage function - Rework execute_stage function Refs #2475 --- ush/python/pygfs/task/stage.py | 67 ++++++++++++++++------------------ 1 file changed, 31 insertions(+), 36 deletions(-) diff --git a/ush/python/pygfs/task/stage.py b/ush/python/pygfs/task/stage.py index 459c944319..dbb357a9fe 100644 --- a/ush/python/pygfs/task/stage.py +++ b/ush/python/pygfs/task/stage.py @@ -39,7 +39,7 @@ def __init__(self, config: Dict[str, Any]) -> None: self.task_config = AttrDict(**self.config, **self.runtime_config) @logit(logger) - def determine_stage(self, stage_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str, Any]]): + def determine_stage(self, stage_dict: Dict[str, Any]) -> List[str]: """Determine which initial condition files need to be placed in ROTDIR. Parameters @@ -49,52 +49,39 @@ def determine_stage(self, stage_dict: Dict[str, Any]) -> (Dict[str, Any], List[D Return ------ - stage_set : Dict[str, Any] - Set of FileHandler instructions to copy files to the ROTDIR + stage_sets : List + List of yamls to parse for staging """ - stage_parm = os.path.join(stage_dict.PARMgfs, "stage") - - # Add the os.path.exists function to the dict for yaml parsing - stage_dict['path_exists'] = os.path.exists + stage_sets = [] - if not os.path.isdir(stage_dict.ROTDIR): - raise FileNotFoundError(f"FATAL ERROR: The ROTDIR ({stage_dict.ROTDIR}) does not exist!") - - if stage_dict.RUN == "gfs" or stage_dict.RUN == "gdas": - if stage_dict.MODE == "cycled": - master_yaml = "master_cycled.yaml.j2" - elif stage_dict.MODE == "forecast-only": - #master_yaml = "master_forecast_only.yaml.j2" - master_yaml = "fv3_cold.yaml.j2" - elif stage_dict.RUN == "gefs": - raise NotImplementedError("FATAL ERROR: Staging is not yet set up for GEFS runs") - elif stage_dict.RUN == "sfs": - raise NotImplementedError("FATAL ERROR: Staging is not yet set up for SFS runs") + if stage_dict.EXP_WARM_START: + stage_sets.append("fv3_warm.yaml.j2") else: - raise ValueError(f"FATAL ERROR: Staging is not enabled for {stage_dict.RUN} runs") - - #parsed_sets = parse_j2yaml(os.path.join(stage_parm, master_yaml), stage_dict) - stage_set = parse_j2yaml(os.path.join(stage_parm, master_yaml), stage_dict) - #print(f'parsed_sets = {parsed_sets}') + stage_sets.append("fv3_cold.yaml.j2") - #stage_sets = [] + if stage_dict.DO_WAVE: + stage_sets.append("wave.yaml.j2") - #for dataset in parsed_sets.datasets.values(): + if stage_dict.DO_OCN: + stage_sets.append("ocean.yaml.j2") - # dataset["fileset"] = Stage._create_fileset(dataset) + if stage_dict.DO_ICE: + stage_sets.append("ice.yaml.j2") - # stage_sets.append(dataset) + if stage_dict.DO_NEST: + stage_sets.append("fv3_nest.yaml.j2") - #return stage_sets - return stage_set + return stage_sets @logit(logger) - def execute_stage(self, stage_set: Dict[str, Any]) -> None: + def execute_stage(self, stage_dict: Dict[str, Any], stage_sets: List) -> None: """Perform local staging of initial condition files. Parameters ---------- + stage_sets : List + List of stage sets to send to FileHandler stage_set : Dict[str, Any] FileHandler instructions to populate ROTDIR with @@ -103,7 +90,15 @@ def execute_stage(self, stage_set: Dict[str, Any]) -> None: None """ - # Copy files to ROTDIR - for key in stage_set.keys(): - # print(f'key = {key}') - FileHandler(stage_set[key]).sync() + stage_parm = os.path.join(stage_dict.PARMgfs, "stage") + + if not os.path.isdir(stage_dict.ROTDIR): + raise FileNotFoundError(f"FATAL ERROR: The ROTDIR ({stage_dict.ROTDIR}) does not exist!") + + for set_yaml in stage_sets: + + stage_set = parse_j2yaml(os.path.join(stage_parm, set_yaml), stage_dict) + + # Copy files to ROTDIR + for key in stage_set.keys(): + FileHandler(stage_set[key]).sync() From 4e947348f4106d1bd2627e218650e8bc76619917 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Thu, 30 May 2024 19:39:19 +0000 Subject: [PATCH 23/89] Yaml updates - Delete da.yaml.j2; will remake in follow-up work - Update fv3_warm.yaml.j2 to not use src/head variables Refs #2475 --- parm/stage/da.yaml.j2 | 6 ------ parm/stage/fv3_warm.yaml.j2 | 5 ++--- 2 files changed, 2 insertions(+), 9 deletions(-) delete mode 100644 parm/stage/da.yaml.j2 diff --git a/parm/stage/da.yaml.j2 b/parm/stage/da.yaml.j2 deleted file mode 100644 index b5aede5a7a..0000000000 --- a/parm/stage/da.yaml.j2 +++ /dev/null @@ -1,6 +0,0 @@ -#for ftype in ['abias', 'abias_pc', 'abias_air', 'radstat']: -da: - name: "DA" - source: "{{ BASE_CPLIC }}/{{ CPL_ICEIC }}/{{ cycle_YMD }}{{ cycle_HH }}/{{ MEMDIR }}/ice" - required: - - "{{ COM_ICE_RESTART_PREV | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.cice_model.res.nc" diff --git a/parm/stage/fv3_warm.yaml.j2 b/parm/stage/fv3_warm.yaml.j2 index efd2a3a4d8..7f97d6c571 100644 --- a/parm/stage/fv3_warm.yaml.j2 +++ b/parm/stage/fv3_warm.yaml.j2 @@ -5,12 +5,11 @@ fv3_warm: mkdir: - "{{ COM_ATMOS_RESTART_PREV }}" copy: - {% set head = "{{ BASE_CPLIC }}/{{ CPL_ATMIC }}/{{ cycle_YMDH }}/{{ MEMDIR }}/atmos/{{ cycle_YMD }}.{{ cycle_HH }}0000." %} {% for ftype in ["coupler.res", "fv_core.res.nc"] %} - - ["{{ head }}{{ ftype }}", "{{ COM_ATMOS_RESTART_PREV }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.{{ ftype }}"] + - ["{{ BASE_CPLIC }}/{{ CPL_ATMIC }}/{{ cycle_YMDH }}/{{ MEMDIR }}/atmos/{{ cycle_YMD }}.{{ cycle_HH }}0000.{{ ftype }}", "{{ COM_ATMOS_RESTART_PREV }}"] {% endfor %} {% for ftype in ["ca_data", "fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data sfc_data"] %} {% for ntile in range(1, ntiles) %} - - ["{{ head }}{{ ftype }}.tile{{ ntile }}.nc", "{{ COM_ATMOS_RESTART_PREV }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.{{ ftype }}.tile{{ ntile }}.nc"] + - ["{{ BASE_CPLIC }}/{{ CPL_ATMIC }}/{{ cycle_YMDH }}/{{ MEMDIR }}/atmos/{{ cycle_YMD }}.{{ cycle_HH }}0000.{{ ftype }}.tile{{ ntile }}.nc", "{{ COM_ATMOS_RESTART_PREV }}"] {% endfor %} # ntile {% endfor %} # ftype From b7301f025c7101f1a560dc5727e49f545058baa2 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Fri, 31 May 2024 14:40:23 +0000 Subject: [PATCH 24/89] Move COM declares inside MEMDIR loop Refs #2475 --- jobs/JGLOBAL_STAGE_IC | 54 +++++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/jobs/JGLOBAL_STAGE_IC b/jobs/JGLOBAL_STAGE_IC index 61e437e278..63ae1aff5e 100755 --- a/jobs/JGLOBAL_STAGE_IC +++ b/jobs/JGLOBAL_STAGE_IC @@ -15,28 +15,6 @@ GDATE=$(date --utc -d "${PDY} ${cyc} - ${assim_freq} hours" +%Y%m%d%H) gPDY="${GDATE:0:8}" gcyc="${GDATE:8:2}" -# Declare COMs -if [[ ${EXP_WARM_START:-".false."} = ".true." ]]; then - RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL - RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl COM_MED_RESTART_PREV:COM_MED_RESTART_TMPL - export COM_ATMOS_RESTART_PREV COM_MED_RESTART_PREV -else - YMD=${PDY} HH=${cyc} declare_from_tmpl COM_ATMOS_INPUT - export COM_ATMOS_INPUT -fi -if [[ "${DO_OCN:-}" = "YES" ]]; then - RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl COM_OCEAN_RESTART_PREV:COM_OCEAN_RESTART_TMPL - export COM_OCEAN_RESTART_PREV -fi -if [[ "${DO_ICE:-}" = "YES" ]]; then - RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl COM_ICE_RESTART_PREV:COM_ICE_RESTART_TMPL - export COM_ICE_RESTART_PREV -fi -if [[ "${DO_WAVE:-}" = "YES" ]]; then - YMD=${PDY} HH=${cyc} declare_from_tmpl COM_WAVE_RESTART - export COM_WAVE_RESTART -fi - # Define MEMDIR_ARRAY MEMDIR_ARRAY=() if [[ "${RUN:-}" = "gefs" ]]; then @@ -51,15 +29,35 @@ fi # Initialize return code err=0 -error_message() { - echo "FATAL ERROR: Unable to copy ${1} to ${2} (Error code ${3})" -} - ############################################################### for MEMDIR in "${MEMDIR_ARRAY[@]}"; do + # Export MEMDIR; need even if empty export MEMDIR=${MEMDIR:-} - # Execute the Script + + # Declare COMs + if [[ ${EXP_WARM_START:-".false."} = ".true." ]]; then + RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL + RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl COM_MED_RESTART_PREV:COM_MED_RESTART_TMPL + export COM_ATMOS_RESTART_PREV COM_MED_RESTART_PREV + else + YMD=${PDY} HH=${cyc} declare_from_tmpl COM_ATMOS_INPUT + export COM_ATMOS_INPUT + fi + if [[ "${DO_OCN:-}" = "YES" ]]; then + RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl COM_OCEAN_RESTART_PREV:COM_OCEAN_RESTART_TMPL + export COM_OCEAN_RESTART_PREV + fi + if [[ "${DO_ICE:-}" = "YES" ]]; then + RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl COM_ICE_RESTART_PREV:COM_ICE_RESTART_TMPL + export COM_ICE_RESTART_PREV + fi + if [[ "${DO_WAVE:-}" = "YES" ]]; then + YMD=${PDY} HH=${cyc} declare_from_tmpl COM_WAVE_RESTART + export COM_WAVE_RESTART + fi + + # Execute staging "${SCRgfs}/exglobal_stage_ic.py" done # for MEMDIR in "${MEMDIR_ARRAY[@]}"; do @@ -67,7 +65,7 @@ done # for MEMDIR in "${MEMDIR_ARRAY[@]}"; do ############################################################### # Check for errors and exit if any of the above failed if [[ "${err}" -ne 0 ]]; then - echo "FATAL ERROR: Unable to copy ICs from ${BASE_CPLIC} to ${ROTDIR}; ABORT!" + echo "FATAL ERROR: Unable to copy ICs to ${ROTDIR}; ABORT!" exit "${err}" fi From 4a92a107e0604f803a2d96beca0d2b85fb59b729 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Fri, 31 May 2024 18:06:04 +0000 Subject: [PATCH 25/89] Remove yamls not yet needed Also clean up unneeded python imports Refs #2475 --- parm/stage/fv3_nest.yaml.j2 | 44 ------------------------------------ parm/stage/fv3_warm.yaml.j2 | 15 ------------ parm/stage/mediator.yaml.j2 | 8 ------- scripts/exglobal_stage_ic.py | 7 +----- 4 files changed, 1 insertion(+), 73 deletions(-) delete mode 100644 parm/stage/fv3_nest.yaml.j2 delete mode 100644 parm/stage/fv3_warm.yaml.j2 delete mode 100644 parm/stage/mediator.yaml.j2 diff --git a/parm/stage/fv3_nest.yaml.j2 b/parm/stage/fv3_nest.yaml.j2 deleted file mode 100644 index bc33eb729a..0000000000 --- a/parm/stage/fv3_nest.yaml.j2 +++ /dev/null @@ -1,44 +0,0 @@ -#WARM -# for ftype in ca_data fv_core.res fv_srf_wnd.res fv_tracer.res phy_data sfc_data; do -# src="${BASE_CPLIC}/${CPL_ATMIC:-}/${PDY}${cyc}/${MEMDIR}/atmos/${PDY}.${cyc}0000.${ftype}.tile${tt}.nc" -# if (( tt > 6 )) ; then -# tgt="${COM_ATMOS_RESTART_PREV}/${PDY}.${cyc}0000.${ftype}.nest0$((tt-5)).tile${tt}.nc" -# fi -#COLD -# for ftype in gfs_data sfc_data; do -# for ((tt = 1; tt <= ntiles; tt++)); do -# src="${BASE_CPLIC}/${CPL_ATMIC:-}/${PDY}${cyc}/${MEMDIR}/atmos/${ftype}.tile${tt}.nc" -# tgt="${COM_ATMOS_INPUT}/${ftype}.tile${tt}.nc" -# if (( ntiles > 6 )); then -# ${NLN} "${COM_ATMOS_INPUT}/${ftype}.tile7.nc" "${COM_ATMOS_INPUT}/${ftype}.nest02.tile7.nc" -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMD = current_cycle | to_YMD %} -{% set cycle_YMDH = current_cycle | to_YMDH %} -{% set ntile = 7 %} -fv3_nest: - mkdir: - - "{{ COM_ATMOS_RESTART_PREV }}" - copy: - #--------------- - # WARM START - #--------------- - {% if EXP_WARM_START %} - {% set head = "{{ BASE_CPLIC }}/{{ CPL_ATMIC }}/{{ cycle_YMD }}{{ cycle_HH }}/{{ MEMDIR }}/atmos/{{ cycle_YMD }}.{{ cycle_HH }}0000." %} - {% set ntile = 7 %} - {% for ftype in ca_data fv_core.res fv_srf_wnd.res fv_tracer.res phy_data sfc_data %} - #TODO: figure out how to copy one name to the other in here - - "{{ head }}{{ ftype }}.nest0{{ ntile-5 }}.tile{{ ntile }}.nc" - {% endfor %} # ftype - #--------------- # end warm start - {% else %} - #--------------- - # COLD START - #--------------- - {% set head = "{{ COM_ATMOS_INPUT | relpath(ROTDIR) }}" %} - - "{{ head }}/gfs_ctrl.nc" - {% for ftype in gfs_data sfc_data %} - #TODO: figure out how to copy one name to the other in here - - "{{ head }}/{{ ftype }}.tile{{ ntile }}.nc" - {% endfor %} # ftype - #--------------- # end cold start - {% endif %} diff --git a/parm/stage/fv3_warm.yaml.j2 b/parm/stage/fv3_warm.yaml.j2 deleted file mode 100644 index 7f97d6c571..0000000000 --- a/parm/stage/fv3_warm.yaml.j2 +++ /dev/null @@ -1,15 +0,0 @@ -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMD = current_cycle | to_YMD %} -{% set cycle_YMDH = current_cycle | to_YMDH %} -fv3_warm: - mkdir: - - "{{ COM_ATMOS_RESTART_PREV }}" - copy: - {% for ftype in ["coupler.res", "fv_core.res.nc"] %} - - ["{{ BASE_CPLIC }}/{{ CPL_ATMIC }}/{{ cycle_YMDH }}/{{ MEMDIR }}/atmos/{{ cycle_YMD }}.{{ cycle_HH }}0000.{{ ftype }}", "{{ COM_ATMOS_RESTART_PREV }}"] - {% endfor %} - {% for ftype in ["ca_data", "fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data sfc_data"] %} - {% for ntile in range(1, ntiles) %} - - ["{{ BASE_CPLIC }}/{{ CPL_ATMIC }}/{{ cycle_YMDH }}/{{ MEMDIR }}/atmos/{{ cycle_YMD }}.{{ cycle_HH }}0000.{{ ftype }}.tile{{ ntile }}.nc", "{{ COM_ATMOS_RESTART_PREV }}"] - {% endfor %} # ntile - {% endfor %} # ftype diff --git a/parm/stage/mediator.yaml.j2 b/parm/stage/mediator.yaml.j2 deleted file mode 100644 index 5f1e4d20ba..0000000000 --- a/parm/stage/mediator.yaml.j2 +++ /dev/null @@ -1,8 +0,0 @@ -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMD = current_cycle | to_YMD %} -{% set cycle_YMDH = current_cycle | to_YMDH %} -mediator: - mkdir: - - "{{ COM_MED_RESTART_PREV }}" - copy: - - ["{{ BASE_CPLIC }}/{{ CPL_MEDIC }}/{{ cycle_YMDH }}/{{ MEMDIR }}/med/{{ cycle_YMD }}.{{ cycle_HH }}0000.ufs.cpld.cpl.r.nc", "{{ COM_MED_RESTART_PREV }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.ufs.cpld.cpl.r.nc"] diff --git a/scripts/exglobal_stage_ic.py b/scripts/exglobal_stage_ic.py index aad20252ea..4fbcf1f8cf 100755 --- a/scripts/exglobal_stage_ic.py +++ b/scripts/exglobal_stage_ic.py @@ -3,7 +3,7 @@ import os from pygfs.task.stage import Stage -from wxflow import AttrDict, Logger, cast_strdict_as_dtypedict, chdir, logit +from wxflow import AttrDict, Logger, cast_strdict_as_dtypedict, logit # Initialize root logger logger = Logger(level=os.environ.get("LOGGING_LEVEL", "DEBUG"), colored_log=True) @@ -21,7 +21,6 @@ def main(): 'current_cycle','EXP_WARM_START','CDUMP','rCDUMP', 'ROTDIR','PARMgfs','ntiles','MEMDIR', 'BASE_CPLIC','waveGRD','OCNRES','USE_OCN_PERTURB_FILES', - #TODO: Need this for mediator#'CPL_MEDIC', 'CPL_ATMIC','CPL_ICEIC','CPL_OCNIC','CPL_WAVIC'] stage_dict = AttrDict() @@ -33,10 +32,6 @@ def main(): if key.startswith("COM"): stage_dict[key] = stage.task_config[key] - #TEST PRINT - for key in stage_dict: - print(f'{key} = {stage_dict[key]}') - # Add the os.path.exists function to the dict for yaml parsing stage_dict['path_exists'] = os.path.exists From 35cb29d952d284eed7c9e3f26568919b710a5942 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Fri, 31 May 2024 18:07:13 +0000 Subject: [PATCH 26/89] Remove nest option and cleanup python imports Will add nest option in future task Refs #2475 --- ush/python/pygfs/task/stage.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/ush/python/pygfs/task/stage.py b/ush/python/pygfs/task/stage.py index dbb357a9fe..895306bcde 100644 --- a/ush/python/pygfs/task/stage.py +++ b/ush/python/pygfs/task/stage.py @@ -1,14 +1,11 @@ #!/usr/bin/env python3 -import glob import os -import shutil -from datetime import timedelta from logging import getLogger from typing import Any, Dict, List -from wxflow import (AttrDict, FileHandler, Hsi, Htar, Task, cast_strdict_as_dtypedict, - chgrp, get_gid, logit, mkdir_p, parse_j2yaml, rm_p, strftime, +from wxflow import (AttrDict, FileHandler, Task, cast_strdict_as_dtypedict, + logit, parse_j2yaml, strftime, to_YMD, to_YMDH, Template, TemplateConstants) logger = getLogger(__name__.split('.')[-1]) @@ -69,9 +66,6 @@ def determine_stage(self, stage_dict: Dict[str, Any]) -> List[str]: if stage_dict.DO_ICE: stage_sets.append("ice.yaml.j2") - if stage_dict.DO_NEST: - stage_sets.append("fv3_nest.yaml.j2") - return stage_sets @logit(logger) From 21c1d225bbb2e455e523a25bc6b9679684a70453 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Fri, 31 May 2024 18:09:58 +0000 Subject: [PATCH 27/89] Undo change to setup_expt.py Refs #2475 --- workflow/setup_expt.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/workflow/setup_expt.py b/workflow/setup_expt.py index 19235f9486..97d25dc15a 100755 --- a/workflow/setup_expt.py +++ b/workflow/setup_expt.py @@ -468,7 +468,7 @@ def _gefs_args(parser): description = """ Setup files and directories to start a GFS parallel.\n Create EXPDIR, copy config files.\n - Create ROTDIR experiment directory, + Create ROTDIR experiment directory structure, """ parser = ArgumentParser(description=description, @@ -580,6 +580,7 @@ def main(*argv): if create_rotdir: makedirs_if_missing(rotdir) + fill_ROTDIR(host, user_inputs) if create_expdir: makedirs_if_missing(expdir) From a4cec795d75d323c03c62e335a01cd7dbbf2dc82 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Mon, 3 Jun 2024 13:59:36 +0000 Subject: [PATCH 28/89] Adjust MEMDIR export in JGLOBAL_STAGE_IC Refs #2475 --- jobs/JGLOBAL_STAGE_IC | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/JGLOBAL_STAGE_IC b/jobs/JGLOBAL_STAGE_IC index 63ae1aff5e..17a0c2c847 100755 --- a/jobs/JGLOBAL_STAGE_IC +++ b/jobs/JGLOBAL_STAGE_IC @@ -33,7 +33,7 @@ err=0 for MEMDIR in "${MEMDIR_ARRAY[@]}"; do # Export MEMDIR; need even if empty - export MEMDIR=${MEMDIR:-} + export MEMDIR # Declare COMs if [[ ${EXP_WARM_START:-".false."} = ".true." ]]; then From 87302c2a1abfa08cbce14ea38818e01ba4f75ed9 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Mon, 3 Jun 2024 14:25:51 +0000 Subject: [PATCH 29/89] Add fv3_warm and fv3_nest yamls back in - Add DO_NEST back into stage.py - Add back in reworked yamls for fv3_warm and the nest Refs #2475 --- parm/stage/fv3_nest.yaml.j2 | 17 +++++++++++++++++ parm/stage/fv3_warm.yaml.j2 | 15 +++++++++++++++ ush/python/pygfs/task/stage.py | 3 +++ 3 files changed, 35 insertions(+) create mode 100644 parm/stage/fv3_nest.yaml.j2 create mode 100644 parm/stage/fv3_warm.yaml.j2 diff --git a/parm/stage/fv3_nest.yaml.j2 b/parm/stage/fv3_nest.yaml.j2 new file mode 100644 index 0000000000..22d5c98490 --- /dev/null +++ b/parm/stage/fv3_nest.yaml.j2 @@ -0,0 +1,17 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMD = current_cycle | to_YMD %} +{% set cycle_YMDH = current_cycle | to_YMDH %} +{% set ntile = 7 %} +fv3_nest: + mkdir: + - "{{ COM_ATMOS_RESTART_PREV }}" + copy: + {% if EXP_WARM_START %} + {% for ftype in ca_data fv_core.res fv_srf_wnd.res fv_tracer.res phy_data sfc_data %} + - ["{{ BASE_CPLIC }}/{{ CPL_ATMIC }}/{{ cycle_YMDH }}/{{ MEMDIR }}/atmos/{{ cycle_YMD }}.{{ cycle_HH }}0000.{{ ftype }}.tile{{ ntile }}.nc", "{{ COM_ATMOS_RESTART_PREV }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.{{ ftype }}.nest0{{ ntile-5 }}.tile{{ ntile }}.nc"] + {% endfor %} # ftype + {% else %} + {% for ftype in gfs_data sfc_data %} + - ["{{ COM_ATMOS_INPUT }}/{{ ftype }}.tile{{ ntile }}.nc", "{{ COM_ATMOS_INPUT }}/{{ ftype }}.nest02.tile{{ ntile }}.nc"] + {% endfor %} # ftype + {% endif %} # cold-start diff --git a/parm/stage/fv3_warm.yaml.j2 b/parm/stage/fv3_warm.yaml.j2 new file mode 100644 index 0000000000..9a460b78f1 --- /dev/null +++ b/parm/stage/fv3_warm.yaml.j2 @@ -0,0 +1,15 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMD = current_cycle | to_YMD %} +{% set cycle_YMDH = current_cycle | to_YMDH %} +fv3_warm: + mkdir: + - "{{ COM_ATMOS_RESTART_PREV }}" + copy: + {% for ftype in ["coupler.res", "fv_core.res.nc"] %} + - ["{{ BASE_CPLIC }}/{{ CPL_ATMIC }}/{{ cycle_YMDH }}/{{ MEMDIR }}/atmos/{{ cycle_YMD }}.{{ cycle_HH }}0000.{{ ftype }}", "{{ COM_ATMOS_RESTART_PREV }}"] + {% endfor %} + {% for ftype in ["ca_data", "fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data sfc_data"] %} + {% for ntile in range(1, ntiles + 1) %} + - ["{{ BASE_CPLIC }}/{{ CPL_ATMIC }}/{{ cycle_YMDH }}/{{ MEMDIR }}/atmos/{{ cycle_YMD }}.{{ cycle_HH }}0000.{{ ftype }}.tile{{ ntile }}.nc", "{{ COM_ATMOS_RESTART_PREV }}"] + {% endfor %} # ntile + {% endfor %} # ftype diff --git a/ush/python/pygfs/task/stage.py b/ush/python/pygfs/task/stage.py index 895306bcde..9a1bd977fe 100644 --- a/ush/python/pygfs/task/stage.py +++ b/ush/python/pygfs/task/stage.py @@ -66,6 +66,9 @@ def determine_stage(self, stage_dict: Dict[str, Any]) -> List[str]: if stage_dict.DO_ICE: stage_sets.append("ice.yaml.j2") + if stage_dict.DO_NEST: + stage_sets.append("fv3_nest.yaml.j2") + return stage_sets @logit(logger) From 9be34aab14fd3ffc4a8517cc482cd79ffc780ea9 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Mon, 3 Jun 2024 15:14:42 +0000 Subject: [PATCH 30/89] Add whitespaces to address pynorms errors Add extra whitespaces in exglobal_stage_ic.py to address E231 error. Refs #2475 --- scripts/exglobal_stage_ic.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/exglobal_stage_ic.py b/scripts/exglobal_stage_ic.py index 4fbcf1f8cf..ce0eabeb0b 100755 --- a/scripts/exglobal_stage_ic.py +++ b/scripts/exglobal_stage_ic.py @@ -17,11 +17,11 @@ def main(): stage = Stage(config) #Pull out all the configuration keys needed to run stage job - keys = ['RUN','MODE','DO_WAVE','DO_OCN','DO_ICE','DO_NEST', - 'current_cycle','EXP_WARM_START','CDUMP','rCDUMP', - 'ROTDIR','PARMgfs','ntiles','MEMDIR', - 'BASE_CPLIC','waveGRD','OCNRES','USE_OCN_PERTURB_FILES', - 'CPL_ATMIC','CPL_ICEIC','CPL_OCNIC','CPL_WAVIC'] + keys = ['RUN', 'MODE', 'DO_WAVE', 'DO_OCN', 'DO_ICE', 'DO_NEST', + 'current_cycle', 'EXP_WARM_START', 'CDUMP', 'rCDUMP', + 'ROTDIR', 'PARMgfs', 'ntiles', 'MEMDIR', + 'BASE_CPLIC', 'waveGRD', 'OCNRES', 'USE_OCN_PERTURB_FILES', + 'CPL_ATMIC', 'CPL_ICEIC', 'CPL_OCNIC', 'CPL_WAVIC'] stage_dict = AttrDict() for key in keys: @@ -39,7 +39,7 @@ def main(): stage_sets = stage.determine_stage(stage_dict) # Stage ICs - stage.execute_stage(stage_dict,stage_sets) + stage.execute_stage(stage_dict, stage_sets) if __name__ == '__main__': main() From c2232cdc62533516a990f3f65bba3f49b4a9600b Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Mon, 3 Jun 2024 15:22:37 +0000 Subject: [PATCH 31/89] Address pynorm errors in exglobal_stage_ic.py Refs #2475 --- scripts/exglobal_stage_ic.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/exglobal_stage_ic.py b/scripts/exglobal_stage_ic.py index ce0eabeb0b..ea79af1565 100755 --- a/scripts/exglobal_stage_ic.py +++ b/scripts/exglobal_stage_ic.py @@ -8,6 +8,7 @@ # Initialize root logger logger = Logger(level=os.environ.get("LOGGING_LEVEL", "DEBUG"), colored_log=True) + @logit(logger) def main(): @@ -16,7 +17,7 @@ def main(): # Instantiate the Stage object stage = Stage(config) - #Pull out all the configuration keys needed to run stage job + # Pull out all the configuration keys needed to run stage job keys = ['RUN', 'MODE', 'DO_WAVE', 'DO_OCN', 'DO_ICE', 'DO_NEST', 'current_cycle', 'EXP_WARM_START', 'CDUMP', 'rCDUMP', 'ROTDIR', 'PARMgfs', 'ntiles', 'MEMDIR', @@ -41,5 +42,6 @@ def main(): # Stage ICs stage.execute_stage(stage_dict, stage_sets) + if __name__ == '__main__': main() From 561be58e666bb287eb52aa6c7623c2813616cab2 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Mon, 3 Jun 2024 17:03:17 +0000 Subject: [PATCH 32/89] Resolve pynorms error in stage.py Refs #2475 --- ush/python/pygfs/task/stage.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ush/python/pygfs/task/stage.py b/ush/python/pygfs/task/stage.py index 9a1bd977fe..bd300c4bc3 100644 --- a/ush/python/pygfs/task/stage.py +++ b/ush/python/pygfs/task/stage.py @@ -10,6 +10,7 @@ logger = getLogger(__name__.split('.')[-1]) + class Stage(Task): """Task to stage initial conditions """ From 3f08f035d6c8e51fbc8ebba6e55767e15864cbb1 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Mon, 3 Jun 2024 17:20:05 +0000 Subject: [PATCH 33/89] Indent secondary yaml loops Improve readability of the stage yamls Refs #2475 --- parm/stage/fv3_cold.yaml.j2 | 4 ++-- parm/stage/fv3_nest.yaml.j2 | 10 +++++----- parm/stage/fv3_warm.yaml.j2 | 4 ++-- parm/stage/ocean.yaml.j2 | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/parm/stage/fv3_cold.yaml.j2 b/parm/stage/fv3_cold.yaml.j2 index f4d80131ae..3e8d96d726 100644 --- a/parm/stage/fv3_cold.yaml.j2 +++ b/parm/stage/fv3_cold.yaml.j2 @@ -5,7 +5,7 @@ fv3_cold: copy: - ["{{ BASE_CPLIC }}/{{ CPL_ATMIC }}/{{ cycle_YMDH }}/{{ MEMDIR }}/atmos/gfs_ctrl.nc", "{{ COM_ATMOS_INPUT }}"] {% for ftype in ["gfs_data", "sfc_data"] %} - {% for ntile in range(1, ntiles + 1) %} + {% for ntile in range(1, ntiles + 1) %} - ["{{ BASE_CPLIC }}/{{ CPL_ATMIC }}/{{ cycle_YMDH }}/{{ MEMDIR }}/atmos/{{ ftype }}.tile{{ ntile }}.nc", "{{ COM_ATMOS_INPUT }}"] - {% endfor %} # ntile + {% endfor %} # ntile {% endfor %} # ftype diff --git a/parm/stage/fv3_nest.yaml.j2 b/parm/stage/fv3_nest.yaml.j2 index 22d5c98490..9e2c4b66ab 100644 --- a/parm/stage/fv3_nest.yaml.j2 +++ b/parm/stage/fv3_nest.yaml.j2 @@ -6,12 +6,12 @@ fv3_nest: mkdir: - "{{ COM_ATMOS_RESTART_PREV }}" copy: - {% if EXP_WARM_START %} - {% for ftype in ca_data fv_core.res fv_srf_wnd.res fv_tracer.res phy_data sfc_data %} + {% if EXP_WARM_START == True %} + {% for ftype in ca_data fv_core.res fv_srf_wnd.res fv_tracer.res phy_data sfc_data %} - ["{{ BASE_CPLIC }}/{{ CPL_ATMIC }}/{{ cycle_YMDH }}/{{ MEMDIR }}/atmos/{{ cycle_YMD }}.{{ cycle_HH }}0000.{{ ftype }}.tile{{ ntile }}.nc", "{{ COM_ATMOS_RESTART_PREV }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.{{ ftype }}.nest0{{ ntile-5 }}.tile{{ ntile }}.nc"] - {% endfor %} # ftype + {% endfor %} # ftype {% else %} - {% for ftype in gfs_data sfc_data %} + {% for ftype in gfs_data sfc_data %} - ["{{ COM_ATMOS_INPUT }}/{{ ftype }}.tile{{ ntile }}.nc", "{{ COM_ATMOS_INPUT }}/{{ ftype }}.nest02.tile{{ ntile }}.nc"] - {% endfor %} # ftype + {% endfor %} # ftype {% endif %} # cold-start diff --git a/parm/stage/fv3_warm.yaml.j2 b/parm/stage/fv3_warm.yaml.j2 index 9a460b78f1..3ca9f7c023 100644 --- a/parm/stage/fv3_warm.yaml.j2 +++ b/parm/stage/fv3_warm.yaml.j2 @@ -9,7 +9,7 @@ fv3_warm: - ["{{ BASE_CPLIC }}/{{ CPL_ATMIC }}/{{ cycle_YMDH }}/{{ MEMDIR }}/atmos/{{ cycle_YMD }}.{{ cycle_HH }}0000.{{ ftype }}", "{{ COM_ATMOS_RESTART_PREV }}"] {% endfor %} {% for ftype in ["ca_data", "fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data sfc_data"] %} - {% for ntile in range(1, ntiles + 1) %} + {% for ntile in range(1, ntiles + 1) %} - ["{{ BASE_CPLIC }}/{{ CPL_ATMIC }}/{{ cycle_YMDH }}/{{ MEMDIR }}/atmos/{{ cycle_YMD }}.{{ cycle_HH }}0000.{{ ftype }}.tile{{ ntile }}.nc", "{{ COM_ATMOS_RESTART_PREV }}"] - {% endfor %} # ntile + {% endfor %} # ntile {% endfor %} # ftype diff --git a/parm/stage/ocean.yaml.j2 b/parm/stage/ocean.yaml.j2 index a7fee221ba..6597cb0d85 100644 --- a/parm/stage/ocean.yaml.j2 +++ b/parm/stage/ocean.yaml.j2 @@ -13,13 +13,13 @@ ocean: # Resolution based initial conditions #------------------------------------ {% if OCNRES == "025" %} - {% for nn in range(1, 3) %} + {% for nn in range(1, 3) %} - ["{{ BASE_CPLIC }}/{{ CPL_OCNIC }}/{{ cycle_YMDH }}/{{ MEMDIR }}/ocean/{{ cycle_YMD }}.{{ cycle_HH }}0000.MOM.res_{{ nn }}.nc", "{{ COM_OCEAN_RESTART_PREV }}"] - {% endfor %} + {% endfor %} {% endif %} #------------------------- # Ocean Perturbation Files #------------------------- - {% if MEMDIR and USE_OCN_PERTURB_FILES %} + {% if MEMDIR and USE_OCN_PERTURB_FILES == True %} - ["{{ BASE_CPLIC }}/{{ CPL_OCNIC }}/{{ cycle_YMDH }}/{{ MEMDIR }}/ocean/{{ cycle_YMD }}.{{ cycle_HH }}0000.mom6_increment.nc", "{{ COM_OCEAN_RESTART_PREV }}"] {% endif %} From f793d0bde9033fece5f2252e1393695a4879b80d Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Wed, 5 Jun 2024 14:03:09 +0000 Subject: [PATCH 34/89] Rename fv3 yamls to atmosphere yamls Refs #2475 --- parm/stage/{fv3_cold.yaml.j2 => atmosphere_cold.yaml.j2} | 0 parm/stage/{fv3_nest.yaml.j2 => atmosphere_nest.yaml.j2} | 0 parm/stage/{fv3_warm.yaml.j2 => atmosphere_warm.yaml.j2} | 0 ush/python/pygfs/task/stage.py | 6 +++--- 4 files changed, 3 insertions(+), 3 deletions(-) rename parm/stage/{fv3_cold.yaml.j2 => atmosphere_cold.yaml.j2} (100%) rename parm/stage/{fv3_nest.yaml.j2 => atmosphere_nest.yaml.j2} (100%) rename parm/stage/{fv3_warm.yaml.j2 => atmosphere_warm.yaml.j2} (100%) diff --git a/parm/stage/fv3_cold.yaml.j2 b/parm/stage/atmosphere_cold.yaml.j2 similarity index 100% rename from parm/stage/fv3_cold.yaml.j2 rename to parm/stage/atmosphere_cold.yaml.j2 diff --git a/parm/stage/fv3_nest.yaml.j2 b/parm/stage/atmosphere_nest.yaml.j2 similarity index 100% rename from parm/stage/fv3_nest.yaml.j2 rename to parm/stage/atmosphere_nest.yaml.j2 diff --git a/parm/stage/fv3_warm.yaml.j2 b/parm/stage/atmosphere_warm.yaml.j2 similarity index 100% rename from parm/stage/fv3_warm.yaml.j2 rename to parm/stage/atmosphere_warm.yaml.j2 diff --git a/ush/python/pygfs/task/stage.py b/ush/python/pygfs/task/stage.py index bd300c4bc3..31b23f10a5 100644 --- a/ush/python/pygfs/task/stage.py +++ b/ush/python/pygfs/task/stage.py @@ -54,9 +54,9 @@ def determine_stage(self, stage_dict: Dict[str, Any]) -> List[str]: stage_sets = [] if stage_dict.EXP_WARM_START: - stage_sets.append("fv3_warm.yaml.j2") + stage_sets.append("atmosphere_warm.yaml.j2") else: - stage_sets.append("fv3_cold.yaml.j2") + stage_sets.append("atmosphere_cold.yaml.j2") if stage_dict.DO_WAVE: stage_sets.append("wave.yaml.j2") @@ -68,7 +68,7 @@ def determine_stage(self, stage_dict: Dict[str, Any]) -> List[str]: stage_sets.append("ice.yaml.j2") if stage_dict.DO_NEST: - stage_sets.append("fv3_nest.yaml.j2") + stage_sets.append("atmosphere_nest.yaml.j2") return stage_sets From 60511e96054c92fdf86e94d2f77faa3e753a5cf2 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Fri, 14 Jun 2024 11:02:07 +0000 Subject: [PATCH 35/89] Replace BASE_CPLIC with BASE_IC Also add ICSDIR to config.base Refs #2475 --- parm/config/gefs/config.base | 3 ++- parm/config/gfs/config.base | 3 ++- workflow/hosts/gaea.yaml | 2 +- workflow/hosts/hera.yaml | 2 +- workflow/hosts/hercules.yaml | 2 +- workflow/hosts/jet.yaml | 2 +- workflow/hosts/orion.yaml | 2 +- workflow/hosts/s4.yaml | 2 +- workflow/hosts/wcoss2.yaml | 2 +- 9 files changed, 11 insertions(+), 9 deletions(-) diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base index 1b4f948349..a7687651d6 100644 --- a/parm/config/gefs/config.base +++ b/parm/config/gefs/config.base @@ -37,7 +37,8 @@ export USHgfs=${HOMEgfs}/ush export PACKAGEROOT="@PACKAGEROOT@" # TODO: set via prod_envir in Ops export COMROOT="@COMROOT@" # TODO: set via prod_envir in Ops export COMINsyn="@COMINsyn@" -export BASE_CPLIC="@BASE_CPLIC@" +export ICSDIR="@ICSDIR@" +export BASE_IC="@BASE_IC@" # USER specific paths export HOMEDIR="@HOMEDIR@" diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index 42fa3fd063..e61e261864 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -49,7 +49,8 @@ export PACKAGEROOT="@PACKAGEROOT@" # TODO: set via prod_envir in Ops export COMROOT="@COMROOT@" # TODO: set via prod_envir in Ops export COMINsyn="@COMINsyn@" export DMPDIR="@DMPDIR@" -export BASE_CPLIC="@BASE_CPLIC@" +export ICSDIR="@ICSDIR@" +export BASE_IC="@BASE_IC@" # Gempak from external models # Default locations are to dummy locations for testing diff --git a/workflow/hosts/gaea.yaml b/workflow/hosts/gaea.yaml index 7ca8420997..a05aad4f66 100644 --- a/workflow/hosts/gaea.yaml +++ b/workflow/hosts/gaea.yaml @@ -1,6 +1,6 @@ BASE_GIT: '/gpfs/f5/epic/proj-shared/global/glopara/data/git' DMPDIR: '/gpfs/f5/epic/proj-shared/global/glopara/data/dump' -BASE_CPLIC: '/gpfs/f5/epic/proj-shared/global/glopara/data/ICSDIR/prototype_ICs' +BASE_IC: '/gpfs/f5/epic/proj-shared/global/glopara/data/ICSDIR' PACKAGEROOT: '/gpfs/f5/epic/proj-shared/global/glopara/data/nwpara' COMROOT: '/gpfs/f5/epic/proj-shared/global/glopara/data/com' COMINsyn: '${COMROOT}/gfs/prod/syndat' diff --git a/workflow/hosts/hera.yaml b/workflow/hosts/hera.yaml index 8cf7363605..cfd5e7e657 100644 --- a/workflow/hosts/hera.yaml +++ b/workflow/hosts/hera.yaml @@ -1,6 +1,6 @@ BASE_GIT: '/scratch1/NCEPDEV/global/glopara/git' DMPDIR: '/scratch1/NCEPDEV/global/glopara/dump' -BASE_CPLIC: '/scratch1/NCEPDEV/global/glopara/data/ICSDIR/prototype_ICs' +BASE_IC: '/scratch1/NCEPDEV/global/glopara/data/ICSDIR' PACKAGEROOT: '/scratch1/NCEPDEV/global/glopara/nwpara' COMINsyn: '/scratch1/NCEPDEV/global/glopara/com/gfs/prod/syndat' HOMEDIR: '/scratch1/NCEPDEV/global/${USER}' diff --git a/workflow/hosts/hercules.yaml b/workflow/hosts/hercules.yaml index adebdfe23d..3b10f27e18 100644 --- a/workflow/hosts/hercules.yaml +++ b/workflow/hosts/hercules.yaml @@ -1,6 +1,6 @@ BASE_GIT: '/work/noaa/global/glopara/git_rocky9' DMPDIR: '/work/noaa/rstprod/dump' -BASE_CPLIC: '/work/noaa/global/glopara/data/ICSDIR/prototype_ICs' +BASE_IC: '/work/noaa/global/glopara/data/ICSDIR' PACKAGEROOT: '/work/noaa/global/glopara/nwpara' COMINsyn: '/work/noaa/global/glopara/com/gfs/prod/syndat' HOMEDIR: '/work/noaa/global/${USER}' diff --git a/workflow/hosts/jet.yaml b/workflow/hosts/jet.yaml index fd556fadc7..e3c3dab3e8 100644 --- a/workflow/hosts/jet.yaml +++ b/workflow/hosts/jet.yaml @@ -1,6 +1,6 @@ BASE_GIT: '/lfs4/HFIP/hfv3gfs/glopara/git' DMPDIR: '/lfs4/HFIP/hfv3gfs/glopara/dump' -BASE_CPLIC: '/mnt/lfs4/HFIP/hfv3gfs/glopara/data/ICSDIR/prototype_ICs' +BASE_IC: '/mnt/lfs4/HFIP/hfv3gfs/glopara/data/ICSDIR' PACKAGEROOT: '/lfs4/HFIP/hfv3gfs/glopara/nwpara' COMINsyn: '/lfs4/HFIP/hfv3gfs/glopara/com/gfs/prod/syndat' HOMEDIR: '/lfs4/HFIP/hfv3gfs/${USER}' diff --git a/workflow/hosts/orion.yaml b/workflow/hosts/orion.yaml index ba289df1e3..f1e1282495 100644 --- a/workflow/hosts/orion.yaml +++ b/workflow/hosts/orion.yaml @@ -1,6 +1,6 @@ BASE_GIT: '/work/noaa/global/glopara/git' DMPDIR: '/work/noaa/rstprod/dump' -BASE_CPLIC: '/work/noaa/global/glopara/data/ICSDIR/prototype_ICs' +BASE_IC: '/work/noaa/global/glopara/data/ICSDIR' PACKAGEROOT: '/work/noaa/global/glopara/nwpara' COMINsyn: '/work/noaa/global/glopara/com/gfs/prod/syndat' HOMEDIR: '/work/noaa/global/${USER}' diff --git a/workflow/hosts/s4.yaml b/workflow/hosts/s4.yaml index 543912cf23..d852450b8f 100644 --- a/workflow/hosts/s4.yaml +++ b/workflow/hosts/s4.yaml @@ -1,6 +1,6 @@ BASE_GIT: '/data/prod/glopara/git' DMPDIR: '/data/prod/glopara/dump' -BASE_CPLIC: '/data/prod/glopara/coupled_ICs' +BASE_IC: '/data/prod/glopara/coupled_ICs' PACKAGEROOT: '/data/prod/glopara/nwpara' COMINsyn: '/data/prod/glopara/com/gfs/prod/syndat' HOMEDIR: '/data/users/${USER}' diff --git a/workflow/hosts/wcoss2.yaml b/workflow/hosts/wcoss2.yaml index 4943495289..49fd37c72a 100644 --- a/workflow/hosts/wcoss2.yaml +++ b/workflow/hosts/wcoss2.yaml @@ -1,6 +1,6 @@ BASE_GIT: '/lfs/h2/emc/global/save/emc.global/git' DMPDIR: '/lfs/h2/emc/dump/noscrub/dump' -BASE_CPLIC: '/lfs/h2/emc/global/noscrub/emc.global/data/ICSDIR/prototype_ICs' +BASE_IC: '/lfs/h2/emc/global/noscrub/emc.global/data/ICSDIR' PACKAGEROOT: '${PACKAGEROOT:-"/lfs/h1/ops/prod/packages"}' COMINsyn: '/lfs/h1/ops/prod/com/gfs/v16.3/syndat' HOMEDIR: '/lfs/h2/emc/global/noscrub/${USER}' From f9e3a9cee12c99c794be8c2a6116f3199a662985 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Fri, 14 Jun 2024 15:19:51 +0000 Subject: [PATCH 36/89] Update JGLOBAL_STAGE_IC for cycled mode Also update wave COM variable Refs #2475 --- jobs/JGLOBAL_STAGE_IC | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/jobs/JGLOBAL_STAGE_IC b/jobs/JGLOBAL_STAGE_IC index 17a0c2c847..a7fb5a0e58 100755 --- a/jobs/JGLOBAL_STAGE_IC +++ b/jobs/JGLOBAL_STAGE_IC @@ -17,9 +17,14 @@ gcyc="${GDATE:8:2}" # Define MEMDIR_ARRAY MEMDIR_ARRAY=() -if [[ "${RUN:-}" = "gefs" ]]; then +if [[ "${RUN:-}" = "enkfgdas" || "${RUN:-}" = "gefs" ]]; then + if [[ "${RUN:-}" = "gefs" ]]; then + ii_start=0 + elif [[ "${RUN:-}" = "enkfgdas" ]]; then + ii_start=1 + fi # Populate the member_dirs array based on the value of NMEM_ENS - for ((ii = 0; ii <= "${NMEM_ENS:-0}"; ii++)); do + for ((ii = "${ii_start}"; ii <= "${NMEM_ENS:-0}"; ii++)); do MEMDIR_ARRAY+=("mem$(printf "%03d" "${ii}")") done else @@ -36,6 +41,11 @@ for MEMDIR in "${MEMDIR_ARRAY[@]}"; do export MEMDIR # Declare COMs + if [[ "${MODE}" = "cycled" && "${RUN:-}" = "gdas" ]]; then + YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_ATMOS_ANALYSIS + export COM_ATMOS_ANALYSIS + fi + if [[ ${EXP_WARM_START:-".false."} = ".true." ]]; then RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl COM_MED_RESTART_PREV:COM_MED_RESTART_TMPL @@ -53,8 +63,8 @@ for MEMDIR in "${MEMDIR_ARRAY[@]}"; do export COM_ICE_RESTART_PREV fi if [[ "${DO_WAVE:-}" = "YES" ]]; then - YMD=${PDY} HH=${cyc} declare_from_tmpl COM_WAVE_RESTART - export COM_WAVE_RESTART + YMD=${gPDY} HH=${gcyc} declare_from_tmpl COM_WAVE_RESTART_PREV:COM_WAVE_RESTART_TMPL + export COM_WAVE_RESTART_PREV fi # Execute staging From 81d070c71dcb3162895139e4fc11feb08efad015 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Fri, 14 Jun 2024 15:22:48 +0000 Subject: [PATCH 37/89] Add ICSDIR setting to stage configs Refs #2475 --- parm/config/gefs/config.stage_ic | 6 ++++++ parm/config/gfs/config.stage_ic | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/parm/config/gefs/config.stage_ic b/parm/config/gefs/config.stage_ic index b332ee1826..e0c4c77dc9 100644 --- a/parm/config/gefs/config.stage_ic +++ b/parm/config/gefs/config.stage_ic @@ -26,4 +26,10 @@ case "${CASE}" in ;; esac +if [[ -z "${ICSDIR}" ]] ; then + + export ICSDIR="${BASE_IC}/${CPL_ATMIC}" + +fi + echo "END: config.stage_ic" diff --git a/parm/config/gfs/config.stage_ic b/parm/config/gfs/config.stage_ic index 9956e8af6a..fa06e26de0 100644 --- a/parm/config/gfs/config.stage_ic +++ b/parm/config/gfs/config.stage_ic @@ -42,4 +42,22 @@ if [[ "${DO_NEST:-NO}" == "YES" ]] ; then export CPL_ATMIC="GLOBAL-NEST_${CASE}" fi +# Set ICSDIR + +if [[ -z "${ICSDIR}" ]] ; then + + IC_VER="20240610" + + if [[ "${CASE_ENS}" != "@CASEENS@" ]] ; then + ENSIC="${CASE_ENS}" + fi + + if [[ "${DO_OCN:-NO}" == "YES" ]] ; then + OCNIC="mx${OCNRES}" + fi + + export ICSDIR="${BASE_IC}/${CASE}${ENSIC:-}${OCNIC:-}/${IC_VER}" + +fi + echo "END: config.stage_ic" From bb941117d5dba24a487abb1c20a939373d73c79a Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Fri, 14 Jun 2024 15:24:55 +0000 Subject: [PATCH 38/89] Add cycled keys to exglobal_stage_ic.py Refs #2475 --- scripts/exglobal_stage_ic.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scripts/exglobal_stage_ic.py b/scripts/exglobal_stage_ic.py index ea79af1565..d9e7a9021d 100755 --- a/scripts/exglobal_stage_ic.py +++ b/scripts/exglobal_stage_ic.py @@ -18,10 +18,11 @@ def main(): stage = Stage(config) # Pull out all the configuration keys needed to run stage job - keys = ['RUN', 'MODE', 'DO_WAVE', 'DO_OCN', 'DO_ICE', 'DO_NEST', - 'current_cycle', 'EXP_WARM_START', 'CDUMP', 'rCDUMP', - 'ROTDIR', 'PARMgfs', 'ntiles', 'MEMDIR', - 'BASE_CPLIC', 'waveGRD', 'OCNRES', 'USE_OCN_PERTURB_FILES', + keys = ['RUN', 'MODE', 'CASE', 'CASE_ENS', 'OCNRES', 'ICERES', 'waveGRD', + 'EXP_WARM_START', 'current_cycle', 'CDUMP', 'rCDUMP', + 'ROTDIR', 'PARMgfs', 'ICSDIR', + 'ntiles', 'MEMDIR', 'USE_OCN_PERTURB_FILES', + 'DO_WAVE', 'DO_OCN', 'DO_ICE', 'DO_NEST', # TODO: Add DO_MED 'CPL_ATMIC', 'CPL_ICEIC', 'CPL_OCNIC', 'CPL_WAVIC'] stage_dict = AttrDict() From a675a3d1b17737f2a01adae68cf8288dea64dfa9 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Fri, 14 Jun 2024 15:25:29 +0000 Subject: [PATCH 39/89] Add cycled analysis yaml to stage.py Refs #2475 --- ush/python/pygfs/task/stage.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ush/python/pygfs/task/stage.py b/ush/python/pygfs/task/stage.py index 31b23f10a5..b58ffa47d7 100644 --- a/ush/python/pygfs/task/stage.py +++ b/ush/python/pygfs/task/stage.py @@ -53,6 +53,9 @@ def determine_stage(self, stage_dict: Dict[str, Any]) -> List[str]: stage_sets = [] + if stage_dict.MODE == "cycled" and stage_dict.RUN == "gdas": + stage_sets.append("analysis.yaml.j2") + if stage_dict.EXP_WARM_START: stage_sets.append("atmosphere_warm.yaml.j2") else: From b97a4829c6559b73064c028ad7d867d3d038a222 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Fri, 14 Jun 2024 15:27:26 +0000 Subject: [PATCH 40/89] Add cycled staging job to mesh - set job for gdas_half cycledef only - update fcst jobs dependencies Refs #2475 --- workflow/applications/gfs_cycled.py | 6 +++--- workflow/rocoto/gfs_tasks.py | 9 ++++++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index f7f9b5b5e6..10ff6bb966 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -53,7 +53,7 @@ def _get_app_configs(self): if self.do_ocean or self.do_ice: configs += ['oceanice_products'] - configs += ['sfcanl', 'analcalc', 'fcst', 'upp', 'atmos_products', 'arch', 'cleanup'] + configs += ['stage_ic', 'sfcanl', 'analcalc', 'fcst', 'upp', 'atmos_products', 'arch', 'cleanup'] if self.do_hybvar: if self.do_jediatmens: @@ -163,7 +163,7 @@ def get_task_names(self): if self.do_jediatmens: hybrid_tasks += ['atmensanlinit', 'atmensanlletkf', 'atmensanlfv3inc', 'atmensanlfinal', 'echgres'] else: - hybrid_tasks += ['eobs', 'eupd', 'echgres'] + hybrid_tasks += ['stage_ic', 'eobs', 'eupd', 'echgres'] hybrid_tasks += ['ediag'] if self.lobsdiag_forenkf else ['eomg'] hybrid_after_eupd_tasks += ['ecen', 'esfc', 'efcs', 'epos', 'earc', 'cleanup'] @@ -179,7 +179,7 @@ def get_task_names(self): if self.do_aero and 'gdas' in self.aero_anl_cdumps: gdas_tasks += ['aeroanlinit', 'aeroanlrun', 'aeroanlfinal'] - gdas_tasks += ['atmanlupp', 'atmanlprod', 'fcst'] + gdas_tasks += ['stage_ic', 'atmanlupp', 'atmanlprod', 'fcst'] if self.do_upp: gdas_tasks += ['atmupp'] diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 3ce5febcc1..3992dcd6ee 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -18,6 +18,8 @@ def _is_this_a_gdas_task(cdump, task_name): # Specific Tasks begin here def stage_ic(self): + cycledef = 'gdas_half' if self.cdump in ['gdas', 'enkfgdas'] else self.cdump + cpl_ic = self._configs['stage_ic'] resources = self.get_resource('stage_ic') @@ -25,7 +27,7 @@ def stage_ic(self): task_dict = {'task_name': task_name, 'resources': resources, 'envars': self.envars, - 'cycledef': self.cdump, + 'cycledef': cycledef, 'command': f'{self.HOMEgfs}/jobs/rocoto/stage_ic.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', @@ -833,10 +835,11 @@ def _fcst_cycled(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies) if self.cdump in ['gdas']: - dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'task', 'name': f'{self.cdump}stage_ic'} dependencies.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='or', dep=dependencies) + if self.app_config.do_wave and self.cdump in self.app_config.wave_cdumps: dep_dict = {'type': 'task', 'name': f'{self.cdump}waveprep'} dependencies.append(rocoto.add_dependency(dep_dict)) @@ -2520,7 +2523,7 @@ def efcs(self): dep_dict = {'type': 'task', 'name': f'{self.cdump}esfc'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'task', 'name': f'{self.cdump}stage_ic'} dependencies.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='or', dep=dependencies) From 937a2451b779674f56b004dfbfddc5c472c71d93 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Fri, 14 Jun 2024 15:28:27 +0000 Subject: [PATCH 41/89] Remove fill_ROTDIR function and update icsdir option - Delete the fill_ROTDIR functions used in half cycle setup; functions are replaced by half cycle stage_ic jobs - Update --icsdir setup flag to allow user provided path for all RUNs Refs #2475 --- workflow/setup_expt.py | 235 +---------------------------------------- 1 file changed, 2 insertions(+), 233 deletions(-) diff --git a/workflow/setup_expt.py b/workflow/setup_expt.py index 97d25dc15a..7234862f80 100755 --- a/workflow/setup_expt.py +++ b/workflow/setup_expt.py @@ -29,235 +29,6 @@ def makedirs_if_missing(dirname): os.makedirs(dirname) -def fill_ROTDIR(host, inputs): - """ - Method to populate the ROTDIR for supported modes. - INPUTS: - host: host object from class Host - inputs: user inputs to setup_expt.py - """ - - fill_modes = { - 'cycled': fill_ROTDIR_cycled, - 'forecast-only': fill_ROTDIR_forecasts - } - - try: - fill_modes[inputs.mode](host, inputs) - except KeyError: - raise NotImplementedError(f'{inputs.mode} is not a supported mode.\n' + - 'Currently supported modes are:\n' + - f'{" | ".join(fill_modes.keys())}') - - return - - -def fill_ROTDIR_cycled(host, inputs): - """ - Implementation of 'fill_ROTDIR' for cycled mode - """ - - rotdir = os.path.join(inputs.comroot, inputs.pslot) - - do_ocean = do_ice = do_med = False - - if 'S2S' in inputs.app: - do_ocean = do_ice = do_med = True - - if inputs.icsdir is None: - warnings.warn("User did not provide '--icsdir' to stage initial conditions") - return - - rdatestr = datetime_to_YMDH(inputs.idate - to_timedelta('T06H')) - idatestr = datetime_to_YMDH(inputs.idate) - - # Test if we are using the new COM structure or the old flat one for ICs - if inputs.start in ['warm']: - pathstr = os.path.join(inputs.icsdir, f'{inputs.cdump}.{rdatestr[:8]}', - rdatestr[8:], 'model_data', 'atmos') - else: - pathstr = os.path.join(inputs.icsdir, f'{inputs.cdump}.{idatestr[:8]}', - idatestr[8:], 'model_data', 'atmos') - - if os.path.isdir(pathstr): - flat_structure = False - else: - flat_structure = True - - # Destination always uses the new COM structure - # These should match the templates defined in config.com - if inputs.start in ['warm']: - dst_atm_dir = os.path.join('model_data', 'atmos', 'restart') - dst_med_dir = os.path.join('model_data', 'med', 'restart') - else: - dst_atm_dir = os.path.join('model_data', 'atmos', 'input') - dst_med_dir = '' # no mediator files for a "cold start" - do_med = False - dst_ocn_rst_dir = os.path.join('model_data', 'ocean', 'restart') - dst_ocn_anl_dir = os.path.join('analysis', 'ocean') - dst_ice_rst_dir = os.path.join('model_data', 'ice', 'restart') - dst_ice_anl_dir = os.path.join('analysis', 'ice') - dst_atm_anl_dir = os.path.join('analysis', 'atmos') - - if flat_structure: - # ICs are in the old flat COM structure - if inputs.start in ['warm']: # This is warm start experiment - src_atm_dir = os.path.join('atmos', 'RESTART') - src_med_dir = os.path.join('med', 'RESTART') - elif inputs.start in ['cold']: # This is a cold start experiment - src_atm_dir = os.path.join('atmos', 'INPUT') - src_med_dir = '' # no mediator files for a "cold start" - do_med = False - # ocean and ice have the same filenames for warm and cold - src_ocn_rst_dir = os.path.join('ocean', 'RESTART') - src_ocn_anl_dir = 'ocean' - src_ice_rst_dir = os.path.join('ice', 'RESTART') - src_ice_anl_dir = dst_ice_anl_dir - src_atm_anl_dir = 'atmos' - else: - src_atm_dir = dst_atm_dir - src_med_dir = dst_med_dir - src_ocn_rst_dir = dst_ocn_rst_dir - src_ocn_anl_dir = dst_ocn_anl_dir - src_ice_rst_dir = dst_ice_rst_dir - src_ice_anl_dir = dst_ice_anl_dir - src_atm_anl_dir = dst_atm_anl_dir - - def link_files_from_src_to_dst(src_dir, dst_dir): - files = os.listdir(src_dir) - for fname in files: - os.symlink(os.path.join(src_dir, fname), - os.path.join(dst_dir, fname)) - return - - # Link ensemble member initial conditions - if inputs.nens > 0: - previous_cycle_dir = f'enkf{inputs.cdump}.{rdatestr[:8]}/{rdatestr[8:]}' - current_cycle_dir = f'enkf{inputs.cdump}.{idatestr[:8]}/{idatestr[8:]}' - - for ii in range(1, inputs.nens + 1): - memdir = f'mem{ii:03d}' - # Link atmospheric files - if inputs.start in ['warm']: - dst_dir = os.path.join(rotdir, previous_cycle_dir, memdir, dst_atm_dir) - src_dir = os.path.join(inputs.icsdir, previous_cycle_dir, memdir, src_atm_dir) - elif inputs.start in ['cold']: - dst_dir = os.path.join(rotdir, current_cycle_dir, memdir, dst_atm_dir) - src_dir = os.path.join(inputs.icsdir, current_cycle_dir, memdir, src_atm_dir) - makedirs_if_missing(dst_dir) - link_files_from_src_to_dst(src_dir, dst_dir) - - # Link ocean files - if do_ocean: - dst_dir = os.path.join(rotdir, previous_cycle_dir, memdir, dst_ocn_rst_dir) - src_dir = os.path.join(inputs.icsdir, previous_cycle_dir, memdir, src_ocn_rst_dir) - makedirs_if_missing(dst_dir) - link_files_from_src_to_dst(src_dir, dst_dir) - - # First 1/2 cycle needs a MOM6 increment - incfile = f'enkf{inputs.cdump}.t{idatestr[8:]}z.ocninc.nc' - src_file = os.path.join(inputs.icsdir, current_cycle_dir, memdir, src_ocn_anl_dir, incfile) - dst_file = os.path.join(rotdir, current_cycle_dir, memdir, dst_ocn_anl_dir, incfile) - makedirs_if_missing(os.path.join(rotdir, current_cycle_dir, memdir, dst_ocn_anl_dir)) - os.symlink(src_file, dst_file) - - # Link ice files - if do_ice: - dst_dir = os.path.join(rotdir, previous_cycle_dir, memdir, dst_ice_rst_dir) - src_dir = os.path.join(inputs.icsdir, previous_cycle_dir, memdir, src_ice_rst_dir) - makedirs_if_missing(dst_dir) - link_files_from_src_to_dst(src_dir, dst_dir) - - # Link mediator files - if do_med: - dst_dir = os.path.join(rotdir, previous_cycle_dir, memdir, dst_med_dir) - src_dir = os.path.join(inputs.icsdir, previous_cycle_dir, memdir, src_med_dir) - makedirs_if_missing(dst_dir) - link_files_from_src_to_dst(src_dir, dst_dir) - - # Link deterministic initial conditions - previous_cycle_dir = f'{inputs.cdump}.{rdatestr[:8]}/{rdatestr[8:]}' - current_cycle_dir = f'{inputs.cdump}.{idatestr[:8]}/{idatestr[8:]}' - - # Link atmospheric files - if inputs.start in ['warm']: - dst_dir = os.path.join(rotdir, previous_cycle_dir, dst_atm_dir) - src_dir = os.path.join(inputs.icsdir, previous_cycle_dir, src_atm_dir) - elif inputs.start in ['cold']: - dst_dir = os.path.join(rotdir, current_cycle_dir, dst_atm_dir) - src_dir = os.path.join(inputs.icsdir, current_cycle_dir, src_atm_dir) - - makedirs_if_missing(dst_dir) - link_files_from_src_to_dst(src_dir, dst_dir) - - # Link ocean files - if do_ocean: - dst_dir = os.path.join(rotdir, previous_cycle_dir, dst_ocn_rst_dir) - src_dir = os.path.join(inputs.icsdir, previous_cycle_dir, src_ocn_rst_dir) - makedirs_if_missing(dst_dir) - link_files_from_src_to_dst(src_dir, dst_dir) - - # First 1/2 cycle needs a MOM6 increment - incfile = f'{inputs.cdump}.t{idatestr[8:]}z.ocninc.nc' - src_file = os.path.join(inputs.icsdir, current_cycle_dir, src_ocn_anl_dir, incfile) - dst_file = os.path.join(rotdir, current_cycle_dir, dst_ocn_anl_dir, incfile) - makedirs_if_missing(os.path.join(rotdir, current_cycle_dir, dst_ocn_anl_dir)) - os.symlink(src_file, dst_file) - - # Link ice files - if do_ice: - # First 1/2 cycle needs a CICE6 analysis restart - src_dir = os.path.join(inputs.icsdir, current_cycle_dir, src_ice_anl_dir) - dst_dir = os.path.join(rotdir, current_cycle_dir, src_ice_anl_dir) - makedirs_if_missing(dst_dir) - link_files_from_src_to_dst(src_dir, dst_dir) - - # Link mediator files - if do_med: - dst_dir = os.path.join(rotdir, previous_cycle_dir, dst_med_dir) - src_dir = os.path.join(inputs.icsdir, previous_cycle_dir, src_med_dir) - makedirs_if_missing(dst_dir) - link_files_from_src_to_dst(src_dir, dst_dir) - - # Link bias correction and radiance diagnostics files - src_dir = os.path.join(inputs.icsdir, current_cycle_dir, src_atm_anl_dir) - dst_dir = os.path.join(rotdir, current_cycle_dir, dst_atm_anl_dir) - makedirs_if_missing(dst_dir) - for ftype in ['abias', 'abias_pc', 'abias_air', 'radstat']: - fname = f'{inputs.cdump}.t{idatestr[8:]}z.{ftype}' - src_file = os.path.join(src_dir, fname) - if os.path.exists(src_file): - os.symlink(src_file, os.path.join(dst_dir, fname)) - # First 1/2 cycle also needs a atmos increment if doing warm start - if inputs.start in ['warm']: - for ftype in ['atmi003.nc', 'atminc.nc', 'atmi009.nc']: - fname = f'{inputs.cdump}.t{idatestr[8:]}z.{ftype}' - src_file = os.path.join(src_dir, fname) - if os.path.exists(src_file): - os.symlink(src_file, os.path.join(dst_dir, fname)) - if inputs.nens > 0: - current_cycle_dir = f'enkf{inputs.cdump}.{idatestr[:8]}/{idatestr[8:]}' - for ii in range(1, inputs.nens + 1): - memdir = f'mem{ii:03d}' - src_dir = os.path.join(inputs.icsdir, current_cycle_dir, memdir, src_atm_anl_dir) - dst_dir = os.path.join(rotdir, current_cycle_dir, memdir, dst_atm_anl_dir) - makedirs_if_missing(dst_dir) - for ftype in ['ratmi003.nc', 'ratminc.nc', 'ratmi009.nc']: - fname = f'enkf{inputs.cdump}.t{idatestr[8:]}z.{ftype}' - src_file = os.path.join(src_dir, fname) - if os.path.exists(src_file): - os.symlink(src_file, os.path.join(dst_dir, fname)) - - return - - -def fill_ROTDIR_forecasts(host, inputs): - """ - Implementation of 'fill_ROTDIR' for forecast-only mode - """ - print('forecast-only mode treats ICs differently and cannot be staged here') - - def fill_EXPDIR(inputs): """ Method to copy config files from workflow to experiment directory @@ -331,6 +102,7 @@ def edit_baseconfig(host, inputs, yaml_dict): "@OCNRES@": f"{int(100.*inputs.resdetocean):03d}", "@EXPDIR@": inputs.expdir, "@COMROOT@": inputs.comroot, + "@ICSDIR@": inputs.icsdir, "@EXP_WARM_START@": is_warm_start, "@MODE@": inputs.mode, "@gfs_cyc@": inputs.gfs_cyc, @@ -417,6 +189,7 @@ def _common_args(parser): parser.add_argument('--idate', help='starting date of experiment, initial conditions must exist!', required=True, type=lambda dd: to_datetime(dd)) parser.add_argument('--edate', help='end date experiment', required=True, type=lambda dd: to_datetime(dd)) + parser.add_argument('--icsdir', help='full path to user initial condition directory', type=str, required=False, default='') parser.add_argument('--overwrite', help='overwrite previously created experiment (if it exists)', action='store_true', required=False) return parser @@ -433,7 +206,6 @@ def _gfs_args(parser): return parser def _gfs_cycled_args(parser): - parser.add_argument('--icsdir', help='full path to initial condition directory', type=str, required=False, default=None) parser.add_argument('--app', help='UFS application', type=str, choices=ufs_apps, required=False, default='ATM') parser.add_argument('--gfs_cyc', help='cycles to run forecast', type=int, @@ -461,8 +233,6 @@ def _gefs_args(parser): default=os.path.join(_top, 'parm/config/gefs')) parser.add_argument('--yaml', help='Defaults to substitute from', type=str, required=False, default=os.path.join(_top, 'parm/config/gefs/yaml/defaults.yaml')) - parser.add_argument('--icsdir', help='full path to initial condition directory [temporary hack in place for testing]', - type=str, required=False, default=None) return parser description = """ @@ -580,7 +350,6 @@ def main(*argv): if create_rotdir: makedirs_if_missing(rotdir) - fill_ROTDIR(host, user_inputs) if create_expdir: makedirs_if_missing(expdir) From d0945dae7d9d9e354f456bd1518fedaf71a723fb Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Mon, 17 Jun 2024 13:37:26 +0000 Subject: [PATCH 42/89] Introduce relpaths to stage yamls Makes yamls more generic and uses paths set previously Refs #2475 --- parm/stage/atmosphere_cold.yaml.j2 | 7 +++---- parm/stage/atmosphere_nest.yaml.j2 | 4 ++-- parm/stage/atmosphere_warm.yaml.j2 | 6 +++--- parm/stage/ice.yaml.j2 | 2 +- parm/stage/ocean.yaml.j2 | 6 +++--- parm/stage/wave.yaml.j2 | 4 ++-- 6 files changed, 14 insertions(+), 15 deletions(-) diff --git a/parm/stage/atmosphere_cold.yaml.j2 b/parm/stage/atmosphere_cold.yaml.j2 index 3e8d96d726..f1a057f6e8 100644 --- a/parm/stage/atmosphere_cold.yaml.j2 +++ b/parm/stage/atmosphere_cold.yaml.j2 @@ -1,11 +1,10 @@ -{% set cycle_YMDH = current_cycle | to_YMDH %} -fv3_cold: +atmosphere_cold: mkdir: - "{{ COM_ATMOS_INPUT }}" copy: - - ["{{ BASE_CPLIC }}/{{ CPL_ATMIC }}/{{ cycle_YMDH }}/{{ MEMDIR }}/atmos/gfs_ctrl.nc", "{{ COM_ATMOS_INPUT }}"] + - ["{{ ICSDIR }}/{{ COM_ATMOS_INPUT | relpath(ROTDIR) }}/gfs_ctrl.nc", "{{ COM_ATMOS_INPUT }}"] {% for ftype in ["gfs_data", "sfc_data"] %} {% for ntile in range(1, ntiles + 1) %} - - ["{{ BASE_CPLIC }}/{{ CPL_ATMIC }}/{{ cycle_YMDH }}/{{ MEMDIR }}/atmos/{{ ftype }}.tile{{ ntile }}.nc", "{{ COM_ATMOS_INPUT }}"] + - ["{{ ICSDIR }}/{{ COM_ATMOS_INPUT | relpath(ROTDIR) }}/{{ ftype }}.tile{{ ntile }}.nc", "{{ COM_ATMOS_INPUT }}"] {% endfor %} # ntile {% endfor %} # ftype diff --git a/parm/stage/atmosphere_nest.yaml.j2 b/parm/stage/atmosphere_nest.yaml.j2 index 9e2c4b66ab..bcf36f0cf8 100644 --- a/parm/stage/atmosphere_nest.yaml.j2 +++ b/parm/stage/atmosphere_nest.yaml.j2 @@ -2,13 +2,13 @@ {% set cycle_YMD = current_cycle | to_YMD %} {% set cycle_YMDH = current_cycle | to_YMDH %} {% set ntile = 7 %} -fv3_nest: +atmosphere_nest: mkdir: - "{{ COM_ATMOS_RESTART_PREV }}" copy: {% if EXP_WARM_START == True %} {% for ftype in ca_data fv_core.res fv_srf_wnd.res fv_tracer.res phy_data sfc_data %} - - ["{{ BASE_CPLIC }}/{{ CPL_ATMIC }}/{{ cycle_YMDH }}/{{ MEMDIR }}/atmos/{{ cycle_YMD }}.{{ cycle_HH }}0000.{{ ftype }}.tile{{ ntile }}.nc", "{{ COM_ATMOS_RESTART_PREV }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.{{ ftype }}.nest0{{ ntile-5 }}.tile{{ ntile }}.nc"] + - ["{{ ICSDIR }}/{{ COM_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.{{ ftype }}.tile{{ ntile }}.nc", "{{ COM_ATMOS_RESTART_PREV }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.{{ ftype }}.nest0{{ ntile-5 }}.tile{{ ntile }}.nc"] {% endfor %} # ftype {% else %} {% for ftype in gfs_data sfc_data %} diff --git a/parm/stage/atmosphere_warm.yaml.j2 b/parm/stage/atmosphere_warm.yaml.j2 index 3ca9f7c023..759f04dafa 100644 --- a/parm/stage/atmosphere_warm.yaml.j2 +++ b/parm/stage/atmosphere_warm.yaml.j2 @@ -1,15 +1,15 @@ {% set cycle_HH = current_cycle | strftime("%H") %} {% set cycle_YMD = current_cycle | to_YMD %} {% set cycle_YMDH = current_cycle | to_YMDH %} -fv3_warm: +atmosphere_warm: mkdir: - "{{ COM_ATMOS_RESTART_PREV }}" copy: {% for ftype in ["coupler.res", "fv_core.res.nc"] %} - - ["{{ BASE_CPLIC }}/{{ CPL_ATMIC }}/{{ cycle_YMDH }}/{{ MEMDIR }}/atmos/{{ cycle_YMD }}.{{ cycle_HH }}0000.{{ ftype }}", "{{ COM_ATMOS_RESTART_PREV }}"] + - ["{{ ICSDIR }}/{{ COM_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.{{ ftype }}", "{{ COM_ATMOS_RESTART_PREV }}"] {% endfor %} {% for ftype in ["ca_data", "fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data sfc_data"] %} {% for ntile in range(1, ntiles + 1) %} - - ["{{ BASE_CPLIC }}/{{ CPL_ATMIC }}/{{ cycle_YMDH }}/{{ MEMDIR }}/atmos/{{ cycle_YMD }}.{{ cycle_HH }}0000.{{ ftype }}.tile{{ ntile }}.nc", "{{ COM_ATMOS_RESTART_PREV }}"] + - ["{{ ICSDIR }}/{{ COM_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.{{ ftype }}.tile{{ ntile }}.nc", "{{ COM_ATMOS_RESTART_PREV }}"] {% endfor %} # ntile {% endfor %} # ftype diff --git a/parm/stage/ice.yaml.j2 b/parm/stage/ice.yaml.j2 index d5a47951b9..494f2853bc 100644 --- a/parm/stage/ice.yaml.j2 +++ b/parm/stage/ice.yaml.j2 @@ -5,4 +5,4 @@ ice: mkdir: - "{{ COM_ICE_RESTART_PREV }}" copy: - - ["{{ BASE_CPLIC }}/{{ CPL_ICEIC }}/{{ cycle_YMDH }}/{{ MEMDIR }}/ice/{{ cycle_YMD }}.{{ cycle_HH }}0000.cice_model.res.nc", "{{ COM_ICE_RESTART_PREV }}"] + - ["{{ ICSDIR }}/{{ COM_ICE_RESTART_PREV | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.cice_model.res.nc", "{{ COM_ICE_RESTART_PREV }}"] diff --git a/parm/stage/ocean.yaml.j2 b/parm/stage/ocean.yaml.j2 index 6597cb0d85..23bbb2a47c 100644 --- a/parm/stage/ocean.yaml.j2 +++ b/parm/stage/ocean.yaml.j2 @@ -8,18 +8,18 @@ ocean: #------------------------- # Ocean initial conditions #------------------------- - - ["{{ BASE_CPLIC }}/{{ CPL_OCNIC }}/{{ cycle_YMDH }}/{{ MEMDIR }}/ocean/{{ cycle_YMD }}.{{ cycle_HH }}0000.MOM.res.nc", "{{ COM_OCEAN_RESTART_PREV }}"] + - ["{{ ICSDIR }}/{{ COM_OCEAN_RESTART_PREV | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.MOM.res.nc", "{{ COM_OCEAN_RESTART_PREV }}"] #------------------------------------ # Resolution based initial conditions #------------------------------------ {% if OCNRES == "025" %} {% for nn in range(1, 3) %} - - ["{{ BASE_CPLIC }}/{{ CPL_OCNIC }}/{{ cycle_YMDH }}/{{ MEMDIR }}/ocean/{{ cycle_YMD }}.{{ cycle_HH }}0000.MOM.res_{{ nn }}.nc", "{{ COM_OCEAN_RESTART_PREV }}"] + - ["{{ ICSDIR }}/{{ COM_OCEAN_RESTART_PREV | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.MOM.res_{{ nn }}.nc", "{{ COM_OCEAN_RESTART_PREV }}"] {% endfor %} {% endif %} #------------------------- # Ocean Perturbation Files #------------------------- {% if MEMDIR and USE_OCN_PERTURB_FILES == True %} - - ["{{ BASE_CPLIC }}/{{ CPL_OCNIC }}/{{ cycle_YMDH }}/{{ MEMDIR }}/ocean/{{ cycle_YMD }}.{{ cycle_HH }}0000.mom6_increment.nc", "{{ COM_OCEAN_RESTART_PREV }}"] + - ["{{ ICSDIR }}/{{ COM_OCEAN_RESTART_PREV | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.mom6_increment.nc", "{{ COM_OCEAN_RESTART_PREV }}"] {% endif %} diff --git a/parm/stage/wave.yaml.j2 b/parm/stage/wave.yaml.j2 index db6f6502aa..b40d2e304c 100644 --- a/parm/stage/wave.yaml.j2 +++ b/parm/stage/wave.yaml.j2 @@ -3,6 +3,6 @@ {% set cycle_YMDH = current_cycle | to_YMDH %} wave: mkdir: - - "{{ COM_WAVE_RESTART }}" + - "{{ COM_WAVE_RESTART_PREV }}" copy: - - ["{{ BASE_CPLIC }}/{{ CPL_WAVIC }}/{{ cycle_YMDH }}/{{ MEMDIR }}/wave/{{ cycle_YMD }}.{{ cycle_HH }}0000.restart.{{ waveGRD }}", "{{ COM_WAVE_RESTART }}"] + - ["{{ ICSDIR }}/{{ COM_WAVE_RESTART_PREV | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.restart.{{ waveGRD }}", "{{ COM_WAVE_RESTART_PREV }}"] From 3c45d58c60f9be7568157cf4470489ca3c9b32a5 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Mon, 17 Jun 2024 13:38:45 +0000 Subject: [PATCH 43/89] Create analysis stage yaml Refs #2475 --- parm/stage/analysis.yaml.j2 | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 parm/stage/analysis.yaml.j2 diff --git a/parm/stage/analysis.yaml.j2 b/parm/stage/analysis.yaml.j2 new file mode 100644 index 0000000000..44559ce791 --- /dev/null +++ b/parm/stage/analysis.yaml.j2 @@ -0,0 +1,9 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +analysis: + mkdir: + - "{{ COM_ATMOS_ANALYSIS }}" + copy: + # TODO: Add abias_int? + {% for ftype in ["abias", "abias_air", "abias_pc", "radstat"] %} + - ["{{ ICSDIR }}/{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ RUN }}.t{{ cycle_HH }}z.{{ ftype }}", "{{ COM_ATMOS_ANALYSIS }}"] + {% endfor %} From 03b50244b6306b7bddaed7e50a17529e009d3518 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Mon, 17 Jun 2024 15:12:00 +0000 Subject: [PATCH 44/89] Combine stage yamls into one Refs #2475 --- parm/stage/analysis.yaml.j2 | 9 --- parm/stage/atmosphere_cold.yaml.j2 | 10 --- parm/stage/atmosphere_nest.yaml.j2 | 17 ------ parm/stage/atmosphere_warm.yaml.j2 | 15 ----- parm/stage/ice.yaml.j2 | 8 --- parm/stage/ocean.yaml.j2 | 25 -------- parm/stage/stage.yaml.j2 | 98 ++++++++++++++++++++++++++++++ parm/stage/wave.yaml.j2 | 8 --- 8 files changed, 98 insertions(+), 92 deletions(-) delete mode 100644 parm/stage/analysis.yaml.j2 delete mode 100644 parm/stage/atmosphere_cold.yaml.j2 delete mode 100644 parm/stage/atmosphere_nest.yaml.j2 delete mode 100644 parm/stage/atmosphere_warm.yaml.j2 delete mode 100644 parm/stage/ice.yaml.j2 delete mode 100644 parm/stage/ocean.yaml.j2 create mode 100644 parm/stage/stage.yaml.j2 delete mode 100644 parm/stage/wave.yaml.j2 diff --git a/parm/stage/analysis.yaml.j2 b/parm/stage/analysis.yaml.j2 deleted file mode 100644 index 44559ce791..0000000000 --- a/parm/stage/analysis.yaml.j2 +++ /dev/null @@ -1,9 +0,0 @@ -{% set cycle_HH = current_cycle | strftime("%H") %} -analysis: - mkdir: - - "{{ COM_ATMOS_ANALYSIS }}" - copy: - # TODO: Add abias_int? - {% for ftype in ["abias", "abias_air", "abias_pc", "radstat"] %} - - ["{{ ICSDIR }}/{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ RUN }}.t{{ cycle_HH }}z.{{ ftype }}", "{{ COM_ATMOS_ANALYSIS }}"] - {% endfor %} diff --git a/parm/stage/atmosphere_cold.yaml.j2 b/parm/stage/atmosphere_cold.yaml.j2 deleted file mode 100644 index f1a057f6e8..0000000000 --- a/parm/stage/atmosphere_cold.yaml.j2 +++ /dev/null @@ -1,10 +0,0 @@ -atmosphere_cold: - mkdir: - - "{{ COM_ATMOS_INPUT }}" - copy: - - ["{{ ICSDIR }}/{{ COM_ATMOS_INPUT | relpath(ROTDIR) }}/gfs_ctrl.nc", "{{ COM_ATMOS_INPUT }}"] - {% for ftype in ["gfs_data", "sfc_data"] %} - {% for ntile in range(1, ntiles + 1) %} - - ["{{ ICSDIR }}/{{ COM_ATMOS_INPUT | relpath(ROTDIR) }}/{{ ftype }}.tile{{ ntile }}.nc", "{{ COM_ATMOS_INPUT }}"] - {% endfor %} # ntile - {% endfor %} # ftype diff --git a/parm/stage/atmosphere_nest.yaml.j2 b/parm/stage/atmosphere_nest.yaml.j2 deleted file mode 100644 index bcf36f0cf8..0000000000 --- a/parm/stage/atmosphere_nest.yaml.j2 +++ /dev/null @@ -1,17 +0,0 @@ -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMD = current_cycle | to_YMD %} -{% set cycle_YMDH = current_cycle | to_YMDH %} -{% set ntile = 7 %} -atmosphere_nest: - mkdir: - - "{{ COM_ATMOS_RESTART_PREV }}" - copy: - {% if EXP_WARM_START == True %} - {% for ftype in ca_data fv_core.res fv_srf_wnd.res fv_tracer.res phy_data sfc_data %} - - ["{{ ICSDIR }}/{{ COM_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.{{ ftype }}.tile{{ ntile }}.nc", "{{ COM_ATMOS_RESTART_PREV }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.{{ ftype }}.nest0{{ ntile-5 }}.tile{{ ntile }}.nc"] - {% endfor %} # ftype - {% else %} - {% for ftype in gfs_data sfc_data %} - - ["{{ COM_ATMOS_INPUT }}/{{ ftype }}.tile{{ ntile }}.nc", "{{ COM_ATMOS_INPUT }}/{{ ftype }}.nest02.tile{{ ntile }}.nc"] - {% endfor %} # ftype - {% endif %} # cold-start diff --git a/parm/stage/atmosphere_warm.yaml.j2 b/parm/stage/atmosphere_warm.yaml.j2 deleted file mode 100644 index 759f04dafa..0000000000 --- a/parm/stage/atmosphere_warm.yaml.j2 +++ /dev/null @@ -1,15 +0,0 @@ -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMD = current_cycle | to_YMD %} -{% set cycle_YMDH = current_cycle | to_YMDH %} -atmosphere_warm: - mkdir: - - "{{ COM_ATMOS_RESTART_PREV }}" - copy: - {% for ftype in ["coupler.res", "fv_core.res.nc"] %} - - ["{{ ICSDIR }}/{{ COM_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.{{ ftype }}", "{{ COM_ATMOS_RESTART_PREV }}"] - {% endfor %} - {% for ftype in ["ca_data", "fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data sfc_data"] %} - {% for ntile in range(1, ntiles + 1) %} - - ["{{ ICSDIR }}/{{ COM_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.{{ ftype }}.tile{{ ntile }}.nc", "{{ COM_ATMOS_RESTART_PREV }}"] - {% endfor %} # ntile - {% endfor %} # ftype diff --git a/parm/stage/ice.yaml.j2 b/parm/stage/ice.yaml.j2 deleted file mode 100644 index 494f2853bc..0000000000 --- a/parm/stage/ice.yaml.j2 +++ /dev/null @@ -1,8 +0,0 @@ -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMD = current_cycle | to_YMD %} -{% set cycle_YMDH = current_cycle | to_YMDH %} -ice: - mkdir: - - "{{ COM_ICE_RESTART_PREV }}" - copy: - - ["{{ ICSDIR }}/{{ COM_ICE_RESTART_PREV | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.cice_model.res.nc", "{{ COM_ICE_RESTART_PREV }}"] diff --git a/parm/stage/ocean.yaml.j2 b/parm/stage/ocean.yaml.j2 deleted file mode 100644 index 23bbb2a47c..0000000000 --- a/parm/stage/ocean.yaml.j2 +++ /dev/null @@ -1,25 +0,0 @@ -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMD = current_cycle | to_YMD %} -{% set cycle_YMDH = current_cycle | to_YMDH %} -ocean: - mkdir: - - "{{ COM_OCEAN_RESTART_PREV }}" - copy: - #------------------------- - # Ocean initial conditions - #------------------------- - - ["{{ ICSDIR }}/{{ COM_OCEAN_RESTART_PREV | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.MOM.res.nc", "{{ COM_OCEAN_RESTART_PREV }}"] - #------------------------------------ - # Resolution based initial conditions - #------------------------------------ - {% if OCNRES == "025" %} - {% for nn in range(1, 3) %} - - ["{{ ICSDIR }}/{{ COM_OCEAN_RESTART_PREV | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.MOM.res_{{ nn }}.nc", "{{ COM_OCEAN_RESTART_PREV }}"] - {% endfor %} - {% endif %} - #------------------------- - # Ocean Perturbation Files - #------------------------- - {% if MEMDIR and USE_OCN_PERTURB_FILES == True %} - - ["{{ ICSDIR }}/{{ COM_OCEAN_RESTART_PREV | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.mom6_increment.nc", "{{ COM_OCEAN_RESTART_PREV }}"] - {% endif %} diff --git a/parm/stage/stage.yaml.j2 b/parm/stage/stage.yaml.j2 new file mode 100644 index 0000000000..8f51023ae5 --- /dev/null +++ b/parm/stage/stage.yaml.j2 @@ -0,0 +1,98 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMD = current_cycle | to_YMD %} +{% set cycle_YMDH = current_cycle | to_YMDH %} + +{% if MODE == "cycled" %} +analysis: + mkdir: + - "{{ COM_ATMOS_ANALYSIS }}" + copy: + # TODO: Add abias_int? + {% for ftype in ["abias", "abias_air", "abias_pc", "radstat"] %} + - ["{{ ICSDIR }}/{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ RUN }}.t{{ cycle_HH }}z.{{ ftype }}", "{{ COM_ATMOS_ANALYSIS }}"] + {% endfor %} +{% endif %} + +{% if EXP_WARM_START == True %} +atmosphere_warm: + mkdir: + - "{{ COM_ATMOS_RESTART_PREV }}" + copy: + {% for ftype in ["coupler.res", "fv_core.res.nc"] %} + - ["{{ ICSDIR }}/{{ COM_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ DTG_PREFIX }}.{{ ftype }}", "{{ COM_ATMOS_RESTART_PREV }}"] + {% endfor %} + {% for ftype in ["ca_data", "fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data", "sfc_data"] %} + {% for ntile in range(1, ntiles + 1) %} + - ["{{ ICSDIR }}/{{ COM_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ DTG_PREFIX }}.{{ ftype }}.tile{{ ntile }}.nc", "{{ COM_ATMOS_RESTART_PREV }}"] + {% endfor %} # ntile + {% endfor %} # ftype +{% else %} +atmosphere_cold: + mkdir: + - "{{ COM_ATMOS_INPUT }}" + copy: + - ["{{ ICSDIR }}/{{ COM_ATMOS_INPUT | relpath(ROTDIR) }}/gfs_ctrl.nc", "{{ COM_ATMOS_INPUT }}"] + {% for ftype in ["gfs_data", "sfc_data"] %} + {% for ntile in range(1, ntiles + 1) %} + - ["{{ ICSDIR }}/{{ COM_ATMOS_INPUT | relpath(ROTDIR) }}/{{ ftype }}.tile{{ ntile }}.nc", "{{ COM_ATMOS_INPUT }}"] + {% endfor %} # ntile + {% endfor %} # ftype +{% endif %} + +{% if DO_NEST %} +atmosphere_nest: + {% set ntile = 7 %} + mkdir: + - "{{ COM_ATMOS_RESTART_PREV }}" + copy: + {% if EXP_WARM_START == True %} + {% for ftype in ["ca_data", "fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data", "sfc_data"] %} + - ["{{ ICSDIR }}/{{ COM_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.{{ ftype }}.tile{{ ntile }}.nc", "{{ COM_ATMOS_RESTART_PREV }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.{{ ftype }}.nest0{{ ntile-5 }}.tile{{ ntile }}.nc"] + {% endfor %} # ftype + {% else %} + {% for ftype in ["gfs_data", "sfc_data"] %} + - ["{{ COM_ATMOS_INPUT }}/{{ ftype }}.tile{{ ntile }}.nc", "{{ COM_ATMOS_INPUT }}/{{ ftype }}.nest0{{ ntile-5 }}.tile{{ ntile }}.nc"] + {% endfor %} # ftype + {% endif %} # cold-start +{% endif %} + +{% if DO_WAVE %} +wave: + mkdir: + - "{{ COM_WAVE_RESTART_PREV }}" + copy: + - ["{{ ICSDIR }}/{{ COM_WAVE_RESTART_PREV | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.restart.{{ waveGRD }}", "{{ COM_WAVE_RESTART_PREV }}"] +{% endif %} + +{% if DO_OCN %} +ocean: + mkdir: + - "{{ COM_OCEAN_RESTART_PREV }}" + copy: + #------------------------- + # Ocean initial conditions + #------------------------- + - ["{{ ICSDIR }}/{{ COM_OCEAN_RESTART_PREV | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.MOM.res.nc", "{{ COM_OCEAN_RESTART_PREV }}"] + #------------------------------------ + # Resolution based initial conditions + #------------------------------------ + {% if OCNRES == "025" %} + {% for nn in range(1, 3) %} + - ["{{ ICSDIR }}/{{ COM_OCEAN_RESTART_PREV | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.MOM.res_{{ nn }}.nc", "{{ COM_OCEAN_RESTART_PREV }}"] + {% endfor %} + {% endif %} + #------------------------- + # Ocean Perturbation Files + #------------------------- + {% if MEMDIR and REPLAY_ICS == "YES" %} + - ["{{ ICSDIR }}/{{ COM_OCEAN_RESTART_PREV | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.mom6_increment.nc", "{{ COM_OCEAN_RESTART_PREV }}"] + {% endif %} +{% endif %} + +{% if DO_ICE %} +ice: + mkdir: + - "{{ COM_ICE_RESTART_PREV }}" + copy: + - ["{{ ICSDIR }}/{{ COM_ICE_RESTART_PREV | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.cice_model.res.nc", "{{ COM_ICE_RESTART_PREV }}"] +{% endif %} diff --git a/parm/stage/wave.yaml.j2 b/parm/stage/wave.yaml.j2 deleted file mode 100644 index b40d2e304c..0000000000 --- a/parm/stage/wave.yaml.j2 +++ /dev/null @@ -1,8 +0,0 @@ -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMD = current_cycle | to_YMD %} -{% set cycle_YMDH = current_cycle | to_YMDH %} -wave: - mkdir: - - "{{ COM_WAVE_RESTART_PREV }}" - copy: - - ["{{ ICSDIR }}/{{ COM_WAVE_RESTART_PREV | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.restart.{{ waveGRD }}", "{{ COM_WAVE_RESTART_PREV }}"] From f3c72b237b2fa4cff74a62c277b0262b65af6034 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Mon, 17 Jun 2024 15:22:59 +0000 Subject: [PATCH 45/89] Staging job updates - Add new keys - Remove determine function - Update execute function to use single stage.yaml.j2 Refs #2475 --- scripts/exglobal_stage_ic.py | 10 ++---- ush/python/pygfs/task/stage.py | 64 ++++------------------------------ 2 files changed, 9 insertions(+), 65 deletions(-) diff --git a/scripts/exglobal_stage_ic.py b/scripts/exglobal_stage_ic.py index d9e7a9021d..acc7ea4a92 100755 --- a/scripts/exglobal_stage_ic.py +++ b/scripts/exglobal_stage_ic.py @@ -20,8 +20,8 @@ def main(): # Pull out all the configuration keys needed to run stage job keys = ['RUN', 'MODE', 'CASE', 'CASE_ENS', 'OCNRES', 'ICERES', 'waveGRD', 'EXP_WARM_START', 'current_cycle', 'CDUMP', 'rCDUMP', - 'ROTDIR', 'PARMgfs', 'ICSDIR', - 'ntiles', 'MEMDIR', 'USE_OCN_PERTURB_FILES', + 'ROTDIR', 'PARMgfs', 'ICSDIR', 'DTG_PREFIX', + 'ntiles', 'MEMDIR', 'REPLAY_ICS', 'DO_WAVE', 'DO_OCN', 'DO_ICE', 'DO_NEST', # TODO: Add DO_MED 'CPL_ATMIC', 'CPL_ICEIC', 'CPL_OCNIC', 'CPL_WAVIC'] @@ -37,12 +37,8 @@ def main(): # Add the os.path.exists function to the dict for yaml parsing stage_dict['path_exists'] = os.path.exists - # Determine which ICs to stage - stage_sets = stage.determine_stage(stage_dict) - # Stage ICs - stage.execute_stage(stage_dict, stage_sets) - + stage.execute_stage(stage_dict) if __name__ == '__main__': main() diff --git a/ush/python/pygfs/task/stage.py b/ush/python/pygfs/task/stage.py index b58ffa47d7..c00248c784 100644 --- a/ush/python/pygfs/task/stage.py +++ b/ush/python/pygfs/task/stage.py @@ -37,69 +37,17 @@ def __init__(self, config: Dict[str, Any]) -> None: self.task_config = AttrDict(**self.config, **self.runtime_config) @logit(logger) - def determine_stage(self, stage_dict: Dict[str, Any]) -> List[str]: - """Determine which initial condition files need to be placed in ROTDIR. - - Parameters - ---------- - stage_dict : Dict[str, Any] - Task specific keys, e.g. runtime options (DO_WAVE, DO_ICE, etc) - - Return - ------ - stage_sets : List - List of yamls to parse for staging - """ - - stage_sets = [] - - if stage_dict.MODE == "cycled" and stage_dict.RUN == "gdas": - stage_sets.append("analysis.yaml.j2") - - if stage_dict.EXP_WARM_START: - stage_sets.append("atmosphere_warm.yaml.j2") - else: - stage_sets.append("atmosphere_cold.yaml.j2") - - if stage_dict.DO_WAVE: - stage_sets.append("wave.yaml.j2") - - if stage_dict.DO_OCN: - stage_sets.append("ocean.yaml.j2") - - if stage_dict.DO_ICE: - stage_sets.append("ice.yaml.j2") - - if stage_dict.DO_NEST: - stage_sets.append("atmosphere_nest.yaml.j2") - - return stage_sets - - @logit(logger) - def execute_stage(self, stage_dict: Dict[str, Any], stage_sets: List) -> None: + def execute_stage(self, stage_dict: Dict[str, Any]) -> None: """Perform local staging of initial condition files. - - Parameters - ---------- - stage_sets : List - List of stage sets to send to FileHandler - stage_set : Dict[str, Any] - FileHandler instructions to populate ROTDIR with - - Return - ------ - None """ - stage_parm = os.path.join(stage_dict.PARMgfs, "stage") - if not os.path.isdir(stage_dict.ROTDIR): raise FileNotFoundError(f"FATAL ERROR: The ROTDIR ({stage_dict.ROTDIR}) does not exist!") - for set_yaml in stage_sets: + stage_parm = os.path.join(stage_dict.PARMgfs, "stage") - stage_set = parse_j2yaml(os.path.join(stage_parm, set_yaml), stage_dict) + stage_set = parse_j2yaml(os.path.join(stage_parm, "stage.yaml.j2"), stage_dict) - # Copy files to ROTDIR - for key in stage_set.keys(): - FileHandler(stage_set[key]).sync() + # Copy files to ROTDIR + for key in stage_set.keys(): + FileHandler(stage_set[key]).sync() From 31bb29ca5e513e8f31a56c4fa800ebf0e3d6a47f Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Mon, 17 Jun 2024 15:24:18 +0000 Subject: [PATCH 46/89] Staging variable updates - Add RDATE and DTG_PREFIX - Remove USE_OCN_PERTURB_FILES - Always declare COM_ATMOS_ANALYSIS Refs #2475 --- jobs/JGLOBAL_STAGE_IC | 9 +++++---- parm/config/gfs/config.base | 4 ---- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/jobs/JGLOBAL_STAGE_IC b/jobs/JGLOBAL_STAGE_IC index a7fb5a0e58..80d650917f 100755 --- a/jobs/JGLOBAL_STAGE_IC +++ b/jobs/JGLOBAL_STAGE_IC @@ -15,6 +15,9 @@ GDATE=$(date --utc -d "${PDY} ${cyc} - ${assim_freq} hours" +%Y%m%d%H) gPDY="${GDATE:0:8}" gcyc="${GDATE:8:2}" +RDATE=$(date --utc -d "${PDY} ${cyc} + ${OFFSET_START_HOUR} hours" +%Y%m%d%H) +export DTG_PREFIX="${RDATE:0:8}.${RDATE:8:2}0000" + # Define MEMDIR_ARRAY MEMDIR_ARRAY=() if [[ "${RUN:-}" = "enkfgdas" || "${RUN:-}" = "gefs" ]]; then @@ -41,10 +44,8 @@ for MEMDIR in "${MEMDIR_ARRAY[@]}"; do export MEMDIR # Declare COMs - if [[ "${MODE}" = "cycled" && "${RUN:-}" = "gdas" ]]; then - YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_ATMOS_ANALYSIS - export COM_ATMOS_ANALYSIS - fi + YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_ATMOS_ANALYSIS + export COM_ATMOS_ANALYSIS if [[ ${EXP_WARM_START:-".false."} = ".true." ]]; then RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index d32ec509f7..7e5e6054e5 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -445,10 +445,6 @@ export INCVARS_EFOLD="5" export netcdf_diag=".true." export binary_diag=".false." -# Initialize ocean ensemble members with perturbations -# if true, only occurs for members greater than zero -export USE_OCN_PERTURB_FILES=".false." - # Verification options export DO_METP="NO" # Run METPLUS jobs - set METPLUS settings in config.metp; not supported with spack-stack export DO_FIT2OBS="YES" # Run fit to observations package From e833c2a9f39af5704e211cadb84ef85388230c4f Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Thu, 20 Jun 2024 13:34:05 +0000 Subject: [PATCH 47/89] Update for COMOUT, replay, and add mediator - Convert COM variables to COMOUT variables - Add mediator IC copy - Add DTG_PREFIX to filename constructor - Add replay updates Refs #2475 --- jobs/JGLOBAL_STAGE_IC | 30 ++++++++++-------- parm/stage/stage.yaml.j2 | 60 ++++++++++++++++++------------------ scripts/exglobal_stage_ic.py | 2 +- 3 files changed, 48 insertions(+), 44 deletions(-) diff --git a/jobs/JGLOBAL_STAGE_IC b/jobs/JGLOBAL_STAGE_IC index 80d650917f..2557b32f87 100755 --- a/jobs/JGLOBAL_STAGE_IC +++ b/jobs/JGLOBAL_STAGE_IC @@ -44,28 +44,32 @@ for MEMDIR in "${MEMDIR_ARRAY[@]}"; do export MEMDIR # Declare COMs - YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_ATMOS_ANALYSIS - export COM_ATMOS_ANALYSIS + YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL + export COMOUT_ATMOS_ANALYSIS if [[ ${EXP_WARM_START:-".false."} = ".true." ]]; then - RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL - RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl COM_MED_RESTART_PREV:COM_MED_RESTART_TMPL - export COM_ATMOS_RESTART_PREV COM_MED_RESTART_PREV + RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl COMOUT_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL + RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl COMOUT_MED_RESTART_PREV:COM_MED_RESTART_TMPL + export COMOUT_ATMOS_RESTART_PREV COMOUT_MED_RESTART_PREV else - YMD=${PDY} HH=${cyc} declare_from_tmpl COM_ATMOS_INPUT - export COM_ATMOS_INPUT + YMD=${PDY} HH=${cyc} declare_from_tmpl COMOUT_ATMOS_INPUT:COM_ATMOS_INPUT_TMPL + export COMOUT_ATMOS_INPUT fi if [[ "${DO_OCN:-}" = "YES" ]]; then - RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl COM_OCEAN_RESTART_PREV:COM_OCEAN_RESTART_TMPL - export COM_OCEAN_RESTART_PREV + RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl COMOUT_OCEAN_RESTART_PREV:COM_OCEAN_RESTART_TMPL + export COMOUT_OCEAN_RESTART_PREV + fi + if [[ "${REPLAY_ICS:-NO}" = "YES" ]]; then + YMD=${PDY} HH=${cyc} declare_from_tmpl COMOUT_OCEAN_ANALYSIS:COM_OCEAN_ANALYSIS_TMPL + export COMOUT_OCEAN_ANALYSIS fi if [[ "${DO_ICE:-}" = "YES" ]]; then - RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl COM_ICE_RESTART_PREV:COM_ICE_RESTART_TMPL - export COM_ICE_RESTART_PREV + RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl COMOUT_ICE_RESTART_PREV:COM_ICE_RESTART_TMPL + export COMOUT_ICE_RESTART_PREV fi if [[ "${DO_WAVE:-}" = "YES" ]]; then - YMD=${gPDY} HH=${gcyc} declare_from_tmpl COM_WAVE_RESTART_PREV:COM_WAVE_RESTART_TMPL - export COM_WAVE_RESTART_PREV + YMD=${gPDY} HH=${gcyc} declare_from_tmpl COMOUT_WAVE_RESTART_PREV:COM_WAVE_RESTART_TMPL + export COMOUT_WAVE_RESTART_PREV fi # Execute staging diff --git a/parm/stage/stage.yaml.j2 b/parm/stage/stage.yaml.j2 index 8f51023ae5..59830d2720 100644 --- a/parm/stage/stage.yaml.j2 +++ b/parm/stage/stage.yaml.j2 @@ -1,57 +1,63 @@ {% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMD = current_cycle | to_YMD %} -{% set cycle_YMDH = current_cycle | to_YMDH %} {% if MODE == "cycled" %} analysis: mkdir: - - "{{ COM_ATMOS_ANALYSIS }}" + - "{{ COMOUT_ATMOS_ANALYSIS }}" copy: # TODO: Add abias_int? {% for ftype in ["abias", "abias_air", "abias_pc", "radstat"] %} - - ["{{ ICSDIR }}/{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ RUN }}.t{{ cycle_HH }}z.{{ ftype }}", "{{ COM_ATMOS_ANALYSIS }}"] + - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ RUN }}.t{{ cycle_HH }}z.{{ ftype }}", "{{ COMOUT_ATMOS_ANALYSIS }}"] {% endfor %} {% endif %} {% if EXP_WARM_START == True %} atmosphere_warm: mkdir: - - "{{ COM_ATMOS_RESTART_PREV }}" + - "{{ COMOUT_ATMOS_RESTART_PREV }}" copy: {% for ftype in ["coupler.res", "fv_core.res.nc"] %} - - ["{{ ICSDIR }}/{{ COM_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ DTG_PREFIX }}.{{ ftype }}", "{{ COM_ATMOS_RESTART_PREV }}"] + - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ DTG_PREFIX }}.{{ ftype }}", "{{ COMOUT_ATMOS_RESTART_PREV }}"] {% endfor %} {% for ftype in ["ca_data", "fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data", "sfc_data"] %} {% for ntile in range(1, ntiles + 1) %} - - ["{{ ICSDIR }}/{{ COM_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ DTG_PREFIX }}.{{ ftype }}.tile{{ ntile }}.nc", "{{ COM_ATMOS_RESTART_PREV }}"] + - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ DTG_PREFIX }}.{{ ftype }}.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_RESTART_PREV }}"] {% endfor %} # ntile {% endfor %} # ftype {% else %} atmosphere_cold: mkdir: - - "{{ COM_ATMOS_INPUT }}" + - "{{ COMOUT_ATMOS_INPUT }}" copy: - - ["{{ ICSDIR }}/{{ COM_ATMOS_INPUT | relpath(ROTDIR) }}/gfs_ctrl.nc", "{{ COM_ATMOS_INPUT }}"] + - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_INPUT | relpath(ROTDIR) }}/gfs_ctrl.nc", "{{ COMOUT_ATMOS_INPUT }}"] {% for ftype in ["gfs_data", "sfc_data"] %} {% for ntile in range(1, ntiles + 1) %} - - ["{{ ICSDIR }}/{{ COM_ATMOS_INPUT | relpath(ROTDIR) }}/{{ ftype }}.tile{{ ntile }}.nc", "{{ COM_ATMOS_INPUT }}"] + - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_INPUT | relpath(ROTDIR) }}/{{ ftype }}.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_INPUT }}"] {% endfor %} # ntile {% endfor %} # ftype {% endif %} +{% if MEMDIR and REPLAY_ICS == "YES" %} +atmosphere_perturbation: + mkdir: + - "{{ COMOUT_ATMOS_ANALYSIS }}" + copy: + - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ DTG_PREFIX }}.fv3_perturbation.nc", "{{ COMOUT_ATMOS_ANALYSIS }}/{{ RUN }}.t{{ cycle_HH }}z.atminc.nc"] +{% endif %} + {% if DO_NEST %} atmosphere_nest: {% set ntile = 7 %} mkdir: - - "{{ COM_ATMOS_RESTART_PREV }}" + - "{{ COMOUT_ATMOS_RESTART_PREV }}" copy: {% if EXP_WARM_START == True %} {% for ftype in ["ca_data", "fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data", "sfc_data"] %} - - ["{{ ICSDIR }}/{{ COM_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.{{ ftype }}.tile{{ ntile }}.nc", "{{ COM_ATMOS_RESTART_PREV }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.{{ ftype }}.nest0{{ ntile-5 }}.tile{{ ntile }}.nc"] + - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ DTG_PREFIX }}.{{ ftype }}.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_RESTART_PREV }}/{{ DTG_PREFIX }}.{{ ftype }}.nest0{{ ntile-5 }}.tile{{ ntile }}.nc"] {% endfor %} # ftype {% else %} {% for ftype in ["gfs_data", "sfc_data"] %} - - ["{{ COM_ATMOS_INPUT }}/{{ ftype }}.tile{{ ntile }}.nc", "{{ COM_ATMOS_INPUT }}/{{ ftype }}.nest0{{ ntile-5 }}.tile{{ ntile }}.nc"] + - ["{{ COMOUT_ATMOS_INPUT }}/{{ ftype }}.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_INPUT }}/{{ ftype }}.nest0{{ ntile-5 }}.tile{{ ntile }}.nc"] {% endfor %} # ftype {% endif %} # cold-start {% endif %} @@ -59,40 +65,34 @@ atmosphere_nest: {% if DO_WAVE %} wave: mkdir: - - "{{ COM_WAVE_RESTART_PREV }}" + - "{{ COMOUT_WAVE_RESTART_PREV }}" copy: - - ["{{ ICSDIR }}/{{ COM_WAVE_RESTART_PREV | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.restart.{{ waveGRD }}", "{{ COM_WAVE_RESTART_PREV }}"] + - ["{{ ICSDIR }}/{{ COMOUT_WAVE_RESTART_PREV | relpath(ROTDIR) }}/{{ DTG_PREFIX }}.restart.{{ waveGRD }}", "{{ COMOUT_WAVE_RESTART_PREV }}"] {% endif %} {% if DO_OCN %} ocean: mkdir: - - "{{ COM_OCEAN_RESTART_PREV }}" + - "{{ COMOUT_OCEAN_RESTART_PREV }}" copy: - #------------------------- - # Ocean initial conditions - #------------------------- - - ["{{ ICSDIR }}/{{ COM_OCEAN_RESTART_PREV | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.MOM.res.nc", "{{ COM_OCEAN_RESTART_PREV }}"] - #------------------------------------ - # Resolution based initial conditions - #------------------------------------ + - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_RESTART_PREV | relpath(ROTDIR) }}/{{ DTG_PREFIX }}.MOM.res.nc", "{{ COMOUT_OCEAN_RESTART_PREV }}"] {% if OCNRES == "025" %} {% for nn in range(1, 3) %} - - ["{{ ICSDIR }}/{{ COM_OCEAN_RESTART_PREV | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.MOM.res_{{ nn }}.nc", "{{ COM_OCEAN_RESTART_PREV }}"] + - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_RESTART_PREV | relpath(ROTDIR) }}/{{ DTG_PREFIX }}.MOM.res_{{ nn }}.nc", "{{ COMOUT_OCEAN_RESTART_PREV }}"] {% endfor %} {% endif %} - #------------------------- - # Ocean Perturbation Files - #------------------------- {% if MEMDIR and REPLAY_ICS == "YES" %} - - ["{{ ICSDIR }}/{{ COM_OCEAN_RESTART_PREV | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.mom6_increment.nc", "{{ COM_OCEAN_RESTART_PREV }}"] + - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ DTG_PREFIX }}.mom6_perturbation.nc", "{{ COMOUT_OCEAN_ANALYSIS }}/mom6_increment.nc"] + {% endif %} + {% if EXP_WARM_START == True %} + - ["{{ ICSDIR }}/{{ COMOUT_MED_RESTART_PREV | relpath(ROTDIR) }}/{{ DTG_PREFIX }}.ufs.cpld.cpl.r.nc", "{{ COMOUT_MED_RESTART_PREV }}"] {% endif %} {% endif %} {% if DO_ICE %} ice: mkdir: - - "{{ COM_ICE_RESTART_PREV }}" + - "{{ COMOUT_ICE_RESTART_PREV }}" copy: - - ["{{ ICSDIR }}/{{ COM_ICE_RESTART_PREV | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.cice_model.res.nc", "{{ COM_ICE_RESTART_PREV }}"] + - ["{{ ICSDIR }}/{{ COMOUT_ICE_RESTART_PREV | relpath(ROTDIR) }}/{{ DTG_PREFIX }}.cice_model.res.nc", "{{ COMOUT_ICE_RESTART_PREV }}"] {% endif %} diff --git a/scripts/exglobal_stage_ic.py b/scripts/exglobal_stage_ic.py index acc7ea4a92..ab90f0f1b2 100755 --- a/scripts/exglobal_stage_ic.py +++ b/scripts/exglobal_stage_ic.py @@ -22,7 +22,7 @@ def main(): 'EXP_WARM_START', 'current_cycle', 'CDUMP', 'rCDUMP', 'ROTDIR', 'PARMgfs', 'ICSDIR', 'DTG_PREFIX', 'ntiles', 'MEMDIR', 'REPLAY_ICS', - 'DO_WAVE', 'DO_OCN', 'DO_ICE', 'DO_NEST', # TODO: Add DO_MED + 'DO_WAVE', 'DO_OCN', 'DO_ICE', 'DO_NEST', 'CPL_ATMIC', 'CPL_ICEIC', 'CPL_OCNIC', 'CPL_WAVIC'] stage_dict = AttrDict() From df0c933c505e213030f8b8c8e4508be5e7b670e3 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Thu, 20 Jun 2024 13:49:13 +0000 Subject: [PATCH 48/89] Resolve pynorms errors Refs #2475 --- scripts/exglobal_stage_ic.py | 1 + workflow/rocoto/gfs_tasks.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/exglobal_stage_ic.py b/scripts/exglobal_stage_ic.py index ab90f0f1b2..4fa2ac97d6 100755 --- a/scripts/exglobal_stage_ic.py +++ b/scripts/exglobal_stage_ic.py @@ -40,5 +40,6 @@ def main(): # Stage ICs stage.execute_stage(stage_dict) + if __name__ == '__main__': main() diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 7e49ceb55e..1ab98522f5 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -864,7 +864,6 @@ def _fcst_cycled(self): dependencies.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='or', dep=dependencies) - if self.app_config.do_wave and self.cdump in self.app_config.wave_cdumps: dep_dict = {'type': 'task', 'name': f'{self.cdump}waveprep'} dependencies.append(rocoto.add_dependency(dep_dict)) From 0812baf5530bc4a072c931ec815ed4e72025769d Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Mon, 1 Jul 2024 14:39:48 +0000 Subject: [PATCH 49/89] Some updates - replace DTG_PREFIX with RDATE - only declare COMOUT_ATMOS_ANALYSIS when cycled and gdas - add ICSDIR settings to gefs config.stage_ic Refs #2475 --- jobs/JGLOBAL_STAGE_IC | 8 +++++--- parm/config/gefs/config.stage_ic | 25 ++++++++++++++++------- parm/stage/stage.yaml.j2 | 35 ++++++++++++++++---------------- scripts/exglobal_stage_ic.py | 4 ++-- ush/python/pygfs/task/stage.py | 2 +- 5 files changed, 44 insertions(+), 30 deletions(-) diff --git a/jobs/JGLOBAL_STAGE_IC b/jobs/JGLOBAL_STAGE_IC index 2557b32f87..5bec2031cb 100755 --- a/jobs/JGLOBAL_STAGE_IC +++ b/jobs/JGLOBAL_STAGE_IC @@ -16,7 +16,7 @@ gPDY="${GDATE:0:8}" gcyc="${GDATE:8:2}" RDATE=$(date --utc -d "${PDY} ${cyc} + ${OFFSET_START_HOUR} hours" +%Y%m%d%H) -export DTG_PREFIX="${RDATE:0:8}.${RDATE:8:2}0000" +export RDATE # Define MEMDIR_ARRAY MEMDIR_ARRAY=() @@ -44,8 +44,10 @@ for MEMDIR in "${MEMDIR_ARRAY[@]}"; do export MEMDIR # Declare COMs - YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL - export COMOUT_ATMOS_ANALYSIS + if [[ "${MODE}" = "cycled" && "${RUN}" = "gdas" ]]; then + YMD=${PDY} HH=${cyc} declare_from_tmpl COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL + export COMOUT_ATMOS_ANALYSIS + fi if [[ ${EXP_WARM_START:-".false."} = ".true." ]]; then RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl COMOUT_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL diff --git a/parm/config/gefs/config.stage_ic b/parm/config/gefs/config.stage_ic index ac3903b731..f79f33e3a4 100644 --- a/parm/config/gefs/config.stage_ic +++ b/parm/config/gefs/config.stage_ic @@ -23,11 +23,11 @@ case "${CASE}" in export CPL_MEDIC="" ;; "C48") - export CPL_ATMIC="gefs_test" - export CPL_ICEIC="gefs_test" - export CPL_OCNIC="gefs_test" - export CPL_WAVIC="gefs_test" - export CPL_MEDIC="gefs_test" + export CPL_ATMIC="" + export CPL_ICEIC="" + export CPL_OCNIC="" + export CPL_WAVIC="" + export CPL_MEDIC="" ;; *) echo "FATAL ERROR Unrecognized resolution: ${CASE}" @@ -35,10 +35,21 @@ case "${CASE}" in ;; esac +# Set ICSDIR + if [[ -z "${ICSDIR}" ]] ; then - export ICSDIR="${BASE_IC}/${CPL_ATMIC}" + IC_VER="20240610" -fi + if [[ "${CASE_ENS}" != "@CASEENS@" ]] ; then + ENSIC="${CASE_ENS}" + fi + + if [[ "${DO_OCN:-NO}" == "YES" ]] ; then + OCNIC="mx${OCNRES}" + fi + export ICSDIR="${BASE_IC}/${CASE}${ENSIC:-}${OCNIC:-}/${IC_VER}" + +fi echo "END: config.stage_ic" diff --git a/parm/stage/stage.yaml.j2 b/parm/stage/stage.yaml.j2 index 59830d2720..87653a3075 100644 --- a/parm/stage/stage.yaml.j2 +++ b/parm/stage/stage.yaml.j2 @@ -1,6 +1,7 @@ {% set cycle_HH = current_cycle | strftime("%H") %} +{% set r_prefix = RDATE | to_YMD + "." + RDATE | strftime("%H") + "0000" %} -{% if MODE == "cycled" %} +{% if MODE == "cycled" and RUN == "gdas" %} analysis: mkdir: - "{{ COMOUT_ATMOS_ANALYSIS }}" @@ -17,11 +18,11 @@ atmosphere_warm: - "{{ COMOUT_ATMOS_RESTART_PREV }}" copy: {% for ftype in ["coupler.res", "fv_core.res.nc"] %} - - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ DTG_PREFIX }}.{{ ftype }}", "{{ COMOUT_ATMOS_RESTART_PREV }}"] + - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ r_prefix }}.{{ ftype }}", "{{ COMOUT_ATMOS_RESTART_PREV }}"] {% endfor %} {% for ftype in ["ca_data", "fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data", "sfc_data"] %} {% for ntile in range(1, ntiles + 1) %} - - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ DTG_PREFIX }}.{{ ftype }}.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_RESTART_PREV }}"] + - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ r_prefix }}.{{ ftype }}.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_RESTART_PREV }}"] {% endfor %} # ntile {% endfor %} # ftype {% else %} @@ -42,7 +43,7 @@ atmosphere_perturbation: mkdir: - "{{ COMOUT_ATMOS_ANALYSIS }}" copy: - - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ DTG_PREFIX }}.fv3_perturbation.nc", "{{ COMOUT_ATMOS_ANALYSIS }}/{{ RUN }}.t{{ cycle_HH }}z.atminc.nc"] + - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ r_prefix }}.fv3_perturbation.nc", "{{ COMOUT_ATMOS_ANALYSIS }}/{{ RUN }}.t{{ cycle_HH }}z.atminc.nc"] {% endif %} {% if DO_NEST %} @@ -53,7 +54,7 @@ atmosphere_nest: copy: {% if EXP_WARM_START == True %} {% for ftype in ["ca_data", "fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data", "sfc_data"] %} - - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ DTG_PREFIX }}.{{ ftype }}.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_RESTART_PREV }}/{{ DTG_PREFIX }}.{{ ftype }}.nest0{{ ntile-5 }}.tile{{ ntile }}.nc"] + - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ r_prefix }}.{{ ftype }}.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_RESTART_PREV }}/{{ r_prefix }}.{{ ftype }}.nest0{{ ntile-5 }}.tile{{ ntile }}.nc"] {% endfor %} # ftype {% else %} {% for ftype in ["gfs_data", "sfc_data"] %} @@ -62,12 +63,12 @@ atmosphere_nest: {% endif %} # cold-start {% endif %} -{% if DO_WAVE %} -wave: +{% if DO_ICE %} +ice: mkdir: - - "{{ COMOUT_WAVE_RESTART_PREV }}" + - "{{ COMOUT_ICE_RESTART_PREV }}" copy: - - ["{{ ICSDIR }}/{{ COMOUT_WAVE_RESTART_PREV | relpath(ROTDIR) }}/{{ DTG_PREFIX }}.restart.{{ waveGRD }}", "{{ COMOUT_WAVE_RESTART_PREV }}"] + - ["{{ ICSDIR }}/{{ COMOUT_ICE_RESTART_PREV | relpath(ROTDIR) }}/{{ r_prefix }}.cice_model.res.nc", "{{ COMOUT_ICE_RESTART_PREV }}"] {% endif %} {% if DO_OCN %} @@ -75,24 +76,24 @@ ocean: mkdir: - "{{ COMOUT_OCEAN_RESTART_PREV }}" copy: - - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_RESTART_PREV | relpath(ROTDIR) }}/{{ DTG_PREFIX }}.MOM.res.nc", "{{ COMOUT_OCEAN_RESTART_PREV }}"] + - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_RESTART_PREV | relpath(ROTDIR) }}/{{ r_prefix }}.MOM.res.nc", "{{ COMOUT_OCEAN_RESTART_PREV }}"] {% if OCNRES == "025" %} {% for nn in range(1, 3) %} - - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_RESTART_PREV | relpath(ROTDIR) }}/{{ DTG_PREFIX }}.MOM.res_{{ nn }}.nc", "{{ COMOUT_OCEAN_RESTART_PREV }}"] + - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_RESTART_PREV | relpath(ROTDIR) }}/{{ r_prefix }}.MOM.res_{{ nn }}.nc", "{{ COMOUT_OCEAN_RESTART_PREV }}"] {% endfor %} {% endif %} {% if MEMDIR and REPLAY_ICS == "YES" %} - - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ DTG_PREFIX }}.mom6_perturbation.nc", "{{ COMOUT_OCEAN_ANALYSIS }}/mom6_increment.nc"] + - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ r_prefix }}.mom6_perturbation.nc", "{{ COMOUT_OCEAN_ANALYSIS }}/mom6_increment.nc"] {% endif %} {% if EXP_WARM_START == True %} - - ["{{ ICSDIR }}/{{ COMOUT_MED_RESTART_PREV | relpath(ROTDIR) }}/{{ DTG_PREFIX }}.ufs.cpld.cpl.r.nc", "{{ COMOUT_MED_RESTART_PREV }}"] + - ["{{ ICSDIR }}/{{ COMOUT_MED_RESTART_PREV | relpath(ROTDIR) }}/{{ r_prefix }}.ufs.cpld.cpl.r.nc", "{{ COMOUT_MED_RESTART_PREV }}"] {% endif %} {% endif %} -{% if DO_ICE %} -ice: +{% if DO_WAVE %} +wave: mkdir: - - "{{ COMOUT_ICE_RESTART_PREV }}" + - "{{ COMOUT_WAVE_RESTART_PREV }}" copy: - - ["{{ ICSDIR }}/{{ COMOUT_ICE_RESTART_PREV | relpath(ROTDIR) }}/{{ DTG_PREFIX }}.cice_model.res.nc", "{{ COMOUT_ICE_RESTART_PREV }}"] + - ["{{ ICSDIR }}/{{ COMOUT_WAVE_RESTART_PREV | relpath(ROTDIR) }}/{{ r_prefix }}.restart.{{ waveGRD }}", "{{ COMOUT_WAVE_RESTART_PREV }}"] {% endif %} diff --git a/scripts/exglobal_stage_ic.py b/scripts/exglobal_stage_ic.py index 4fa2ac97d6..0eb1f03b17 100755 --- a/scripts/exglobal_stage_ic.py +++ b/scripts/exglobal_stage_ic.py @@ -19,8 +19,8 @@ def main(): # Pull out all the configuration keys needed to run stage job keys = ['RUN', 'MODE', 'CASE', 'CASE_ENS', 'OCNRES', 'ICERES', 'waveGRD', - 'EXP_WARM_START', 'current_cycle', 'CDUMP', 'rCDUMP', - 'ROTDIR', 'PARMgfs', 'ICSDIR', 'DTG_PREFIX', + 'EXP_WARM_START', 'current_cycle', 'RDATE', + 'ROTDIR', 'PARMgfs', 'ICSDIR', 'CDUMP', 'rCDUMP', 'ntiles', 'MEMDIR', 'REPLAY_ICS', 'DO_WAVE', 'DO_OCN', 'DO_ICE', 'DO_NEST', 'CPL_ATMIC', 'CPL_ICEIC', 'CPL_OCNIC', 'CPL_WAVIC'] diff --git a/ush/python/pygfs/task/stage.py b/ush/python/pygfs/task/stage.py index c00248c784..5ad9ee572d 100644 --- a/ush/python/pygfs/task/stage.py +++ b/ush/python/pygfs/task/stage.py @@ -18,7 +18,7 @@ class Stage(Task): @logit(logger, name="Stage") def __init__(self, config: Dict[str, Any]) -> None: """Constructor for the Stage task - The constructor is responsible for collecting necessary yamls based on + The constructor is responsible for collecting necessary settings based on the runtime options and RUN. Parameters From c564933ff4f39020dae841879e9660e1ebb08f0f Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Wed, 17 Jul 2024 18:30:49 +0000 Subject: [PATCH 50/89] Update stage.py for recent jinja2 updates Refs #2475 --- ush/python/pygfs/task/stage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/stage.py b/ush/python/pygfs/task/stage.py index 5ad9ee572d..14b3f3bf3d 100644 --- a/ush/python/pygfs/task/stage.py +++ b/ush/python/pygfs/task/stage.py @@ -32,9 +32,9 @@ def __init__(self, config: Dict[str, Any]) -> None: """ super().__init__(config) - rotdir = self.config.ROTDIR + os.sep + rotdir = self.task_config.ROTDIR + os.sep - self.task_config = AttrDict(**self.config, **self.runtime_config) + self.task_config = AttrDict(**self.task_config) @logit(logger) def execute_stage(self, stage_dict: Dict[str, Any]) -> None: From 6f5c7725e0a00fc234f347d41eb56320406b4302 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Mon, 22 Jul 2024 09:48:54 -0500 Subject: [PATCH 51/89] Change model_data to model Refs #2686 --- docs/source/init.rst | 10 +++++----- parm/config/gfs/config.com | 30 +++++++++++++++--------------- scripts/exgfs_aero_init_aerosol.py | 4 ++-- ush/check_ice_netcdf.sh | 6 +++--- 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/docs/source/init.rst b/docs/source/init.rst index ac900e1be2..03afa9aff2 100644 --- a/docs/source/init.rst +++ b/docs/source/init.rst @@ -125,14 +125,14 @@ Start date = 2021032312 ├── enkfgdas.20210323 │   ├── 06 │   │   ├── mem001 - │   │   │   └── model_data -> ../../../gdas.20210323/06/model_data + │   │   │   └── model -> ../../../gdas.20210323/06/model │   │   ├── mem002 - │   │   │   └── model_data -> ../../../gdas.20210323/06/model_data + │   │   │   └── model -> ../../../gdas.20210323/06/model │   │   ├── mem003 - │   │   │   └── model_data -> ../../../gdas.20210323/06/model_data + │   │   │   └── model -> ../../../gdas.20210323/06/model ... │   │   └── mem080 - │   │   └── model_data -> ../../../gdas.20210323/06/model_data + │   │   └── model -> ../../../gdas.20210323/06/model │   └── 12 │   ├── mem001 │   │   └── analysis @@ -153,7 +153,7 @@ Start date = 2021032312 │   └── gdas.t12z.ocninc.nc -> ../../../../../gdas.20210323/12/analysis/ocean/gdas.t12z.ocninc.nc └── gdas.20210323 ├── 06 - │   └── model_data + │   └── model │   ├── atmos │   │   └── restart │   │   ├── 20210323.120000.ca_data.tile1.nc diff --git a/parm/config/gfs/config.com b/parm/config/gfs/config.com index ec867e64ba..6500bb69de 100644 --- a/parm/config/gfs/config.com +++ b/parm/config/gfs/config.com @@ -51,12 +51,12 @@ declare -rx COM_TOP_TMPL='${ROTDIR}/${RUN}.${YMD}/${HH}' declare -rx COM_CONF_TMPL=${COM_BASE}'/conf' declare -rx COM_OBS_JEDI=${COM_BASE}'/obs_jedi' -declare -rx COM_ATMOS_INPUT_TMPL=${COM_BASE}'/model_data/atmos/input' -declare -rx COM_ATMOS_RESTART_TMPL=${COM_BASE}'/model_data/atmos/restart' +declare -rx COM_ATMOS_INPUT_TMPL=${COM_BASE}'/model/atmos/input' +declare -rx COM_ATMOS_RESTART_TMPL=${COM_BASE}'/model/atmos/restart' declare -rx COM_ATMOS_ANALYSIS_TMPL=${COM_BASE}'/analysis/atmos' declare -rx COM_SNOW_ANALYSIS_TMPL=${COM_BASE}'/analysis/snow' -declare -rx COM_ATMOS_HISTORY_TMPL=${COM_BASE}'/model_data/atmos/history' -declare -rx COM_ATMOS_MASTER_TMPL=${COM_BASE}'/model_data/atmos/master' +declare -rx COM_ATMOS_HISTORY_TMPL=${COM_BASE}'/model/atmos/history' +declare -rx COM_ATMOS_MASTER_TMPL=${COM_BASE}'/model/atmos/master' declare -rx COM_ATMOS_GRIB_TMPL=${COM_BASE}'/products/atmos/grib2' declare -rx COM_ATMOS_GRIB_GRID_TMPL=${COM_ATMOS_GRIB_TMPL}'/${GRID}' declare -rx COM_ATMOS_BUFR_TMPL=${COM_BASE}'/products/atmos/bufr' @@ -70,31 +70,31 @@ declare -rx COM_ATMOS_RADMON_TMPL=${COM_BASE}'/products/atmos/radmon' declare -rx COM_ATMOS_MINMON_TMPL=${COM_BASE}'/products/atmos/minmon' declare -rx COM_ATMOS_WMO_TMPL=${COM_BASE}'/products/atmos/wmo' -declare -rx COM_WAVE_RESTART_TMPL=${COM_BASE}'/model_data/wave/restart' -declare -rx COM_WAVE_PREP_TMPL=${COM_BASE}'/model_data/wave/prep' -declare -rx COM_WAVE_HISTORY_TMPL=${COM_BASE}'/model_data/wave/history' +declare -rx COM_WAVE_RESTART_TMPL=${COM_BASE}'/model/wave/restart' +declare -rx COM_WAVE_PREP_TMPL=${COM_BASE}'/model/wave/prep' +declare -rx COM_WAVE_HISTORY_TMPL=${COM_BASE}'/model/wave/history' declare -rx COM_WAVE_GRID_TMPL=${COM_BASE}'/products/wave/gridded' declare -rx COM_WAVE_STATION_TMPL=${COM_BASE}'/products/wave/station' declare -rx COM_WAVE_GEMPAK_TMPL=${COM_BASE}'/products/wave/gempak' declare -rx COM_WAVE_WMO_TMPL=${COM_BASE}'/products/wave/wmo' -declare -rx COM_OCEAN_HISTORY_TMPL=${COM_BASE}'/model_data/ocean/history' -declare -rx COM_OCEAN_RESTART_TMPL=${COM_BASE}'/model_data/ocean/restart' -declare -rx COM_OCEAN_INPUT_TMPL=${COM_BASE}'/model_data/ocean/input' +declare -rx COM_OCEAN_HISTORY_TMPL=${COM_BASE}'/model/ocean/history' +declare -rx COM_OCEAN_RESTART_TMPL=${COM_BASE}'/model/ocean/restart' +declare -rx COM_OCEAN_INPUT_TMPL=${COM_BASE}'/model/ocean/input' declare -rx COM_OCEAN_ANALYSIS_TMPL=${COM_BASE}'/analysis/ocean' declare -rx COM_OCEAN_NETCDF_TMPL=${COM_BASE}'/products/ocean/netcdf' declare -rx COM_OCEAN_GRIB_TMPL=${COM_BASE}'/products/ocean/grib2' declare -rx COM_OCEAN_GRIB_GRID_TMPL=${COM_OCEAN_GRIB_TMPL}'/${GRID}' declare -rx COM_ICE_ANALYSIS_TMPL=${COM_BASE}'/analysis/ice' -declare -rx COM_ICE_INPUT_TMPL=${COM_BASE}'/model_data/ice/input' -declare -rx COM_ICE_HISTORY_TMPL=${COM_BASE}'/model_data/ice/history' -declare -rx COM_ICE_RESTART_TMPL=${COM_BASE}'/model_data/ice/restart' +declare -rx COM_ICE_INPUT_TMPL=${COM_BASE}'/model/ice/input' +declare -rx COM_ICE_HISTORY_TMPL=${COM_BASE}'/model/ice/history' +declare -rx COM_ICE_RESTART_TMPL=${COM_BASE}'/model/ice/restart' declare -rx COM_ICE_NETCDF_TMPL=${COM_BASE}'/products/ice/netcdf' declare -rx COM_ICE_GRIB_TMPL=${COM_BASE}'/products/ice/grib2' declare -rx COM_ICE_GRIB_GRID_TMPL=${COM_ICE_GRIB_TMPL}'/${GRID}' -declare -rx COM_CHEM_HISTORY_TMPL=${COM_BASE}'/model_data/chem/history' +declare -rx COM_CHEM_HISTORY_TMPL=${COM_BASE}'/model/chem/history' declare -rx COM_CHEM_ANALYSIS_TMPL=${COM_BASE}'/analysis/chem' -declare -rx COM_MED_RESTART_TMPL=${COM_BASE}'/model_data/med/restart' +declare -rx COM_MED_RESTART_TMPL=${COM_BASE}'/model/med/restart' diff --git a/scripts/exgfs_aero_init_aerosol.py b/scripts/exgfs_aero_init_aerosol.py index 1c81880ca9..957a03f7ad 100755 --- a/scripts/exgfs_aero_init_aerosol.py +++ b/scripts/exgfs_aero_init_aerosol.py @@ -41,10 +41,10 @@ from functools import partial # Constants -atm_base_pattern = "{rot_dir}/{cdump}.%Y%m%d/%H/model_data/atmos/input" # Location of atmosphere ICs +atm_base_pattern = "{rot_dir}/{cdump}.%Y%m%d/%H/model/atmos/input" # Location of atmosphere ICs atm_file_pattern = "{path}/gfs_data.{tile}.nc" # Atm IC file names atm_ctrl_pattern = "{path}/gfs_ctrl.nc" # Atm IC control file name -restart_base_pattern = "{rot_dir}/{cdump}.%Y%m%d/%H/model_data/atmos/restart" # Location of restart files (time of previous run) +restart_base_pattern = "{rot_dir}/{cdump}.%Y%m%d/%H/model/atmos/restart" # Location of restart files (time of previous run) restart_file_pattern = "{file_base}/{timestamp}fv_core.res.{tile}.nc" # Name of restart data files (time when restart is valid) tracer_file_pattern = "{file_base}/{timestamp}fv_tracer.res.{tile}.nc" # Name of restart tracer files (time when restart is valid) dycore_file_pattern = "{file_base}/{timestamp}fv_core.res.nc" # Name of restart dycore file (time when restart is valid) diff --git a/ush/check_ice_netcdf.sh b/ush/check_ice_netcdf.sh index 02ca4dae80..9d2d945a8b 100755 --- a/ush/check_ice_netcdf.sh +++ b/ush/check_ice_netcdf.sh @@ -19,12 +19,12 @@ if (( offset != 0 )); then fhr3=$(printf %03i "${fhri}") if (( fhri <= FHOUT_ICE_GFS )); then (( interval = FHOUT_ICE_GFS - cyc )) - ncfile=${ROTDIR}/gefs.${yyyy}${mm}${dd}/${cyc}/mem${member}/model_data/ice/history/gefs.ice.t${cyc}z.${interval}hr_avg.f${fhr3}.nc + ncfile=${ROTDIR}/gefs.${yyyy}${mm}${dd}/${cyc}/mem${member}/model/ice/history/gefs.ice.t${cyc}z.${interval}hr_avg.f${fhr3}.nc else - ncfile=${ROTDIR}/gefs.${yyyy}${mm}${dd}/${cyc}/mem${member}/model_data/ice/history/gefs.ice.t${cyc}z.${FHOUT_ICE_GFS}hr_avg.f${fhr3}.nc + ncfile=${ROTDIR}/gefs.${yyyy}${mm}${dd}/${cyc}/mem${member}/model/ice/history/gefs.ice.t${cyc}z.${FHOUT_ICE_GFS}hr_avg.f${fhr3}.nc fi else - ncfile=${ROTDIR}/gefs.${yyyy}${mm}${dd}/${cyc}/mem${member}/model_data/ice/history/gefs.ice.t${cyc}z.${FHOUT_ICE_GFS}hr_avg.f${fhr}.nc + ncfile=${ROTDIR}/gefs.${yyyy}${mm}${dd}/${cyc}/mem${member}/model/ice/history/gefs.ice.t${cyc}z.${FHOUT_ICE_GFS}hr_avg.f${fhr}.nc fi #Check if netcdf file exists. From faa96132bc74dfc6b9812e8d26279f535ee85bc0 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Wed, 24 Jul 2024 14:32:44 +0000 Subject: [PATCH 52/89] Remove CASE blocks in stage configs - No longer need the CASE case blocks that set CPL_* variables based on resolution Refs #2475 --- parm/config/gefs/config.stage_ic | 29 +------------------------- parm/config/gfs/config.stage_ic | 35 -------------------------------- 2 files changed, 1 insertion(+), 63 deletions(-) diff --git a/parm/config/gefs/config.stage_ic b/parm/config/gefs/config.stage_ic index f79f33e3a4..03b852c8d5 100644 --- a/parm/config/gefs/config.stage_ic +++ b/parm/config/gefs/config.stage_ic @@ -7,34 +7,6 @@ echo "BEGIN: config.stage_ic" # Get task specific resources source "${EXPDIR}/config.resources" stage_ic -case "${CASE}" in - "C384") - export CPL_ATMIC="" - export CPL_ICEIC="" - export CPL_OCNIC="" - export CPL_WAVIC="" - export CPL_MEDIC="" - ;; - "C96") - export CPL_ATMIC="" - export CPL_ICEIC="" - export CPL_OCNIC="" - export CPL_WAVIC="" - export CPL_MEDIC="" - ;; - "C48") - export CPL_ATMIC="" - export CPL_ICEIC="" - export CPL_OCNIC="" - export CPL_WAVIC="" - export CPL_MEDIC="" - ;; - *) - echo "FATAL ERROR Unrecognized resolution: ${CASE}" - exit 1 - ;; -esac - # Set ICSDIR if [[ -z "${ICSDIR}" ]] ; then @@ -52,4 +24,5 @@ if [[ -z "${ICSDIR}" ]] ; then export ICSDIR="${BASE_IC}/${CASE}${ENSIC:-}${OCNIC:-}/${IC_VER}" fi + echo "END: config.stage_ic" diff --git a/parm/config/gfs/config.stage_ic b/parm/config/gfs/config.stage_ic index fa06e26de0..03b852c8d5 100644 --- a/parm/config/gfs/config.stage_ic +++ b/parm/config/gfs/config.stage_ic @@ -7,41 +7,6 @@ echo "BEGIN: config.stage_ic" # Get task specific resources source "${EXPDIR}/config.resources" stage_ic -case "${CASE}" in - "C48" | "C96" | "C192") - export CPL_ATMIC="workflow_${CASE}_refactored" - export CPL_ICEIC="workflow_${CASE}_refactored" - export CPL_OCNIC="workflow_${CASE}_refactored" - export CPL_WAVIC="workflow_${CASE}_refactored" - ;; - "C384") - export CPL_ATMIC=GEFS-NoahMP-aerosols-p8c_refactored - export CPL_ICEIC=CPC_refactored - export CPL_OCNIC=CPC3Dvar_refactored - export CPL_WAVIC=workflow_C384_refactored - ;; - "C768") - export CPL_ATMIC=HR3C768 - export CPL_ICEIC=HR3marine - export CPL_OCNIC=HR3marine - export CPL_WAVIC=HR3marine - ;; - "C1152") - export CPL_ATMIC=HR3C1152 - export CPL_ICEIC=HR3marine - export CPL_OCNIC=HR3marine - export CPL_WAVIC=HR3marine - ;; - *) - echo "FATAL ERROR Unrecognized resolution: ${CASE}" - exit 1 - ;; -esac - -if [[ "${DO_NEST:-NO}" == "YES" ]] ; then - export CPL_ATMIC="GLOBAL-NEST_${CASE}" -fi - # Set ICSDIR if [[ -z "${ICSDIR}" ]] ; then From 2696940d473e48f2114d21b09d91f6594920b577 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Thu, 1 Aug 2024 13:52:57 +0000 Subject: [PATCH 53/89] Add STAGE_IC_YAML_TMPL variable - Set it in configs so users can change the path to their own yaml - Update python to use variable from configs Refs #2475 --- parm/config/gefs/config.stage_ic | 2 ++ parm/config/gfs/config.stage_ic | 2 ++ scripts/exglobal_stage_ic.py | 7 +++---- ush/python/pygfs/task/stage.py | 4 +--- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/parm/config/gefs/config.stage_ic b/parm/config/gefs/config.stage_ic index 03b852c8d5..ff749bdf87 100644 --- a/parm/config/gefs/config.stage_ic +++ b/parm/config/gefs/config.stage_ic @@ -7,6 +7,8 @@ echo "BEGIN: config.stage_ic" # Get task specific resources source "${EXPDIR}/config.resources" stage_ic +export STAGE_IC_YAML_TMPL="${PARMgfs}/stage/stage.yaml.j2" + # Set ICSDIR if [[ -z "${ICSDIR}" ]] ; then diff --git a/parm/config/gfs/config.stage_ic b/parm/config/gfs/config.stage_ic index 03b852c8d5..ff749bdf87 100644 --- a/parm/config/gfs/config.stage_ic +++ b/parm/config/gfs/config.stage_ic @@ -7,6 +7,8 @@ echo "BEGIN: config.stage_ic" # Get task specific resources source "${EXPDIR}/config.resources" stage_ic +export STAGE_IC_YAML_TMPL="${PARMgfs}/stage/stage.yaml.j2" + # Set ICSDIR if [[ -z "${ICSDIR}" ]] ; then diff --git a/scripts/exglobal_stage_ic.py b/scripts/exglobal_stage_ic.py index 0eb1f03b17..b297840a72 100755 --- a/scripts/exglobal_stage_ic.py +++ b/scripts/exglobal_stage_ic.py @@ -19,11 +19,10 @@ def main(): # Pull out all the configuration keys needed to run stage job keys = ['RUN', 'MODE', 'CASE', 'CASE_ENS', 'OCNRES', 'ICERES', 'waveGRD', - 'EXP_WARM_START', 'current_cycle', 'RDATE', - 'ROTDIR', 'PARMgfs', 'ICSDIR', 'CDUMP', 'rCDUMP', + 'EXP_WARM_START', 'current_cycle', 'RDATE', 'CDUMP', 'rCDUMP', + 'ROTDIR', 'PARMgfs', 'ICSDIR', 'STAGE_IC_YAML_TMPL', 'ntiles', 'MEMDIR', 'REPLAY_ICS', - 'DO_WAVE', 'DO_OCN', 'DO_ICE', 'DO_NEST', - 'CPL_ATMIC', 'CPL_ICEIC', 'CPL_OCNIC', 'CPL_WAVIC'] + 'DO_WAVE', 'DO_OCN', 'DO_ICE', 'DO_NEST'] stage_dict = AttrDict() for key in keys: diff --git a/ush/python/pygfs/task/stage.py b/ush/python/pygfs/task/stage.py index 14b3f3bf3d..d5f6af28ee 100644 --- a/ush/python/pygfs/task/stage.py +++ b/ush/python/pygfs/task/stage.py @@ -44,9 +44,7 @@ def execute_stage(self, stage_dict: Dict[str, Any]) -> None: if not os.path.isdir(stage_dict.ROTDIR): raise FileNotFoundError(f"FATAL ERROR: The ROTDIR ({stage_dict.ROTDIR}) does not exist!") - stage_parm = os.path.join(stage_dict.PARMgfs, "stage") - - stage_set = parse_j2yaml(os.path.join(stage_parm, "stage.yaml.j2"), stage_dict) + stage_set = parse_j2yaml(self.task_config.STAGE_IC_YAML_TMPL, stage_dict) # Copy files to ROTDIR for key in stage_set.keys(): From 7b2621d8850bd58c1d24b56a76d9da09b4988e25 Mon Sep 17 00:00:00 2001 From: Kate Friedman Date: Thu, 1 Aug 2024 10:36:24 -0400 Subject: [PATCH 54/89] Update parm/config/gfs/config.stage_ic Add comment to ICSDIR section title Refs #2475 Co-authored-by: Rahul Mahajan --- parm/config/gfs/config.stage_ic | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/parm/config/gfs/config.stage_ic b/parm/config/gfs/config.stage_ic index ff749bdf87..dcfe2654db 100644 --- a/parm/config/gfs/config.stage_ic +++ b/parm/config/gfs/config.stage_ic @@ -9,8 +9,7 @@ source "${EXPDIR}/config.resources" stage_ic export STAGE_IC_YAML_TMPL="${PARMgfs}/stage/stage.yaml.j2" -# Set ICSDIR - +# Set ICSDIR (if not defined) if [[ -z "${ICSDIR}" ]] ; then IC_VER="20240610" From ed3be1ba13e76202ee484eb2797f79506f752ede Mon Sep 17 00:00:00 2001 From: Kate Friedman Date: Thu, 1 Aug 2024 10:38:25 -0400 Subject: [PATCH 55/89] Add documentation to stage.yaml.j2 Add documentation for users who may edit stage yaml Refs #2475 Co-authored-by: Rahul Mahajan --- parm/stage/stage.yaml.j2 | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/parm/stage/stage.yaml.j2 b/parm/stage/stage.yaml.j2 index 87653a3075..bb77b69a4c 100644 --- a/parm/stage/stage.yaml.j2 +++ b/parm/stage/stage.yaml.j2 @@ -1,3 +1,20 @@ +# This yaml is intended to be of the following structure: +key1: + mkdir: + - "COM directory to create" + copy: + - ["source_file", "destination_file"] +key2: + mkdir: + - "COM directory to create" + copy: + - ["source_file", "destination_file"] +# Any number of keys with nested mkdir and copy are permitted +# Jinja is permitted in this yaml, as long as the keys are: +# - COM_ +# - DO_ATM, DO_OCN, DO_ICE +# For a full list see exglobal_stage_ic.py + {% set cycle_HH = current_cycle | strftime("%H") %} {% set r_prefix = RDATE | to_YMD + "." + RDATE | strftime("%H") + "0000" %} From a3752806146cc0a18e37683933a008cc9a5911cd Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Thu, 1 Aug 2024 14:45:52 +0000 Subject: [PATCH 56/89] Updates to stage.yaml.j2 - Update documentation - Remove MEMDIR references Refs #2475 --- parm/stage/stage.yaml.j2 | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/parm/stage/stage.yaml.j2 b/parm/stage/stage.yaml.j2 index bb77b69a4c..c04b69fc51 100644 --- a/parm/stage/stage.yaml.j2 +++ b/parm/stage/stage.yaml.j2 @@ -1,19 +1,22 @@ +############################################################# # This yaml is intended to be of the following structure: -key1: - mkdir: - - "COM directory to create" - copy: - - ["source_file", "destination_file"] -key2: - mkdir: - - "COM directory to create" - copy: - - ["source_file", "destination_file"] +# key1: +# mkdir: +# - "COM directory to create" +# copy: +# - ["source_file", "destination_file"] +# key2: +# mkdir: +# - "COM directory to create" +# copy: +# - ["source_file", "destination_file"] +# # Any number of keys with nested mkdir and copy are permitted # Jinja is permitted in this yaml, as long as the keys are: -# - COM_ -# - DO_ATM, DO_OCN, DO_ICE -# For a full list see exglobal_stage_ic.py +# - COMOUT_ +# - DO_ATM, DO_OCN, DO_ICE, etc. +# For a full list see scripts/exglobal_stage_ic.py +############################################################# {% set cycle_HH = current_cycle | strftime("%H") %} {% set r_prefix = RDATE | to_YMD + "." + RDATE | strftime("%H") + "0000" %} @@ -55,7 +58,7 @@ atmosphere_cold: {% endfor %} # ftype {% endif %} -{% if MEMDIR and REPLAY_ICS == "YES" %} +{% if REPLAY_ICS == "YES" %} atmosphere_perturbation: mkdir: - "{{ COMOUT_ATMOS_ANALYSIS }}" @@ -99,7 +102,7 @@ ocean: - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_RESTART_PREV | relpath(ROTDIR) }}/{{ r_prefix }}.MOM.res_{{ nn }}.nc", "{{ COMOUT_OCEAN_RESTART_PREV }}"] {% endfor %} {% endif %} - {% if MEMDIR and REPLAY_ICS == "YES" %} + {% if REPLAY_ICS == "YES" %} - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ r_prefix }}.mom6_perturbation.nc", "{{ COMOUT_OCEAN_ANALYSIS }}/mom6_increment.nc"] {% endif %} {% if EXP_WARM_START == True %} From 6d5a8024c8a6e630c4d4b27d94f35a1605f2ec94 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Mon, 5 Aug 2024 13:38:54 +0000 Subject: [PATCH 57/89] Use newer cycle variables and RUN - Replace GDATE with previous_cycle - Replace RDATE with model_start_date_current_cycle - Replace CDUMP with RUN and rCDUMP with rRUN - Add if-block with half_window variable for determining the value of model_start_date_current_cycle - Update staging job to use these variables Refs #2475 --- jobs/JGLOBAL_STAGE_IC | 44 ++++++++++++++++++++++-------------- parm/stage/stage.yaml.j2 | 2 +- scripts/exglobal_stage_ic.py | 10 ++++---- 3 files changed, 33 insertions(+), 23 deletions(-) diff --git a/jobs/JGLOBAL_STAGE_IC b/jobs/JGLOBAL_STAGE_IC index 6234ad3d58..a92ffafb01 100755 --- a/jobs/JGLOBAL_STAGE_IC +++ b/jobs/JGLOBAL_STAGE_IC @@ -5,19 +5,29 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "stage_ic" -c "base stage_ic" # Restart conditions for GFS cycle come from GDAS # shellcheck disable=SC2153 -rCDUMP=${RUN} +rRUN=${RUN} # shellcheck disable=SC2153 -[[ ${RUN} = "gfs" ]] && export rCDUMP="gdas" -export rCDUMP +[[ ${RUN} = "gfs" ]] && export rRUN="gdas" +# Define significant cycles # Locally scoped variables and functions # shellcheck disable=SC2153 -GDATE=$(date --utc -d "${PDY} ${cyc} - ${assim_freq} hours" +%Y%m%d%H) -gPDY="${GDATE:0:8}" -gcyc="${GDATE:8:2}" - -RDATE=$(date --utc -d "${PDY} ${cyc} + ${OFFSET_START_HOUR} hours" +%Y%m%d%H) -export RDATE +half_window=$(( assim_freq / 2 )) +current_cycle="${PDY}${cyc}" +previous_cycle=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} - ${assim_freq} hours" +%Y%m%d%H) +current_cycle_begin=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} - ${half_window} hours" +%Y%m%d%H) +current_cycle_end=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + ${half_window} hours" +%Y%m%d%H) +# Define model start date for current_cycle as the time the forecast will start +if [[ "${DOIAU:-NO}" == "YES" ]]; then + model_start_date_current_cycle="${current_cycle_begin}" +else + if [[ "${REPLAY_ICS:-NO}" == "YES" ]]; then + model_start_date_current_cycle=${current_cycle_end} + else + model_start_date_current_cycle=${current_cycle} + fi +fi +export model_start_date_current_cycle # Define MEMDIR_ARRAY MEMDIR_ARRAY=() @@ -46,32 +56,32 @@ for MEMDIR in "${MEMDIR_ARRAY[@]}"; do # Declare COMs if [[ "${MODE}" = "cycled" && "${RUN}" = "gdas" ]]; then - YMD=${PDY} HH=${cyc} declare_from_tmpl COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL + YMD=${current_cycle:0:8} HH=${current_cycle:8:2} declare_from_tmpl COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL export COMOUT_ATMOS_ANALYSIS fi if [[ ${EXP_WARM_START:-".false."} = ".true." ]]; then - RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl COMOUT_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL - RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl COMOUT_MED_RESTART_PREV:COM_MED_RESTART_TMPL + RUN=${rRUN} YMD=${previous_cycle:0:8} HH=${previous_cycle:8:2} declare_from_tmpl COMOUT_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL + RUN=${rRUN} YMD=${previous_cycle:0:8} HH=${previous_cycle:8:2} declare_from_tmpl COMOUT_MED_RESTART_PREV:COM_MED_RESTART_TMPL export COMOUT_ATMOS_RESTART_PREV COMOUT_MED_RESTART_PREV else - YMD=${PDY} HH=${cyc} declare_from_tmpl COMOUT_ATMOS_INPUT:COM_ATMOS_INPUT_TMPL + YMD=${current_cycle:0:8} HH=${current_cycle:8:2} declare_from_tmpl COMOUT_ATMOS_INPUT:COM_ATMOS_INPUT_TMPL export COMOUT_ATMOS_INPUT fi if [[ "${DO_OCN:-}" = "YES" ]]; then - RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl COMOUT_OCEAN_RESTART_PREV:COM_OCEAN_RESTART_TMPL + RUN=${rRUN} YMD=${previous_cycle:0:8} HH=${previous_cycle:8:2} declare_from_tmpl COMOUT_OCEAN_RESTART_PREV:COM_OCEAN_RESTART_TMPL export COMOUT_OCEAN_RESTART_PREV fi if [[ "${REPLAY_ICS:-NO}" = "YES" ]]; then - YMD=${PDY} HH=${cyc} declare_from_tmpl COMOUT_OCEAN_ANALYSIS:COM_OCEAN_ANALYSIS_TMPL + YMD=${current_cycle:0:8} HH=${current_cycle:8:2} declare_from_tmpl COMOUT_OCEAN_ANALYSIS:COM_OCEAN_ANALYSIS_TMPL export COMOUT_OCEAN_ANALYSIS fi if [[ "${DO_ICE:-}" = "YES" ]]; then - RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl COMOUT_ICE_RESTART_PREV:COM_ICE_RESTART_TMPL + RUN=${rRUN} YMD=${previous_cycle:0:8} HH=${previous_cycle:8:2} declare_from_tmpl COMOUT_ICE_RESTART_PREV:COM_ICE_RESTART_TMPL export COMOUT_ICE_RESTART_PREV fi if [[ "${DO_WAVE:-}" = "YES" ]]; then - YMD=${gPDY} HH=${gcyc} declare_from_tmpl COMOUT_WAVE_RESTART_PREV:COM_WAVE_RESTART_TMPL + YMD=${previous_cycle:0:8} HH=${previous_cycle:8:2} declare_from_tmpl COMOUT_WAVE_RESTART_PREV:COM_WAVE_RESTART_TMPL export COMOUT_WAVE_RESTART_PREV fi diff --git a/parm/stage/stage.yaml.j2 b/parm/stage/stage.yaml.j2 index c04b69fc51..d5d20e9322 100644 --- a/parm/stage/stage.yaml.j2 +++ b/parm/stage/stage.yaml.j2 @@ -19,7 +19,7 @@ ############################################################# {% set cycle_HH = current_cycle | strftime("%H") %} -{% set r_prefix = RDATE | to_YMD + "." + RDATE | strftime("%H") + "0000" %} +{% set r_prefix = model_start_date_current_cycle | to_YMD + "." + model_start_date_current_cycle | strftime("%H") + "0000" %} {% if MODE == "cycled" and RUN == "gdas" %} analysis: diff --git a/scripts/exglobal_stage_ic.py b/scripts/exglobal_stage_ic.py index b297840a72..e70eab0130 100755 --- a/scripts/exglobal_stage_ic.py +++ b/scripts/exglobal_stage_ic.py @@ -18,11 +18,11 @@ def main(): stage = Stage(config) # Pull out all the configuration keys needed to run stage job - keys = ['RUN', 'MODE', 'CASE', 'CASE_ENS', 'OCNRES', 'ICERES', 'waveGRD', - 'EXP_WARM_START', 'current_cycle', 'RDATE', 'CDUMP', 'rCDUMP', - 'ROTDIR', 'PARMgfs', 'ICSDIR', 'STAGE_IC_YAML_TMPL', - 'ntiles', 'MEMDIR', 'REPLAY_ICS', - 'DO_WAVE', 'DO_OCN', 'DO_ICE', 'DO_NEST'] + keys = ['RUN', 'MODE', 'EXP_WARM_START', + 'current_cycle', 'model_start_date_current_cycle', + 'ROTDIR', 'ICSDIR', 'STAGE_IC_YAML_TMPL', + 'OCNRES', 'waveGRD', 'ntiles', + 'REPLAY_ICS', 'DO_WAVE', 'DO_OCN', 'DO_ICE', 'DO_NEST'] stage_dict = AttrDict() for key in keys: From 7af7f5455a672e6cc65fc7339f303bf97b2c7336 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Mon, 5 Aug 2024 15:26:26 +0000 Subject: [PATCH 58/89] Move path_exists into stage task Refs #2475 --- scripts/exglobal_stage_ic.py | 3 --- ush/python/pygfs/task/stage.py | 3 +++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/exglobal_stage_ic.py b/scripts/exglobal_stage_ic.py index e70eab0130..b294ebb018 100755 --- a/scripts/exglobal_stage_ic.py +++ b/scripts/exglobal_stage_ic.py @@ -33,9 +33,6 @@ def main(): if key.startswith("COM"): stage_dict[key] = stage.task_config[key] - # Add the os.path.exists function to the dict for yaml parsing - stage_dict['path_exists'] = os.path.exists - # Stage ICs stage.execute_stage(stage_dict) diff --git a/ush/python/pygfs/task/stage.py b/ush/python/pygfs/task/stage.py index d5f6af28ee..9f21c63060 100644 --- a/ush/python/pygfs/task/stage.py +++ b/ush/python/pygfs/task/stage.py @@ -44,6 +44,9 @@ def execute_stage(self, stage_dict: Dict[str, Any]) -> None: if not os.path.isdir(stage_dict.ROTDIR): raise FileNotFoundError(f"FATAL ERROR: The ROTDIR ({stage_dict.ROTDIR}) does not exist!") + # Add the os.path.exists function to the dict for yaml parsing + stage_dict['path_exists'] = os.path.exists + stage_set = parse_j2yaml(self.task_config.STAGE_IC_YAML_TMPL, stage_dict) # Copy files to ROTDIR From ccf59456161d397560e3f5f7369eef58acdf7fd4 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Mon, 5 Aug 2024 16:45:32 +0000 Subject: [PATCH 59/89] Update ocean ICs and add path_exists Refs #2475 --- parm/stage/stage.yaml.j2 | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/parm/stage/stage.yaml.j2 b/parm/stage/stage.yaml.j2 index d5d20e9322..125d111a17 100644 --- a/parm/stage/stage.yaml.j2 +++ b/parm/stage/stage.yaml.j2 @@ -18,15 +18,18 @@ # For a full list see scripts/exglobal_stage_ic.py ############################################################# +# Set variables used below {% set cycle_HH = current_cycle | strftime("%H") %} {% set r_prefix = model_start_date_current_cycle | to_YMD + "." + model_start_date_current_cycle | strftime("%H") + "0000" %} +############################################################# +# Initial condition to stage + {% if MODE == "cycled" and RUN == "gdas" %} analysis: mkdir: - "{{ COMOUT_ATMOS_ANALYSIS }}" copy: - # TODO: Add abias_int? {% for ftype in ["abias", "abias_air", "abias_pc", "radstat"] %} - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ RUN }}.t{{ cycle_HH }}z.{{ ftype }}", "{{ COMOUT_ATMOS_ANALYSIS }}"] {% endfor %} @@ -102,14 +105,27 @@ ocean: - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_RESTART_PREV | relpath(ROTDIR) }}/{{ r_prefix }}.MOM.res_{{ nn }}.nc", "{{ COMOUT_OCEAN_RESTART_PREV }}"] {% endfor %} {% endif %} - {% if REPLAY_ICS == "YES" %} + +{% if REPLAY_ICS == "YES" %} +replay: + mkdir: + - "{{ COMOUT_OCEAN_ANALYSIS }}" + copy: - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ r_prefix }}.mom6_perturbation.nc", "{{ COMOUT_OCEAN_ANALYSIS }}/mom6_increment.nc"] - {% endif %} - {% if EXP_WARM_START == True %} - - ["{{ ICSDIR }}/{{ COMOUT_MED_RESTART_PREV | relpath(ROTDIR) }}/{{ r_prefix }}.ufs.cpld.cpl.r.nc", "{{ COMOUT_MED_RESTART_PREV }}"] - {% endif %} {% endif %} +{% if EXP_WARM_START == True %} +{% if path_exists(ICSDIR ~ "/" ~ COMOUT_MED_RESTART_PREV | relpath(ROTDIR) ~ "/" ~ r_prefix ~ ".ufs.cpld.cpl.r.nc") %} +mediator: + mkdir: + - "{{ COMOUT_MED_RESTART_PREV }}" + copy: + - ["{{ ICSDIR }}/{{ COMOUT_MED_RESTART_PREV | relpath(ROTDIR) }}/{{ r_prefix }}.ufs.cpld.cpl.r.nc", "{{ COMOUT_MED_RESTART_PREV }}"] +{% endif %} # path exists +{% endif %} # warm start true + +{% endif %} # DO_OCN=YES + {% if DO_WAVE %} wave: mkdir: From 68c121f37cd482942127bc3953dd5aa814f891b9 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Mon, 5 Aug 2024 17:26:01 +0000 Subject: [PATCH 60/89] Add allow_missing=False to stage task Refs #2475 --- ush/python/pygfs/task/stage.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/stage.py b/ush/python/pygfs/task/stage.py index 9f21c63060..f5f587adea 100644 --- a/ush/python/pygfs/task/stage.py +++ b/ush/python/pygfs/task/stage.py @@ -47,7 +47,8 @@ def execute_stage(self, stage_dict: Dict[str, Any]) -> None: # Add the os.path.exists function to the dict for yaml parsing stage_dict['path_exists'] = os.path.exists - stage_set = parse_j2yaml(self.task_config.STAGE_IC_YAML_TMPL, stage_dict) + stage_set = parse_j2yaml(self.task_config.STAGE_IC_YAML_TMPL, stage_dict, + allow_missing=False) # Copy files to ROTDIR for key in stage_set.keys(): From deec1af3e04ed2c377fd2e375cd67a020624c31b Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Mon, 5 Aug 2024 19:59:30 +0000 Subject: [PATCH 61/89] Update cycle prefixes and add analysis file checks - Replace r_prefix with either m_prefix or o_prefix - Add path_exists checks for analysis ICs Refs #2475 --- jobs/JGLOBAL_STAGE_IC | 3 +++ parm/stage/stage.yaml.j2 | 29 ++++++++++++++++------------- scripts/exglobal_stage_ic.py | 4 ++-- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/jobs/JGLOBAL_STAGE_IC b/jobs/JGLOBAL_STAGE_IC index a92ffafb01..a1707b18a0 100755 --- a/jobs/JGLOBAL_STAGE_IC +++ b/jobs/JGLOBAL_STAGE_IC @@ -17,6 +17,9 @@ current_cycle="${PDY}${cyc}" previous_cycle=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} - ${assim_freq} hours" +%Y%m%d%H) current_cycle_begin=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} - ${half_window} hours" +%Y%m%d%H) current_cycle_end=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + ${half_window} hours" +%Y%m%d%H) +current_cycle_offset=$(date --utc -d "${PDY} ${cyc} + ${OFFSET_START_HOUR} hours" +%Y%m%d%H) +export current_cycle_offset + # Define model start date for current_cycle as the time the forecast will start if [[ "${DOIAU:-NO}" == "YES" ]]; then model_start_date_current_cycle="${current_cycle_begin}" diff --git a/parm/stage/stage.yaml.j2 b/parm/stage/stage.yaml.j2 index 125d111a17..bfa1da217f 100644 --- a/parm/stage/stage.yaml.j2 +++ b/parm/stage/stage.yaml.j2 @@ -20,7 +20,8 @@ # Set variables used below {% set cycle_HH = current_cycle | strftime("%H") %} -{% set r_prefix = model_start_date_current_cycle | to_YMD + "." + model_start_date_current_cycle | strftime("%H") + "0000" %} +{% set m_prefix = model_start_date_current_cycle | to_YMD + "." + model_start_date_current_cycle | strftime("%H") + "0000" %} +{% set o_prefix = current_cycle_offset | to_YMD + "." + current_cycle_offset | strftime("%H") + "0000" %} ############################################################# # Initial condition to stage @@ -30,8 +31,10 @@ analysis: mkdir: - "{{ COMOUT_ATMOS_ANALYSIS }}" copy: - {% for ftype in ["abias", "abias_air", "abias_pc", "radstat"] %} + {% for ftype in ["abias", "abias_air", "abias_int", "abias_pc", "atminc.nc", "radstat"] %} + {% if path_exists(ICSDIR ~ "/" ~ COMOUT_ATMOS_ANALYSIS | relpath(ROTDIR) ~ "/" ~ RUN ~ ".t" ~ cycle_HH ~ "z." ~ ftype) %} - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ RUN }}.t{{ cycle_HH }}z.{{ ftype }}", "{{ COMOUT_ATMOS_ANALYSIS }}"] + {% endif %} {% endfor %} {% endif %} @@ -41,11 +44,11 @@ atmosphere_warm: - "{{ COMOUT_ATMOS_RESTART_PREV }}" copy: {% for ftype in ["coupler.res", "fv_core.res.nc"] %} - - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ r_prefix }}.{{ ftype }}", "{{ COMOUT_ATMOS_RESTART_PREV }}"] + - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ m_prefix }}.{{ ftype }}", "{{ COMOUT_ATMOS_RESTART_PREV }}"] {% endfor %} {% for ftype in ["ca_data", "fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data", "sfc_data"] %} {% for ntile in range(1, ntiles + 1) %} - - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ r_prefix }}.{{ ftype }}.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_RESTART_PREV }}"] + - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ m_prefix }}.{{ ftype }}.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_RESTART_PREV }}"] {% endfor %} # ntile {% endfor %} # ftype {% else %} @@ -66,7 +69,7 @@ atmosphere_perturbation: mkdir: - "{{ COMOUT_ATMOS_ANALYSIS }}" copy: - - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ r_prefix }}.fv3_perturbation.nc", "{{ COMOUT_ATMOS_ANALYSIS }}/{{ RUN }}.t{{ cycle_HH }}z.atminc.nc"] + - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ m_prefix }}.fv3_perturbation.nc", "{{ COMOUT_ATMOS_ANALYSIS }}/{{ RUN }}.t{{ cycle_HH }}z.atminc.nc"] {% endif %} {% if DO_NEST %} @@ -77,7 +80,7 @@ atmosphere_nest: copy: {% if EXP_WARM_START == True %} {% for ftype in ["ca_data", "fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data", "sfc_data"] %} - - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ r_prefix }}.{{ ftype }}.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_RESTART_PREV }}/{{ r_prefix }}.{{ ftype }}.nest0{{ ntile-5 }}.tile{{ ntile }}.nc"] + - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ m_prefix }}.{{ ftype }}.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_RESTART_PREV }}/{{ m_prefix }}.{{ ftype }}.nest0{{ ntile-5 }}.tile{{ ntile }}.nc"] {% endfor %} # ftype {% else %} {% for ftype in ["gfs_data", "sfc_data"] %} @@ -91,7 +94,7 @@ ice: mkdir: - "{{ COMOUT_ICE_RESTART_PREV }}" copy: - - ["{{ ICSDIR }}/{{ COMOUT_ICE_RESTART_PREV | relpath(ROTDIR) }}/{{ r_prefix }}.cice_model.res.nc", "{{ COMOUT_ICE_RESTART_PREV }}"] + - ["{{ ICSDIR }}/{{ COMOUT_ICE_RESTART_PREV | relpath(ROTDIR) }}/{{ o_prefix }}.cice_model.res.nc", "{{ COMOUT_ICE_RESTART_PREV }}"] {% endif %} {% if DO_OCN %} @@ -99,10 +102,10 @@ ocean: mkdir: - "{{ COMOUT_OCEAN_RESTART_PREV }}" copy: - - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_RESTART_PREV | relpath(ROTDIR) }}/{{ r_prefix }}.MOM.res.nc", "{{ COMOUT_OCEAN_RESTART_PREV }}"] + - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_RESTART_PREV | relpath(ROTDIR) }}/{{ o_prefix }}.MOM.res.nc", "{{ COMOUT_OCEAN_RESTART_PREV }}"] {% if OCNRES == "025" %} {% for nn in range(1, 3) %} - - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_RESTART_PREV | relpath(ROTDIR) }}/{{ r_prefix }}.MOM.res_{{ nn }}.nc", "{{ COMOUT_OCEAN_RESTART_PREV }}"] + - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_RESTART_PREV | relpath(ROTDIR) }}/{{ o_prefix }}.MOM.res_{{ nn }}.nc", "{{ COMOUT_OCEAN_RESTART_PREV }}"] {% endfor %} {% endif %} @@ -111,16 +114,16 @@ replay: mkdir: - "{{ COMOUT_OCEAN_ANALYSIS }}" copy: - - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ r_prefix }}.mom6_perturbation.nc", "{{ COMOUT_OCEAN_ANALYSIS }}/mom6_increment.nc"] + - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ o_prefix }}.mom6_perturbation.nc", "{{ COMOUT_OCEAN_ANALYSIS }}/mom6_increment.nc"] {% endif %} {% if EXP_WARM_START == True %} -{% if path_exists(ICSDIR ~ "/" ~ COMOUT_MED_RESTART_PREV | relpath(ROTDIR) ~ "/" ~ r_prefix ~ ".ufs.cpld.cpl.r.nc") %} +{% if path_exists(ICSDIR ~ "/" ~ COMOUT_MED_RESTART_PREV | relpath(ROTDIR) ~ "/" ~ m_prefix ~ ".ufs.cpld.cpl.r.nc") %} mediator: mkdir: - "{{ COMOUT_MED_RESTART_PREV }}" copy: - - ["{{ ICSDIR }}/{{ COMOUT_MED_RESTART_PREV | relpath(ROTDIR) }}/{{ r_prefix }}.ufs.cpld.cpl.r.nc", "{{ COMOUT_MED_RESTART_PREV }}"] + - ["{{ ICSDIR }}/{{ COMOUT_MED_RESTART_PREV | relpath(ROTDIR) }}/{{ m_prefix }}.ufs.cpld.cpl.r.nc", "{{ COMOUT_MED_RESTART_PREV }}"] {% endif %} # path exists {% endif %} # warm start true @@ -131,5 +134,5 @@ wave: mkdir: - "{{ COMOUT_WAVE_RESTART_PREV }}" copy: - - ["{{ ICSDIR }}/{{ COMOUT_WAVE_RESTART_PREV | relpath(ROTDIR) }}/{{ r_prefix }}.restart.{{ waveGRD }}", "{{ COMOUT_WAVE_RESTART_PREV }}"] + - ["{{ ICSDIR }}/{{ COMOUT_WAVE_RESTART_PREV | relpath(ROTDIR) }}/{{ o_prefix }}.restart.{{ waveGRD }}", "{{ COMOUT_WAVE_RESTART_PREV }}"] {% endif %} diff --git a/scripts/exglobal_stage_ic.py b/scripts/exglobal_stage_ic.py index b294ebb018..02b3b62973 100755 --- a/scripts/exglobal_stage_ic.py +++ b/scripts/exglobal_stage_ic.py @@ -18,8 +18,8 @@ def main(): stage = Stage(config) # Pull out all the configuration keys needed to run stage job - keys = ['RUN', 'MODE', 'EXP_WARM_START', - 'current_cycle', 'model_start_date_current_cycle', + keys = ['RUN', 'MODE', 'EXP_WARM_START', 'current_cycle', + 'current_cycle_offset', 'model_start_date_current_cycle', 'ROTDIR', 'ICSDIR', 'STAGE_IC_YAML_TMPL', 'OCNRES', 'waveGRD', 'ntiles', 'REPLAY_ICS', 'DO_WAVE', 'DO_OCN', 'DO_ICE', 'DO_NEST'] From 5f755aeed72b115eb2bf6aa32af86ab1695b7698 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Mon, 5 Aug 2024 20:28:10 +0000 Subject: [PATCH 62/89] Add previous_cycle to staging yaml for sfcanl_data - Add previous_cycle as p_prefix - Add sfcanl_data tile files to warm start file pickup - Change some m_prefix to o_prefix Refs #2475 --- jobs/JGLOBAL_STAGE_IC | 3 +-- parm/stage/stage.yaml.j2 | 10 +++++++--- scripts/exglobal_stage_ic.py | 3 ++- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/jobs/JGLOBAL_STAGE_IC b/jobs/JGLOBAL_STAGE_IC index a1707b18a0..0d3fe2e11d 100755 --- a/jobs/JGLOBAL_STAGE_IC +++ b/jobs/JGLOBAL_STAGE_IC @@ -18,7 +18,6 @@ previous_cycle=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} - ${as current_cycle_begin=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} - ${half_window} hours" +%Y%m%d%H) current_cycle_end=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + ${half_window} hours" +%Y%m%d%H) current_cycle_offset=$(date --utc -d "${PDY} ${cyc} + ${OFFSET_START_HOUR} hours" +%Y%m%d%H) -export current_cycle_offset # Define model start date for current_cycle as the time the forecast will start if [[ "${DOIAU:-NO}" == "YES" ]]; then @@ -30,7 +29,7 @@ else model_start_date_current_cycle=${current_cycle} fi fi -export model_start_date_current_cycle +export previous_cycle current_cycle_offset model_start_date_current_cycle # Define MEMDIR_ARRAY MEMDIR_ARRAY=() diff --git a/parm/stage/stage.yaml.j2 b/parm/stage/stage.yaml.j2 index bfa1da217f..a8e86a8899 100644 --- a/parm/stage/stage.yaml.j2 +++ b/parm/stage/stage.yaml.j2 @@ -22,6 +22,7 @@ {% set cycle_HH = current_cycle | strftime("%H") %} {% set m_prefix = model_start_date_current_cycle | to_YMD + "." + model_start_date_current_cycle | strftime("%H") + "0000" %} {% set o_prefix = current_cycle_offset | to_YMD + "." + current_cycle_offset | strftime("%H") + "0000" %} +{% set p_prefix = previous_cycle | to_YMD + "." + previous_cycle | strftime("%H") + "0000" %} ############################################################# # Initial condition to stage @@ -44,13 +45,16 @@ atmosphere_warm: - "{{ COMOUT_ATMOS_RESTART_PREV }}" copy: {% for ftype in ["coupler.res", "fv_core.res.nc"] %} - - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ m_prefix }}.{{ ftype }}", "{{ COMOUT_ATMOS_RESTART_PREV }}"] + - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ o_prefix }}.{{ ftype }}", "{{ COMOUT_ATMOS_RESTART_PREV }}"] {% endfor %} {% for ftype in ["ca_data", "fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data", "sfc_data"] %} {% for ntile in range(1, ntiles + 1) %} - - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ m_prefix }}.{{ ftype }}.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_RESTART_PREV }}"] + - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ o_prefix }}.{{ ftype }}.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_RESTART_PREV }}"] {% endfor %} # ntile {% endfor %} # ftype + {% for ntile in range(1, ntiles + 1) %} + - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ p_prefix }}.sfcanl_data.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_RESTART_PREV }}"] + {% endfor %} # ntile {% else %} atmosphere_cold: mkdir: @@ -80,7 +84,7 @@ atmosphere_nest: copy: {% if EXP_WARM_START == True %} {% for ftype in ["ca_data", "fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data", "sfc_data"] %} - - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ m_prefix }}.{{ ftype }}.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_RESTART_PREV }}/{{ m_prefix }}.{{ ftype }}.nest0{{ ntile-5 }}.tile{{ ntile }}.nc"] + - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ o_prefix }}.{{ ftype }}.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_RESTART_PREV }}/{{ o_prefix }}.{{ ftype }}.nest0{{ ntile-5 }}.tile{{ ntile }}.nc"] {% endfor %} # ftype {% else %} {% for ftype in ["gfs_data", "sfc_data"] %} diff --git a/scripts/exglobal_stage_ic.py b/scripts/exglobal_stage_ic.py index 02b3b62973..d5be8e8ee1 100755 --- a/scripts/exglobal_stage_ic.py +++ b/scripts/exglobal_stage_ic.py @@ -18,7 +18,8 @@ def main(): stage = Stage(config) # Pull out all the configuration keys needed to run stage job - keys = ['RUN', 'MODE', 'EXP_WARM_START', 'current_cycle', + keys = ['RUN', 'MODE', 'EXP_WARM_START', + 'previous_cycle', 'current_cycle', 'current_cycle_offset', 'model_start_date_current_cycle', 'ROTDIR', 'ICSDIR', 'STAGE_IC_YAML_TMPL', 'OCNRES', 'waveGRD', 'ntiles', From 256b8e2c1af4a1cd8011dd556a0394c6636fcaf7 Mon Sep 17 00:00:00 2001 From: Kate Friedman Date: Tue, 6 Aug 2024 08:44:14 -0400 Subject: [PATCH 63/89] Discontinue exporting rRUN Refs #2475 Co-authored-by: Walter Kolczynski - NOAA --- jobs/JGLOBAL_STAGE_IC | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/JGLOBAL_STAGE_IC b/jobs/JGLOBAL_STAGE_IC index 0d3fe2e11d..6aafa993e3 100755 --- a/jobs/JGLOBAL_STAGE_IC +++ b/jobs/JGLOBAL_STAGE_IC @@ -7,7 +7,7 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "stage_ic" -c "base stage_ic" # shellcheck disable=SC2153 rRUN=${RUN} # shellcheck disable=SC2153 -[[ ${RUN} = "gfs" ]] && export rRUN="gdas" +[[ ${RUN} == "gfs" ]] && rRUN="gdas" # Define significant cycles # Locally scoped variables and functions From da0e39c3e1b870b75a9370a9f012cb076a6e1040 Mon Sep 17 00:00:00 2001 From: Kate Friedman Date: Tue, 6 Aug 2024 08:53:30 -0400 Subject: [PATCH 64/89] Update cycle variables to use PDYcyc Refs #2475 Co-authored-by: Walter Kolczynski - NOAA --- jobs/JGLOBAL_STAGE_IC | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/jobs/JGLOBAL_STAGE_IC b/jobs/JGLOBAL_STAGE_IC index 6aafa993e3..5bbc8adef3 100755 --- a/jobs/JGLOBAL_STAGE_IC +++ b/jobs/JGLOBAL_STAGE_IC @@ -14,9 +14,9 @@ rRUN=${RUN} # shellcheck disable=SC2153 half_window=$(( assim_freq / 2 )) current_cycle="${PDY}${cyc}" -previous_cycle=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} - ${assim_freq} hours" +%Y%m%d%H) -current_cycle_begin=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} - ${half_window} hours" +%Y%m%d%H) -current_cycle_end=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + ${half_window} hours" +%Y%m%d%H) +previous_cycle=$(date --utc -d "${PDY} ${cyc} - ${assim_freq} hours" +%Y%m%d%H) +current_cycle_begin=$(date --utc -d "${PDY} ${cyc} - ${half_window} hours" +%Y%m%d%H) +current_cycle_end=$(date --utc -d "${PDY} ${cyc} + ${half_window} hours" +%Y%m%d%H) current_cycle_offset=$(date --utc -d "${PDY} ${cyc} + ${OFFSET_START_HOUR} hours" +%Y%m%d%H) # Define model start date for current_cycle as the time the forecast will start From 0ee847fd67717b7e41a138853348d5f4c09018ff Mon Sep 17 00:00:00 2001 From: Kate Friedman Date: Tue, 6 Aug 2024 08:56:03 -0400 Subject: [PATCH 65/89] Add -x for COMOUT_ATMOS_ANALYSIS declare Refs #2475 Co-authored-by: Walter Kolczynski - NOAA --- jobs/JGLOBAL_STAGE_IC | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/jobs/JGLOBAL_STAGE_IC b/jobs/JGLOBAL_STAGE_IC index 5bbc8adef3..f3964088c9 100755 --- a/jobs/JGLOBAL_STAGE_IC +++ b/jobs/JGLOBAL_STAGE_IC @@ -58,8 +58,7 @@ for MEMDIR in "${MEMDIR_ARRAY[@]}"; do # Declare COMs if [[ "${MODE}" = "cycled" && "${RUN}" = "gdas" ]]; then - YMD=${current_cycle:0:8} HH=${current_cycle:8:2} declare_from_tmpl COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL - export COMOUT_ATMOS_ANALYSIS + YMD=${current_cycle:0:8} HH=${current_cycle:8:2} declare_from_tmpl -x COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL fi if [[ ${EXP_WARM_START:-".false."} = ".true." ]]; then From 3fee34c877f14b3615d4de3b9de00c449dd22b5c Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Tue, 6 Aug 2024 13:01:09 +0000 Subject: [PATCH 66/89] Update staging job declares - Remove exports and add "-x" to commands - Add current_cycle to exports - Move REPLAY_ICS if inside DO_OCN if Refs #2475 --- jobs/JGLOBAL_STAGE_IC | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/jobs/JGLOBAL_STAGE_IC b/jobs/JGLOBAL_STAGE_IC index f3964088c9..6c6304526e 100755 --- a/jobs/JGLOBAL_STAGE_IC +++ b/jobs/JGLOBAL_STAGE_IC @@ -29,7 +29,7 @@ else model_start_date_current_cycle=${current_cycle} fi fi -export previous_cycle current_cycle_offset model_start_date_current_cycle +export current_cycle previous_cycle current_cycle_offset model_start_date_current_cycle # Define MEMDIR_ARRAY MEMDIR_ARRAY=() @@ -62,28 +62,22 @@ for MEMDIR in "${MEMDIR_ARRAY[@]}"; do fi if [[ ${EXP_WARM_START:-".false."} = ".true." ]]; then - RUN=${rRUN} YMD=${previous_cycle:0:8} HH=${previous_cycle:8:2} declare_from_tmpl COMOUT_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL - RUN=${rRUN} YMD=${previous_cycle:0:8} HH=${previous_cycle:8:2} declare_from_tmpl COMOUT_MED_RESTART_PREV:COM_MED_RESTART_TMPL - export COMOUT_ATMOS_RESTART_PREV COMOUT_MED_RESTART_PREV + RUN=${rRUN} YMD=${previous_cycle:0:8} HH=${previous_cycle:8:2} declare_from_tmpl -x COMOUT_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL + RUN=${rRUN} YMD=${previous_cycle:0:8} HH=${previous_cycle:8:2} declare_from_tmpl -x COMOUT_MED_RESTART_PREV:COM_MED_RESTART_TMPL else - YMD=${current_cycle:0:8} HH=${current_cycle:8:2} declare_from_tmpl COMOUT_ATMOS_INPUT:COM_ATMOS_INPUT_TMPL - export COMOUT_ATMOS_INPUT + YMD=${current_cycle:0:8} HH=${current_cycle:8:2} declare_from_tmpl -x COMOUT_ATMOS_INPUT:COM_ATMOS_INPUT_TMPL fi if [[ "${DO_OCN:-}" = "YES" ]]; then - RUN=${rRUN} YMD=${previous_cycle:0:8} HH=${previous_cycle:8:2} declare_from_tmpl COMOUT_OCEAN_RESTART_PREV:COM_OCEAN_RESTART_TMPL - export COMOUT_OCEAN_RESTART_PREV - fi - if [[ "${REPLAY_ICS:-NO}" = "YES" ]]; then - YMD=${current_cycle:0:8} HH=${current_cycle:8:2} declare_from_tmpl COMOUT_OCEAN_ANALYSIS:COM_OCEAN_ANALYSIS_TMPL - export COMOUT_OCEAN_ANALYSIS + RUN=${rRUN} YMD=${previous_cycle:0:8} HH=${previous_cycle:8:2} declare_from_tmpl -x COMOUT_OCEAN_RESTART_PREV:COM_OCEAN_RESTART_TMPL + if [[ "${REPLAY_ICS:-NO}" = "YES" ]]; then + YMD=${current_cycle:0:8} HH=${current_cycle:8:2} declare_from_tmpl -x COMOUT_OCEAN_ANALYSIS:COM_OCEAN_ANALYSIS_TMPL + fi fi if [[ "${DO_ICE:-}" = "YES" ]]; then - RUN=${rRUN} YMD=${previous_cycle:0:8} HH=${previous_cycle:8:2} declare_from_tmpl COMOUT_ICE_RESTART_PREV:COM_ICE_RESTART_TMPL - export COMOUT_ICE_RESTART_PREV + RUN=${rRUN} YMD=${previous_cycle:0:8} HH=${previous_cycle:8:2} declare_from_tmpl -x COMOUT_ICE_RESTART_PREV:COM_ICE_RESTART_TMPL fi if [[ "${DO_WAVE:-}" = "YES" ]]; then - YMD=${previous_cycle:0:8} HH=${previous_cycle:8:2} declare_from_tmpl COMOUT_WAVE_RESTART_PREV:COM_WAVE_RESTART_TMPL - export COMOUT_WAVE_RESTART_PREV + YMD=${previous_cycle:0:8} HH=${previous_cycle:8:2} declare_from_tmpl -x COMOUT_WAVE_RESTART_PREV:COM_WAVE_RESTART_TMPL fi # Execute staging From 8f096b77c3d54ff0091f515c186a1ad1c050cf05 Mon Sep 17 00:00:00 2001 From: Kate Friedman Date: Tue, 6 Aug 2024 09:05:28 -0400 Subject: [PATCH 67/89] Update ENS check in config.stage_ic for gefs Refs #2475 Co-authored-by: Walter Kolczynski - NOAA --- parm/config/gefs/config.stage_ic | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/gefs/config.stage_ic b/parm/config/gefs/config.stage_ic index ff749bdf87..73807ff1c7 100644 --- a/parm/config/gefs/config.stage_ic +++ b/parm/config/gefs/config.stage_ic @@ -15,7 +15,7 @@ if [[ -z "${ICSDIR}" ]] ; then IC_VER="20240610" - if [[ "${CASE_ENS}" != "@CASEENS@" ]] ; then + if (( NMEM_ENS > 0 )) ; then ENSIC="${CASE_ENS}" fi From 052726709cfd0caa61d32ce18b3bcfdc57daf5ae Mon Sep 17 00:00:00 2001 From: Kate Friedman Date: Tue, 6 Aug 2024 09:06:30 -0400 Subject: [PATCH 68/89] Remove PYTHONPATH export in stage_ic.sh Refs #2475 Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> --- jobs/rocoto/stage_ic.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/jobs/rocoto/stage_ic.sh b/jobs/rocoto/stage_ic.sh index 5b79bb9e7d..5e7b3395d2 100755 --- a/jobs/rocoto/stage_ic.sh +++ b/jobs/rocoto/stage_ic.sh @@ -7,11 +7,6 @@ source "${HOMEgfs}/ush/preamble.sh" status=$? [[ "${status}" -ne 0 ]] && exit "${status}" -############################################################### -# setup python path for workflow utilities and tasks -PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${HOMEgfs}/ush/python" -export PYTHONPATH - export job="stage_ic" export jobid="${job}.$$" From 00d0a8cab0ab98893eca6265053c756eb177028e Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Tue, 6 Aug 2024 13:12:29 +0000 Subject: [PATCH 69/89] Change non-exported variables to lowercase Do so in stage configs Refs #2475 --- parm/config/gefs/config.stage_ic | 8 ++++---- parm/config/gfs/config.stage_ic | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/parm/config/gefs/config.stage_ic b/parm/config/gefs/config.stage_ic index 73807ff1c7..5bbf483a1a 100644 --- a/parm/config/gefs/config.stage_ic +++ b/parm/config/gefs/config.stage_ic @@ -13,17 +13,17 @@ export STAGE_IC_YAML_TMPL="${PARMgfs}/stage/stage.yaml.j2" if [[ -z "${ICSDIR}" ]] ; then - IC_VER="20240610" + ic_ver="20240610" if (( NMEM_ENS > 0 )) ; then - ENSIC="${CASE_ENS}" + ensic="${CASE_ENS}" fi if [[ "${DO_OCN:-NO}" == "YES" ]] ; then - OCNIC="mx${OCNRES}" + ocnic="mx${OCNRES}" fi - export ICSDIR="${BASE_IC}/${CASE}${ENSIC:-}${OCNIC:-}/${IC_VER}" + export ICSDIR="${BASE_IC}/${CASE}${ensic:-}${ocnic:-}/${ic_ver}" fi diff --git a/parm/config/gfs/config.stage_ic b/parm/config/gfs/config.stage_ic index dcfe2654db..54f65df7c4 100644 --- a/parm/config/gfs/config.stage_ic +++ b/parm/config/gfs/config.stage_ic @@ -12,17 +12,17 @@ export STAGE_IC_YAML_TMPL="${PARMgfs}/stage/stage.yaml.j2" # Set ICSDIR (if not defined) if [[ -z "${ICSDIR}" ]] ; then - IC_VER="20240610" + ic_ver="20240610" - if [[ "${CASE_ENS}" != "@CASEENS@" ]] ; then - ENSIC="${CASE_ENS}" + if (( NMEM_ENS > 0 )) ; then + ensic="${CASE_ENS}" fi if [[ "${DO_OCN:-NO}" == "YES" ]] ; then - OCNIC="mx${OCNRES}" + ocnic="mx${OCNRES}" fi - export ICSDIR="${BASE_IC}/${CASE}${ENSIC:-}${OCNIC:-}/${IC_VER}" + export ICSDIR="${BASE_IC}/${CASE}${ensic:-}${ocnic:-}/${ic_ver}" fi From e14e623287709910e1063052f88b8b38e13bc72a Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Tue, 6 Aug 2024 14:54:56 +0000 Subject: [PATCH 70/89] Add missing documentation to execute_stage function Refs #2475 --- ush/python/pygfs/task/stage.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ush/python/pygfs/task/stage.py b/ush/python/pygfs/task/stage.py index f5f587adea..e1459fdee0 100644 --- a/ush/python/pygfs/task/stage.py +++ b/ush/python/pygfs/task/stage.py @@ -39,6 +39,15 @@ def __init__(self, config: Dict[str, Any]) -> None: @logit(logger) def execute_stage(self, stage_dict: Dict[str, Any]) -> None: """Perform local staging of initial condition files. + + Parameters + ---------- + stage_dict : Dict[str, Any] + Configuration dictionary + + Returns + ------- + None """ if not os.path.isdir(stage_dict.ROTDIR): From a0b9af0720b4684e1b7f939c3f32991afb4f9bea Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Tue, 6 Aug 2024 16:52:19 +0000 Subject: [PATCH 71/89] Move IC variables from base to stage config - Move ICSDIR and BASE_IC variables to config.stage_ic - Update workflow/setup_expt.py to fill in templates in config.stage_ic based on user input and host Refs #2475 --- parm/config/gefs/config.base | 2 -- parm/config/gefs/config.stage_ic | 3 +++ parm/config/gfs/config.base | 2 -- parm/config/gfs/config.stage_ic | 3 +++ workflow/setup_expt.py | 13 +++++++++++-- 5 files changed, 17 insertions(+), 6 deletions(-) diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base index 282c230194..3aacf62f01 100644 --- a/parm/config/gefs/config.base +++ b/parm/config/gefs/config.base @@ -38,8 +38,6 @@ export FIXugwd=${FIXgfs}/ugwd export PACKAGEROOT="@PACKAGEROOT@" # TODO: set via prod_envir in Ops export COMROOT="@COMROOT@" # TODO: set via prod_envir in Ops export COMINsyn="@COMINsyn@" -export ICSDIR="@ICSDIR@" -export BASE_IC="@BASE_IC@" # USER specific paths export HOMEDIR="@HOMEDIR@" diff --git a/parm/config/gefs/config.stage_ic b/parm/config/gefs/config.stage_ic index 5bbf483a1a..a7c57591d2 100644 --- a/parm/config/gefs/config.stage_ic +++ b/parm/config/gefs/config.stage_ic @@ -7,6 +7,9 @@ echo "BEGIN: config.stage_ic" # Get task specific resources source "${EXPDIR}/config.resources" stage_ic +export ICSDIR="@ICSDIR@" +export BASE_IC="@BASE_IC@" + export STAGE_IC_YAML_TMPL="${PARMgfs}/stage/stage.yaml.j2" # Set ICSDIR diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index fa892bb579..866da30569 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -47,8 +47,6 @@ export PACKAGEROOT="@PACKAGEROOT@" # TODO: set via prod_envir in Ops export COMROOT="@COMROOT@" # TODO: set via prod_envir in Ops export COMINsyn="@COMINsyn@" export DMPDIR="@DMPDIR@" -export ICSDIR="@ICSDIR@" -export BASE_IC="@BASE_IC@" # Gempak from external models # Default locations are to dummy locations for testing diff --git a/parm/config/gfs/config.stage_ic b/parm/config/gfs/config.stage_ic index 54f65df7c4..d81bbf3ac6 100644 --- a/parm/config/gfs/config.stage_ic +++ b/parm/config/gfs/config.stage_ic @@ -7,6 +7,9 @@ echo "BEGIN: config.stage_ic" # Get task specific resources source "${EXPDIR}/config.resources" stage_ic +export ICSDIR="@ICSDIR@" +export BASE_IC="@BASE_IC@" + export STAGE_IC_YAML_TMPL="${PARMgfs}/stage/stage.yaml.j2" # Set ICSDIR (if not defined) diff --git a/workflow/setup_expt.py b/workflow/setup_expt.py index 5e9bd1f197..e213394e20 100755 --- a/workflow/setup_expt.py +++ b/workflow/setup_expt.py @@ -65,7 +65,17 @@ def _update_defaults(dict_in: dict) -> dict: # First update config.base edit_baseconfig(host, inputs, yaml_dict) - # loop over other configs and update them + # Update stage config + stage_dict = { + "@ICSDIR@": inputs.icsdir + } + host_dict = get_template_dict(host.info) + stage_dict = dict(stage_dict, **host_dict) + stage_input = f'{inputs.configdir}/config.stage_ic' + stage_output = f'{inputs.expdir}/{inputs.pslot}/config.stage_ic' + edit_config(stage_input, stage_output, stage_dict) + + # Loop over other configs and update them with defaults for cfg in yaml_dict.keys(): if cfg == 'base': continue @@ -104,7 +114,6 @@ def edit_baseconfig(host, inputs, yaml_dict): "@OCNRES@": f"{int(100.*inputs.resdetocean):03d}", "@EXPDIR@": inputs.expdir, "@COMROOT@": inputs.comroot, - "@ICSDIR@": inputs.icsdir, "@EXP_WARM_START@": is_warm_start, "@MODE@": inputs.mode, "@gfs_cyc@": inputs.gfs_cyc, From bb54087c11d87ce4b24f3e41f02415c01323b267 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Tue, 6 Aug 2024 16:57:38 +0000 Subject: [PATCH 72/89] Correct spelling mistake in config.com - Change "decalre" to "declare" Refs #2475 --- parm/config/gfs/config.com | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/gfs/config.com b/parm/config/gfs/config.com index 85b8cd3188..02a5b1edf5 100644 --- a/parm/config/gfs/config.com +++ b/parm/config/gfs/config.com @@ -12,7 +12,7 @@ echo "BEGIN: config.com" # declare_from_tmpl [-rx] $var1[:$tmpl1] [$var2[:$tmpl2]] [...]] # # options: -# -r: Make variable read-only (same as `decalre -r`) +# -r: Make variable read-only (same as `declare -r`) # -x: Mark variable for declare -rx (same as `declare -x`) # var1, var2, etc: Variable names whose values will be generated from a template # and declared From 6f2b75a75f83931c2cec1c7e28abea198d936f15 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Tue, 6 Aug 2024 17:00:42 +0000 Subject: [PATCH 73/89] Add comments in config.stage_ic Short descriptions for ICSDIR and BASE_IC Refs #2475 --- parm/config/gefs/config.stage_ic | 4 ++-- parm/config/gfs/config.stage_ic | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/parm/config/gefs/config.stage_ic b/parm/config/gefs/config.stage_ic index a7c57591d2..7cf64648bc 100644 --- a/parm/config/gefs/config.stage_ic +++ b/parm/config/gefs/config.stage_ic @@ -7,8 +7,8 @@ echo "BEGIN: config.stage_ic" # Get task specific resources source "${EXPDIR}/config.resources" stage_ic -export ICSDIR="@ICSDIR@" -export BASE_IC="@BASE_IC@" +export ICSDIR="@ICSDIR@" # User provided ICSDIR; blank if not provided +export BASE_IC="@BASE_IC@" # Platform home for staged ICs export STAGE_IC_YAML_TMPL="${PARMgfs}/stage/stage.yaml.j2" diff --git a/parm/config/gfs/config.stage_ic b/parm/config/gfs/config.stage_ic index d81bbf3ac6..78eba84962 100644 --- a/parm/config/gfs/config.stage_ic +++ b/parm/config/gfs/config.stage_ic @@ -7,8 +7,8 @@ echo "BEGIN: config.stage_ic" # Get task specific resources source "${EXPDIR}/config.resources" stage_ic -export ICSDIR="@ICSDIR@" -export BASE_IC="@BASE_IC@" +export ICSDIR="@ICSDIR@" # User provided ICSDIR; blank if not provided +export BASE_IC="@BASE_IC@" # Platform home for staged ICs export STAGE_IC_YAML_TMPL="${PARMgfs}/stage/stage.yaml.j2" From 55e47c67cd26c1b12f65d5278a09b851c63d8d78 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Tue, 6 Aug 2024 17:05:28 +0000 Subject: [PATCH 74/89] MEMDIR updates in JGLOBAL_STAGE_IC - Remove MEMDIR export - Add MEMDIR input for all declare commands that may need it; if not gefs or enkfgdas then MEMDIR will be empty and COM won't include - Remove incorrect comment about locally scoped variables Refs #2475 --- jobs/JGLOBAL_STAGE_IC | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/jobs/JGLOBAL_STAGE_IC b/jobs/JGLOBAL_STAGE_IC index 6c6304526e..33b44f2a70 100755 --- a/jobs/JGLOBAL_STAGE_IC +++ b/jobs/JGLOBAL_STAGE_IC @@ -10,7 +10,6 @@ rRUN=${RUN} [[ ${RUN} == "gfs" ]] && rRUN="gdas" # Define significant cycles -# Locally scoped variables and functions # shellcheck disable=SC2153 half_window=$(( assim_freq / 2 )) current_cycle="${PDY}${cyc}" @@ -53,31 +52,28 @@ err=0 ############################################################### for MEMDIR in "${MEMDIR_ARRAY[@]}"; do - # Export MEMDIR; need even if empty - export MEMDIR - # Declare COMs if [[ "${MODE}" = "cycled" && "${RUN}" = "gdas" ]]; then YMD=${current_cycle:0:8} HH=${current_cycle:8:2} declare_from_tmpl -x COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL fi if [[ ${EXP_WARM_START:-".false."} = ".true." ]]; then - RUN=${rRUN} YMD=${previous_cycle:0:8} HH=${previous_cycle:8:2} declare_from_tmpl -x COMOUT_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL - RUN=${rRUN} YMD=${previous_cycle:0:8} HH=${previous_cycle:8:2} declare_from_tmpl -x COMOUT_MED_RESTART_PREV:COM_MED_RESTART_TMPL + MEMDIR=${MEMDIR} RUN=${rRUN} YMD=${previous_cycle:0:8} HH=${previous_cycle:8:2} declare_from_tmpl -x COMOUT_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL + MEMDIR=${MEMDIR} RUN=${rRUN} YMD=${previous_cycle:0:8} HH=${previous_cycle:8:2} declare_from_tmpl -x COMOUT_MED_RESTART_PREV:COM_MED_RESTART_TMPL else - YMD=${current_cycle:0:8} HH=${current_cycle:8:2} declare_from_tmpl -x COMOUT_ATMOS_INPUT:COM_ATMOS_INPUT_TMPL + MEMDIR=${MEMDIR} YMD=${current_cycle:0:8} HH=${current_cycle:8:2} declare_from_tmpl -x COMOUT_ATMOS_INPUT:COM_ATMOS_INPUT_TMPL fi if [[ "${DO_OCN:-}" = "YES" ]]; then - RUN=${rRUN} YMD=${previous_cycle:0:8} HH=${previous_cycle:8:2} declare_from_tmpl -x COMOUT_OCEAN_RESTART_PREV:COM_OCEAN_RESTART_TMPL + MEMDIR=${MEMDIR} RUN=${rRUN} YMD=${previous_cycle:0:8} HH=${previous_cycle:8:2} declare_from_tmpl -x COMOUT_OCEAN_RESTART_PREV:COM_OCEAN_RESTART_TMPL if [[ "${REPLAY_ICS:-NO}" = "YES" ]]; then - YMD=${current_cycle:0:8} HH=${current_cycle:8:2} declare_from_tmpl -x COMOUT_OCEAN_ANALYSIS:COM_OCEAN_ANALYSIS_TMPL + MEMDIR=${MEMDIR} YMD=${current_cycle:0:8} HH=${current_cycle:8:2} declare_from_tmpl -x COMOUT_OCEAN_ANALYSIS:COM_OCEAN_ANALYSIS_TMPL fi fi if [[ "${DO_ICE:-}" = "YES" ]]; then - RUN=${rRUN} YMD=${previous_cycle:0:8} HH=${previous_cycle:8:2} declare_from_tmpl -x COMOUT_ICE_RESTART_PREV:COM_ICE_RESTART_TMPL + MEMDIR=${MEMDIR} RUN=${rRUN} YMD=${previous_cycle:0:8} HH=${previous_cycle:8:2} declare_from_tmpl -x COMOUT_ICE_RESTART_PREV:COM_ICE_RESTART_TMPL fi if [[ "${DO_WAVE:-}" = "YES" ]]; then - YMD=${previous_cycle:0:8} HH=${previous_cycle:8:2} declare_from_tmpl -x COMOUT_WAVE_RESTART_PREV:COM_WAVE_RESTART_TMPL + MEMDIR=${MEMDIR} YMD=${previous_cycle:0:8} HH=${previous_cycle:8:2} declare_from_tmpl -x COMOUT_WAVE_RESTART_PREV:COM_WAVE_RESTART_TMPL fi # Execute staging From 2913b5815b5b4f4039ec539f6e1a9af7e3758344 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Mon, 12 Aug 2024 13:59:33 +0000 Subject: [PATCH 75/89] Add enkfgdas into stage_ic job checks Refs #2475 --- workflow/rocoto/gfs_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 630506e5b0..5bb928f20a 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -868,7 +868,7 @@ def _fcst_cycled(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies) - if self.run in ['gdas']: + if self.run in ['gdas','enkfgdas']: dep_dict = {'type': 'task', 'name': f'{self.run}stage_ic'} dependencies.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='or', dep=dependencies) From b74e6965726f7cbaf52807b00e429131c5a83500 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Mon, 12 Aug 2024 14:03:47 +0000 Subject: [PATCH 76/89] Refactor staging job for mem loop and COM declare - Move rRUN, mem list, and COM declares inside python/yaml - Move mem loop inside python - Add path_exists checks to some initial conditions - Create dicts for replace_tmpl COM declares Refs #2475 --- jobs/JGLOBAL_STAGE_IC | 55 +----------------------- parm/stage/stage.yaml.j2 | 77 +++++++++++++++++++++++++++------- scripts/exglobal_stage_ic.py | 2 +- ush/python/pygfs/task/stage.py | 34 ++++++++++++--- 4 files changed, 92 insertions(+), 76 deletions(-) diff --git a/jobs/JGLOBAL_STAGE_IC b/jobs/JGLOBAL_STAGE_IC index 33b44f2a70..3346db0827 100755 --- a/jobs/JGLOBAL_STAGE_IC +++ b/jobs/JGLOBAL_STAGE_IC @@ -3,12 +3,6 @@ source "${HOMEgfs}/ush/preamble.sh" source "${HOMEgfs}/ush/jjob_header.sh" -e "stage_ic" -c "base stage_ic" -# Restart conditions for GFS cycle come from GDAS -# shellcheck disable=SC2153 -rRUN=${RUN} -# shellcheck disable=SC2153 -[[ ${RUN} == "gfs" ]] && rRUN="gdas" - # Define significant cycles # shellcheck disable=SC2153 half_window=$(( assim_freq / 2 )) @@ -30,56 +24,11 @@ else fi export current_cycle previous_cycle current_cycle_offset model_start_date_current_cycle -# Define MEMDIR_ARRAY -MEMDIR_ARRAY=() -if [[ "${RUN:-}" = "enkfgdas" || "${RUN:-}" = "gefs" ]]; then - if [[ "${RUN:-}" = "gefs" ]]; then - ii_start=0 - elif [[ "${RUN:-}" = "enkfgdas" ]]; then - ii_start=1 - fi - # Populate the member_dirs array based on the value of NMEM_ENS - for ((ii = "${ii_start}"; ii <= "${NMEM_ENS:-0}"; ii++)); do - MEMDIR_ARRAY+=("mem$(printf "%03d" "${ii}")") - done -else - MEMDIR_ARRAY+=("") -fi - # Initialize return code err=0 -############################################################### -for MEMDIR in "${MEMDIR_ARRAY[@]}"; do - - # Declare COMs - if [[ "${MODE}" = "cycled" && "${RUN}" = "gdas" ]]; then - YMD=${current_cycle:0:8} HH=${current_cycle:8:2} declare_from_tmpl -x COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL - fi - - if [[ ${EXP_WARM_START:-".false."} = ".true." ]]; then - MEMDIR=${MEMDIR} RUN=${rRUN} YMD=${previous_cycle:0:8} HH=${previous_cycle:8:2} declare_from_tmpl -x COMOUT_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL - MEMDIR=${MEMDIR} RUN=${rRUN} YMD=${previous_cycle:0:8} HH=${previous_cycle:8:2} declare_from_tmpl -x COMOUT_MED_RESTART_PREV:COM_MED_RESTART_TMPL - else - MEMDIR=${MEMDIR} YMD=${current_cycle:0:8} HH=${current_cycle:8:2} declare_from_tmpl -x COMOUT_ATMOS_INPUT:COM_ATMOS_INPUT_TMPL - fi - if [[ "${DO_OCN:-}" = "YES" ]]; then - MEMDIR=${MEMDIR} RUN=${rRUN} YMD=${previous_cycle:0:8} HH=${previous_cycle:8:2} declare_from_tmpl -x COMOUT_OCEAN_RESTART_PREV:COM_OCEAN_RESTART_TMPL - if [[ "${REPLAY_ICS:-NO}" = "YES" ]]; then - MEMDIR=${MEMDIR} YMD=${current_cycle:0:8} HH=${current_cycle:8:2} declare_from_tmpl -x COMOUT_OCEAN_ANALYSIS:COM_OCEAN_ANALYSIS_TMPL - fi - fi - if [[ "${DO_ICE:-}" = "YES" ]]; then - MEMDIR=${MEMDIR} RUN=${rRUN} YMD=${previous_cycle:0:8} HH=${previous_cycle:8:2} declare_from_tmpl -x COMOUT_ICE_RESTART_PREV:COM_ICE_RESTART_TMPL - fi - if [[ "${DO_WAVE:-}" = "YES" ]]; then - MEMDIR=${MEMDIR} YMD=${previous_cycle:0:8} HH=${previous_cycle:8:2} declare_from_tmpl -x COMOUT_WAVE_RESTART_PREV:COM_WAVE_RESTART_TMPL - fi - - # Execute staging - "${SCRgfs}/exglobal_stage_ic.py" - -done # for MEMDIR in "${MEMDIR_ARRAY[@]}"; do +# Execute staging +"${SCRgfs}/exglobal_stage_ic.py" ############################################################### # Check for errors and exit if any of the above failed diff --git a/parm/stage/stage.yaml.j2 b/parm/stage/stage.yaml.j2 index a8e86a8899..56f9eda35b 100644 --- a/parm/stage/stage.yaml.j2 +++ b/parm/stage/stage.yaml.j2 @@ -19,43 +19,77 @@ ############################################################# # Set variables used below -{% set cycle_HH = current_cycle | strftime("%H") %} +{% set current_cycle_YMD = current_cycle | to_YMD %} +{% set current_cycle_HH = current_cycle | strftime("%H") %} +{% set previous_cycle_YMD = previous_cycle | to_YMD %} +{% set previous_cycle_HH = previous_cycle | strftime("%H") %} {% set m_prefix = model_start_date_current_cycle | to_YMD + "." + model_start_date_current_cycle | strftime("%H") + "0000" %} {% set o_prefix = current_cycle_offset | to_YMD + "." + current_cycle_offset | strftime("%H") + "0000" %} {% set p_prefix = previous_cycle | to_YMD + "." + previous_cycle | strftime("%H") + "0000" %} -############################################################# +#################################################################### # Initial condition to stage +#################################################################### + +# Declare a dict of search and replace terms to run on each template + +{% if mem >= 0 %} + {% set mem_char = 'mem%03d' | format(mem) %} +{% else %} + {% set mem_char = '' %} +{% endif %} +{% set current_dict = ({ '${ROTDIR}':ROTDIR, + '${RUN}':RUN, + '${YMD}':current_cycle_YMD, + '${HH}':current_cycle_HH, + '${MEMDIR}': mem_char }) %} +{% set previous_dict = ({ '${ROTDIR}':ROTDIR, + '${RUN}':rRUN, + '${YMD}':previous_cycle_YMD, + '${HH}':previous_cycle_HH, + '${MEMDIR}': mem_char }) %} +{% set previous_run_dict = ({ '${ROTDIR}':ROTDIR, + '${RUN}':RUN, + '${YMD}':previous_cycle_YMD, + '${HH}':previous_cycle_HH, + '${MEMDIR}': mem_char }) %} + +# Initial condition definitions {% if MODE == "cycled" and RUN == "gdas" %} +{% set COMOUT_ATMOS_ANALYSIS = COM_ATMOS_ANALYSIS_TMPL | replace_tmpl(current_dict) %} analysis: mkdir: - "{{ COMOUT_ATMOS_ANALYSIS }}" copy: {% for ftype in ["abias", "abias_air", "abias_int", "abias_pc", "atminc.nc", "radstat"] %} - {% if path_exists(ICSDIR ~ "/" ~ COMOUT_ATMOS_ANALYSIS | relpath(ROTDIR) ~ "/" ~ RUN ~ ".t" ~ cycle_HH ~ "z." ~ ftype) %} - - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ RUN }}.t{{ cycle_HH }}z.{{ ftype }}", "{{ COMOUT_ATMOS_ANALYSIS }}"] + {% if path_exists(ICSDIR ~ "/" ~ COMOUT_ATMOS_ANALYSIS | relpath(ROTDIR) ~ "/" ~ RUN ~ ".t" ~ current_cycle_HH ~ "z." ~ ftype) %} + - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ RUN }}.t{{ current_cycle_HH }}z.{{ ftype }}", "{{ COMOUT_ATMOS_ANALYSIS }}"] {% endif %} {% endfor %} {% endif %} {% if EXP_WARM_START == True %} +{% set COMOUT_ATMOS_RESTART_PREV = COM_ATMOS_RESTART_TMPL | replace_tmpl(previous_dict) %} atmosphere_warm: mkdir: - "{{ COMOUT_ATMOS_RESTART_PREV }}" copy: {% for ftype in ["coupler.res", "fv_core.res.nc"] %} - - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ o_prefix }}.{{ ftype }}", "{{ COMOUT_ATMOS_RESTART_PREV }}"] + - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ m_prefix }}.{{ ftype }}", "{{ COMOUT_ATMOS_RESTART_PREV }}"] {% endfor %} {% for ftype in ["ca_data", "fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data", "sfc_data"] %} {% for ntile in range(1, ntiles + 1) %} - - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ o_prefix }}.{{ ftype }}.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_RESTART_PREV }}"] + - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ m_prefix }}.{{ ftype }}.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_RESTART_PREV }}"] {% endfor %} # ntile {% endfor %} # ftype {% for ntile in range(1, ntiles + 1) %} + {% if path_exists(ICSDIR ~ "/" ~ COMOUT_ATMOS_RESTART_PREV | relpath(ROTDIR) ~ "/" ~ p_prefix ~ ".sfcanl_data.tile" ~ ntile ~ ".nc") %} - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ p_prefix }}.sfcanl_data.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_RESTART_PREV }}"] + {% endif %} # path_exists {% endfor %} # ntile -{% else %} +{% else %} # cold start +{% set COMOUT_ATMOS_INPUT = COM_ATMOS_INPUT_TMPL | replace_tmpl(current_dict) %} atmosphere_cold: mkdir: - "{{ COMOUT_ATMOS_INPUT }}" @@ -69,31 +103,38 @@ atmosphere_cold: {% endif %} {% if REPLAY_ICS == "YES" %} +{% set COMOUT_ATMOS_ANALYSIS = COM_ATMOS_ANALYSIS_TMPL | replace_tmpl(current_dict) %} atmosphere_perturbation: mkdir: - "{{ COMOUT_ATMOS_ANALYSIS }}" copy: - - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ m_prefix }}.fv3_perturbation.nc", "{{ COMOUT_ATMOS_ANALYSIS }}/{{ RUN }}.t{{ cycle_HH }}z.atminc.nc"] + - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ m_prefix }}.fv3_perturbation.nc", "{{ COMOUT_ATMOS_ANALYSIS }}/{{ RUN }}.t{{ current_cycle_HH }}z.atminc.nc"] {% endif %} {% if DO_NEST %} atmosphere_nest: - {% set ntile = 7 %} + {% set ntile = 7 %} + {% if EXP_WARM_START == True %} + {% set COMOUT_ATMOS_RESTART_PREV = COM_ATMOS_RESTART_TMPL | replace_tmpl(previous_dict) %} mkdir: - "{{ COMOUT_ATMOS_RESTART_PREV }}" copy: - {% if EXP_WARM_START == True %} - {% for ftype in ["ca_data", "fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data", "sfc_data"] %} + {% for ftype in ["ca_data", "fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data", "sfc_data"] %} - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ o_prefix }}.{{ ftype }}.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_RESTART_PREV }}/{{ o_prefix }}.{{ ftype }}.nest0{{ ntile-5 }}.tile{{ ntile }}.nc"] - {% endfor %} # ftype - {% else %} - {% for ftype in ["gfs_data", "sfc_data"] %} + {% endfor %} + {% else %} # cold start + {% set COMOUT_ATMOS_INPUT = COM_ATMOS_INPUT_TMPL | replace_tmpl(current_dict) %} + mkdir: + - "{{ COMOUT_ATMOS_INPUT }}" + copy: + {% for ftype in ["gfs_data", "sfc_data"] %} - ["{{ COMOUT_ATMOS_INPUT }}/{{ ftype }}.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_INPUT }}/{{ ftype }}.nest0{{ ntile-5 }}.tile{{ ntile }}.nc"] - {% endfor %} # ftype - {% endif %} # cold-start + {% endfor %} + {% endif %} {% endif %} {% if DO_ICE %} +{% set COMOUT_ICE_RESTART_PREV = COM_ICE_RESTART_TMPL | replace_tmpl(previous_dict) %} ice: mkdir: - "{{ COMOUT_ICE_RESTART_PREV }}" @@ -102,6 +143,7 @@ ice: {% endif %} {% if DO_OCN %} +{% set COMOUT_OCEAN_RESTART_PREV = COM_OCEAN_RESTART_TMPL | replace_tmpl(previous_dict) %} ocean: mkdir: - "{{ COMOUT_OCEAN_RESTART_PREV }}" @@ -114,6 +156,7 @@ ocean: {% endif %} {% if REPLAY_ICS == "YES" %} +{% set COMOUT_OCEAN_ANALYSIS = COM_OCEAN_ANALYSIS_TMPL | replace_tmpl(current_dict) %} replay: mkdir: - "{{ COMOUT_OCEAN_ANALYSIS }}" @@ -122,6 +165,7 @@ replay: {% endif %} {% if EXP_WARM_START == True %} +{% set COMOUT_MED_RESTART_PREV = COM_MED_RESTART_TMPL | replace_tmpl(previous_dict) %} {% if path_exists(ICSDIR ~ "/" ~ COMOUT_MED_RESTART_PREV | relpath(ROTDIR) ~ "/" ~ m_prefix ~ ".ufs.cpld.cpl.r.nc") %} mediator: mkdir: @@ -134,6 +178,7 @@ mediator: {% endif %} # DO_OCN=YES {% if DO_WAVE %} +{% set COMOUT_WAVE_RESTART_PREV = COM_WAVE_RESTART_TMPL | replace_tmpl(previous_run_dict) %} wave: mkdir: - "{{ COMOUT_WAVE_RESTART_PREV }}" diff --git a/scripts/exglobal_stage_ic.py b/scripts/exglobal_stage_ic.py index d5be8e8ee1..8e0832de4b 100755 --- a/scripts/exglobal_stage_ic.py +++ b/scripts/exglobal_stage_ic.py @@ -18,7 +18,7 @@ def main(): stage = Stage(config) # Pull out all the configuration keys needed to run stage job - keys = ['RUN', 'MODE', 'EXP_WARM_START', + keys = ['RUN', 'MODE', 'EXP_WARM_START', 'NMEM_ENS', 'previous_cycle', 'current_cycle', 'current_cycle_offset', 'model_start_date_current_cycle', 'ROTDIR', 'ICSDIR', 'STAGE_IC_YAML_TMPL', diff --git a/ush/python/pygfs/task/stage.py b/ush/python/pygfs/task/stage.py index e1459fdee0..edc12540cb 100644 --- a/ush/python/pygfs/task/stage.py +++ b/ush/python/pygfs/task/stage.py @@ -56,9 +56,31 @@ def execute_stage(self, stage_dict: Dict[str, Any]) -> None: # Add the os.path.exists function to the dict for yaml parsing stage_dict['path_exists'] = os.path.exists - stage_set = parse_j2yaml(self.task_config.STAGE_IC_YAML_TMPL, stage_dict, - allow_missing=False) - - # Copy files to ROTDIR - for key in stage_set.keys(): - FileHandler(stage_set[key]).sync() + # Determine restart RUN + rRUN = self.task_config.RUN + if self.task_config.RUN == "gfs": + rRUN = "gdas" + stage_dict['rRUN'] = rRUN + + # Determine ensemble member settings + MEM_START = -1 # Deterministic default, no members + if self.task_config.NMEM_ENS > 0: + if self.task_config.RUN == "gefs": + MEM_START = 0 + elif self.task_config.RUN == "enkfgdas": + MEM_START = 1 + + if MEM_START >= 0: # Ensemble RUN + first_mem = MEM_START + last_mem = self.task_config.NMEM_ENS + else: # Deteministic RUN + first_mem = MEM_START + last_mem = MEM_START + + # Loop over members + for mem in range(first_mem, last_mem + 1): + stage_dict['mem'] = mem + stage_set = parse_j2yaml(self.task_config.STAGE_IC_YAML_TMPL, stage_dict, allow_missing=False) + # Copy files to ROTDIR + for key in stage_set.keys(): + FileHandler(stage_set[key]).sync() From 9a0c642926603760e7d8f20705b420c42c279984 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Mon, 12 Aug 2024 14:23:39 +0000 Subject: [PATCH 77/89] Resolve pynorms errors Refs #2475 --- ush/python/pygfs/task/stage.py | 6 +++--- workflow/rocoto/gfs_tasks.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ush/python/pygfs/task/stage.py b/ush/python/pygfs/task/stage.py index edc12540cb..377ce1faae 100644 --- a/ush/python/pygfs/task/stage.py +++ b/ush/python/pygfs/task/stage.py @@ -63,17 +63,17 @@ def execute_stage(self, stage_dict: Dict[str, Any]) -> None: stage_dict['rRUN'] = rRUN # Determine ensemble member settings - MEM_START = -1 # Deterministic default, no members + MEM_START = -1 # Deterministic default, no members if self.task_config.NMEM_ENS > 0: if self.task_config.RUN == "gefs": MEM_START = 0 elif self.task_config.RUN == "enkfgdas": MEM_START = 1 - if MEM_START >= 0: # Ensemble RUN + if MEM_START >= 0: # Ensemble RUN first_mem = MEM_START last_mem = self.task_config.NMEM_ENS - else: # Deteministic RUN + else: # Deteministic RUN first_mem = MEM_START last_mem = MEM_START diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 5bb928f20a..b737affbae 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -868,7 +868,7 @@ def _fcst_cycled(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies) - if self.run in ['gdas','enkfgdas']: + if self.run in ['gdas', 'enkfgdas']: dep_dict = {'type': 'task', 'name': f'{self.run}stage_ic'} dependencies.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='or', dep=dependencies) From 863ecee838db626b2add6eaf510bdfc4d546b27e Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Mon, 12 Aug 2024 16:42:54 +0000 Subject: [PATCH 78/89] Some fixes and updates to conditions - Replace all o_prefix with m_prefix and remove offset time - Fix condition check for REPLAY_ICS (YES -> True) - Add check against DO_JEDIOCNVAR for ice ICs - Update model_start_date_current_cycle setting for MODE Refs #2475 --- jobs/JGLOBAL_STAGE_IC | 5 ++--- parm/stage/stage.yaml.j2 | 28 ++++++++++++++++++---------- scripts/exglobal_stage_ic.py | 4 ++-- 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/jobs/JGLOBAL_STAGE_IC b/jobs/JGLOBAL_STAGE_IC index 3346db0827..0113b6c224 100755 --- a/jobs/JGLOBAL_STAGE_IC +++ b/jobs/JGLOBAL_STAGE_IC @@ -10,10 +10,9 @@ current_cycle="${PDY}${cyc}" previous_cycle=$(date --utc -d "${PDY} ${cyc} - ${assim_freq} hours" +%Y%m%d%H) current_cycle_begin=$(date --utc -d "${PDY} ${cyc} - ${half_window} hours" +%Y%m%d%H) current_cycle_end=$(date --utc -d "${PDY} ${cyc} + ${half_window} hours" +%Y%m%d%H) -current_cycle_offset=$(date --utc -d "${PDY} ${cyc} + ${OFFSET_START_HOUR} hours" +%Y%m%d%H) # Define model start date for current_cycle as the time the forecast will start -if [[ "${DOIAU:-NO}" == "YES" ]]; then +if [[ "${DOIAU:-NO}" == "YES" ]] && [[ "${MODE}" == "cycled" ]] ; then model_start_date_current_cycle="${current_cycle_begin}" else if [[ "${REPLAY_ICS:-NO}" == "YES" ]]; then @@ -22,7 +21,7 @@ else model_start_date_current_cycle=${current_cycle} fi fi -export current_cycle previous_cycle current_cycle_offset model_start_date_current_cycle +export current_cycle previous_cycle model_start_date_current_cycle # Initialize return code err=0 diff --git a/parm/stage/stage.yaml.j2 b/parm/stage/stage.yaml.j2 index 56f9eda35b..1d223f80a1 100644 --- a/parm/stage/stage.yaml.j2 +++ b/parm/stage/stage.yaml.j2 @@ -24,7 +24,6 @@ {% set previous_cycle_YMD = previous_cycle | to_YMD %} {% set previous_cycle_HH = previous_cycle | strftime("%H") %} {% set m_prefix = model_start_date_current_cycle | to_YMD + "." + model_start_date_current_cycle | strftime("%H") + "0000" %} -{% set o_prefix = current_cycle_offset | to_YMD + "." + current_cycle_offset | strftime("%H") + "0000" %} {% set p_prefix = previous_cycle | to_YMD + "." + previous_cycle | strftime("%H") + "0000" %} #################################################################### @@ -102,7 +101,7 @@ atmosphere_cold: {% endfor %} # ftype {% endif %} -{% if REPLAY_ICS == "YES" %} +{% if REPLAY_ICS == True %} {% set COMOUT_ATMOS_ANALYSIS = COM_ATMOS_ANALYSIS_TMPL | replace_tmpl(current_dict) %} atmosphere_perturbation: mkdir: @@ -120,7 +119,7 @@ atmosphere_nest: - "{{ COMOUT_ATMOS_RESTART_PREV }}" copy: {% for ftype in ["ca_data", "fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data", "sfc_data"] %} - - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ o_prefix }}.{{ ftype }}.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_RESTART_PREV }}/{{ o_prefix }}.{{ ftype }}.nest0{{ ntile-5 }}.tile{{ ntile }}.nc"] + - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ m_prefix }}.{{ ftype }}.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_RESTART_PREV }}/{{ m_prefix }}.{{ ftype }}.nest0{{ ntile-5 }}.tile{{ ntile }}.nc"] {% endfor %} {% else %} # cold start {% set COMOUT_ATMOS_INPUT = COM_ATMOS_INPUT_TMPL | replace_tmpl(current_dict) %} @@ -134,12 +133,21 @@ atmosphere_nest: {% endif %} {% if DO_ICE %} +{% if DO_JEDIOCNVAR == True %} +{% set COMOUT_ICE_ANALYSIS = COM_ICE_ANALYSIS_TMPL | replace_tmpl(current_dict) %} +ice: + mkdir: + - "{{ COMOUT_ICE_ANALYSIS }}" + copy: + - ["{{ ICSDIR }}/{{ COMOUT_ICE_ANALYSIS | relpath(ROTDIR) }}/{{ m_prefix }}.cice_model_anl.res.nc", "{{ COMOUT_ICE_ANALYSIS }}"] +{% else %} {% set COMOUT_ICE_RESTART_PREV = COM_ICE_RESTART_TMPL | replace_tmpl(previous_dict) %} ice: mkdir: - "{{ COMOUT_ICE_RESTART_PREV }}" copy: - - ["{{ ICSDIR }}/{{ COMOUT_ICE_RESTART_PREV | relpath(ROTDIR) }}/{{ o_prefix }}.cice_model.res.nc", "{{ COMOUT_ICE_RESTART_PREV }}"] + - ["{{ ICSDIR }}/{{ COMOUT_ICE_RESTART_PREV | relpath(ROTDIR) }}/{{ m_prefix }}.cice_model.res.nc", "{{ COMOUT_ICE_RESTART_PREV }}"] +{% endif %} {% endif %} {% if DO_OCN %} @@ -148,20 +156,20 @@ ocean: mkdir: - "{{ COMOUT_OCEAN_RESTART_PREV }}" copy: - - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_RESTART_PREV | relpath(ROTDIR) }}/{{ o_prefix }}.MOM.res.nc", "{{ COMOUT_OCEAN_RESTART_PREV }}"] + - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_RESTART_PREV | relpath(ROTDIR) }}/{{ m_prefix }}.MOM.res.nc", "{{ COMOUT_OCEAN_RESTART_PREV }}"] {% if OCNRES == "025" %} {% for nn in range(1, 3) %} - - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_RESTART_PREV | relpath(ROTDIR) }}/{{ o_prefix }}.MOM.res_{{ nn }}.nc", "{{ COMOUT_OCEAN_RESTART_PREV }}"] + - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_RESTART_PREV | relpath(ROTDIR) }}/{{ m_prefix }}.MOM.res_{{ nn }}.nc", "{{ COMOUT_OCEAN_RESTART_PREV }}"] {% endfor %} {% endif %} -{% if REPLAY_ICS == "YES" %} +{% if REPLAY_ICS == True %} {% set COMOUT_OCEAN_ANALYSIS = COM_OCEAN_ANALYSIS_TMPL | replace_tmpl(current_dict) %} replay: mkdir: - "{{ COMOUT_OCEAN_ANALYSIS }}" copy: - - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ o_prefix }}.mom6_perturbation.nc", "{{ COMOUT_OCEAN_ANALYSIS }}/mom6_increment.nc"] + - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ m_prefix }}.mom6_perturbation.nc", "{{ COMOUT_OCEAN_ANALYSIS }}/mom6_increment.nc"] {% endif %} {% if EXP_WARM_START == True %} @@ -175,7 +183,7 @@ mediator: {% endif %} # path exists {% endif %} # warm start true -{% endif %} # DO_OCN=YES +{% endif %} # DO_OCN=True {% if DO_WAVE %} {% set COMOUT_WAVE_RESTART_PREV = COM_WAVE_RESTART_TMPL | replace_tmpl(previous_run_dict) %} @@ -183,5 +191,5 @@ wave: mkdir: - "{{ COMOUT_WAVE_RESTART_PREV }}" copy: - - ["{{ ICSDIR }}/{{ COMOUT_WAVE_RESTART_PREV | relpath(ROTDIR) }}/{{ o_prefix }}.restart.{{ waveGRD }}", "{{ COMOUT_WAVE_RESTART_PREV }}"] + - ["{{ ICSDIR }}/{{ COMOUT_WAVE_RESTART_PREV | relpath(ROTDIR) }}/{{ m_prefix }}.restart.{{ waveGRD }}", "{{ COMOUT_WAVE_RESTART_PREV }}"] {% endif %} diff --git a/scripts/exglobal_stage_ic.py b/scripts/exglobal_stage_ic.py index 8e0832de4b..7757caf879 100755 --- a/scripts/exglobal_stage_ic.py +++ b/scripts/exglobal_stage_ic.py @@ -20,9 +20,9 @@ def main(): # Pull out all the configuration keys needed to run stage job keys = ['RUN', 'MODE', 'EXP_WARM_START', 'NMEM_ENS', 'previous_cycle', 'current_cycle', - 'current_cycle_offset', 'model_start_date_current_cycle', + 'model_start_date_current_cycle', 'ROTDIR', 'ICSDIR', 'STAGE_IC_YAML_TMPL', - 'OCNRES', 'waveGRD', 'ntiles', + 'OCNRES', 'waveGRD', 'ntiles', 'DO_JEDIOCNVAR', 'REPLAY_ICS', 'DO_WAVE', 'DO_OCN', 'DO_ICE', 'DO_NEST'] stage_dict = AttrDict() From 7b3edf008c9b8e48298622d88a5501053ef0d5f9 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Mon, 12 Aug 2024 20:34:32 +0000 Subject: [PATCH 79/89] Add ocninc.nc copy to stage.yaml.j2 - when DO_OCN=YES and DO_JEDIOCNVAR=YES Refs #2475 --- parm/stage/stage.yaml.j2 | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/parm/stage/stage.yaml.j2 b/parm/stage/stage.yaml.j2 index 1d223f80a1..84e79da980 100644 --- a/parm/stage/stage.yaml.j2 +++ b/parm/stage/stage.yaml.j2 @@ -163,6 +163,15 @@ ocean: {% endfor %} {% endif %} +{% if DO_JEDIOCNVAR == True %} +{% set COMOUT_OCEAN_ANALYSIS = COM_OCEAN_ANALYSIS_TMPL | replace_tmpl(current_dict) %} +rerun: + mkdir: + - "{{ COMOUT_OCEAN_ANALYSIS }}" + copy: + - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ RUN }}.t{{ current_cycle_HH }}z.ocninc.nc", "{{ COMOUT_OCEAN_ANALYSIS }}"] +{% endif %} + {% if REPLAY_ICS == True %} {% set COMOUT_OCEAN_ANALYSIS = COM_OCEAN_ANALYSIS_TMPL | replace_tmpl(current_dict) %} replay: From c7ec976b503fa3fc9e498e1323691b7f9fb85ff8 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Mon, 12 Aug 2024 20:36:58 +0000 Subject: [PATCH 80/89] Add IC timestamps to ICSDIR in CI yamls Refs #2475 --- ci/cases/pr/C48mx500_3DVarAOWCDA.yaml | 2 +- ci/cases/pr/C96C48_hybatmDA.yaml | 2 +- ci/cases/pr/C96C48_ufs_hybatmDA.yaml | 2 +- ci/cases/pr/C96_atm3DVar.yaml | 2 +- ci/cases/pr/C96_atm3DVar_extended.yaml | 2 +- ci/cases/pr/C96_atmaerosnowDA.yaml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml b/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml index fd056cf895..59c677595d 100644 --- a/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml +++ b/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml @@ -9,7 +9,7 @@ arguments: resdetocean: 5.0 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C48mx500 + icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C48mx500/20240610 idate: 2021032412 edate: 2021032418 nens: 0 diff --git a/ci/cases/pr/C96C48_hybatmDA.yaml b/ci/cases/pr/C96C48_hybatmDA.yaml index d08374d4e0..7617e39217 100644 --- a/ci/cases/pr/C96C48_hybatmDA.yaml +++ b/ci/cases/pr/C96C48_hybatmDA.yaml @@ -10,7 +10,7 @@ arguments: resensatmos: 48 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48 + icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20240610 idate: 2021122018 edate: 2021122106 nens: 2 diff --git a/ci/cases/pr/C96C48_ufs_hybatmDA.yaml b/ci/cases/pr/C96C48_ufs_hybatmDA.yaml index d1556dc1d0..8050b75f99 100644 --- a/ci/cases/pr/C96C48_ufs_hybatmDA.yaml +++ b/ci/cases/pr/C96C48_ufs_hybatmDA.yaml @@ -9,7 +9,7 @@ arguments: resensatmos: 48 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48 + icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20240610 idate: 2024022318 edate: 2024022400 nens: 2 diff --git a/ci/cases/pr/C96_atm3DVar.yaml b/ci/cases/pr/C96_atm3DVar.yaml index 8a89ff25ec..e9e6c2b31c 100644 --- a/ci/cases/pr/C96_atm3DVar.yaml +++ b/ci/cases/pr/C96_atm3DVar.yaml @@ -8,7 +8,7 @@ arguments: resdetatmos: 96 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48 + icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20240610 idate: 2021122018 edate: 2021122106 nens: 0 diff --git a/ci/cases/pr/C96_atm3DVar_extended.yaml b/ci/cases/pr/C96_atm3DVar_extended.yaml index 994d3ef3a0..0bf6ab3677 100644 --- a/ci/cases/pr/C96_atm3DVar_extended.yaml +++ b/ci/cases/pr/C96_atm3DVar_extended.yaml @@ -8,7 +8,7 @@ arguments: resdetatmos: 96 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48 + icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20240610 idate: 2021122018 edate: 2021122118 nens: 0 diff --git a/ci/cases/pr/C96_atmaerosnowDA.yaml b/ci/cases/pr/C96_atmaerosnowDA.yaml index 7e22955a37..3a0c85c68e 100644 --- a/ci/cases/pr/C96_atmaerosnowDA.yaml +++ b/ci/cases/pr/C96_atmaerosnowDA.yaml @@ -8,7 +8,7 @@ arguments: resdetatmos: 96 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48 + icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20240610 idate: 2021122012 edate: 2021122100 nens: 0 From ff1246b5ff93c6fbf7ccb9012bd4d388aa00e3a4 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Tue, 13 Aug 2024 13:57:56 +0000 Subject: [PATCH 81/89] Update AWSPW host BASE_IC path Refs #2475 --- workflow/hosts/awspw.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/hosts/awspw.yaml b/workflow/hosts/awspw.yaml index f925f54008..a9c708253e 100644 --- a/workflow/hosts/awspw.yaml +++ b/workflow/hosts/awspw.yaml @@ -18,7 +18,7 @@ CHGRP_RSTPROD: 'YES' CHGRP_CMD: 'chgrp rstprod' # TODO: This is not yet supported. HPSSARCH: 'NO' HPSS_PROJECT: emc-global #TODO: See `ATARDIR` below. -BASE_CPLIC: '/bucket/global-workflow-shared-data/ICSDIR/prototype_ICs' +BASE_IC: '/bucket/global-workflow-shared-data/ICSDIR' LOCALARCH: 'NO' ATARDIR: '' # TODO: This will not yet work from AWS. MAKE_NSSTBUFR: 'NO' From 7ae9ea4912fae8d8759c357b99ad4fe977928e8a Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Tue, 13 Aug 2024 17:53:05 +0000 Subject: [PATCH 82/89] Consolidate staging settings - Move cycle, rRUN, model_start_date_current_cycle, and member start/stop settings into new configure function and out of JJOB. - Address some reviewer feedback and make corrections. Refs #2475 --- jobs/JGLOBAL_STAGE_IC | 21 +------ parm/stage/stage.yaml.j2 | 60 +++++++++--------- scripts/exglobal_stage_ic.py | 8 ++- .../pygfs/task/{stage.py => stage_ic.py} | 63 ++++++++++++++----- 4 files changed, 84 insertions(+), 68 deletions(-) rename ush/python/pygfs/task/{stage.py => stage_ic.py} (55%) diff --git a/jobs/JGLOBAL_STAGE_IC b/jobs/JGLOBAL_STAGE_IC index 0113b6c224..5039c94eec 100755 --- a/jobs/JGLOBAL_STAGE_IC +++ b/jobs/JGLOBAL_STAGE_IC @@ -3,31 +3,12 @@ source "${HOMEgfs}/ush/preamble.sh" source "${HOMEgfs}/ush/jjob_header.sh" -e "stage_ic" -c "base stage_ic" -# Define significant cycles -# shellcheck disable=SC2153 -half_window=$(( assim_freq / 2 )) -current_cycle="${PDY}${cyc}" -previous_cycle=$(date --utc -d "${PDY} ${cyc} - ${assim_freq} hours" +%Y%m%d%H) -current_cycle_begin=$(date --utc -d "${PDY} ${cyc} - ${half_window} hours" +%Y%m%d%H) -current_cycle_end=$(date --utc -d "${PDY} ${cyc} + ${half_window} hours" +%Y%m%d%H) - -# Define model start date for current_cycle as the time the forecast will start -if [[ "${DOIAU:-NO}" == "YES" ]] && [[ "${MODE}" == "cycled" ]] ; then - model_start_date_current_cycle="${current_cycle_begin}" -else - if [[ "${REPLAY_ICS:-NO}" == "YES" ]]; then - model_start_date_current_cycle=${current_cycle_end} - else - model_start_date_current_cycle=${current_cycle} - fi -fi -export current_cycle previous_cycle model_start_date_current_cycle - # Initialize return code err=0 # Execute staging "${SCRgfs}/exglobal_stage_ic.py" +err=$? ############################################################### # Check for errors and exit if any of the above failed diff --git a/parm/stage/stage.yaml.j2 b/parm/stage/stage.yaml.j2 index 84e79da980..49415a3e17 100644 --- a/parm/stage/stage.yaml.j2 +++ b/parm/stage/stage.yaml.j2 @@ -23,8 +23,8 @@ {% set current_cycle_HH = current_cycle | strftime("%H") %} {% set previous_cycle_YMD = previous_cycle | to_YMD %} {% set previous_cycle_HH = previous_cycle | strftime("%H") %} -{% set m_prefix = model_start_date_current_cycle | to_YMD + "." + model_start_date_current_cycle | strftime("%H") + "0000" %} -{% set p_prefix = previous_cycle | to_YMD + "." + previous_cycle | strftime("%H") + "0000" %} +{% set m_prefix = model_start_date_current_cycle | strftime("%Y%m%d.%H0000") %} +{% set p_prefix = previous_cycle | strftime("%Y%m%d.%H0000") %} #################################################################### # Initial condition to stage @@ -37,26 +37,26 @@ {% else %} {% set mem_char = '' %} {% endif %} -{% set current_dict = ({ '${ROTDIR}':ROTDIR, - '${RUN}':RUN, - '${YMD}':current_cycle_YMD, - '${HH}':current_cycle_HH, - '${MEMDIR}': mem_char }) %} -{% set previous_dict = ({ '${ROTDIR}':ROTDIR, - '${RUN}':rRUN, - '${YMD}':previous_cycle_YMD, - '${HH}':previous_cycle_HH, - '${MEMDIR}': mem_char }) %} -{% set previous_run_dict = ({ '${ROTDIR}':ROTDIR, - '${RUN}':RUN, - '${YMD}':previous_cycle_YMD, - '${HH}':previous_cycle_HH, - '${MEMDIR}': mem_char }) %} +{% set current_cycle_dict = ({ '${ROTDIR}':ROTDIR, + '${RUN}':RUN, + '${YMD}':current_cycle_YMD, + '${HH}':current_cycle_HH, + '${MEMDIR}': mem_char }) %} +{% set previous_cycle_dict = ({ '${ROTDIR}':ROTDIR, + '${RUN}':rRUN, + '${YMD}':previous_cycle_YMD, + '${HH}':previous_cycle_HH, + '${MEMDIR}': mem_char }) %} +{% set previous_cycle_and_run_dict = ({ '${ROTDIR}':ROTDIR, + '${RUN}':RUN, + '${YMD}':previous_cycle_YMD, + '${HH}':previous_cycle_HH, + '${MEMDIR}': mem_char }) %} # Initial condition definitions {% if MODE == "cycled" and RUN == "gdas" %} -{% set COMOUT_ATMOS_ANALYSIS = COM_ATMOS_ANALYSIS_TMPL | replace_tmpl(current_dict) %} +{% set COMOUT_ATMOS_ANALYSIS = COM_ATMOS_ANALYSIS_TMPL | replace_tmpl(current_cycle_dict) %} analysis: mkdir: - "{{ COMOUT_ATMOS_ANALYSIS }}" @@ -69,7 +69,7 @@ analysis: {% endif %} {% if EXP_WARM_START == True %} -{% set COMOUT_ATMOS_RESTART_PREV = COM_ATMOS_RESTART_TMPL | replace_tmpl(previous_dict) %} +{% set COMOUT_ATMOS_RESTART_PREV = COM_ATMOS_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} atmosphere_warm: mkdir: - "{{ COMOUT_ATMOS_RESTART_PREV }}" @@ -88,7 +88,7 @@ atmosphere_warm: {% endif %} # path_exists {% endfor %} # ntile {% else %} # cold start -{% set COMOUT_ATMOS_INPUT = COM_ATMOS_INPUT_TMPL | replace_tmpl(current_dict) %} +{% set COMOUT_ATMOS_INPUT = COM_ATMOS_INPUT_TMPL | replace_tmpl(current_cycle_dict) %} atmosphere_cold: mkdir: - "{{ COMOUT_ATMOS_INPUT }}" @@ -102,7 +102,7 @@ atmosphere_cold: {% endif %} {% if REPLAY_ICS == True %} -{% set COMOUT_ATMOS_ANALYSIS = COM_ATMOS_ANALYSIS_TMPL | replace_tmpl(current_dict) %} +{% set COMOUT_ATMOS_ANALYSIS = COM_ATMOS_ANALYSIS_TMPL | replace_tmpl(current_cycle_dict) %} atmosphere_perturbation: mkdir: - "{{ COMOUT_ATMOS_ANALYSIS }}" @@ -114,7 +114,7 @@ atmosphere_perturbation: atmosphere_nest: {% set ntile = 7 %} {% if EXP_WARM_START == True %} - {% set COMOUT_ATMOS_RESTART_PREV = COM_ATMOS_RESTART_TMPL | replace_tmpl(previous_dict) %} + {% set COMOUT_ATMOS_RESTART_PREV = COM_ATMOS_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} mkdir: - "{{ COMOUT_ATMOS_RESTART_PREV }}" copy: @@ -122,7 +122,7 @@ atmosphere_nest: - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ m_prefix }}.{{ ftype }}.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_RESTART_PREV }}/{{ m_prefix }}.{{ ftype }}.nest0{{ ntile-5 }}.tile{{ ntile }}.nc"] {% endfor %} {% else %} # cold start - {% set COMOUT_ATMOS_INPUT = COM_ATMOS_INPUT_TMPL | replace_tmpl(current_dict) %} + {% set COMOUT_ATMOS_INPUT = COM_ATMOS_INPUT_TMPL | replace_tmpl(current_cycle_dict) %} mkdir: - "{{ COMOUT_ATMOS_INPUT }}" copy: @@ -134,14 +134,14 @@ atmosphere_nest: {% if DO_ICE %} {% if DO_JEDIOCNVAR == True %} -{% set COMOUT_ICE_ANALYSIS = COM_ICE_ANALYSIS_TMPL | replace_tmpl(current_dict) %} +{% set COMOUT_ICE_ANALYSIS = COM_ICE_ANALYSIS_TMPL | replace_tmpl(current_cycle_dict) %} ice: mkdir: - "{{ COMOUT_ICE_ANALYSIS }}" copy: - ["{{ ICSDIR }}/{{ COMOUT_ICE_ANALYSIS | relpath(ROTDIR) }}/{{ m_prefix }}.cice_model_anl.res.nc", "{{ COMOUT_ICE_ANALYSIS }}"] {% else %} -{% set COMOUT_ICE_RESTART_PREV = COM_ICE_RESTART_TMPL | replace_tmpl(previous_dict) %} +{% set COMOUT_ICE_RESTART_PREV = COM_ICE_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} ice: mkdir: - "{{ COMOUT_ICE_RESTART_PREV }}" @@ -151,7 +151,7 @@ ice: {% endif %} {% if DO_OCN %} -{% set COMOUT_OCEAN_RESTART_PREV = COM_OCEAN_RESTART_TMPL | replace_tmpl(previous_dict) %} +{% set COMOUT_OCEAN_RESTART_PREV = COM_OCEAN_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} ocean: mkdir: - "{{ COMOUT_OCEAN_RESTART_PREV }}" @@ -164,7 +164,7 @@ ocean: {% endif %} {% if DO_JEDIOCNVAR == True %} -{% set COMOUT_OCEAN_ANALYSIS = COM_OCEAN_ANALYSIS_TMPL | replace_tmpl(current_dict) %} +{% set COMOUT_OCEAN_ANALYSIS = COM_OCEAN_ANALYSIS_TMPL | replace_tmpl(current_cycle_dict) %} rerun: mkdir: - "{{ COMOUT_OCEAN_ANALYSIS }}" @@ -173,7 +173,7 @@ rerun: {% endif %} {% if REPLAY_ICS == True %} -{% set COMOUT_OCEAN_ANALYSIS = COM_OCEAN_ANALYSIS_TMPL | replace_tmpl(current_dict) %} +{% set COMOUT_OCEAN_ANALYSIS = COM_OCEAN_ANALYSIS_TMPL | replace_tmpl(current_cycle_dict) %} replay: mkdir: - "{{ COMOUT_OCEAN_ANALYSIS }}" @@ -182,7 +182,7 @@ replay: {% endif %} {% if EXP_WARM_START == True %} -{% set COMOUT_MED_RESTART_PREV = COM_MED_RESTART_TMPL | replace_tmpl(previous_dict) %} +{% set COMOUT_MED_RESTART_PREV = COM_MED_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} {% if path_exists(ICSDIR ~ "/" ~ COMOUT_MED_RESTART_PREV | relpath(ROTDIR) ~ "/" ~ m_prefix ~ ".ufs.cpld.cpl.r.nc") %} mediator: mkdir: @@ -195,7 +195,7 @@ mediator: {% endif %} # DO_OCN=True {% if DO_WAVE %} -{% set COMOUT_WAVE_RESTART_PREV = COM_WAVE_RESTART_TMPL | replace_tmpl(previous_run_dict) %} +{% set COMOUT_WAVE_RESTART_PREV = COM_WAVE_RESTART_TMPL | replace_tmpl(previous_cycle_and_run_dict) %} wave: mkdir: - "{{ COMOUT_WAVE_RESTART_PREV }}" diff --git a/scripts/exglobal_stage_ic.py b/scripts/exglobal_stage_ic.py index 7757caf879..4a7b034efa 100755 --- a/scripts/exglobal_stage_ic.py +++ b/scripts/exglobal_stage_ic.py @@ -2,7 +2,7 @@ import os -from pygfs.task.stage import Stage +from pygfs.task.stage_ic import Stage from wxflow import AttrDict, Logger, cast_strdict_as_dtypedict, logit # Initialize root logger @@ -19,8 +19,7 @@ def main(): # Pull out all the configuration keys needed to run stage job keys = ['RUN', 'MODE', 'EXP_WARM_START', 'NMEM_ENS', - 'previous_cycle', 'current_cycle', - 'model_start_date_current_cycle', + 'current_cycle', 'previous_cycle', 'ROTDIR', 'ICSDIR', 'STAGE_IC_YAML_TMPL', 'OCNRES', 'waveGRD', 'ntiles', 'DO_JEDIOCNVAR', 'REPLAY_ICS', 'DO_WAVE', 'DO_OCN', 'DO_ICE', 'DO_NEST'] @@ -34,6 +33,9 @@ def main(): if key.startswith("COM"): stage_dict[key] = stage.task_config[key] + # Configure staging + stage_dict = stage.configure(stage_dict) + # Stage ICs stage.execute_stage(stage_dict) diff --git a/ush/python/pygfs/task/stage.py b/ush/python/pygfs/task/stage_ic.py similarity index 55% rename from ush/python/pygfs/task/stage.py rename to ush/python/pygfs/task/stage_ic.py index 377ce1faae..32bc7efdf1 100644 --- a/ush/python/pygfs/task/stage.py +++ b/ush/python/pygfs/task/stage_ic.py @@ -5,8 +5,8 @@ from typing import Any, Dict, List from wxflow import (AttrDict, FileHandler, Task, cast_strdict_as_dtypedict, - logit, parse_j2yaml, strftime, - to_YMD, to_YMDH, Template, TemplateConstants) + logit, parse_j2yaml, strftime, to_YMD, + add_to_datetime, to_timedelta, Template, TemplateConstants) logger = getLogger(__name__.split('.')[-1]) @@ -32,13 +32,11 @@ def __init__(self, config: Dict[str, Any]) -> None: """ super().__init__(config) - rotdir = self.task_config.ROTDIR + os.sep - self.task_config = AttrDict(**self.task_config) @logit(logger) - def execute_stage(self, stage_dict: Dict[str, Any]) -> None: - """Perform local staging of initial condition files. + def configure(self, stage_dict: Dict[str, Any]) -> (Dict[str, Any]): + """Determine stage settings based on configuration and runtime options. Parameters ---------- @@ -47,22 +45,38 @@ def execute_stage(self, stage_dict: Dict[str, Any]) -> None: Returns ------- - None + stage_dict : Dict[str, Any] + Configuration dictionary updated """ - if not os.path.isdir(stage_dict.ROTDIR): - raise FileNotFoundError(f"FATAL ERROR: The ROTDIR ({stage_dict.ROTDIR}) does not exist!") - # Add the os.path.exists function to the dict for yaml parsing + #------------------------------------------------------------- stage_dict['path_exists'] = os.path.exists + # Determine model start date + #--------------------------- + current_cycle_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config['assim_freq']}H") / 2) + current_cycle_end = add_to_datetime(self.task_config.current_cycle, to_timedelta(f"{self.task_config['assim_freq']}H") / 2) + + if self.task_config.DOIAU == True and self.task_config.MODE == "cycled": + model_start_date_current_cycle = current_cycle_begin + else: + if self.task_config.REPLAY_ICS == True: + model_start_date_current_cycle = current_cycle_end + else: + model_start_date_current_cycle = self.task_config.current_cycle + + stage_dict['model_start_date_current_cycle'] = model_start_date_current_cycle + # Determine restart RUN + #---------------------- rRUN = self.task_config.RUN if self.task_config.RUN == "gfs": rRUN = "gdas" stage_dict['rRUN'] = rRUN # Determine ensemble member settings + #----------------------------------- MEM_START = -1 # Deterministic default, no members if self.task_config.NMEM_ENS > 0: if self.task_config.RUN == "gefs": @@ -71,14 +85,33 @@ def execute_stage(self, stage_dict: Dict[str, Any]) -> None: MEM_START = 1 if MEM_START >= 0: # Ensemble RUN - first_mem = MEM_START - last_mem = self.task_config.NMEM_ENS + stage_dict['first_mem'] = MEM_START + stage_dict['last_mem'] = self.task_config.NMEM_ENS else: # Deteministic RUN - first_mem = MEM_START - last_mem = MEM_START + stage_dict['first_mem'] = MEM_START + stage_dict['last_mem'] = MEM_START + + return stage_dict + + @logit(logger) + def execute_stage(self, stage_dict: Dict[str, Any]) -> None: + """Perform local staging of initial condition files. + + Parameters + ---------- + stage_dict : Dict[str, Any] + Configuration dictionary + + Returns + ------- + None + """ + + if not os.path.isdir(stage_dict.ROTDIR): + raise FileNotFoundError(f"FATAL ERROR: The ROTDIR ({stage_dict.ROTDIR}) does not exist!") # Loop over members - for mem in range(first_mem, last_mem + 1): + for mem in range(stage_dict.first_mem, stage_dict.last_mem + 1): stage_dict['mem'] = mem stage_set = parse_j2yaml(self.task_config.STAGE_IC_YAML_TMPL, stage_dict, allow_missing=False) # Copy files to ROTDIR From 5c1f0c83771ab6628a89706650141100039fdc27 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Tue, 13 Aug 2024 18:02:49 +0000 Subject: [PATCH 83/89] Address pynorms errors Refs #2475 --- ush/python/pygfs/task/stage_ic.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/ush/python/pygfs/task/stage_ic.py b/ush/python/pygfs/task/stage_ic.py index 32bc7efdf1..68e37822f6 100644 --- a/ush/python/pygfs/task/stage_ic.py +++ b/ush/python/pygfs/task/stage_ic.py @@ -50,18 +50,16 @@ def configure(self, stage_dict: Dict[str, Any]) -> (Dict[str, Any]): """ # Add the os.path.exists function to the dict for yaml parsing - #------------------------------------------------------------- stage_dict['path_exists'] = os.path.exists # Determine model start date - #--------------------------- current_cycle_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config['assim_freq']}H") / 2) current_cycle_end = add_to_datetime(self.task_config.current_cycle, to_timedelta(f"{self.task_config['assim_freq']}H") / 2) - if self.task_config.DOIAU == True and self.task_config.MODE == "cycled": + if self.task_config.DOIAU and self.task_config.MODE == "cycled": model_start_date_current_cycle = current_cycle_begin else: - if self.task_config.REPLAY_ICS == True: + if self.task_config.REPLAY_ICS: model_start_date_current_cycle = current_cycle_end else: model_start_date_current_cycle = self.task_config.current_cycle @@ -69,14 +67,12 @@ def configure(self, stage_dict: Dict[str, Any]) -> (Dict[str, Any]): stage_dict['model_start_date_current_cycle'] = model_start_date_current_cycle # Determine restart RUN - #---------------------- rRUN = self.task_config.RUN if self.task_config.RUN == "gfs": rRUN = "gdas" stage_dict['rRUN'] = rRUN # Determine ensemble member settings - #----------------------------------- MEM_START = -1 # Deterministic default, no members if self.task_config.NMEM_ENS > 0: if self.task_config.RUN == "gefs": From c6bb9696610b8dc74c55d10abf3a7ddaf489ce46 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Tue, 13 Aug 2024 18:50:52 +0000 Subject: [PATCH 84/89] Compress first[last]_mem logic and remove unused variable Refs #2475 --- ush/python/pygfs/task/stage_ic.py | 19 ++++++++----------- workflow/rocoto/gefs_tasks.py | 1 - workflow/rocoto/gfs_tasks.py | 2 -- 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/ush/python/pygfs/task/stage_ic.py b/ush/python/pygfs/task/stage_ic.py index 68e37822f6..c808937e8f 100644 --- a/ush/python/pygfs/task/stage_ic.py +++ b/ush/python/pygfs/task/stage_ic.py @@ -73,19 +73,16 @@ def configure(self, stage_dict: Dict[str, Any]) -> (Dict[str, Any]): stage_dict['rRUN'] = rRUN # Determine ensemble member settings - MEM_START = -1 # Deterministic default, no members - if self.task_config.NMEM_ENS > 0: + if self.task_config.NMEM_ENS > 0: # Ensemble RUN if self.task_config.RUN == "gefs": - MEM_START = 0 + stage_dict['first_mem'] = 0 + stage_dict['last_mem'] = self.task_config.NMEM_ENS elif self.task_config.RUN == "enkfgdas": - MEM_START = 1 - - if MEM_START >= 0: # Ensemble RUN - stage_dict['first_mem'] = MEM_START - stage_dict['last_mem'] = self.task_config.NMEM_ENS - else: # Deteministic RUN - stage_dict['first_mem'] = MEM_START - stage_dict['last_mem'] = MEM_START + stage_dict['first_mem'] = 1 + stage_dict['last_mem'] = self.task_config.NMEM_ENS + else: # Deterministic RUN + stage_dict['first_mem'] = -1 + stage_dict['last_mem'] = -1 return stage_dict diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index 8940a4ff6f..5d706071b6 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -10,7 +10,6 @@ def __init__(self, app_config: AppConfig, run: str) -> None: super().__init__(app_config, run) def stage_ic(self): - cpl_ic = self._configs['stage_ic'] resources = self.get_resource('stage_ic') task_name = f'stage_ic' diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 29caacb7ee..44ab9aea49 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -20,8 +20,6 @@ def stage_ic(self): cycledef = 'gdas_half' if self.run in ['gdas', 'enkfgdas'] else self.run - cpl_ic = self._configs['stage_ic'] - resources = self.get_resource('stage_ic') task_name = f'{self.run}stage_ic' task_dict = {'task_name': task_name, From 302265013f335be55d912c30f4937a78f3aa2fc2 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Tue, 13 Aug 2024 19:57:39 +0000 Subject: [PATCH 85/89] Remove unneeded lines and update mem setup - Check for "enkfgdas" or "gefs" when NMEM_ENS>=0 for first[last]_mem configuration Refs #2475 --- jobs/JGLOBAL_STAGE_IC | 3 --- ush/python/pygfs/task/stage_ic.py | 4 +--- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/jobs/JGLOBAL_STAGE_IC b/jobs/JGLOBAL_STAGE_IC index 5039c94eec..b8126f8efe 100755 --- a/jobs/JGLOBAL_STAGE_IC +++ b/jobs/JGLOBAL_STAGE_IC @@ -3,9 +3,6 @@ source "${HOMEgfs}/ush/preamble.sh" source "${HOMEgfs}/ush/jjob_header.sh" -e "stage_ic" -c "base stage_ic" -# Initialize return code -err=0 - # Execute staging "${SCRgfs}/exglobal_stage_ic.py" err=$? diff --git a/ush/python/pygfs/task/stage_ic.py b/ush/python/pygfs/task/stage_ic.py index c808937e8f..6211da5504 100644 --- a/ush/python/pygfs/task/stage_ic.py +++ b/ush/python/pygfs/task/stage_ic.py @@ -32,8 +32,6 @@ def __init__(self, config: Dict[str, Any]) -> None: """ super().__init__(config) - self.task_config = AttrDict(**self.task_config) - @logit(logger) def configure(self, stage_dict: Dict[str, Any]) -> (Dict[str, Any]): """Determine stage settings based on configuration and runtime options. @@ -73,7 +71,7 @@ def configure(self, stage_dict: Dict[str, Any]) -> (Dict[str, Any]): stage_dict['rRUN'] = rRUN # Determine ensemble member settings - if self.task_config.NMEM_ENS > 0: # Ensemble RUN + if self.task_config.NMEM_ENS > 0 and self.task_config.RUN in ['enkfgdas', 'gefs']: # Ensemble RUN if self.task_config.RUN == "gefs": stage_dict['first_mem'] = 0 stage_dict['last_mem'] = self.task_config.NMEM_ENS From 54f8326df17338e0271a175a1c09f87a9d0e70c0 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Thu, 15 Aug 2024 16:49:28 +0000 Subject: [PATCH 86/89] Separate stage yaml into component ones - Create master yamls for gfs and gefs - Create yamls for each component/group - Update python to move all logic inside yamls - Update yaml names in gfs/gefs configs Refs #2475 --- parm/config/gefs/config.stage_ic | 2 +- parm/config/gfs/config.stage_ic | 2 +- parm/stage/analysis.yaml.j2 | 10 + parm/stage/atmosphere_cold.yaml.j2 | 18 ++ parm/stage/atmosphere_nest.yaml.j2 | 33 ++++ parm/stage/atmosphere_perturbation.yaml.j2 | 13 ++ parm/stage/atmosphere_warm.yaml.j2 | 28 +++ parm/stage/ice.yaml.j2 | 28 +++ parm/stage/master_gefs.yaml.j2 | 154 ++++++++++++++++ parm/stage/master_gfs.yaml.j2 | 189 +++++++++++++++++++ parm/stage/ocean.yaml.j2 | 18 ++ parm/stage/ocean_mediator.yaml.j2 | 15 ++ parm/stage/ocean_replay.yaml.j2 | 13 ++ parm/stage/ocean_rerun.yaml.j2 | 13 ++ parm/stage/stage.yaml.j2 | 204 --------------------- parm/stage/wave.yaml.j2 | 13 ++ scripts/exglobal_stage_ic.py | 7 +- ush/python/pygfs/task/stage_ic.py | 68 +------ 18 files changed, 558 insertions(+), 270 deletions(-) create mode 100644 parm/stage/analysis.yaml.j2 create mode 100644 parm/stage/atmosphere_cold.yaml.j2 create mode 100644 parm/stage/atmosphere_nest.yaml.j2 create mode 100644 parm/stage/atmosphere_perturbation.yaml.j2 create mode 100644 parm/stage/atmosphere_warm.yaml.j2 create mode 100644 parm/stage/ice.yaml.j2 create mode 100644 parm/stage/master_gefs.yaml.j2 create mode 100644 parm/stage/master_gfs.yaml.j2 create mode 100644 parm/stage/ocean.yaml.j2 create mode 100644 parm/stage/ocean_mediator.yaml.j2 create mode 100644 parm/stage/ocean_replay.yaml.j2 create mode 100644 parm/stage/ocean_rerun.yaml.j2 delete mode 100644 parm/stage/stage.yaml.j2 create mode 100644 parm/stage/wave.yaml.j2 diff --git a/parm/config/gefs/config.stage_ic b/parm/config/gefs/config.stage_ic index 7cf64648bc..cac65c74b9 100644 --- a/parm/config/gefs/config.stage_ic +++ b/parm/config/gefs/config.stage_ic @@ -10,7 +10,7 @@ source "${EXPDIR}/config.resources" stage_ic export ICSDIR="@ICSDIR@" # User provided ICSDIR; blank if not provided export BASE_IC="@BASE_IC@" # Platform home for staged ICs -export STAGE_IC_YAML_TMPL="${PARMgfs}/stage/stage.yaml.j2" +export STAGE_IC_YAML_TMPL="${PARMgfs}/stage/master_gefs.yaml.j2" # Set ICSDIR diff --git a/parm/config/gfs/config.stage_ic b/parm/config/gfs/config.stage_ic index 78eba84962..7aa0c25f32 100644 --- a/parm/config/gfs/config.stage_ic +++ b/parm/config/gfs/config.stage_ic @@ -10,7 +10,7 @@ source "${EXPDIR}/config.resources" stage_ic export ICSDIR="@ICSDIR@" # User provided ICSDIR; blank if not provided export BASE_IC="@BASE_IC@" # Platform home for staged ICs -export STAGE_IC_YAML_TMPL="${PARMgfs}/stage/stage.yaml.j2" +export STAGE_IC_YAML_TMPL="${PARMgfs}/stage/master_gfs.yaml.j2" # Set ICSDIR (if not defined) if [[ -z "${ICSDIR}" ]] ; then diff --git a/parm/stage/analysis.yaml.j2 b/parm/stage/analysis.yaml.j2 new file mode 100644 index 0000000000..f6beada6b4 --- /dev/null +++ b/parm/stage/analysis.yaml.j2 @@ -0,0 +1,10 @@ +analysis: + {% set COMOUT_ATMOS_ANALYSIS_MEM = COMOUT_ATMOS_ANALYSIS_MEM_list[0] %} + mkdir: + - "{{ COMOUT_ATMOS_ANALYSIS_MEM }}" + copy: + {% for ftype in ["abias", "abias_air", "abias_int", "abias_pc", "atminc.nc", "radstat"] %} + {% if path_exists(ICSDIR ~ "/" ~ COMOUT_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) ~ "/" ~ RUN ~ ".t" ~ current_cycle_HH ~ "z." ~ ftype) %} + - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ RUN }}.t{{ current_cycle_HH }}z.{{ ftype }}", "{{ COMOUT_ATMOS_ANALYSIS_MEM }}"] + {% endif %} + {% endfor %} diff --git a/parm/stage/atmosphere_cold.yaml.j2 b/parm/stage/atmosphere_cold.yaml.j2 new file mode 100644 index 0000000000..9eeaaf4b9e --- /dev/null +++ b/parm/stage/atmosphere_cold.yaml.j2 @@ -0,0 +1,18 @@ +atmosphere_cold: + mkdir: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_ATMOS_INPUT_MEM = COMOUT_ATMOS_INPUT_MEM_list[imem] %} + - "{{ COMOUT_ATMOS_INPUT_MEM }}" + {% endfor %} # mem loop + copy: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_ATMOS_INPUT_MEM = COMOUT_ATMOS_INPUT_MEM_list[imem] %} + - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_INPUT_MEM | relpath(ROTDIR) }}/gfs_ctrl.nc", "{{ COMOUT_ATMOS_INPUT_MEM }}"] + {% for ftype in ["gfs_data", "sfc_data"] %} + {% for ntile in range(1, ntiles + 1) %} + - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_INPUT_MEM | relpath(ROTDIR) }}/{{ ftype }}.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_INPUT_MEM }}"] + {% endfor %} # ntile + {% endfor %} # ftype + {% endfor %} # mem loop diff --git a/parm/stage/atmosphere_nest.yaml.j2 b/parm/stage/atmosphere_nest.yaml.j2 new file mode 100644 index 0000000000..13ec0ed8c5 --- /dev/null +++ b/parm/stage/atmosphere_nest.yaml.j2 @@ -0,0 +1,33 @@ +atmosphere_nest: + {% set ntile = 7 %} + {% if EXP_WARM_START == True %} + mkdir: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_ATMOS_RESTART_PREV_MEM = COMOUT_ATMOS_RESTART_PREV_MEM_list[imem] %} + - "{{ COMOUT_ATMOS_RESTART_PREV_MEM }}" + {% endfor %} # mem loop + copy: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_ATMOS_RESTART_PREV_MEM = COMOUT_ATMOS_RESTART_PREV_MEM_list[imem] %} + {% for ftype in ["ca_data", "fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data", "sfc_data"] %} + - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.{{ ftype }}.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_RESTART_PREV_MEM }}/{{ m_prefix }}.{{ ftype }}.nest0{{ ntile-5 }}.tile{{ ntile }}.nc"] + {% endfor %} + {% endfor %} # mem loop + {% else %} # cold start + mkdir: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_ATMOS_INPUT_MEM = COMOUT_ATMOS_INPUT_MEM_list[imem] %} + - "{{ COMOUT_ATMOS_INPUT_MEM }}" + {% endfor %} # mem loop + copy: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_ATMOS_INPUT_MEM = COMOUT_ATMOS_INPUT_MEM_list[imem] %} + {% for ftype in ["gfs_data", "sfc_data"] %} + - ["{{ COMOUT_ATMOS_INPUT_MEM }}/{{ ftype }}.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_INPUT_MEM }}/{{ ftype }}.nest0{{ ntile-5 }}.tile{{ ntile }}.nc"] + {% endfor %} + {% endfor %} # mem loop + {% endif %} diff --git a/parm/stage/atmosphere_perturbation.yaml.j2 b/parm/stage/atmosphere_perturbation.yaml.j2 new file mode 100644 index 0000000000..0e097b71dc --- /dev/null +++ b/parm/stage/atmosphere_perturbation.yaml.j2 @@ -0,0 +1,13 @@ +atmosphere_perturbation: + mkdir: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_ATMOS_ANALYSIS_MEM = COMOUT_ATMOS_ANALYSIS_MEM_list[imem] %} + - "{{ COMOUT_ATMOS_ANALYSIS_MEM }}" + {% endfor %} # mem loop + copy: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_ATMOS_ANALYSIS_MEM = COMOUT_ATMOS_ANALYSIS_MEM_list[imem] %} + - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.fv3_perturbation.nc", "{{ COMOUT_ATMOS_ANALYSIS_MEM }}/{{ RUN }}.t{{ current_cycle_HH }}z.atminc.nc"] + {% endfor %} # mem loop diff --git a/parm/stage/atmosphere_warm.yaml.j2 b/parm/stage/atmosphere_warm.yaml.j2 new file mode 100644 index 0000000000..14c8615262 --- /dev/null +++ b/parm/stage/atmosphere_warm.yaml.j2 @@ -0,0 +1,28 @@ +atmosphere_warm: + mkdir: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_ATMOS_RESTART_PREV_MEM = COMOUT_ATMOS_RESTART_PREV_MEM_list[imem] %} + - "{{ COMOUT_ATMOS_RESTART_PREV_MEM }}" + {% endfor %} # mem loop + copy: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_ATMOS_RESTART_PREV_MEM = COMOUT_ATMOS_RESTART_PREV_MEM_list[imem] %} + {% if path_exists(ICSDIR ~ "/" ~ COMOUT_ATMOS_RESTART_PREV_MEM | relpath(ROTDIR) ~ "/" ~ m_prefix ~ ".atm_stoch.res.nc") %} + - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.atm_stoch.res.nc", "{{ COMOUT_ATMOS_RESTART_PREV_MEM }}"] + {% endif %} # path_exists + {% for ftype in ["coupler.res", "fv_core.res.nc"] %} + - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.{{ ftype }}", "{{ COMOUT_ATMOS_RESTART_PREV_MEM }}"] + {% endfor %} + {% for ftype in ["ca_data", "fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data", "sfc_data"] %} + {% for ntile in range(1, ntiles + 1) %} + - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.{{ ftype }}.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_RESTART_PREV_MEM }}"] + {% endfor %} # ntile + {% endfor %} # ftype + {% for ntile in range(1, ntiles + 1) %} + {% if path_exists(ICSDIR ~ "/" ~ COMOUT_ATMOS_RESTART_PREV_MEM | relpath(ROTDIR) ~ "/" ~ p_prefix ~ ".sfcanl_data.tile" ~ ntile ~ ".nc") %} + - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ p_prefix }}.sfcanl_data.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_RESTART_PREV_MEM }}"] + {% endif %} # path_exists + {% endfor %} # ntile + {% endfor %} # mem loop diff --git a/parm/stage/ice.yaml.j2 b/parm/stage/ice.yaml.j2 new file mode 100644 index 0000000000..0e0aa40c7f --- /dev/null +++ b/parm/stage/ice.yaml.j2 @@ -0,0 +1,28 @@ +ice: + {% if DO_JEDIOCNVAR == True %} + mkdir: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_ICE_ANALYSIS_MEM = COMOUT_ICE_ANALYSIS_MEM_list[imem] %} + - "{{ COMOUT_ICE_ANALYSIS_MEM }}" + {% endfor %} # mem loop + copy: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_ICE_ANALYSIS_MEM = COMOUT_ICE_ANALYSIS_MEM_list[imem] %} + - ["{{ ICSDIR }}/{{ COMOUT_ICE_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.cice_model_anl.res.nc", "{{ COMOUT_ICE_ANALYSIS_MEM }}"] + {% endfor %} # mem loop + {% else %} + mkdir: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_ICE_RESTART_PREV_MEM = COMOUT_ICE_RESTART_PREV_MEM_list[imem] %} + - "{{ COMOUT_ICE_RESTART_PREV_MEM }}" + {% endfor %} # mem loop + copy: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_ICE_RESTART_PREV_MEM = COMOUT_ICE_RESTART_PREV_MEM_list[imem] %} + - ["{{ ICSDIR }}/{{ COMOUT_ICE_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.cice_model.res.nc", "{{ COMOUT_ICE_RESTART_PREV_MEM }}"] + {% endfor %} # mem loop + {% endif %} diff --git a/parm/stage/master_gefs.yaml.j2 b/parm/stage/master_gefs.yaml.j2 new file mode 100644 index 0000000000..bdd4c8de5f --- /dev/null +++ b/parm/stage/master_gefs.yaml.j2 @@ -0,0 +1,154 @@ +################################################################### +# This is the master yaml for the GEFS +# +# Cycle, member, and RUN settings are set before including each +# component yaml based on DO switches +# +# The included yamls are intended to be of the following structure: +# key1: +# mkdir: +# - "COM directory to create" +# copy: +# - ["source_file", "destination_file"] +# key2: +# mkdir: +# - "COM directory to create" +# copy: +# - ["source_file", "destination_file"] +# +# Any number of keys with nested mkdir and copy are permitted +# Jinja is permitted in this yaml, as long as the keys are: +# - COMOUT_ +# - DO_ATM, DO_OCN, DO_ICE, etc. +# For a full list see scripts/exglobal_stage_ic.py +################################################################### + +# Set cycle variables +# ------------------------ +{% set half_window = assim_freq // 2 %} +{% set half_window_begin = (-half_window | string + "H") | to_timedelta %} +{% set half_window_end = (half_window | string + "H") | to_timedelta %} +{% if DOIAU and MODE == "cycled" %} + {% set model_start_date_current_cycle = current_cycle | add_to_datetime(half_window_begin) %} +{% else %} + {% if REPLAY_ICS %} + {% set model_start_date_current_cycle = current_cycle | add_to_datetime(half_window_end) %} + {% else %} + {% set model_start_date_current_cycle = current_cycle %} + {% endif %} +{% endif %} + +{% set current_cycle_YMD = current_cycle | to_YMD %} +{% set current_cycle_HH = current_cycle | strftime("%H") %} +{% set previous_cycle_YMD = previous_cycle | to_YMD %} +{% set previous_cycle_HH = previous_cycle | strftime("%H") %} +{% set p_prefix = previous_cycle | strftime("%Y%m%d.%H0000") %} +{% set m_prefix = model_start_date_current_cycle | strftime("%Y%m%d.%H0000") %} + +# Set first/last mem for loop +# --------------------------- +{% set first_mem = 0 %} +{% set last_mem = NMEM_ENS %} + +# Declare to-be-filled lists of member COM directories +# ---------------------------------------------------- +{% set COMOUT_ATMOS_INPUT_MEM_list = [] %} +{% set COMOUT_ATMOS_RESTART_PREV_MEM_list = [] %} +{% set COMOUT_ATMOS_ANALYSIS_MEM_list = [] %} +{% set COMOUT_ICE_ANALYSIS_MEM_list = [] %} +{% set COMOUT_ICE_RESTART_PREV_MEM_list = [] %} +{% set COMOUT_OCEAN_RESTART_PREV_MEM_list = [] %} +{% set COMOUT_OCEAN_ANALYSIS_MEM_list = [] %} +{% set COMOUT_MED_RESTART_PREV_MEM_list = [] %} +{% set COMOUT_WAVE_RESTART_PREV_MEM_list = [] %} + +# Construct member COM directory lists +# ------------------------------------ +{% for mem in range(first_mem, last_mem + 1) %} + + {% set current_cycle_dict = ({ '${ROTDIR}':ROTDIR, + '${RUN}':RUN, + '${YMD}':current_cycle_YMD, + '${HH}':current_cycle_HH, + '${MEMDIR}': 'mem%03d' | format(mem) }) %} + {% set previous_cycle_dict = ({ '${ROTDIR}':ROTDIR, + '${RUN}':RUN, + '${YMD}':previous_cycle_YMD, + '${HH}':previous_cycle_HH, + '${MEMDIR}': 'mem%03d' | format(mem) }) %} + + {% set COMOUT_ATMOS_INPUT_MEM = COM_ATMOS_INPUT_TMPL | replace_tmpl(current_cycle_dict) %} + {% set COMOUT_ATMOS_RESTART_PREV_MEM = COM_ATMOS_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} + {% set COMOUT_ATMOS_ANALYSIS_MEM = COM_ATMOS_ANALYSIS_TMPL | replace_tmpl(current_cycle_dict) %} + {% set COMOUT_ICE_ANALYSIS_MEM = COM_ICE_ANALYSIS_TMPL | replace_tmpl(current_cycle_dict) %} + {% set COMOUT_ICE_RESTART_PREV_MEM = COM_ICE_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} + {% set COMOUT_OCEAN_RESTART_PREV_MEM = COM_OCEAN_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} + {% set COMOUT_OCEAN_ANALYSIS_MEM = COM_OCEAN_ANALYSIS_TMPL | replace_tmpl(current_cycle_dict) %} + {% set COMOUT_MED_RESTART_PREV_MEM = COM_MED_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} + {% set COMOUT_WAVE_RESTART_PREV_MEM = COM_WAVE_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} + + # Append the member COM directories + {% do COMOUT_ATMOS_INPUT_MEM_list.append(COMOUT_ATMOS_INPUT_MEM)%} + {% do COMOUT_ATMOS_RESTART_PREV_MEM_list.append(COMOUT_ATMOS_RESTART_PREV_MEM)%} + {% do COMOUT_ATMOS_ANALYSIS_MEM_list.append(COMOUT_ATMOS_ANALYSIS_MEM)%} + {% do COMOUT_ICE_ANALYSIS_MEM_list.append(COMOUT_ICE_ANALYSIS_MEM)%} + {% do COMOUT_ICE_RESTART_PREV_MEM_list.append(COMOUT_ICE_RESTART_PREV_MEM)%} + {% do COMOUT_OCEAN_RESTART_PREV_MEM_list.append(COMOUT_OCEAN_RESTART_PREV_MEM)%} + {% do COMOUT_OCEAN_ANALYSIS_MEM_list.append(COMOUT_OCEAN_ANALYSIS_MEM)%} + {% do COMOUT_MED_RESTART_PREV_MEM_list.append(COMOUT_MED_RESTART_PREV_MEM)%} + {% do COMOUT_WAVE_RESTART_PREV_MEM_list.append(COMOUT_WAVE_RESTART_PREV_MEM)%} + +{% endfor %} + +################################################################### +# Initial condition to stage - include components based on switches +################################################################### + +{% if EXP_WARM_START %} +{% filter indent(width=4) %} +{% include "atmosphere_warm.yaml.j2" %} +{% endfilter %} +{% else %} # cold start +{% filter indent(width=4) %} +{% include "atmosphere_cold.yaml.j2" %} +{% endfilter %} +{% endif %} + +{% if REPLAY_ICS %} +{% filter indent(width=4) %} +{% include "atmosphere_perturbation.yaml.j2" %} +{% endfilter %} +{% endif %} + +{% if DO_ICE %} +{% filter indent(width=4) %} +{% include "ice.yaml.j2" %} +{% endfilter %} +{% endif %} + +{% if DO_OCN %} +{% filter indent(width=4) %} +{% include "ocean.yaml.j2" %} +{% endfilter %} +{% if DO_JEDIOCNVAR %} +{% filter indent(width=4) %} +{% include "ocean_rerun.yaml.j2" %} +{% endfilter %} +{% endif %} +{% if REPLAY_ICS %} +{% filter indent(width=4) %} +{% include "ocean_replay.yaml.j2" %} +{% endfilter %} +{% endif %} +{% if EXP_WARM_START %} +{% filter indent(width=4) %} +{% include "ocean_mediator.yaml.j2" %} +{% endfilter %} +{% endif %} +{% endif %} # DO_OCN + +{% if DO_WAVE %} +{% filter indent(width=4) %} +{% include "wave.yaml.j2" %} +{% endfilter %} +{% endif %} diff --git a/parm/stage/master_gfs.yaml.j2 b/parm/stage/master_gfs.yaml.j2 new file mode 100644 index 0000000000..0f27365ba9 --- /dev/null +++ b/parm/stage/master_gfs.yaml.j2 @@ -0,0 +1,189 @@ +################################################################### +# This is the master yaml for the GFS +# +# Cycle, member, and RUN settings are set before including each +# component yaml based on DO switches +# +# The included yamls are intended to be of the following structure: +# key1: +# mkdir: +# - "COM directory to create" +# copy: +# - ["source_file", "destination_file"] +# key2: +# mkdir: +# - "COM directory to create" +# copy: +# - ["source_file", "destination_file"] +# +# Any number of keys with nested mkdir and copy are permitted +# Jinja is permitted in this yaml, as long as the keys are: +# - COMOUT_ +# - DO_ATM, DO_OCN, DO_ICE, etc. +# For a full list see scripts/exglobal_stage_ic.py +################################################################### + +# Set cycle date variables +# ------------------------ +{% set half_window = assim_freq // 2 %} +{% set half_window_begin = (-half_window | string + "H") | to_timedelta %} +{% set half_window_end = (half_window | string + "H") | to_timedelta %} +{% if DOIAU and MODE == "cycled" %} + {% set model_start_date_current_cycle = current_cycle | add_to_datetime(half_window_begin) %} +{% else %} + {% if REPLAY_ICS %} + {% set model_start_date_current_cycle = current_cycle | add_to_datetime(half_window_end) %} + {% else %} + {% set model_start_date_current_cycle = current_cycle %} + {% endif %} +{% endif %} + +{% set current_cycle_YMD = current_cycle | to_YMD %} +{% set current_cycle_HH = current_cycle | strftime("%H") %} +{% set previous_cycle_YMD = previous_cycle | to_YMD %} +{% set previous_cycle_HH = previous_cycle | strftime("%H") %} +{% set p_prefix = previous_cycle | strftime("%Y%m%d.%H0000") %} +{% set m_prefix = model_start_date_current_cycle | strftime("%Y%m%d.%H0000") %} + +# Determine restart RUN +# --------------------- +{% set rRUN = RUN %} +{% if RUN == "gfs" %} + {% set rRUN = "gdas" %} +{% endif %} + +# Set first/last mem for loop +# --------------------------- +{% if NMEM_ENS > 0 and RUN == "enkfgdas" %} # Ensemble RUN + {% set first_mem = 1 %} + {% set last_mem = NMEM_ENS %} +{% else %} # Deterministic RUN + {% set first_mem = -1 %} + {% set last_mem = -1 %} +{% endif %} + +# Declare to-be-filled lists of member COM directories +# ---------------------------------------------------- +{% set COMOUT_ATMOS_INPUT_MEM_list = [] %} +{% set COMOUT_ATMOS_RESTART_PREV_MEM_list = [] %} +{% set COMOUT_ATMOS_ANALYSIS_MEM_list = [] %} +{% set COMOUT_ICE_ANALYSIS_MEM_list = [] %} +{% set COMOUT_ICE_RESTART_PREV_MEM_list = [] %} +{% set COMOUT_OCEAN_RESTART_PREV_MEM_list = [] %} +{% set COMOUT_OCEAN_ANALYSIS_MEM_list = [] %} +{% set COMOUT_MED_RESTART_PREV_MEM_list = [] %} +{% set COMOUT_WAVE_RESTART_PREV_MEM_list = [] %} + +# Construct member COM directory lists +# ------------------------------------ +{% for mem in range(first_mem, last_mem + 1) %} + + {% if mem >= 0 %} + {% set mem_char = 'mem%03d' | format(mem) %} + {% else %} + {% set mem_char = '' %} + {% endif %} + + {% set current_cycle_dict = ({ '${ROTDIR}':ROTDIR, + '${RUN}':RUN, + '${YMD}':current_cycle_YMD, + '${HH}':current_cycle_HH, + '${MEMDIR}': mem_char }) %} + {% set previous_cycle_dict = ({ '${ROTDIR}':ROTDIR, + '${RUN}':rRUN, + '${YMD}':previous_cycle_YMD, + '${HH}':previous_cycle_HH, + '${MEMDIR}': mem_char }) %} + {% set previous_cycle_and_run_dict = ({ '${ROTDIR}':ROTDIR, + '${RUN}':RUN, + '${YMD}':previous_cycle_YMD, + '${HH}':previous_cycle_HH, + '${MEMDIR}': mem_char }) %} + + {% set COMOUT_ATMOS_INPUT_MEM = COM_ATMOS_INPUT_TMPL | replace_tmpl(current_cycle_dict) %} + {% set COMOUT_ATMOS_RESTART_PREV_MEM = COM_ATMOS_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} + {% set COMOUT_ATMOS_ANALYSIS_MEM = COM_ATMOS_ANALYSIS_TMPL | replace_tmpl(current_cycle_dict) %} + {% set COMOUT_ICE_ANALYSIS_MEM = COM_ICE_ANALYSIS_TMPL | replace_tmpl(current_cycle_dict) %} + {% set COMOUT_ICE_RESTART_PREV_MEM = COM_ICE_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} + {% set COMOUT_OCEAN_RESTART_PREV_MEM = COM_OCEAN_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} + {% set COMOUT_OCEAN_ANALYSIS_MEM = COM_OCEAN_ANALYSIS_TMPL | replace_tmpl(current_cycle_dict) %} + {% set COMOUT_MED_RESTART_PREV_MEM = COM_MED_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} + {% set COMOUT_WAVE_RESTART_PREV_MEM = COM_WAVE_RESTART_TMPL | replace_tmpl(previous_cycle_and_run_dict) %} + + # Append the member COM directories + {% do COMOUT_ATMOS_INPUT_MEM_list.append(COMOUT_ATMOS_INPUT_MEM)%} + {% do COMOUT_ATMOS_RESTART_PREV_MEM_list.append(COMOUT_ATMOS_RESTART_PREV_MEM)%} + {% do COMOUT_ATMOS_ANALYSIS_MEM_list.append(COMOUT_ATMOS_ANALYSIS_MEM)%} + {% do COMOUT_ICE_ANALYSIS_MEM_list.append(COMOUT_ICE_ANALYSIS_MEM)%} + {% do COMOUT_ICE_RESTART_PREV_MEM_list.append(COMOUT_ICE_RESTART_PREV_MEM)%} + {% do COMOUT_OCEAN_RESTART_PREV_MEM_list.append(COMOUT_OCEAN_RESTART_PREV_MEM)%} + {% do COMOUT_OCEAN_ANALYSIS_MEM_list.append(COMOUT_OCEAN_ANALYSIS_MEM)%} + {% do COMOUT_MED_RESTART_PREV_MEM_list.append(COMOUT_MED_RESTART_PREV_MEM)%} + {% do COMOUT_WAVE_RESTART_PREV_MEM_list.append(COMOUT_WAVE_RESTART_PREV_MEM)%} + +{% endfor %} + +################################################################### +# Initial condition to stage - include components based on switches +################################################################### + +{% if MODE == "cycled" and RUN == "gdas" %} +{% filter indent(width=4) %} +{% include "analysis.yaml.j2" %} +{% endfilter %} +{% endif %} + +{% if EXP_WARM_START %} +{% filter indent(width=4) %} +{% include "atmosphere_warm.yaml.j2" %} +{% endfilter %} +{% else %} # cold start +{% filter indent(width=4) %} +{% include "atmosphere_cold.yaml.j2" %} +{% endfilter %} +{% endif %} + +{% if DO_NEST %} +{% filter indent(width=4) %} +{% include "atmosphere_nest.yaml.j2" %} +{% endfilter %} +{% endif %} + +{% if REPLAY_ICS %} +{% filter indent(width=4) %} +{% include "atmosphere_perturbation.yaml.j2" %} +{% endfilter %} +{% endif %} + +{% if DO_ICE %} +{% filter indent(width=4) %} +{% include "ice.yaml.j2" %} +{% endfilter %} +{% endif %} + +{% if DO_OCN %} +{% filter indent(width=4) %} +{% include "ocean.yaml.j2" %} +{% endfilter %} +{% if DO_JEDIOCNVAR %} +{% filter indent(width=4) %} +{% include "ocean_rerun.yaml.j2" %} +{% endfilter %} +{% endif %} +{% if REPLAY_ICS %} +{% filter indent(width=4) %} +{% include "ocean_replay.yaml.j2" %} +{% endfilter %} +{% endif %} +{% if EXP_WARM_START %} +{% filter indent(width=4) %} +{% include "ocean_mediator.yaml.j2" %} +{% endfilter %} +{% endif %} +{% endif %} + +{% if DO_WAVE %} +{% filter indent(width=4) %} +{% include "wave.yaml.j2" %} +{% endfilter %} +{% endif %} diff --git a/parm/stage/ocean.yaml.j2 b/parm/stage/ocean.yaml.j2 new file mode 100644 index 0000000000..75a472dff0 --- /dev/null +++ b/parm/stage/ocean.yaml.j2 @@ -0,0 +1,18 @@ +ocean: + mkdir: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_OCEAN_RESTART_PREV_MEM = COMOUT_OCEAN_RESTART_PREV_MEM_list[imem] %} + - "{{ COMOUT_OCEAN_RESTART_PREV_MEM }}" + {% endfor %} # mem loop + copy: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_OCEAN_RESTART_PREV_MEM = COMOUT_OCEAN_RESTART_PREV_MEM_list[imem] %} + - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.MOM.res.nc", "{{ COMOUT_OCEAN_RESTART_PREV_MEM }}"] + {% if OCNRES == "025" %} + {% for nn in range(1, 3) %} + - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.MOM.res_{{ nn }}.nc", "{{ COMOUT_OCEAN_RESTART_PREV_MEM }}"] + {% endfor %} + {% endif %} + {% endfor %} # mem loop diff --git a/parm/stage/ocean_mediator.yaml.j2 b/parm/stage/ocean_mediator.yaml.j2 new file mode 100644 index 0000000000..c986b2e746 --- /dev/null +++ b/parm/stage/ocean_mediator.yaml.j2 @@ -0,0 +1,15 @@ +{% if path_exists(ICSDIR ~ "/" ~ COMOUT_MED_RESTART_PREV_MEM_list[0] | relpath(ROTDIR) ~ "/" ~ m_prefix ~ ".ufs.cpld.cpl.r.nc") %} +ocean_mediator: + mkdir: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_MED_RESTART_PREV_MEM = COMOUT_MED_RESTART_PREV_MEM_list[imem] %} + - "{{ COMOUT_MED_RESTART_PREV_MEM }}" + {% endfor %} # mem loop + copy: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_MED_RESTART_PREV_MEM = COMOUT_MED_RESTART_PREV_MEM_list[imem] %} + - ["{{ ICSDIR }}/{{ COMOUT_MED_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.ufs.cpld.cpl.r.nc", "{{ COMOUT_MED_RESTART_PREV_MEM }}"] + {% endfor %} # mem loop +{% endif %} # path exists diff --git a/parm/stage/ocean_replay.yaml.j2 b/parm/stage/ocean_replay.yaml.j2 new file mode 100644 index 0000000000..8b52108bec --- /dev/null +++ b/parm/stage/ocean_replay.yaml.j2 @@ -0,0 +1,13 @@ +ocean_replay: + mkdir: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_OCEAN_ANALYSIS_MEM = COMOUT_OCEAN_ANALYSIS_MEM_list[imem] %} + - "{{ COMOUT_OCEAN_ANALYSIS_MEM }}" + {% endfor %} # mem loop + copy: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_OCEAN_ANALYSIS_MEM = COMOUT_OCEAN_ANALYSIS_MEM_list[imem] %} + - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.mom6_perturbation.nc", "{{ COMOUT_OCEAN_ANALYSIS_MEM }}/mom6_increment.nc"] + {% endfor %} # mem loop diff --git a/parm/stage/ocean_rerun.yaml.j2 b/parm/stage/ocean_rerun.yaml.j2 new file mode 100644 index 0000000000..8b4042d730 --- /dev/null +++ b/parm/stage/ocean_rerun.yaml.j2 @@ -0,0 +1,13 @@ +ocean_rerun: + mkdir: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_OCEAN_ANALYSIS_MEM = COMOUT_OCEAN_ANALYSIS_MEM_list[imem] %} + - "{{ COMOUT_OCEAN_ANALYSIS_MEM }}" + {% endfor %} # mem loop + copy: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_OCEAN_ANALYSIS_MEM = COMOUT_OCEAN_ANALYSIS_MEM_list[imem] %} + - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ RUN }}.t{{ current_cycle_HH }}z.ocninc.nc", "{{ COMOUT_OCEAN_ANALYSIS_MEM }}"] + {% endfor %} # mem loop diff --git a/parm/stage/stage.yaml.j2 b/parm/stage/stage.yaml.j2 deleted file mode 100644 index 49415a3e17..0000000000 --- a/parm/stage/stage.yaml.j2 +++ /dev/null @@ -1,204 +0,0 @@ -############################################################# -# This yaml is intended to be of the following structure: -# key1: -# mkdir: -# - "COM directory to create" -# copy: -# - ["source_file", "destination_file"] -# key2: -# mkdir: -# - "COM directory to create" -# copy: -# - ["source_file", "destination_file"] -# -# Any number of keys with nested mkdir and copy are permitted -# Jinja is permitted in this yaml, as long as the keys are: -# - COMOUT_ -# - DO_ATM, DO_OCN, DO_ICE, etc. -# For a full list see scripts/exglobal_stage_ic.py -############################################################# - -# Set variables used below -{% set current_cycle_YMD = current_cycle | to_YMD %} -{% set current_cycle_HH = current_cycle | strftime("%H") %} -{% set previous_cycle_YMD = previous_cycle | to_YMD %} -{% set previous_cycle_HH = previous_cycle | strftime("%H") %} -{% set m_prefix = model_start_date_current_cycle | strftime("%Y%m%d.%H0000") %} -{% set p_prefix = previous_cycle | strftime("%Y%m%d.%H0000") %} - -#################################################################### -# Initial condition to stage -#################################################################### - -# Declare a dict of search and replace terms to run on each template - -{% if mem >= 0 %} - {% set mem_char = 'mem%03d' | format(mem) %} -{% else %} - {% set mem_char = '' %} -{% endif %} -{% set current_cycle_dict = ({ '${ROTDIR}':ROTDIR, - '${RUN}':RUN, - '${YMD}':current_cycle_YMD, - '${HH}':current_cycle_HH, - '${MEMDIR}': mem_char }) %} -{% set previous_cycle_dict = ({ '${ROTDIR}':ROTDIR, - '${RUN}':rRUN, - '${YMD}':previous_cycle_YMD, - '${HH}':previous_cycle_HH, - '${MEMDIR}': mem_char }) %} -{% set previous_cycle_and_run_dict = ({ '${ROTDIR}':ROTDIR, - '${RUN}':RUN, - '${YMD}':previous_cycle_YMD, - '${HH}':previous_cycle_HH, - '${MEMDIR}': mem_char }) %} - -# Initial condition definitions - -{% if MODE == "cycled" and RUN == "gdas" %} -{% set COMOUT_ATMOS_ANALYSIS = COM_ATMOS_ANALYSIS_TMPL | replace_tmpl(current_cycle_dict) %} -analysis: - mkdir: - - "{{ COMOUT_ATMOS_ANALYSIS }}" - copy: - {% for ftype in ["abias", "abias_air", "abias_int", "abias_pc", "atminc.nc", "radstat"] %} - {% if path_exists(ICSDIR ~ "/" ~ COMOUT_ATMOS_ANALYSIS | relpath(ROTDIR) ~ "/" ~ RUN ~ ".t" ~ current_cycle_HH ~ "z." ~ ftype) %} - - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ RUN }}.t{{ current_cycle_HH }}z.{{ ftype }}", "{{ COMOUT_ATMOS_ANALYSIS }}"] - {% endif %} - {% endfor %} -{% endif %} - -{% if EXP_WARM_START == True %} -{% set COMOUT_ATMOS_RESTART_PREV = COM_ATMOS_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} -atmosphere_warm: - mkdir: - - "{{ COMOUT_ATMOS_RESTART_PREV }}" - copy: - {% for ftype in ["coupler.res", "fv_core.res.nc"] %} - - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ m_prefix }}.{{ ftype }}", "{{ COMOUT_ATMOS_RESTART_PREV }}"] - {% endfor %} - {% for ftype in ["ca_data", "fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data", "sfc_data"] %} - {% for ntile in range(1, ntiles + 1) %} - - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ m_prefix }}.{{ ftype }}.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_RESTART_PREV }}"] - {% endfor %} # ntile - {% endfor %} # ftype - {% for ntile in range(1, ntiles + 1) %} - {% if path_exists(ICSDIR ~ "/" ~ COMOUT_ATMOS_RESTART_PREV | relpath(ROTDIR) ~ "/" ~ p_prefix ~ ".sfcanl_data.tile" ~ ntile ~ ".nc") %} - - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ p_prefix }}.sfcanl_data.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_RESTART_PREV }}"] - {% endif %} # path_exists - {% endfor %} # ntile -{% else %} # cold start -{% set COMOUT_ATMOS_INPUT = COM_ATMOS_INPUT_TMPL | replace_tmpl(current_cycle_dict) %} -atmosphere_cold: - mkdir: - - "{{ COMOUT_ATMOS_INPUT }}" - copy: - - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_INPUT | relpath(ROTDIR) }}/gfs_ctrl.nc", "{{ COMOUT_ATMOS_INPUT }}"] - {% for ftype in ["gfs_data", "sfc_data"] %} - {% for ntile in range(1, ntiles + 1) %} - - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_INPUT | relpath(ROTDIR) }}/{{ ftype }}.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_INPUT }}"] - {% endfor %} # ntile - {% endfor %} # ftype -{% endif %} - -{% if REPLAY_ICS == True %} -{% set COMOUT_ATMOS_ANALYSIS = COM_ATMOS_ANALYSIS_TMPL | replace_tmpl(current_cycle_dict) %} -atmosphere_perturbation: - mkdir: - - "{{ COMOUT_ATMOS_ANALYSIS }}" - copy: - - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ m_prefix }}.fv3_perturbation.nc", "{{ COMOUT_ATMOS_ANALYSIS }}/{{ RUN }}.t{{ current_cycle_HH }}z.atminc.nc"] -{% endif %} - -{% if DO_NEST %} -atmosphere_nest: - {% set ntile = 7 %} - {% if EXP_WARM_START == True %} - {% set COMOUT_ATMOS_RESTART_PREV = COM_ATMOS_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} - mkdir: - - "{{ COMOUT_ATMOS_RESTART_PREV }}" - copy: - {% for ftype in ["ca_data", "fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data", "sfc_data"] %} - - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV | relpath(ROTDIR) }}/{{ m_prefix }}.{{ ftype }}.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_RESTART_PREV }}/{{ m_prefix }}.{{ ftype }}.nest0{{ ntile-5 }}.tile{{ ntile }}.nc"] - {% endfor %} - {% else %} # cold start - {% set COMOUT_ATMOS_INPUT = COM_ATMOS_INPUT_TMPL | replace_tmpl(current_cycle_dict) %} - mkdir: - - "{{ COMOUT_ATMOS_INPUT }}" - copy: - {% for ftype in ["gfs_data", "sfc_data"] %} - - ["{{ COMOUT_ATMOS_INPUT }}/{{ ftype }}.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_INPUT }}/{{ ftype }}.nest0{{ ntile-5 }}.tile{{ ntile }}.nc"] - {% endfor %} - {% endif %} -{% endif %} - -{% if DO_ICE %} -{% if DO_JEDIOCNVAR == True %} -{% set COMOUT_ICE_ANALYSIS = COM_ICE_ANALYSIS_TMPL | replace_tmpl(current_cycle_dict) %} -ice: - mkdir: - - "{{ COMOUT_ICE_ANALYSIS }}" - copy: - - ["{{ ICSDIR }}/{{ COMOUT_ICE_ANALYSIS | relpath(ROTDIR) }}/{{ m_prefix }}.cice_model_anl.res.nc", "{{ COMOUT_ICE_ANALYSIS }}"] -{% else %} -{% set COMOUT_ICE_RESTART_PREV = COM_ICE_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} -ice: - mkdir: - - "{{ COMOUT_ICE_RESTART_PREV }}" - copy: - - ["{{ ICSDIR }}/{{ COMOUT_ICE_RESTART_PREV | relpath(ROTDIR) }}/{{ m_prefix }}.cice_model.res.nc", "{{ COMOUT_ICE_RESTART_PREV }}"] -{% endif %} -{% endif %} - -{% if DO_OCN %} -{% set COMOUT_OCEAN_RESTART_PREV = COM_OCEAN_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} -ocean: - mkdir: - - "{{ COMOUT_OCEAN_RESTART_PREV }}" - copy: - - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_RESTART_PREV | relpath(ROTDIR) }}/{{ m_prefix }}.MOM.res.nc", "{{ COMOUT_OCEAN_RESTART_PREV }}"] - {% if OCNRES == "025" %} - {% for nn in range(1, 3) %} - - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_RESTART_PREV | relpath(ROTDIR) }}/{{ m_prefix }}.MOM.res_{{ nn }}.nc", "{{ COMOUT_OCEAN_RESTART_PREV }}"] - {% endfor %} - {% endif %} - -{% if DO_JEDIOCNVAR == True %} -{% set COMOUT_OCEAN_ANALYSIS = COM_OCEAN_ANALYSIS_TMPL | replace_tmpl(current_cycle_dict) %} -rerun: - mkdir: - - "{{ COMOUT_OCEAN_ANALYSIS }}" - copy: - - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ RUN }}.t{{ current_cycle_HH }}z.ocninc.nc", "{{ COMOUT_OCEAN_ANALYSIS }}"] -{% endif %} - -{% if REPLAY_ICS == True %} -{% set COMOUT_OCEAN_ANALYSIS = COM_OCEAN_ANALYSIS_TMPL | replace_tmpl(current_cycle_dict) %} -replay: - mkdir: - - "{{ COMOUT_OCEAN_ANALYSIS }}" - copy: - - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ m_prefix }}.mom6_perturbation.nc", "{{ COMOUT_OCEAN_ANALYSIS }}/mom6_increment.nc"] -{% endif %} - -{% if EXP_WARM_START == True %} -{% set COMOUT_MED_RESTART_PREV = COM_MED_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} -{% if path_exists(ICSDIR ~ "/" ~ COMOUT_MED_RESTART_PREV | relpath(ROTDIR) ~ "/" ~ m_prefix ~ ".ufs.cpld.cpl.r.nc") %} -mediator: - mkdir: - - "{{ COMOUT_MED_RESTART_PREV }}" - copy: - - ["{{ ICSDIR }}/{{ COMOUT_MED_RESTART_PREV | relpath(ROTDIR) }}/{{ m_prefix }}.ufs.cpld.cpl.r.nc", "{{ COMOUT_MED_RESTART_PREV }}"] -{% endif %} # path exists -{% endif %} # warm start true - -{% endif %} # DO_OCN=True - -{% if DO_WAVE %} -{% set COMOUT_WAVE_RESTART_PREV = COM_WAVE_RESTART_TMPL | replace_tmpl(previous_cycle_and_run_dict) %} -wave: - mkdir: - - "{{ COMOUT_WAVE_RESTART_PREV }}" - copy: - - ["{{ ICSDIR }}/{{ COMOUT_WAVE_RESTART_PREV | relpath(ROTDIR) }}/{{ m_prefix }}.restart.{{ waveGRD }}", "{{ COMOUT_WAVE_RESTART_PREV }}"] -{% endif %} diff --git a/parm/stage/wave.yaml.j2 b/parm/stage/wave.yaml.j2 new file mode 100644 index 0000000000..d610430bc7 --- /dev/null +++ b/parm/stage/wave.yaml.j2 @@ -0,0 +1,13 @@ +wave: + mkdir: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_WAVE_RESTART_PREV_MEM = COMOUT_WAVE_RESTART_PREV_MEM_list[imem] %} + - "{{ COMOUT_WAVE_RESTART_PREV_MEM }}" + {% endfor %} # mem loop + copy: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_WAVE_RESTART_PREV_MEM = COMOUT_WAVE_RESTART_PREV_MEM_list[imem] %} + - ["{{ ICSDIR }}/{{ COMOUT_WAVE_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.restart.{{ waveGRD }}", "{{ COMOUT_WAVE_RESTART_PREV_MEM }}"] + {% endfor %} # mem loop diff --git a/scripts/exglobal_stage_ic.py b/scripts/exglobal_stage_ic.py index 4a7b034efa..5efc1bca96 100755 --- a/scripts/exglobal_stage_ic.py +++ b/scripts/exglobal_stage_ic.py @@ -19,9 +19,9 @@ def main(): # Pull out all the configuration keys needed to run stage job keys = ['RUN', 'MODE', 'EXP_WARM_START', 'NMEM_ENS', - 'current_cycle', 'previous_cycle', + 'assim_freq', 'current_cycle', 'previous_cycle', 'ROTDIR', 'ICSDIR', 'STAGE_IC_YAML_TMPL', - 'OCNRES', 'waveGRD', 'ntiles', 'DO_JEDIOCNVAR', + 'OCNRES', 'waveGRD', 'ntiles', 'DOIAU', 'DO_JEDIOCNVAR', 'REPLAY_ICS', 'DO_WAVE', 'DO_OCN', 'DO_ICE', 'DO_NEST'] stage_dict = AttrDict() @@ -33,9 +33,6 @@ def main(): if key.startswith("COM"): stage_dict[key] = stage.task_config[key] - # Configure staging - stage_dict = stage.configure(stage_dict) - # Stage ICs stage.execute_stage(stage_dict) diff --git a/ush/python/pygfs/task/stage_ic.py b/ush/python/pygfs/task/stage_ic.py index 6211da5504..d4d9dc3e19 100644 --- a/ush/python/pygfs/task/stage_ic.py +++ b/ush/python/pygfs/task/stage_ic.py @@ -32,58 +32,6 @@ def __init__(self, config: Dict[str, Any]) -> None: """ super().__init__(config) - @logit(logger) - def configure(self, stage_dict: Dict[str, Any]) -> (Dict[str, Any]): - """Determine stage settings based on configuration and runtime options. - - Parameters - ---------- - stage_dict : Dict[str, Any] - Configuration dictionary - - Returns - ------- - stage_dict : Dict[str, Any] - Configuration dictionary updated - """ - - # Add the os.path.exists function to the dict for yaml parsing - stage_dict['path_exists'] = os.path.exists - - # Determine model start date - current_cycle_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config['assim_freq']}H") / 2) - current_cycle_end = add_to_datetime(self.task_config.current_cycle, to_timedelta(f"{self.task_config['assim_freq']}H") / 2) - - if self.task_config.DOIAU and self.task_config.MODE == "cycled": - model_start_date_current_cycle = current_cycle_begin - else: - if self.task_config.REPLAY_ICS: - model_start_date_current_cycle = current_cycle_end - else: - model_start_date_current_cycle = self.task_config.current_cycle - - stage_dict['model_start_date_current_cycle'] = model_start_date_current_cycle - - # Determine restart RUN - rRUN = self.task_config.RUN - if self.task_config.RUN == "gfs": - rRUN = "gdas" - stage_dict['rRUN'] = rRUN - - # Determine ensemble member settings - if self.task_config.NMEM_ENS > 0 and self.task_config.RUN in ['enkfgdas', 'gefs']: # Ensemble RUN - if self.task_config.RUN == "gefs": - stage_dict['first_mem'] = 0 - stage_dict['last_mem'] = self.task_config.NMEM_ENS - elif self.task_config.RUN == "enkfgdas": - stage_dict['first_mem'] = 1 - stage_dict['last_mem'] = self.task_config.NMEM_ENS - else: # Deterministic RUN - stage_dict['first_mem'] = -1 - stage_dict['last_mem'] = -1 - - return stage_dict - @logit(logger) def execute_stage(self, stage_dict: Dict[str, Any]) -> None: """Perform local staging of initial condition files. @@ -101,10 +49,12 @@ def execute_stage(self, stage_dict: Dict[str, Any]) -> None: if not os.path.isdir(stage_dict.ROTDIR): raise FileNotFoundError(f"FATAL ERROR: The ROTDIR ({stage_dict.ROTDIR}) does not exist!") - # Loop over members - for mem in range(stage_dict.first_mem, stage_dict.last_mem + 1): - stage_dict['mem'] = mem - stage_set = parse_j2yaml(self.task_config.STAGE_IC_YAML_TMPL, stage_dict, allow_missing=False) - # Copy files to ROTDIR - for key in stage_set.keys(): - FileHandler(stage_set[key]).sync() + # Add the os.path.exists function to the dict for yaml parsing + stage_dict['path_exists'] = os.path.exists + + # Parse stage yaml to get list of files to copy + stage_set = parse_j2yaml(self.task_config.STAGE_IC_YAML_TMPL, stage_dict, allow_missing=False) + + # Copy files to ROTDIR + for key in stage_set.keys(): + FileHandler(stage_set[key]).sync() From a53c97e21c463186ca6be44be34c470ff621ded4 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Thu, 15 Aug 2024 18:03:58 +0000 Subject: [PATCH 87/89] Remove enkfgdas from gdas/gfs stage dependency Refs #2475 --- workflow/rocoto/gfs_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 44ab9aea49..3439e537ad 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -883,7 +883,7 @@ def _fcst_cycled(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies) - if self.run in ['gdas', 'enkfgdas']: + if self.run in ['gdas']: dep_dict = {'type': 'task', 'name': f'{self.run}stage_ic'} dependencies.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='or', dep=dependencies) From 13dfad25715cca4bed741ae0dd731187d265baf3 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Mon, 19 Aug 2024 15:25:34 +0000 Subject: [PATCH 88/89] Adjust mem if-block and fix job list for JEDI - Incorporate reviewer feedback and adjust first[last_mem] if-block in master_gfs.yaml.j2 to only check if RUN=enkfgdas when setting ensemble mem values and not check for NMEM_ENS>0 - Update task list when DO_JEDIATMENS=YES to include the stage_ic job in the hybrid_after_eupd_tasks group instead of the hybrid_tasks. Now the enkfgdasstage_ic job shows up in the xml, it was missing before. Refs #2475 --- parm/stage/master_gfs.yaml.j2 | 2 +- workflow/applications/gfs_cycled.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/parm/stage/master_gfs.yaml.j2 b/parm/stage/master_gfs.yaml.j2 index 0f27365ba9..8608d5a40e 100644 --- a/parm/stage/master_gfs.yaml.j2 +++ b/parm/stage/master_gfs.yaml.j2 @@ -54,7 +54,7 @@ # Set first/last mem for loop # --------------------------- -{% if NMEM_ENS > 0 and RUN == "enkfgdas" %} # Ensemble RUN +{% if RUN == "enkfgdas" %} # Ensemble RUN {% set first_mem = 1 %} {% set last_mem = NMEM_ENS %} {% else %} # Deterministic RUN diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index 3ae8f132a6..7aefa8a0f7 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -165,9 +165,9 @@ def get_task_names(self): if self.do_jediatmens: hybrid_tasks += ['atmensanlinit', 'atmensanlletkf', 'atmensanlfv3inc', 'atmensanlfinal', 'echgres'] else: - hybrid_tasks += ['stage_ic', 'eobs', 'eupd', 'echgres'] + hybrid_tasks += ['eobs', 'eupd', 'echgres'] hybrid_tasks += ['ediag'] if self.lobsdiag_forenkf else ['eomg'] - hybrid_after_eupd_tasks += ['ecen', 'esfc', 'efcs', 'epos', 'earc', 'cleanup'] + hybrid_after_eupd_tasks += ['stage_ic', 'ecen', 'esfc', 'efcs', 'epos', 'earc', 'cleanup'] # Collect all "gdas" cycle tasks gdas_tasks = gdas_gfs_common_tasks_before_fcst.copy() From ab35ffce8aa61388f28e8a40f2a460dd24a30502 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Mon, 19 Aug 2024 20:46:23 +0000 Subject: [PATCH 89/89] Update analysis yaml for enkfgdas ratminc.nc - Add ratminc.nc to the list of files in the analysis staging yaml. - Remove "gdas" condition for including analysis yaml. Refs #2475 --- parm/stage/analysis.yaml.j2 | 13 +++++++++++-- parm/stage/master_gfs.yaml.j2 | 2 +- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/parm/stage/analysis.yaml.j2 b/parm/stage/analysis.yaml.j2 index f6beada6b4..e014313b6d 100644 --- a/parm/stage/analysis.yaml.j2 +++ b/parm/stage/analysis.yaml.j2 @@ -1,10 +1,19 @@ +{% if path_exists(ICSDIR ~ "/" ~ COMOUT_ATMOS_ANALYSIS_MEM_list[0] | relpath(ROTDIR)) %} analysis: - {% set COMOUT_ATMOS_ANALYSIS_MEM = COMOUT_ATMOS_ANALYSIS_MEM_list[0] %} mkdir: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_ATMOS_ANALYSIS_MEM = COMOUT_ATMOS_ANALYSIS_MEM_list[imem] %} - "{{ COMOUT_ATMOS_ANALYSIS_MEM }}" + {% endfor %} copy: - {% for ftype in ["abias", "abias_air", "abias_int", "abias_pc", "atminc.nc", "radstat"] %} + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_ATMOS_ANALYSIS_MEM = COMOUT_ATMOS_ANALYSIS_MEM_list[imem] %} + {% for ftype in ["abias", "abias_air", "abias_int", "abias_pc", "atminc.nc", "radstat", "ratminc.nc"] %} {% if path_exists(ICSDIR ~ "/" ~ COMOUT_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) ~ "/" ~ RUN ~ ".t" ~ current_cycle_HH ~ "z." ~ ftype) %} - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ RUN }}.t{{ current_cycle_HH }}z.{{ ftype }}", "{{ COMOUT_ATMOS_ANALYSIS_MEM }}"] {% endif %} {% endfor %} + {% endfor %} # mem loop +{% endif %} diff --git a/parm/stage/master_gfs.yaml.j2 b/parm/stage/master_gfs.yaml.j2 index 8608d5a40e..5204221c9b 100644 --- a/parm/stage/master_gfs.yaml.j2 +++ b/parm/stage/master_gfs.yaml.j2 @@ -127,7 +127,7 @@ # Initial condition to stage - include components based on switches ################################################################### -{% if MODE == "cycled" and RUN == "gdas" %} +{% if MODE == "cycled" %} {% filter indent(width=4) %} {% include "analysis.yaml.j2" %} {% endfilter %}