From 1137531dfc31030f73ef28af8080089c68cfc7e8 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Wed, 26 Jul 2023 19:27:35 +0000 Subject: [PATCH 1/3] Small changes to get cycling going at C384 --- parm/config/gfs/config.resources | 19 ++++++++++--------- parm/config/gfs/config.ufs | 2 +- ush/python/pygfs/task/aero_analysis.py | 8 ++++---- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index bc9edfcb7e..98b3c6b6a8 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -250,12 +250,12 @@ elif [[ "${step}" = "aeroanlinit" ]]; then # below lines are for creating JEDI YAML case ${CASE} in C768) - layout_x=6 - layout_y=6 + layout_x=8 + layout_y=8 ;; C384) - layout_x=5 - layout_y=5 + layout_x=8 + layout_y=8 ;; C192 | C96 | C48) layout_x=8 @@ -280,12 +280,12 @@ elif [[ "${step}" = "aeroanlrun" ]]; then case ${CASE} in C768) - layout_x=6 - layout_y=6 + layout_x=8 + layout_y=8 ;; C384) - layout_x=5 - layout_y=5 + layout_x=8 + layout_y=8 ;; C192 | C96 | C48) layout_x=8 @@ -470,6 +470,7 @@ elif [[ ${step} = "analcalc" ]]; then npe_node_analcalc=$(echo "${npe_node_max} / ${nth_analcalc}" | bc) export npe_node_analcalc export is_exclusive=True + export memory_analcalc="48GB" elif [[ ${step} = "analdiag" ]]; then @@ -885,7 +886,7 @@ elif [[ ${step} = "epos" ]]; then export npe_epos=80 export nth_epos=4 if [[ "${machine}" == "HERA" ]]; then - export nth_epos=6 + export nth_epos=1 fi npe_node_epos=$(echo "${npe_node_max} / ${nth_epos}" | bc) export npe_node_epos diff --git a/parm/config/gfs/config.ufs b/parm/config/gfs/config.ufs index bfbb0a12cb..15b5046d6c 100644 --- a/parm/config/gfs/config.ufs +++ b/parm/config/gfs/config.ufs @@ -151,7 +151,7 @@ case "${fv3_res}" in export layout_y=8 export layout_x_gfs=8 export layout_y_gfs=8 - export nthreads_fv3=1 + export nthreads_fv3=2 export nthreads_fv3_gfs=2 export cdmbgwd="1.1,0.72,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export WRITE_GROUP=2 diff --git a/ush/python/pygfs/task/aero_analysis.py b/ush/python/pygfs/task/aero_analysis.py index 2955baa069..c1afef98f6 100644 --- a/ush/python/pygfs/task/aero_analysis.py +++ b/ush/python/pygfs/task/aero_analysis.py @@ -30,7 +30,7 @@ def __init__(self, config): super().__init__(config) _res = int(self.config['CASE'][1:]) - _res_enkf = int(self.config['CASE_ENS'][1:]) + _res_anl = int(self.config['CASE_ANL'][1:]) _window_begin = add_to_datetime(self.runtime_config.current_cycle, -to_timedelta(f"{self.config['assim_freq']}H") / 2) _fv3jedi_yaml = os.path.join(self.runtime_config.DATA, f"{self.runtime_config.CDUMP}.t{self.runtime_config['cyc']:02d}z.aerovar.yaml") @@ -41,8 +41,8 @@ def __init__(self, config): 'npy_ges': _res + 1, 'npz_ges': self.config.LEVS - 1, 'npz': self.config.LEVS - 1, - 'npx_anl': _res_enkf + 1, - 'npy_anl': _res_enkf + 1, + 'npx_anl': _res_anl + 1, + 'npy_anl': _res_anl + 1, 'npz_anl': self.config['LEVS'] - 1, 'AERO_WINDOW_BEGIN': _window_begin, 'AERO_WINDOW_LENGTH': f"PT{self.config['assim_freq']}H", @@ -284,7 +284,7 @@ def get_berror_dict(self, config: Dict[str, Any]) -> Dict[str, List[str]]: berror_list.append([ os.path.join(b_dir, coupler), os.path.join(config.DATA, 'berror', coupler) ]) - template = '{b_datestr}.{ftype}.fv_tracer.res.tile{{tilenum}}.nc' + template = f'{b_datestr}.{ftype}.fv_tracer.res.tile{{tilenum}}.nc' for itile in range(1, config.ntiles + 1): tracer = template.format(tilenum=itile) berror_list.append([ From 483a112d0f3ba66bfbfd4c5d6b46c95ed9001627 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Wed, 26 Jul 2023 20:22:45 +0000 Subject: [PATCH 2/3] force 1 thread for epos for all platforms --- parm/config/gfs/config.resources | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 98b3c6b6a8..36d4d3f144 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -884,10 +884,7 @@ elif [[ ${step} = "epos" ]]; then export wtime_epos="00:15:00" export npe_epos=80 - export nth_epos=4 - if [[ "${machine}" == "HERA" ]]; then - export nth_epos=1 - fi + export nth_epos=1 npe_node_epos=$(echo "${npe_node_max} / ${nth_epos}" | bc) export npe_node_epos export is_exclusive=True From b926e7a9c3ac73da15b1d917212fef36c11e3cd6 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Thu, 27 Jul 2023 12:43:09 +0000 Subject: [PATCH 3/3] make walltime for esfc 8 mins --- parm/config/gfs/config.resources | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 36d4d3f144..6844802a0b 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -870,7 +870,7 @@ elif [[ ${step} = "ecen" ]]; then elif [[ ${step} = "esfc" ]]; then - export wtime_esfc="00:06:00" + export wtime_esfc="00:08:00" export npe_esfc=80 export nth_esfc=1 npe_node_esfc=$(echo "${npe_node_max} / ${nth_esfc}" | bc)