Skip to content

Commit

Permalink
Issue 694: Upgrade/refactoring for U and V write-out sub for FV3REG G…
Browse files Browse the repository at this point in the history 
…SI failure … (#698)

**DUE DATE for merger of this PR into `develop` is 3/27/2024 (six weeks
after PR creation).**

Resolves #693 (Thanks to @edwardhartnett 's suggestions)
Resolves # 694 ( this PR is not able to provide a stable solution, more
details would be given on the issue page)
Resolves # 697: With larger requested memory for each mpi task, it still
showed, for some time, the differences in the analysis files between
loproc vs hiproc for the control runs on hercules. whether integrating
this with the refactored IO part would provide a stable solution remains
to be seen.

This PR resolved the newly emerged issue with IO of netcdf files in the
continuous storage, with upgraded FV3REG IO for the cold start options.
(Co author Ming Hu @hu5970 )
This PR is being worked on in collaboration with Pete Johnson through
RDHPCS help desk, @RussTreadon-NOAA @DavidHuber-NOAA and thanks to help
from @ed Raghue Reddy through RDHPCS help desk.

---------

Co-authored-by: Ting Lei <tlei@hercules-login-4.hpc.msstate.edu>
Co-authored-by: Ting.Lei-NOAA <ting.lei@dlogin08.dogwood.wcoss2.ncep.noaa.gov>
  • Loading branch information
@TingLei-daprediction
3 people authored Mar 20, 2024
1 parent 4e8107c commit 2167bc9
Show file tree
Hide file tree
Showing 7 changed files with 365 additions and 240 deletions.
2 changes: 2 additions & 0 deletions regression/regression_driver.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,12 @@ for jn in `seq ${RSTART} ${REND}`; do
export scripts=${scripts_updat:-$scripts}
export fixgsi=${fixgsi_updat:-$fixgsi}
export modulefiles=${modulefiles_updat:-$modulefiles}
export ush=${ush_update:-$ush}
else
export scripts=${scripts_contrl:-$scripts}
export fixgsi=${fixgsi_contrl:-$fixgsi}
export modulefiles=${modulefiles_contrl:-$modulefiles}
export ush=${ush_cntrl:-$ush}
fi
rm -f ${job[$jn]}.out

Expand Down
48 changes: 24 additions & 24 deletions regression/regression_param.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,17 +87,17 @@ case $regtest in
rrfs_3denvar_glbens)

if [[ "$machine" = "Hera" ]]; then
topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/1"
topts[1]="0:15:00" ; popts[1]="5/4/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="10/4/" ; ropts[2]="/1"
elif [[ "$machine" = "Orion" ]]; then
topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/2"
topts[1]="0:15:00" ; popts[1]="5/4/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="10/4/" ; ropts[2]="/2"
elif [[ "$machine" = "Hercules" ]]; then
topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/2"
topts[1]="0:15:00" ; popts[1]="5/4/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="10/4/" ; ropts[2]="/2"
elif [[ "$machine" = "Jet" ]]; then
topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/1"
topts[1]="0:15:00" ; popts[1]="5/4/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="10/4/" ; ropts[2]="/1"
elif [[ "$machine" = "Gaea" ]]; then
topts[1]="0:15:00" ; popts[1]="18/1/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="18/2/" ; ropts[2]="/1"
Expand All @@ -117,17 +117,17 @@ case $regtest in
hafs_3denvar_hybens)

if [[ "$machine" = "Hera" ]]; then
topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/1"
topts[1]="0:15:00" ; popts[1]="5/4/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="10/4/" ; ropts[2]="/1"
elif [[ "$machine" = "Orion" ]]; then
topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/2"
topts[1]="0:15:00" ; popts[1]="5/4/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="10/4/" ; ropts[2]="/2"
elif [[ "$machine" = "Hercules" ]]; then
topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/2"
topts[2]="0:15:00" ; popts[2]="5/8/" ; ropts[2]="/2"
elif [[ "$machine" = "Jet" ]]; then
topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/1"
topts[1]="0:15:00" ; popts[1]="5/4/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="10/4/" ; ropts[2]="/1"
elif [[ "$machine" = "Gaea" ]]; then
topts[1]="0:15:00" ; popts[1]="18/1/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="18/2/" ; ropts[2]="/1"
Expand All @@ -146,17 +146,17 @@ case $regtest in

hafs_4denvar_glbens)
if [[ "$machine" = "Hera" ]]; then
topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/1"
topts[1]="0:15:00" ; popts[1]="5/4/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="10/4/" ; ropts[2]="/1"
elif [[ "$machine" = "Orion" ]]; then
topts[1]="0:20:00" ; popts[1]="20/1/" ; ropts[1]="/1"
topts[1]="0:20:00" ; popts[1]="5/4/" ; ropts[1]="/1"
topts[2]="0:20:00" ; popts[2]="20/2/" ; ropts[2]="/1"
elif [[ "$machine" = "Hercules" ]]; then
topts[1]="0:20:00" ; popts[1]="20/1/" ; ropts[1]="/1"
topts[2]="0:20:00" ; popts[2]="20/2/" ; ropts[2]="/1"
topts[1]="0:20:00" ; popts[1]="5/4/" ; ropts[1]="/1"
topts[2]="0:20:00" ; popts[2]="10/4/" ; ropts[2]="/1"
elif [[ "$machine" = "Jet" ]]; then
topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/1"
topts[1]="0:15:00" ; popts[1]="5/4/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="10/4/" ; ropts[2]="/1"
elif [[ "$machine" = "Gaea" ]]; then
topts[1]="0:15:00" ; popts[1]="18/1/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="18/2/" ; ropts[2]="/1"
Expand Down Expand Up @@ -300,10 +300,10 @@ if [[ "$machine" = "Hera" ]]; then
export APRUN="srun"
elif [[ "$machine" = "Orion" ]]; then
export OMP_STACKSIZE=2048M
export APRUN="srun -n \$ntasks --cpus-per-task=\$threads"
export APRUN="srun -n \$ntasks --mem=0 --cpus-per-task=\$threads"
elif [[ "$machine" = "Hercules" ]]; then
export OMP_STACKSIZE=2048M
export APRUN="srun -n \$ntasks --cpus-per-task=\$threads"
export APRUN="srun -n \$ntasks --mem=0 --cpus-per-task=\$threads"
elif [[ "$machine" = "Jet" ]]; then
export OMP_STACKSIZE=1024M
export MPI_BUFS_PER_PROC=256
Expand Down
Loading

0 comments on commit 2167bc9

Please sign in to comment.