From 12d4689c0b96d178a26fb5a3ee2667395912e600 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Fri, 30 Sep 2022 12:58:56 +0000 Subject: [PATCH 01/42] Add WCOSS2 BASE_CPLIC to config.coupled_ic - WCOSS2 BASE_CPLIC is /lfs/h2/emc/global/noscrub/emc.global/IC/COUPLED Refs #419 --- parm/config/config.coupled_ic | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/parm/config/config.coupled_ic b/parm/config/config.coupled_ic index 0df82591d9..ab72def07a 100755 --- a/parm/config/config.coupled_ic +++ b/parm/config/config.coupled_ic @@ -7,7 +7,9 @@ echo "BEGIN: config.coupled_ic" # Get task specific resources source $EXPDIR/config.resources coupled_ic -if [[ "$machine" == "HERA" ]]; then +if [[ "$machine" == "WCOSS2" ]]; then + export BASE_CPLIC="/lfs/h2/emc/global/noscrub/emc.global/IC/COUPLED" +elif [[ "$machine" == "HERA" ]]; then export BASE_CPLIC="/scratch1/NCEPDEV/climate/role.ufscpara/IC" elif [[ "$machine" == "ORION" ]]; then export BASE_CPLIC="/work/noaa/global/wkolczyn/noscrub/global-workflow/IC" From f79dbdbeb54c4509129efaab6e31ffa5850b052d Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Fri, 30 Sep 2022 13:00:02 +0000 Subject: [PATCH 02/42] Remove errant ) from WCOSS2 hosts file for COMINsyn Refs #419 --- workflow/hosts/wcoss2.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/hosts/wcoss2.yaml b/workflow/hosts/wcoss2.yaml index e715ed3e9b..3a1bd623d7 100644 --- a/workflow/hosts/wcoss2.yaml +++ b/workflow/hosts/wcoss2.yaml @@ -2,7 +2,7 @@ base_git: '/lfs/h2/emc/global/save/emc.global/git' dmpdir: '/lfs/h2/emc/global/noscrub/emc.global/dump' packageroot: '${PACKAGEROOT:-"/lfs/h1/ops/prod/packages"}' comroot: '${COMROOT:-"/lfs/h1/ops/prod/com"}' -cominsyn: '${COMROOT}/gfs/${gfs_ver:-"v16.2"})/syndat' +cominsyn: '${COMROOT}/gfs/${gfs_ver:-"v16.2"}/syndat' homedir: '/lfs/h2/emc/global/noscrub/$USER' stmp: '/lfs/h2/emc/stmp/$USER' ptmp: '/lfs/h2/emc/ptmp/$USER' From 686272cf3928eca0a297c12f38c3767cadfcbb77 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Fri, 30 Sep 2022 13:14:13 +0000 Subject: [PATCH 03/42] Updates to config.fv3 from WCOSS2 testing - Add npe_node_max=128 for WCOSS2 in machine if-block. - Change WRTTASK_PER_GROUP and WRTTASK_PER_GROUP_GFS to be 64 for all supported resolutions (good value for WCOSS2) but then add check for whether WRTTTASK_PER_GROUP* variables are greater than npe_node_max value (for R&DS) and if so, set WRTTASK_PER_GROUP* values to equal the npe_node_max value (prior setting). - Change C384 DELTIM to 200 (current GFSv16 ops value). Refs #419 --- parm/config/config.fv3 | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/parm/config/config.fv3 b/parm/config/config.fv3 index c8c959362e..9284fe3faa 100755 --- a/parm/config/config.fv3 +++ b/parm/config/config.fv3 @@ -20,7 +20,9 @@ case_in=$1 echo "BEGIN: config.fv3" -if [[ "$machine" = "JET" ]]; then +if [[ "$machine" = "WCOSS2" ]]; then + export npe_node_max=128 +elif [[ "$machine" = "JET" ]]; then if [[ "$PARTITION_BATCH" = "xjet" ]]; then export npe_node_max=24 elif [[ "$PARTITION_BATCH" = "vjet" || "$PARTITION_BATCH" = "sjet" ]]; then @@ -46,9 +48,11 @@ case $case_in in export nth_fv3_gfs=1 export cdmbgwd="0.071,2.1,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export WRITE_GROUP=1 - export WRTTASK_PER_GROUP=$npe_node_max + export WRTTASK_PER_GROUP=64 + if [[ "$WRTTASK_PER_GROUP" -gt "$npe_node_max" ]]; then export WRTTASK_PER_GROUP=$npe_node_max ; fi export WRITE_GROUP_GFS=1 - export WRTTASK_PER_GROUP_GFS=$npe_node_max + export WRTTASK_PER_GROUP_GFS=64 + if [[ "$WRTTASK_PER_GROUP_GFS" -gt "$npe_node_max" ]]; then export WRTTASK_PER_GROUP_GFS=$npe_node_max ; fi export WRTIOBUF="1M" ;; "C96") @@ -61,9 +65,11 @@ case $case_in in export nth_fv3_gfs=1 export cdmbgwd="0.14,1.8,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export WRITE_GROUP=1 - export WRTTASK_PER_GROUP=$npe_node_max + export WRTTASK_PER_GROUP=64 + if [[ "$WRTTASK_PER_GROUP" -gt "$npe_node_max" ]]; then export WRTTASK_PER_GROUP=$npe_node_max ; fi export WRITE_GROUP_GFS=1 - export WRTTASK_PER_GROUP_GFS=$npe_node_max + export WRTTASK_PER_GROUP_GFS=64 + if [[ "$WRTTASK_PER_GROUP_GFS" -gt "$npe_node_max" ]]; then export WRTTASK_PER_GROUP_GFS=$npe_node_max ; fi export WRTIOBUF="4M" export n_split=6 ;; @@ -77,13 +83,15 @@ case $case_in in export nth_fv3_gfs=2 export cdmbgwd="0.23,1.5,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export WRITE_GROUP=1 - export WRTTASK_PER_GROUP=$npe_node_max + export WRTTASK_PER_GROUP=64 + if [[ "$WRTTASK_PER_GROUP" -gt "$npe_node_max" ]]; then export WRTTASK_PER_GROUP=$npe_node_max ; fi export WRITE_GROUP_GFS=2 - export WRTTASK_PER_GROUP_GFS=$npe_node_max + export WRTTASK_PER_GROUP_GFS=64 + if [[ "$WRTTASK_PER_GROUP_GFS" -gt "$npe_node_max" ]]; then export WRTTASK_PER_GROUP_GFS=$npe_node_max ; fi export WRTIOBUF="8M" ;; "C384") - export DELTIM=${DELTIM:-300} + export DELTIM=${DELTIM:-200} export layout_x=6 export layout_y=8 export layout_x_gfs=${layout_x_gfs:-8} @@ -92,9 +100,11 @@ case $case_in in export nth_fv3_gfs=${nth_fv3_gfs:-2} export cdmbgwd="1.1,0.72,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export WRITE_GROUP=1 - export WRTTASK_PER_GROUP=$npe_node_max + export WRTTASK_PER_GROUP=64 + if [[ "$WRTTASK_PER_GROUP" -gt "$npe_node_max" ]]; then export WRTTASK_PER_GROUP=$npe_node_max ; fi export WRITE_GROUP_GFS=${WRITE_GROUP_GFS:-2} - export WRTTASK_PER_GROUP_GFS=${WRTTASK_PER_GROUP_GFS:-$npe_node_max} + export WRTTASK_PER_GROUP_GFS=${WRTTASK_PER_GROUP_GFS:-64} + if [[ "$WRTTASK_PER_GROUP_GFS" -gt "$npe_node_max" ]]; then export WRTTASK_PER_GROUP_GFS=$npe_node_max ; fi export WRTIOBUF=${WRTIOBUF:-"16M"} ;; "C768") @@ -107,9 +117,11 @@ case $case_in in export nth_fv3_gfs=4 export cdmbgwd="4.0,0.15,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export WRITE_GROUP=2 - export WRTTASK_PER_GROUP=$(echo "2*$npe_node_max" |bc) + export WRTTASK_PER_GROUP=64 + if [[ "$WRTTASK_PER_GROUP" -gt "$npe_node_max" ]]; then export WRTTASK_PER_GROUP=$npe_node_max ; fi export WRITE_GROUP_GFS=4 - export WRTTASK_PER_GROUP_GFS=$(echo "2*$npe_node_max" |bc) + export WRTTASK_PER_GROUP_GFS=64 + if [[ "$WRTTASK_PER_GROUP_GFS" -gt "$npe_node_max" ]]; then export WRTTASK_PER_GROUP_GFS=$npe_node_max ; fi export WRTIOBUF="32M" ;; "C1152") From c41265b8b730c4c81fa4c75d9512b910f8048685 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Fri, 30 Sep 2022 13:24:59 +0000 Subject: [PATCH 04/42] Adjust how S2SW resource defaults are used - Rename the resource variables in config.defaults.s2sw to include "_s2sw" in the names. - Update the C384 block of config.fv3 to use the "_s2sw" variables for the relevant resource variable values (when set). - When not running coupled S2Sw the "_s2sw" variables will not be set and the C384 block of config.fv3 will not use them but instead use the appropriate values for non-coupled. - Resolves issue where S2Sw resource values were being forced onto the C384 atmos-only enkf forecast jobs. Refs #419 --- parm/config/config.defaults.s2sw | 18 +++++++++--------- parm/config/config.fv3 | 14 +++++++------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/parm/config/config.defaults.s2sw b/parm/config/config.defaults.s2sw index 5032a998ad..51693ab3bd 100644 --- a/parm/config/config.defaults.s2sw +++ b/parm/config/config.defaults.s2sw @@ -16,16 +16,16 @@ min_seaice="1.0e-6" use_cice_alb=".true." # config.fv3 -DELTIM=300 -layout_x_gfs=8 -layout_y_gfs=8 -WRITE_GROUP_GFS=1 -WRTTASK_PER_GROUP_GFS=24 +DELTIM_s2sw=300 +layout_x_gfs_s2sw=8 +layout_y_gfs_s2sw=8 +WRITE_GROUP_GFS_s2sw=1 +WRTTASK_PER_GROUP_GFS_s2sw=24 #The settings below will result in S2SWA running 35 days under 8 hours wallclock on hera -#layout_x_gfs=24 -#layout_y_gfs=16 -#WRTTASK_PER_GROUP_GFS=86 -WRTIOBUF="32M" +#layout_x_gfs_s2sw=24 +#layout_y_gfs_s2sw=16 +#WRTTASK_PER_GROUP_GFS_s2sw=86 +WRTIOBUF_s2sw="32M" MEDPETS=300 # config.wave diff --git a/parm/config/config.fv3 b/parm/config/config.fv3 index 9284fe3faa..8537933e81 100755 --- a/parm/config/config.fv3 +++ b/parm/config/config.fv3 @@ -91,21 +91,21 @@ case $case_in in export WRTIOBUF="8M" ;; "C384") - export DELTIM=${DELTIM:-200} + export DELTIM=${DELTIM_s2sw:-200} export layout_x=6 export layout_y=8 - export layout_x_gfs=${layout_x_gfs:-8} - export layout_y_gfs=${layout_y_gfs:-12} + export layout_x_gfs=${layout_x_gfs_s2sw:-8} + export layout_y_gfs=${layout_y_gfs_s2sw:-12} export nth_fv3=2 - export nth_fv3_gfs=${nth_fv3_gfs:-2} + export nth_fv3_gfs=${nth_fv3_gfs_s2sw:-2} export cdmbgwd="1.1,0.72,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export WRITE_GROUP=1 export WRTTASK_PER_GROUP=64 if [[ "$WRTTASK_PER_GROUP" -gt "$npe_node_max" ]]; then export WRTTASK_PER_GROUP=$npe_node_max ; fi - export WRITE_GROUP_GFS=${WRITE_GROUP_GFS:-2} - export WRTTASK_PER_GROUP_GFS=${WRTTASK_PER_GROUP_GFS:-64} + export WRITE_GROUP_GFS=${WRITE_GROUP_GFS_s2sw:-2} + export WRTTASK_PER_GROUP_GFS=${WRTTASK_PER_GROUP_GFS_s2sw:-64} if [[ "$WRTTASK_PER_GROUP_GFS" -gt "$npe_node_max" ]]; then export WRTTASK_PER_GROUP_GFS=$npe_node_max ; fi - export WRTIOBUF=${WRTIOBUF:-"16M"} + export WRTIOBUF=${WRTIOBUF_s2sw:-"16M"} ;; "C768") export DELTIM=150 From 63d144b30ad43d369cc8341ddb46ebf31ba185e6 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Mon, 3 Oct 2022 19:52:23 +0000 Subject: [PATCH 05/42] Add "is_exclusive" setting to config.resources - When job should run exclusively, set is_exclusive=True. - Replace native_*="--exclusive" with is_exclusive=True. Refs #419 --- parm/config/config.resources | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/parm/config/config.resources b/parm/config/config.resources index b723b57916..932b31bfb1 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -46,7 +46,11 @@ if [ $step = "prep" -o $step = "prepbufr" ]; then eval "export npe_$step=4" eval "export npe_node_$step=2" eval "export nth_$step=1" - eval "export memory_$step=40G" + if [[ "$machine" = "WCOSS2" ]]; then + export is_exclusive=True + else + eval "export memory_$step=40G" + fi elif [ $step = "aerosol_init" ]; then export wtime_aerosol_init="00:05:00" @@ -87,6 +91,7 @@ elif [ $step = "wavepostbndpnt" ]; then export nth_wavepostbndpnt=1 export npe_node_wavepostbndpnt=$(echo "$npe_node_max / $nth_wavepostbndpnt" | bc) export NTASKS=${npe_wavepostbndpnt} + export is_exclusive=True elif [ $step = "wavepostbndpntbll" ]; then @@ -95,6 +100,7 @@ elif [ $step = "wavepostbndpntbll" ]; then export nth_wavepostbndpntbll=1 export npe_node_wavepostbndpntbll=$(echo "$npe_node_max / $nth_wavepostbndpntbll" | bc) export NTASKS=${npe_wavepostbndpntbll} + export is_exclusive=True elif [ $step = "wavepostpnt" ]; then @@ -103,6 +109,7 @@ elif [ $step = "wavepostpnt" ]; then export nth_wavepostpnt=1 export npe_node_wavepostpnt=$(echo "$npe_node_max / $nth_wavepostpnt" | bc) export NTASKS=${npe_wavepostpnt} + export is_exclusive=True elif [ $step = "wavegempak" ]; then @@ -119,6 +126,7 @@ elif [ $step = "waveawipsbulls" ]; then export nth_waveawipsbulls=1 export npe_node_waveawipsbulls=$(echo "$npe_node_max / $nth_waveawipsbulls" | bc) export NTASKS=${npe_waveawipsbulls} + export is_exclusive=True elif [ $step = "waveawipsgridded" ]; then @@ -147,8 +155,8 @@ elif [ $step = "atmanalrun" ]; then export npe_atmanalrun_gfs=$(echo "$layout_x * $layout_y * 6" | bc) export nth_atmanalrun=1 export nth_atmanalrun_gfs=$nth_atmanalrun - export native_atmanalrun="--exclusive" export npe_node_atmanalrun=$(echo "$npe_node_max / $nth_atmanalrun" | bc) + export is_exclusive=True elif [ $step = "atmanalpost" ]; then @@ -156,6 +164,7 @@ elif [ $step = "atmanalpost" ]; then export npe_atmanalpost=$npe_node_max export nth_atmanalpost=1 export npe_node_atmanalpost=$(echo "$npe_node_max / $nth_atmanalpost" | bc) + export is_exclusive=True elif [ $step = "aeroanlinit" ]; then @@ -187,8 +196,8 @@ elif [ $step = "aeroanlrun" ]; then export npe_aeroanlrun_gfs=$(echo "$layout_x * $layout_y * 6" | bc) export nth_aeroanlrun=1 export nth_aeroanlrun_gfs=1 - export native_aeroanlrun="--exclusive" export npe_node_aeroanlrun=$(echo "$npe_node_max / $nth_aeroanlrun" | bc) + export is_exclusive=True elif [ $step = "aeroanlfinal" ]; then @@ -215,6 +224,7 @@ elif [ $step = "anal" ]; then export nth_anal_gfs=$nth_anal export npe_node_anal=$(echo "$npe_node_max / $nth_anal" | bc) export nth_cycle=$nth_anal + export is_exclusive=True elif [ $step = "analcalc" ]; then @@ -225,6 +235,7 @@ elif [ $step = "analcalc" ]; then export nth_echgres=4 export nth_echgres_gfs=12 export npe_node_analcalc=$npe_node_max + export is_exclusive=True elif [ $step = "analdiag" ]; then @@ -239,6 +250,7 @@ elif [ $step = "sfcanl" ]; then export npe_sfcanl=6 export nth_sfcanl=1 export npe_node_sfcanl=$(echo "$npe_node_max / $nth_sfcanl" | bc) + export is_exclusive=True elif [ $step = "gldas" ]; then @@ -249,6 +261,7 @@ elif [ $step = "gldas" ]; then export npe_gaussian=96 export nth_gaussian=1 export npe_node_gaussian=24 + export is_exclusive=True elif [ $step = "fcst" ]; then @@ -261,6 +274,8 @@ elif [ $step = "fcst" ]; then export wtime_fcst_gfs="03:00:00" fi + export is_exclusive=True + # During workflow creation, we need resources for all CDUMPs and CDUMP is undefined CDUMP_LIST=${CDUMP:-"gdas gfs"} for CDUMP in $CDUMP_LIST; do @@ -360,6 +375,7 @@ elif [ $step = "post" ]; then export nth_post=1 export npe_node_post=12 export npe_node_dwn=$npe_node_max + export is_exclusive=True elif [ $step = "wafs" ]; then @@ -415,6 +431,7 @@ elif [ $step = "vrfy" ]; then if [[ "$machine" == "HERA" ]]; then export memory_vrfy="16384M" fi + export is_exclusive=True elif [ $step = "metp" ]; then @@ -425,6 +442,7 @@ elif [ $step = "metp" ]; then export wtime_metp_gfs="06:00:00" export npe_metp_gfs=4 export npe_node_metp_gfs=4 + export is_exclusive=True elif [ $step = "echgres" ]; then @@ -446,12 +464,14 @@ elif [ $step = "init_chem" ]; then export wtime_init_chem="00:30:00" export npe_init_chem=1 export npe_node_init_chem=1 + export is_exclusive=True elif [ $step = "mom6ic" ]; then export wtime_mom6ic="00:30:00" export npe_mom6ic=24 export npe_node_mom6ic=24 + export is_exclusive=True elif [ $step = "arch" -o $step = "earc" -o $step = "getic" ]; then @@ -467,6 +487,7 @@ elif [ $step = "coupled_ic" ]; then export npe_coupled_ic=1 export npe_node_coupled_ic=1 export nth_coupled_ic=1 + export is_exclusive=True elif [ $step = "atmensanalprep" ]; then @@ -474,6 +495,7 @@ elif [ $step = "atmensanalprep" ]; then export npe_atmensanalprep=1 export nth_atmensanalprep=1 export npe_node_atmensanalprep=$(echo "$npe_node_max / $nth_atmensanalprep" | bc) + export is_exclusive=True elif [ $step = "atmensanalrun" ]; then @@ -486,8 +508,8 @@ elif [ $step = "atmensanalrun" ]; then export npe_atmensanalrun_gfs=$(echo "$layout_x * $layout_y * 6" | bc) export nth_atmensanalrun=1 export nth_atmensanalrun_gfs=$nth_atmensanalrun - export native_atmensanalrun="--exclusive" export npe_node_atmensanalrun=$(echo "$npe_node_max / $nth_atmensanalrun" | bc) + export is_exclusive=True elif [ $step = "atmensanalpost" ]; then @@ -495,6 +517,7 @@ elif [ $step = "atmensanalpost" ]; then export npe_atmensanalpost=$npe_node_max export nth_atmensanalpost=1 export npe_node_atmensanalpost=$(echo "$npe_node_max / $nth_atmensanalpost" | bc) + export is_exclusive=True elif [ $step = "eobs" -o $step = "eomg" ]; then @@ -514,6 +537,7 @@ elif [ $step = "eobs" -o $step = "eomg" ]; then export nth_eomg=$nth_eobs export npe_node_eobs=$(echo "$npe_node_max / $nth_eobs" | bc) export npe_node_eomg=$npe_node_eobs + export is_exclusive=True elif [ $step = "ediag" ]; then @@ -548,6 +572,7 @@ elif [ $step = "eupd" ]; then fi fi export npe_node_eupd=$(echo "$npe_node_max / $nth_eupd" | bc) + export is_exclusive=True elif [ $step = "ecen" ]; then @@ -557,6 +582,7 @@ elif [ $step = "ecen" ]; then if [ $CASE = "C384" -o $CASE = "C192" -o $CASE = "C96" -o $CASE = "C48" ]; then export nth_ecen=2; fi export npe_node_ecen=$(echo "$npe_node_max / $nth_ecen" | bc) export nth_cycle=$nth_ecen + export is_exclusive=True elif [ $step = "esfc" ]; then @@ -583,6 +609,7 @@ elif [ $step = "epos" ]; then export npe_epos=80 export nth_epos=6 export npe_node_epos=$(echo "$npe_node_max / $nth_epos" | bc) + export is_exclusive=True elif [ $step = "postsnd" ]; then @@ -597,6 +624,7 @@ elif [ $step = "postsnd" ]; then export npe_node_postsnd=4 fi if [[ "$machine" = "HERA" ]]; then export npe_node_postsnd=2; fi + export is_exclusive=True elif [ $step = "awips" ]; then From e1e5e3372032b4fd825b1fba78b311286510fc0b Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Mon, 3 Oct 2022 19:55:16 +0000 Subject: [PATCH 06/42] Add WCOSS2 to machine npe_node_max check in config.resources - Set npe_node_max=128 when on WCOSS2. Refs #419 --- parm/config/config.resources | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/parm/config/config.resources b/parm/config/config.resources index 932b31bfb1..c1e7dd5597 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -27,7 +27,9 @@ step=$1 echo "BEGIN: config.resources" -if [[ "$machine" = "JET" ]]; then +if [[ "$machine" = "WCOSS2" ]]; then + export npe_node_max=128 +elif [[ "$machine" = "JET" ]]; then if [[ "$PARTITION_BATCH" = "xjet" ]]; then export npe_node_max=24 elif [[ "$PARTITION_BATCH" = "vjet" || "$PARTITION_BATCH" = "sjet" ]]; then From 6db1a8176f7e9ee3a52aa034c0bb9eb83bceb7a5 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Mon, 3 Oct 2022 20:04:13 +0000 Subject: [PATCH 07/42] Update wave job resources in config.resources - Pull in wave job resources from GFSv16 port. - Adjust walltimes, tasks, add memory, and add "_gfs" versions of resource variables as needed. Refs #419 --- parm/config/config.resources | 55 +++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 22 deletions(-) diff --git a/parm/config/config.resources b/parm/config/config.resources index c1e7dd5597..97f47496c0 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -67,64 +67,74 @@ elif [ $step = "waveinit" ]; then export wtime_waveinit="00:10:00" export npe_waveinit=12 export nth_waveinit=1 - export npe_node_waveinit=$(echo "$npe_node_max / $nth_waveinit" | bc) + export npe_node_waveinit=$npe_waveinit export NTASKS=${npe_waveinit} + export memory_waveinit="2GB" elif [ $step = "waveprep" ]; then - export wtime_waveprep="00:30:00" - export npe_waveprep=65 + export wtime_waveprep="00:10:00" + export npe_waveprep=5 + export npe_waveprep_gfs=65 export nth_waveprep=1 - export npe_node_waveprep=$(echo "$npe_node_max / $nth_waveprep" | bc) + export npe_node_waveprep=$npe_waveprep + export npe_node_waveprep_gfs=$npe_waveprep_gfs export NTASKS=${npe_waveprep} + export NTASKS_gfs=$npe_waveprep_gfs + export memory_waveprep="100GB" + export memory_waveprep_gfs="220GB" elif [ $step = "wavepostsbs" ]; then - export wtime_wavepostsbs="06:00:00" - export npe_wavepostsbs=10 + export wtime_wavepostsbs="00:20:00" + export wtime_wavepostsbs_gfs="03:00:00" + export npe_wavepostsbs=8 export nth_wavepostsbs=1 - export npe_node_wavepostsbs=$(echo "$npe_node_max / $nth_wavepostsbs" | bc) + export npe_node_wavepostsbs=$npe_wavepostsbs export NTASKS=${npe_wavepostsbs} + export memory_wavepostsbs="10GB" + export memory_wavepostsbs_gfs="40GB" elif [ $step = "wavepostbndpnt" ]; then - export wtime_wavepostbndpnt="02:00:00" - export npe_wavepostbndpnt=280 + export wtime_wavepostbndpnt="01:00:00" + export npe_wavepostbndpnt=240 export nth_wavepostbndpnt=1 - export npe_node_wavepostbndpnt=$(echo "$npe_node_max / $nth_wavepostbndpnt" | bc) + export npe_node_wavepostbndpnt=80 export NTASKS=${npe_wavepostbndpnt} export is_exclusive=True elif [ $step = "wavepostbndpntbll" ]; then export wtime_wavepostbndpntbll="01:00:00" - export npe_wavepostbndpntbll=280 + export npe_wavepostbndpntbll=448 export nth_wavepostbndpntbll=1 - export npe_node_wavepostbndpntbll=$(echo "$npe_node_max / $nth_wavepostbndpntbll" | bc) + export npe_node_wavepostbndpntbll=112 export NTASKS=${npe_wavepostbndpntbll} export is_exclusive=True elif [ $step = "wavepostpnt" ]; then - export wtime_wavepostpnt="02:00:00" - export npe_wavepostpnt=280 + export wtime_wavepostpnt="01:30:00" + export npe_wavepostpnt=200 export nth_wavepostpnt=1 - export npe_node_wavepostpnt=$(echo "$npe_node_max / $nth_wavepostpnt" | bc) + export npe_node_wavepostpnt=50 export NTASKS=${npe_wavepostpnt} export is_exclusive=True elif [ $step = "wavegempak" ]; then - export wtime_wavegempak="01:00:00" - export npe_wavegempak=$npe_node_max + export wtime_wavegempak="02:00:00" + export npe_wavegempak=1 export nth_wavegempak=1 - export npe_node_wavegempak=$(echo "$npe_node_max / $nth_wavegempak" | bc) + export npe_node_wavegempak=$npe_wavegempak export NTASKS=${npe_wavegempak} + export memory_wavegempak="10GB" elif [ $step = "waveawipsbulls" ]; then - export wtime_waveawipsbulls="00:30:00" - export npe_waveawipsbulls=$npe_node_max + export wtime_waveawipsbulls="00:20:00" + export npe_waveawipsbulls=1 export nth_waveawipsbulls=1 export npe_node_waveawipsbulls=$(echo "$npe_node_max / $nth_waveawipsbulls" | bc) export NTASKS=${npe_waveawipsbulls} @@ -132,11 +142,12 @@ elif [ $step = "waveawipsbulls" ]; then elif [ $step = "waveawipsgridded" ]; then - export wtime_waveawipsgridded="00:30:00" - export npe_waveawipsgridded=$npe_node_max + export wtime_waveawipsgridded="02:00:00" + export npe_waveawipsgridded=1 export nth_waveawipsgridded=1 export npe_node_waveawipsgridded=$(echo "$npe_node_max / $nth_waveawipsgridded" | bc) export NTASKS=${npe_waveawipsgridded} + export memory_waveawipsgridded_gfs="2GB" elif [ $step = "atmanalprep" ]; then From 4089bfc808ad240b96ba27baa22e7c0d58058874 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Mon, 3 Oct 2022 20:07:28 +0000 Subject: [PATCH 08/42] Update awips and gempak resources in config.resources - Based on GFSv16 port to WCOSS2. - Add memory settings for both jobs. - Adjust tasks and threads for both jobs. - Add "_gfs" variable versions for gempak job. Refs #419 --- parm/config/config.resources | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/parm/config/config.resources b/parm/config/config.resources index 97f47496c0..e17fd86445 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -642,16 +642,22 @@ elif [ $step = "postsnd" ]; then elif [ $step = "awips" ]; then export wtime_awips="03:30:00" - export npe_awips=4 - export npe_node_awips=4 - export nth_awips=2 + export npe_awips=1 + export npe_node_awips=1 + export nth_awips=1 + export memory_awips="10GB" elif [ $step = "gempak" ]; then - export wtime_gempak="02:00:00" - export npe_gempak=28 - export npe_node_gempak=4 - export nth_gempak=3 + export wtime_gempak="03:00:00" + export wtime_gempak="03:00:00" + export npe_gempak=2 + export npe_gempak_gfs=28 + export npe_node_gempak=2 + export npe_node_gempak_gfs=28 + export nth_gempak=1 + export memory_gempak="20GB" + export memory_gempak_gfs="200GB" else From 38a3d60dbe4ff5e65b3301227c90ed4c658d0332 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Mon, 3 Oct 2022 20:09:13 +0000 Subject: [PATCH 09/42] Remove duplicate wtime_gempak Refs #419 --- parm/config/config.resources | 1 - 1 file changed, 1 deletion(-) diff --git a/parm/config/config.resources b/parm/config/config.resources index e17fd86445..318dd60305 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -649,7 +649,6 @@ elif [ $step = "awips" ]; then elif [ $step = "gempak" ]; then - export wtime_gempak="03:00:00" export wtime_gempak="03:00:00" export npe_gempak=2 export npe_gempak_gfs=28 From 8ff74c68d6cb58458e21a2e5ddfa703b5b853741 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Mon, 3 Oct 2022 20:10:49 +0000 Subject: [PATCH 10/42] Update postsnd job resources in config.resources - Pull in WCOSS2 GFSv16 ops port resource updates into config.resources. Refs #419 --- parm/config/config.resources | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/parm/config/config.resources b/parm/config/config.resources index 318dd60305..1dfc8d52a0 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -628,15 +628,17 @@ elif [ $step = "postsnd" ]; then export wtime_postsnd="02:00:00" export npe_postsnd=40 - export nth_postsnd=1 - export npe_node_postsnd=5 + export nth_postsnd=8 + export npe_node_postsnd=10 export npe_postsndcfp=9 - export npe_node_postsndcfp=3 + export npe_node_postsndcfp=1 if [ $OUTPUT_FILE == "nemsio" ]; then export npe_postsnd=13 export npe_node_postsnd=4 fi - if [[ "$machine" = "HERA" ]]; then export npe_node_postsnd=2; fi + if [[ "$(echo "$npe_node_postsnd * $nth_postsnd" | bc)" -gt "$npe_node_max" ]]; then + export npe_node_postsnd=$(echo "$npe_node_max / $nth_postsnd" | bc) + fi export is_exclusive=True elif [ $step = "awips" ]; then From f58793d7de06e89ed4a58bf45fed88a086ce64a4 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Mon, 3 Oct 2022 20:11:43 +0000 Subject: [PATCH 11/42] Change epos thread # from 6 to 4 Refs #419 --- parm/config/config.resources | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/config.resources b/parm/config/config.resources index 1dfc8d52a0..ba9b5e47ba 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -620,7 +620,7 @@ elif [ $step = "epos" ]; then export wtime_epos="00:15:00" export npe_epos=80 - export nth_epos=6 + export nth_epos=4 export npe_node_epos=$(echo "$npe_node_max / $nth_epos" | bc) export is_exclusive=True From 529dc3d95b197b9bd6f9cd0bc75e084cd70b31a1 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Mon, 3 Oct 2022 20:13:26 +0000 Subject: [PATCH 12/42] Update esfc resources in config.resources - Set $npe_node_esfc to $npe_esfc instead of $npe_node_max; then check if $npe_node_esfc is greater than $npe_node_max and set to $npe_node_max if so. - Set $npe_node_cycle based on $npe_node_max divided by $nth_cycle. - Add memory setting. Refs #419 --- parm/config/config.resources | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/parm/config/config.resources b/parm/config/config.resources index ba9b5e47ba..bc64aba6f6 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -601,9 +601,12 @@ elif [ $step = "esfc" ]; then export wtime_esfc="00:06:00" export npe_esfc=80 - export npe_node_esfc=$npe_node_max + export npe_node_esfc=$npe_esfc + if [[ "$npe_node_esfc" -gt "$npe_node_max" ]]; then export npe_node_esfc=$npe_node_max ; fi export nth_esfc=1 export nth_cycle=$nth_esfc + export npe_node_cycle=$(echo "$npe_node_max / $nth_cycle" | bc) + export memory_esfc="80GB" elif [ $step = "efcs" ]; then From bc0f025769073e178636928b972f6c44a65620ca Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Mon, 3 Oct 2022 20:16:04 +0000 Subject: [PATCH 13/42] Update ecen resources in config.resources - Change nth_ecen from 6 to 4. - Add setting for $npe_node_cycle based on $npe_node_max divided by $nth_cycle. Refs #419 --- parm/config/config.resources | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/parm/config/config.resources b/parm/config/config.resources index bc64aba6f6..0de9faa2ff 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -591,10 +591,11 @@ elif [ $step = "ecen" ]; then export wtime_ecen="00:10:00" export npe_ecen=80 - export nth_ecen=6 + export nth_ecen=4 if [ $CASE = "C384" -o $CASE = "C192" -o $CASE = "C96" -o $CASE = "C48" ]; then export nth_ecen=2; fi export npe_node_ecen=$(echo "$npe_node_max / $nth_ecen" | bc) export nth_cycle=$nth_ecen + export npe_node_cycle=$(echo "$npe_node_max / $nth_cycle" | bc) export is_exclusive=True elif [ $step = "esfc" ]; then From fdc1875160e845c35ca6c15b6583368d19c76447 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Mon, 3 Oct 2022 20:18:07 +0000 Subject: [PATCH 14/42] Update eupd resources in config.resources - Remove HERA block for C768. - Add WCOSS2 block for C768; use GFSv16 ops settings. - Adjust other resolution HERA blocks to remove npe_eupd settings (use defaults for all machines) and reduce nth_eupd values to more appropriate values other than 40 (npe_node_max on R&DS). Refs #419 --- parm/config/config.resources | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/parm/config/config.resources b/parm/config/config.resources index 0de9faa2ff..928af37172 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -565,23 +565,21 @@ elif [ $step = "eupd" ]; then if [ $CASE = "C768" ]; then export npe_eupd=480 export nth_eupd=6 - if [[ "$machine" = "HERA" ]]; then - export npe_eupd=150 - export nth_eupd=40 + if [[ "$machine" = "WCOSS2" ]]; then + export npe_eupd=315 + export nth_eupd=14 fi elif [ $CASE = "C384" ]; then export npe_eupd=270 export nth_eupd=2 if [[ "$machine" = "HERA" ]]; then - export npe_eupd=100 - export nth_eupd=40 + export nth_eupd=8 fi elif [ $CASE = "C192" -o $CASE = "C96" -o $CASE = "C48" ]; then export npe_eupd=42 export nth_eupd=2 if [[ "$machine" = "HERA" ]]; then - export npe_eupd=40 - export nth_eupd=40 + export nth_eupd=4 fi fi export npe_node_eupd=$(echo "$npe_node_max / $nth_eupd" | bc) From 51df4508bd719c89977cf037cf5e3cc949e57512 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Mon, 3 Oct 2022 20:19:58 +0000 Subject: [PATCH 15/42] Update ediag resources in config.resources - Reduce npe_ediag from 56 to 48. - Add memory setting. Refs #419 --- parm/config/config.resources | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/parm/config/config.resources b/parm/config/config.resources index 928af37172..6593e17ad7 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -555,9 +555,10 @@ elif [ $step = "eobs" -o $step = "eomg" ]; then elif [ $step = "ediag" ]; then export wtime_ediag="00:06:00" - export npe_ediag=56 + export npe_ediag=48 export nth_ediag=1 export npe_node_ediag=$npe_node_max + export memory_ediag="28GB" elif [ $step = "eupd" ]; then From 3e32ffdd46369600652d6adeb281d0d36ad6c10a Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Mon, 3 Oct 2022 20:20:55 +0000 Subject: [PATCH 16/42] Reduce eobs walltime to 15 mins in config.resources Refs #419 --- parm/config/config.resources | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/config.resources b/parm/config/config.resources index 6593e17ad7..aef4ea93ce 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -534,7 +534,7 @@ elif [ $step = "atmensanalpost" ]; then elif [ $step = "eobs" -o $step = "eomg" ]; then - export wtime_eobs="00:45:00" + export wtime_eobs="00:15:00" export wtime_eomg="01:00:00" if [ $CASE = "C768" ]; then export npe_eobs=200 From cf3901c63353b7a506dce5ce473e40d8bc438b39 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Mon, 3 Oct 2022 20:22:00 +0000 Subject: [PATCH 17/42] Increase memory for archive jobs Refs #419 --- parm/config/config.resources | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/config.resources b/parm/config/config.resources index aef4ea93ce..7b3c1ce82d 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -492,7 +492,7 @@ elif [ $step = "arch" -o $step = "earc" -o $step = "getic" ]; then eval "export npe_$step=1" eval "export npe_node_$step=1" eval "export nth_$step=1" - eval "export memory_$step=2048M" + eval "export memory_$step=50GB" elif [ $step = "coupled_ic" ]; then From bb8e2fc78a2c5a09b8aaf2c94697d3af796fc4fe Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Mon, 3 Oct 2022 20:23:57 +0000 Subject: [PATCH 18/42] Set memory for echgres in config.resources Refs #419 --- parm/config/config.resources | 1 + 1 file changed, 1 insertion(+) diff --git a/parm/config/config.resources b/parm/config/config.resources index 7b3c1ce82d..9533fa9247 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -463,6 +463,7 @@ elif [ $step = "echgres" ]; then export npe_echgres=3 export nth_echgres=$npe_node_max export npe_node_echgres=1 + export memory_echgres="200GB" elif [ $step = "init" ]; then From fef343e81337a732c0858f42334847e972a6080b Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Mon, 3 Oct 2022 20:26:15 +0000 Subject: [PATCH 19/42] Update wafs jobs resources in config.resources - Add memory settings for wafs jobs. - Adjust task values based on GFSv16 ops port. - Set $npe_node_wafs* to $npe_wafs* values. Refs #419 --- parm/config/config.resources | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/parm/config/config.resources b/parm/config/config.resources index 9533fa9247..64cf4449f8 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -394,43 +394,49 @@ elif [ $step = "wafs" ]; then export wtime_wafs="00:30:00" export npe_wafs=1 - export npe_node_wafs=1 + export npe_node_wafs=$npe_wafs export nth_wafs=1 + export memory_wafs="5GB" elif [ $step = "wafsgcip" ]; then export wtime_wafsgcip="00:30:00" export npe_wafsgcip=2 - export npe_node_wafsgcip=1 + export npe_node_wafsgcip=$npe_wafsgcip export nth_wafsgcip=1 + export memory_wafsgcip="50GB" elif [ $step = "wafsgrib2" ]; then export wtime_wafsgrib2="00:30:00" - export npe_wafsgrib2=1 - export npe_node_wafsgrib2=1 + export npe_wafsgrib2=18 + export npe_node_wafsgrib2=$npe_wafsgrib2 export nth_wafsgrib2=1 + export memory_wafsgrib2="80GB" elif [ $step = "wafsblending" ]; then export wtime_wafsblending="00:30:00" export npe_wafsblending=1 - export npe_node_wafsblending=1 + export npe_node_wafsblending=$npe_wafsblending export nth_wafsblending=1 + export memory_wafsblending="1GB" elif [ $step = "wafsgrib20p25" ]; then export wtime_wafsgrib20p25="00:30:00" - export npe_wafsgrib20p25=1 - export npe_node_wafsgrib20p25=1 + export npe_wafsgrib20p25=11 + export npe_node_wafsgrib20p25=$npe_wafsgrib20p25 export nth_wafsgrib20p25=1 + export memory_wafsgrib20p25="80GB" elif [ $step = "wafsblending0p25" ]; then export wtime_wafsblending0p25="00:30:00" export npe_wafsblending0p25=1 - export npe_node_wafsblending0p25=1 + export npe_node_wafsblending0p25=$npe_wafsblending0p25 export nth_wafsblending0p25=1 + export memory_wafsblending0p25="15GB" elif [ $step = "vrfy" ]; then From bce25ca40238c06020d1ee26c3cfbd4d06c81115 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Mon, 3 Oct 2022 20:28:25 +0000 Subject: [PATCH 20/42] Update post resources in config.resources - Add "_gfs" variables. - Reduce walltimes. - Increase npe_post from 112 to 126 (GFSv16 ops port value). - Check if $npe_node_post[_gfs] is greater than $npe_node_max and set it to $npe_node_max if so. Refs #419 --- parm/config/config.resources | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/parm/config/config.resources b/parm/config/config.resources index 64cf4449f8..5d123a3eb9 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -382,12 +382,15 @@ elif [ $step = "ocnpost" ]; then elif [ $step = "post" ]; then - export wtime_post="02:00:00" - export wtime_post_gfs="06:00:00" - export npe_post=112 + export wtime_post="00:12:00" + export wtime_post_gfs="01:00:00" + export npe_post=126 export nth_post=1 - export npe_node_post=12 + export npe_node_post=$npe_post + export npe_node_post_gfs=$npe_post export npe_node_dwn=$npe_node_max + if [[ "$npe_node_post" -gt "$npe_node_max" ]]; then export npe_node_post=$npe_node_max ; fi + if [[ "$npe_node_post_gfs" -gt "$npe_node_max" ]]; then export npe_node_post_gfs=$npe_node_max ; fi export is_exclusive=True elif [ $step = "wafs" ]; then From ab96ded6fb9763cd3a9fa4745c5ea602bc0421e1 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Mon, 3 Oct 2022 20:30:27 +0000 Subject: [PATCH 21/42] Increase gdasfcst walltime to 40mins in config.resources Refs #419 --- parm/config/config.resources | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/config.resources b/parm/config/config.resources index 5d123a3eb9..348f9fc5bf 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -278,7 +278,7 @@ elif [ $step = "gldas" ]; then elif [ $step = "fcst" ]; then - export wtime_fcst="00:30:00" + export wtime_fcst="00:40:00" if [ $CASE = "C768" ]; then export wtime_fcst_gfs="06:00:00" elif [ $CASE = "C384" ]; then From b9b91c3ba96c199e54363601403729dfc824643e Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Mon, 3 Oct 2022 20:31:43 +0000 Subject: [PATCH 22/42] Update gldas job resources in config.resources - Increase $npe_gldas from 96 to 112. - Set $npe_node_gldas to $npe_gldas instead of $npe_node_max. - Set $npe_node_gaussian based on $npe_node_max divided by $nth_gaussian. Refs #419 --- parm/config/config.resources | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/parm/config/config.resources b/parm/config/config.resources index 348f9fc5bf..6c252037f0 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -268,12 +268,12 @@ elif [ $step = "sfcanl" ]; then elif [ $step = "gldas" ]; then export wtime_gldas="00:10:00" - export npe_gldas=96 + export npe_gldas=112 export nth_gldas=1 - export npe_node_gldas=$npe_node_max + export npe_node_gldas=$npe_gldas export npe_gaussian=96 export nth_gaussian=1 - export npe_node_gaussian=24 + export npe_node_gaussian=$(echo "$npe_node_max / $nth_gaussian" | bc) export is_exclusive=True elif [ $step = "fcst" ]; then From 86836b7a186121bd938c3223f54dfd6e4f267a43 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Mon, 3 Oct 2022 20:33:48 +0000 Subject: [PATCH 23/42] Update analdiag resources in config.resources - Reduce $npe_analdiag from 112 to 96; add note that $npe_analdiag should be at least twice npe_diag. - Set $npe_node_analdiag to $npe_analdiag. - Check if $npe_node_analdiag is greater than $npe_node_max and set to $npe_node_max if so. - Add memory setting. Refs #419 --- parm/config/config.resources | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/parm/config/config.resources b/parm/config/config.resources index 6c252037f0..cf0405b532 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -253,9 +253,11 @@ elif [ $step = "analcalc" ]; then elif [ $step = "analdiag" ]; then export wtime_analdiag="00:10:00" - export npe_analdiag=112 + export npe_analdiag=96 # Should be at least twice npe_ediag export nth_analdiag=1 - export npe_node_analdiag=$npe_node_max + export npe_node_analdiag=$npe_analdiag + if [[ "$npe_node_analdiag" -gt "$npe_node_max" ]]; then export npe_node_analdiag=$npe_node_max ; fi + export memory_analdiag="48GB" elif [ $step = "sfcanl" ]; then From 9772ca27b46189079e972d00abda547f42649656 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Mon, 3 Oct 2022 20:36:32 +0000 Subject: [PATCH 24/42] Update gldas to check $npe_node_gldas - If $npe_node_gldas is greater than $npe_node_max then set it to $npe_node_max. Refs #419 --- parm/config/config.resources | 1 + 1 file changed, 1 insertion(+) diff --git a/parm/config/config.resources b/parm/config/config.resources index cf0405b532..824b2833b1 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -273,6 +273,7 @@ elif [ $step = "gldas" ]; then export npe_gldas=112 export nth_gldas=1 export npe_node_gldas=$npe_gldas + if [[ "$npe_node_gldas" -gt "$npe_node_max" ]]; then export npe_node_gldas=$npe_node_max ; fi export npe_gaussian=96 export nth_gaussian=1 export npe_node_gaussian=$(echo "$npe_node_max / $nth_gaussian" | bc) From 67747be0081d8ae7ac831a66d24c3d50331cd909 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Mon, 3 Oct 2022 20:39:11 +0000 Subject: [PATCH 25/42] Increase efcs walltime Refs #419 --- parm/config/config.resources | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/config.resources b/parm/config/config.resources index 824b2833b1..b50c62725c 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -624,7 +624,7 @@ elif [ $step = "esfc" ]; then elif [ $step = "efcs" ]; then if [ $CASE = "C768" ]; then - export wtime_efcs="01:00:00" + export wtime_efcs="01:20:00" else export wtime_efcs="00:40:00" fi From 8c247c6fcd6855a5ef56344d024659aa9b4747e8 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Mon, 3 Oct 2022 20:42:05 +0000 Subject: [PATCH 26/42] Update anal job resources in config.resources - Bring in GFSv16 ops port values for C768. - Also adjust C384 values. - Add "_gfs" versions of variables so they can be set separately. - Set $npe_node_cycle based on $npe_node_max divided by $nth_cycle. Refs #419 --- parm/config/config.resources | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/parm/config/config.resources b/parm/config/config.resources index b50c62725c..4b1b63680d 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -222,21 +222,25 @@ elif [ $step = "aeroanlfinal" ]; then elif [ $step = "anal" ]; then - export wtime_anal="01:00:00" - export npe_anal=1000 - export nth_anal=5 - export npe_anal_gfs=1000 + export wtime_anal="00:50:00" + export wtime_anal_gfs="00:40:00" + export npe_anal=780 + export nth_anal=8 + export npe_anal_gfs=825 + export nth_anal_gfs=8 if [ $CASE = "C384" ]; then - export npe_anal=400 - export npe_anal_gfs=400 + export npe_anal=160 + export npe_anal_gfs=160 + export nth_anal=10 + export nth_anal_gfs=10 fi if [ $CASE = "C192" -o $CASE = "C96" -o $CASE = "C48" ]; then export npe_anal=84 export npe_anal_gfs=84 fi - export nth_anal_gfs=$nth_anal export npe_node_anal=$(echo "$npe_node_max / $nth_anal" | bc) export nth_cycle=$nth_anal + export npe_node_cycle=$(echo "$npe_node_max / $nth_cycle" | bc) export is_exclusive=True elif [ $step = "analcalc" ]; then From 80f295c3f12479341f44f123845fb080f629231c Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Wed, 12 Oct 2022 14:50:49 +0000 Subject: [PATCH 27/42] Add make_nsstbufr/make_acftbufr to WCOSS2 hosts file - Set both make_nsstbufr and make_acftbufr to NO. Refs #419 --- workflow/hosts/wcoss2.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/workflow/hosts/wcoss2.yaml b/workflow/hosts/wcoss2.yaml index 3a1bd623d7..a729620dde 100644 --- a/workflow/hosts/wcoss2.yaml +++ b/workflow/hosts/wcoss2.yaml @@ -17,4 +17,6 @@ chgrp_cmd: 'chgrp rstprod' hpssarch: 'YES' localarch: 'NO' atardir: '/NCEPDEV/$HPSS_PROJECT/1year/$USER/$machine/scratch/$PSLOT' +make_nsstbufr: 'NO' +make_acftbufr: 'NO' supported_resolutions: ['C768', 'C384', 'C192', 'C96', 'C48'] From 29cf230c3ed9bad07a0504c3ecd83bce6e7aedb2 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Thu, 13 Oct 2022 12:47:50 +0000 Subject: [PATCH 28/42] Resources updates in config.fv3 from WCOSS2 testing - add if-block for WCOSS2 to set export nth_fv3_efcs=1; nth_fv3_efcs is new and used for nth_fv3 in the efcs job - change C384 WRITE_GROUP from 1 to 2 - change C384 to run serially Refs #419 --- parm/config/config.fv3 | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/parm/config/config.fv3 b/parm/config/config.fv3 index 8ea139e2e8..bc22b555ea 100755 --- a/parm/config/config.fv3 +++ b/parm/config/config.fv3 @@ -86,6 +86,9 @@ case $case_in in export layout_x_gfs=4 export layout_y_gfs=6 export nth_fv3=2 + if [[ "$machine" = "WCOSS2" ]]; then + export nth_fv3_efcs=1 + fi export nth_fv3_gfs=2 export cdmbgwd="0.23,1.5,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export WRITE_GROUP=1 @@ -105,7 +108,7 @@ case $case_in in export nth_fv3=2 export nth_fv3_gfs=${nth_fv3_gfs_s2sw:-2} export cdmbgwd="1.1,0.72,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling - export WRITE_GROUP=1 + export WRITE_GROUP=2 export WRTTASK_PER_GROUP=64 if [[ "$WRTTASK_PER_GROUP" -gt "$npe_node_max" ]]; then export WRTTASK_PER_GROUP=$npe_node_max ; fi export WRITE_GROUP_GFS=${WRITE_GROUP_GFS_s2sw:-2} @@ -176,11 +179,11 @@ export kchunk3d=1 # Determine whether to use parallel NetCDF based on resolution case ${case_in} in - "C48" | "C96" | "C192") + "C48" | "C96" | "C192" | "C384") export OUTPUT_FILETYPE_ATM="netcdf" export OUTPUT_FILETYPE_SFC="netcdf" ;; - "C384" | "C768" | "C1152" | "C3072") + "C768" | "C1152" | "C3072") export OUTPUT_FILETYPE_ATM="netcdf_parallel" export OUTPUT_FILETYPE_SFC="netcdf_parallel" ;; From 13ccbc9a03a27744201181dcc1d341b0d7a5de3e Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Thu, 13 Oct 2022 12:49:42 +0000 Subject: [PATCH 29/42] WCOSS2 updates from testing in config.resources - set npe_node_analdiag=$npe_analdiag - add WCOSS2 if-block to set memory_echgres="200GB" only on WCOSS2 - add npe_node_cycle back into ecen block - update nth_efcs to use nth_fv3_efcs if set; matching update in config.fv3 Refs #419 --- parm/config/config.resources | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/parm/config/config.resources b/parm/config/config.resources index 30f2aabb44..3fef4ce140 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -290,7 +290,7 @@ elif [ ${step} = "analdiag" ]; then export wtime_analdiag="00:10:00" export npe_analdiag=96 # Should be at least twice npe_ediag export nth_analdiag=1 - export npe_node_analdiag=${npe_node_max} + export npe_node_analdiag=${npe_analdiag} if [[ "${npe_node_analdiag}" -gt "${npe_node_max}" ]]; then export npe_node_analdiag=${npe_node_max} ; fi export memory_analdiag="48GB" @@ -514,7 +514,9 @@ elif [ ${step} = "echgres" ]; then export npe_echgres=3 export nth_echgres=${npe_node_max} export npe_node_echgres=1 - export memory_echgres="200GB" + if [[ "$machine" = "WCOSS2" ]]; then + export memory_echgres="200GB" + fi elif [ ${step} = "init" ]; then @@ -630,7 +632,10 @@ elif [ ${step} = "eupd" ]; then elif [ ${CASE} = "C384" ]; then export npe_eupd=270 export nth_eupd=2 - if [[ "$machine" = "HERA" ]]; then + if [[ "$machine" = "WCOSS2" ]]; then + export npe_eupd=315 + export nth_eupd=14 + elif [[ "$machine" = "HERA" ]]; then export nth_eupd=8 fi if [[ ${machine} = "S4" ]]; then @@ -656,6 +661,7 @@ elif [ ${step} = "ecen" ]; then if [[ ${CASE} = "C384" || ${CASE} = "C192" || ${CASE} = "C96" || ${CASE} = "C48" ]]; then export nth_ecen=2; fi export npe_node_ecen=$(echo "${npe_node_max} / ${nth_ecen}" | bc) export nth_cycle=${nth_ecen} + export npe_node_cycle=$(echo "${npe_node_max} / ${nth_cycle}" | bc) export is_exclusive=True elif [ ${step} = "esfc" ]; then @@ -677,7 +683,7 @@ elif [ ${step} = "efcs" ]; then export wtime_efcs="00:40:00" fi export npe_efcs=$(echo "${layout_x} * ${layout_y} * 6" | bc) - export nth_efcs=${nth_fv3:-2} + export nth_efcs=${nth_fv3_efcs:-${nth_fv3:-2}} export npe_node_efcs=$(echo "${npe_node_max} / ${nth_efcs}" | bc) elif [ ${step} = "epos" ]; then From 1049e5084ba222c12c67bb2d18768f49f4f9c82c Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Thu, 13 Oct 2022 13:05:56 +0000 Subject: [PATCH 30/42] Updates from R&D testing - remove nth_fv3_efcs and use nth_fv3=1 for C192 everywhere - set nth_epos=6 on Hera Refs #419 --- parm/config/config.fv3 | 5 +---- parm/config/config.resources | 5 ++++- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/parm/config/config.fv3 b/parm/config/config.fv3 index bc22b555ea..46fd18bf07 100755 --- a/parm/config/config.fv3 +++ b/parm/config/config.fv3 @@ -85,10 +85,7 @@ case $case_in in export layout_y=6 export layout_x_gfs=4 export layout_y_gfs=6 - export nth_fv3=2 - if [[ "$machine" = "WCOSS2" ]]; then - export nth_fv3_efcs=1 - fi + export nth_fv3=1 export nth_fv3_gfs=2 export cdmbgwd="0.23,1.5,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export WRITE_GROUP=1 diff --git a/parm/config/config.resources b/parm/config/config.resources index 3fef4ce140..deb9109a9b 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -683,7 +683,7 @@ elif [ ${step} = "efcs" ]; then export wtime_efcs="00:40:00" fi export npe_efcs=$(echo "${layout_x} * ${layout_y} * 6" | bc) - export nth_efcs=${nth_fv3_efcs:-${nth_fv3:-2}} + export nth_efcs=${nth_fv3:-2} export npe_node_efcs=$(echo "${npe_node_max} / ${nth_efcs}" | bc) elif [ ${step} = "epos" ]; then @@ -691,6 +691,9 @@ elif [ ${step} = "epos" ]; then export wtime_epos="00:15:00" export npe_epos=80 export nth_epos=4 + if [[ "$machine" == "HERA" ]]; then + export nth_epos=6 + fi export npe_node_epos=$(echo "${npe_node_max} / ${nth_epos}" | bc) export is_exclusive=True From 835a19a0e8a07c210efc98e4d2992556169888e2 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Thu, 13 Oct 2022 09:32:50 -0500 Subject: [PATCH 31/42] Update anal block in config.resources - add WCOSS2 check to set general threads to 8 and keep it 5 threads on R&Ds to keep nodes the same on R&Ds Refs #419 --- parm/config/config.resources | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/parm/config/config.resources b/parm/config/config.resources index 30f2aabb44..c8e0893d79 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -231,9 +231,13 @@ elif [ ${step} = "anal" ]; then export wtime_anal="00:50:00" export wtime_anal_gfs="00:40:00" export npe_anal=780 - export nth_anal=8 + export nth_anal=5 export npe_anal_gfs=825 - export nth_anal_gfs=8 + export nth_anal_gfs=5 + if [[ "$machine" = "WCOSS2" ]]; then + export nth_anal=8 + export nth_anal_gfs=8 + fi if [ $CASE = "C384" ]; then export npe_anal=160 export npe_anal_gfs=160 From 11ffd146b19a8b8560be62e08ae8c9609e858ce3 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Thu, 13 Oct 2022 14:56:23 +0000 Subject: [PATCH 32/42] Update C384 nth_fv3 and remove _s2sw - Update C384 nth_fv3 from 2 to 1. - Remove added "_s2sw" text to S2SW and C384 resource variables. Refs #419 --- parm/config/config.defaults.s2sw | 18 +++++++++--------- parm/config/config.fv3 | 16 ++++++++-------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/parm/config/config.defaults.s2sw b/parm/config/config.defaults.s2sw index 51693ab3bd..5032a998ad 100644 --- a/parm/config/config.defaults.s2sw +++ b/parm/config/config.defaults.s2sw @@ -16,16 +16,16 @@ min_seaice="1.0e-6" use_cice_alb=".true." # config.fv3 -DELTIM_s2sw=300 -layout_x_gfs_s2sw=8 -layout_y_gfs_s2sw=8 -WRITE_GROUP_GFS_s2sw=1 -WRTTASK_PER_GROUP_GFS_s2sw=24 +DELTIM=300 +layout_x_gfs=8 +layout_y_gfs=8 +WRITE_GROUP_GFS=1 +WRTTASK_PER_GROUP_GFS=24 #The settings below will result in S2SWA running 35 days under 8 hours wallclock on hera -#layout_x_gfs_s2sw=24 -#layout_y_gfs_s2sw=16 -#WRTTASK_PER_GROUP_GFS_s2sw=86 -WRTIOBUF_s2sw="32M" +#layout_x_gfs=24 +#layout_y_gfs=16 +#WRTTASK_PER_GROUP_GFS=86 +WRTIOBUF="32M" MEDPETS=300 # config.wave diff --git a/parm/config/config.fv3 b/parm/config/config.fv3 index 46fd18bf07..07fc468bfe 100755 --- a/parm/config/config.fv3 +++ b/parm/config/config.fv3 @@ -97,21 +97,21 @@ case $case_in in export WRTIOBUF="8M" ;; "C384") - export DELTIM=${DELTIM_s2sw:-200} + export DELTIM=${DELTIM:-200} export layout_x=6 export layout_y=8 - export layout_x_gfs=${layout_x_gfs_s2sw:-8} - export layout_y_gfs=${layout_y_gfs_s2sw:-12} - export nth_fv3=2 - export nth_fv3_gfs=${nth_fv3_gfs_s2sw:-2} + export layout_x_gfs=${layout_x_gfs:-8} + export layout_y_gfs=${layout_y_gfs:-12} + export nth_fv3=1 + export nth_fv3_gfs=${nth_fv3_gfs:-2} export cdmbgwd="1.1,0.72,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export WRITE_GROUP=2 export WRTTASK_PER_GROUP=64 if [[ "$WRTTASK_PER_GROUP" -gt "$npe_node_max" ]]; then export WRTTASK_PER_GROUP=$npe_node_max ; fi - export WRITE_GROUP_GFS=${WRITE_GROUP_GFS_s2sw:-2} - export WRTTASK_PER_GROUP_GFS=${WRTTASK_PER_GROUP_GFS_s2sw:-64} + export WRITE_GROUP_GFS=${WRITE_GROUP_GFS:-2} + export WRTTASK_PER_GROUP_GFS=${WRTTASK_PER_GROUP_GFS:-64} if [[ "$WRTTASK_PER_GROUP_GFS" -gt "$npe_node_max" ]]; then export WRTTASK_PER_GROUP_GFS=$npe_node_max ; fi - export WRTIOBUF=${WRTIOBUF_s2sw:-"16M"} + export WRTIOBUF=${WRTIOBUF:-"16M"} ;; "C768") export DELTIM=150 From cbc655c57bfbaa1abaa66d3e6997fa4bcb879bef Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Thu, 13 Oct 2022 10:04:36 -0500 Subject: [PATCH 33/42] Add npe_node_max check for wavepostpnt Refs #419 --- parm/config/config.resources | 1 + 1 file changed, 1 insertion(+) diff --git a/parm/config/config.resources b/parm/config/config.resources index 1c000b1104..72fface57c 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -125,6 +125,7 @@ elif [ ${step} = "wavepostpnt" ]; then export npe_wavepostpnt=200 export nth_wavepostpnt=1 export npe_node_wavepostpnt=50 + if [[ "${npe_node_wavepostpnt}" -gt "${npe_node_max}" ]]; then export npe_node_wavepostpnt=${npe_node_max} ; fi export NTASKS=${npe_wavepostpnt} export is_exclusive=True From aeaccebb156733f77bfadacd654139ef4563540d Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Wed, 19 Oct 2022 18:29:14 +0000 Subject: [PATCH 34/42] Resource updates - Pull in GFSv16.3 updates from NCO and into ecf scripts. - Adjust npe_node variables for some jobs to better adjust to different npe_node_max values. - Address some linter warnings about {}. Refs #419 --- .../jgfs_atmos_wafs_blending_0p25.ecf | 3 +- .../grib2_wafs/jgfs_atmos_wafs_grib2.ecf | 5 +- .../grib2_wafs/jgfs_atmos_wafs_grib2_0p25.ecf | 5 +- .../post_processing/jgfs_atmos_wafs_gcip.ecf | 1 + parm/config/config.resources | 65 +++++++++---------- 5 files changed, 42 insertions(+), 37 deletions(-) diff --git a/ecf/scripts/gfs/atmos/post_processing/grib2_wafs/jgfs_atmos_wafs_blending_0p25.ecf b/ecf/scripts/gfs/atmos/post_processing/grib2_wafs/jgfs_atmos_wafs_blending_0p25.ecf index f0ce0daacf..83647d9c15 100755 --- a/ecf/scripts/gfs/atmos/post_processing/grib2_wafs/jgfs_atmos_wafs_blending_0p25.ecf +++ b/ecf/scripts/gfs/atmos/post_processing/grib2_wafs/jgfs_atmos_wafs_blending_0p25.ecf @@ -4,7 +4,7 @@ #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:30:00 -#PBS -l select=1:mpiprocs=1:ompthreads=1:ncpus=1:mem=1GB +#PBS -l select=1:mpiprocs=1:ompthreads=1:ncpus=1:mem=15GB #PBS -l place=vscatter #PBS -l debug=true @@ -35,6 +35,7 @@ module list ############################################################# export cyc=%CYC% export cycle=t%CYC%z +export ICAO2023=no ############################################################ # CALL executable job script here diff --git a/ecf/scripts/gfs/atmos/post_processing/grib2_wafs/jgfs_atmos_wafs_grib2.ecf b/ecf/scripts/gfs/atmos/post_processing/grib2_wafs/jgfs_atmos_wafs_grib2.ecf index 1c271e57b8..25d4fc37aa 100755 --- a/ecf/scripts/gfs/atmos/post_processing/grib2_wafs/jgfs_atmos_wafs_grib2.ecf +++ b/ecf/scripts/gfs/atmos/post_processing/grib2_wafs/jgfs_atmos_wafs_grib2.ecf @@ -4,7 +4,7 @@ #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:30:00 -#PBS -l select=1:mpiprocs=1:ompthreads=1:ncpus=1:mem=5GB +#PBS -l select=1:mpiprocs=18:ompthreads=1:ncpus=18:mem=80GB #PBS -l place=vscatter #PBS -l debug=true @@ -28,6 +28,7 @@ module load cray-pals/${cray_pals_ver} module load libjpeg/${libjpeg_ver} module load grib_util/${grib_util_ver} module load wgrib2/${wgrib2_ver} +module load cfp/${cfp_ver} module list @@ -36,6 +37,8 @@ module list ############################################################# export cyc=%CYC% export cycle=t%CYC%z +export USE_CFP=YES +export ICAO2023=no ############################################################ # CALL executable job script here diff --git a/ecf/scripts/gfs/atmos/post_processing/grib2_wafs/jgfs_atmos_wafs_grib2_0p25.ecf b/ecf/scripts/gfs/atmos/post_processing/grib2_wafs/jgfs_atmos_wafs_grib2_0p25.ecf index ce78889886..9beac6f13a 100755 --- a/ecf/scripts/gfs/atmos/post_processing/grib2_wafs/jgfs_atmos_wafs_grib2_0p25.ecf +++ b/ecf/scripts/gfs/atmos/post_processing/grib2_wafs/jgfs_atmos_wafs_grib2_0p25.ecf @@ -4,7 +4,7 @@ #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:30:00 -#PBS -l select=1:mpiprocs=1:ompthreads=1:ncpus=1:mem=1GB +#PBS -l select=1:mpiprocs=11:ompthreads=1:ncpus=11:mem=80GB #PBS -l place=vscatter #PBS -l debug=true @@ -28,6 +28,7 @@ module load cray-pals/${cray_pals_ver} module load libjpeg/${libjpeg_ver} module load grib_util/${grib_util_ver} module load wgrib2/${wgrib2_ver} +module load cfp/${cfp_ver} module list @@ -36,6 +37,8 @@ module list ############################################################# export cyc=%CYC% export cycle=t%CYC%z +export USE_CFP=YES +export ICAO2023=no ############################################################ # CALL executable job script here diff --git a/ecf/scripts/gfs/atmos/post_processing/jgfs_atmos_wafs_gcip.ecf b/ecf/scripts/gfs/atmos/post_processing/jgfs_atmos_wafs_gcip.ecf index cf5d893b6e..00a87f3948 100755 --- a/ecf/scripts/gfs/atmos/post_processing/jgfs_atmos_wafs_gcip.ecf +++ b/ecf/scripts/gfs/atmos/post_processing/jgfs_atmos_wafs_gcip.ecf @@ -40,6 +40,7 @@ module list export cyc=%CYC% export cycle=t%CYC%z export USE_CFP=YES +export ICAO2023=no ############################################################ # CALL executable job script here diff --git a/parm/config/config.resources b/parm/config/config.resources index 72fface57c..d11e88f892 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -73,7 +73,7 @@ elif [ ${step} = "waveinit" ]; then export wtime_waveinit="00:10:00" export npe_waveinit=12 export nth_waveinit=1 - export npe_node_waveinit=$npe_waveinit + export npe_node_waveinit=$(echo "${npe_node_max} / ${nth_waveinit}" | bc) export NTASKS=${npe_waveinit} export memory_waveinit="2GB" @@ -83,12 +83,13 @@ elif [ ${step} = "waveprep" ]; then export npe_waveprep=5 export npe_waveprep_gfs=65 export nth_waveprep=1 - export npe_node_waveprep=$npe_waveprep - export npe_node_waveprep_gfs=$npe_waveprep_gfs + export nth_waveprep_gfs=1 + export npe_node_waveprep=$(echo "${npe_node_max} / ${nth_waveprep}" | bc) + export npe_node_waveprep_gfs=$(echo "${npe_node_max} / ${nth_waveprep_gfs}" | bc) export NTASKS=${npe_waveprep} export NTASKS_gfs=$npe_waveprep_gfs export memory_waveprep="100GB" - export memory_waveprep_gfs="220GB" + export memory_waveprep_gfs="150GB" elif [ ${step} = "wavepostsbs" ]; then @@ -96,17 +97,17 @@ elif [ ${step} = "wavepostsbs" ]; then export wtime_wavepostsbs_gfs="03:00:00" export npe_wavepostsbs=8 export nth_wavepostsbs=1 - export npe_node_wavepostsbs=$npe_wavepostsbs + export npe_node_wavepostsbs=$(echo "${npe_node_max} / ${nth_wavepostsbs}" | bc) export NTASKS=${npe_wavepostsbs} export memory_wavepostsbs="10GB" - export memory_wavepostsbs_gfs="40GB" + export memory_wavepostsbs_gfs="10GB" elif [ ${step} = "wavepostbndpnt" ]; then export wtime_wavepostbndpnt="01:00:00" export npe_wavepostbndpnt=240 export nth_wavepostbndpnt=1 - export npe_node_wavepostbndpnt=80 + export npe_node_wavepostbndpnt=$(echo "${npe_node_max} / ${nth_wavepostbndpnt}" | bc) export NTASKS=${npe_wavepostbndpnt} export is_exclusive=True @@ -115,7 +116,7 @@ elif [ ${step} = "wavepostbndpntbll" ]; then export wtime_wavepostbndpntbll="01:00:00" export npe_wavepostbndpntbll=448 export nth_wavepostbndpntbll=1 - export npe_node_wavepostbndpntbll=112 + export npe_node_wavepostbndpntbll=$(echo "${npe_node_max} / ${nth_wavepostbndpntbll}" | bc) export NTASKS=${npe_wavepostbndpntbll} export is_exclusive=True @@ -124,8 +125,7 @@ elif [ ${step} = "wavepostpnt" ]; then export wtime_wavepostpnt="01:30:00" export npe_wavepostpnt=200 export nth_wavepostpnt=1 - export npe_node_wavepostpnt=50 - if [[ "${npe_node_wavepostpnt}" -gt "${npe_node_max}" ]]; then export npe_node_wavepostpnt=${npe_node_max} ; fi + export npe_node_wavepostpnt=$(echo "${npe_node_max} / ${nth_wavepostpnt}" | bc) export NTASKS=${npe_wavepostpnt} export is_exclusive=True @@ -134,9 +134,9 @@ elif [ ${step} = "wavegempak" ]; then export wtime_wavegempak="02:00:00" export npe_wavegempak=1 export nth_wavegempak=1 - export npe_node_wavegempak=$npe_wavegempak + export npe_node_wavegempak=$(echo "${npe_node_max} / ${nth_wavegempak}" | bc) export NTASKS=${npe_wavegempak} - export memory_wavegempak="10GB" + export memory_wavegempak="1GB" elif [ ${step} = "waveawipsbulls" ]; then @@ -154,7 +154,7 @@ elif [ ${step} = "waveawipsgridded" ]; then export nth_waveawipsgridded=1 export npe_node_waveawipsgridded=$(echo "${npe_node_max} / ${nth_waveawipsgridded}" | bc) export NTASKS=${npe_waveawipsgridded} - export memory_waveawipsgridded_gfs="2GB" + export memory_waveawipsgridded_gfs="1GB" elif [ ${step} = "atmanalprep" ]; then @@ -287,7 +287,7 @@ elif [ ${step} = "analcalc" ]; then export nth_analcalc=1 export nth_echgres=4 export nth_echgres_gfs=12 - export npe_node_analcalc=${npe_node_max} + export npe_node_analcalc=$(echo "${npe_node_max} / ${nth_analcalc}" | bc) export is_exclusive=True elif [ ${step} = "analdiag" ]; then @@ -295,8 +295,7 @@ elif [ ${step} = "analdiag" ]; then export wtime_analdiag="00:10:00" export npe_analdiag=96 # Should be at least twice npe_ediag export nth_analdiag=1 - export npe_node_analdiag=${npe_analdiag} - if [[ "${npe_node_analdiag}" -gt "${npe_node_max}" ]]; then export npe_node_analdiag=${npe_node_max} ; fi + export npe_node_analdiag=$(echo "${npe_node_max} / ${nth_analdiag}" | bc) export memory_analdiag="48GB" elif [ ${step} = "sfcanl" ]; then @@ -312,11 +311,10 @@ elif [ ${step} = "gldas" ]; then export wtime_gldas="00:10:00" export npe_gldas=112 export nth_gldas=1 - export npe_node_gldas=${npe_gldas} - if [[ "${npe_node_gldas}" -gt "${npe_node_max}" ]]; then export npe_node_gldas=${npe_node_max} ; fi + export npe_node_gldas=$(echo "${npe_node_max} / ${nth_gldas}" | bc) export npe_gaussian=96 export nth_gaussian=1 - export npe_node_gaussian=$(echo "$npe_node_max / $nth_gaussian" | bc) + export npe_node_gaussian=$(echo "${npe_node_max} / ${nth_gaussian}" | bc) export is_exclusive=True elif [ ${step} = "fcst" ]; then @@ -446,46 +444,46 @@ elif [ ${step} = "wafs" ]; then export npe_wafs=1 export npe_node_wafs=$npe_wafs export nth_wafs=1 - export memory_wafs="5GB" + export memory_wafs="1GB" elif [ ${step} = "wafsgcip" ]; then export wtime_wafsgcip="00:30:00" export npe_wafsgcip=2 - export npe_node_wafsgcip=$npe_wafsgcip export nth_wafsgcip=1 + export npe_node_wafsgcip=1 export memory_wafsgcip="50GB" elif [ ${step} = "wafsgrib2" ]; then export wtime_wafsgrib2="00:30:00" export npe_wafsgrib2=18 - export npe_node_wafsgrib2=$npe_wafsgrib2 export nth_wafsgrib2=1 + export npe_node_wafsgrib2=$(echo "${npe_node_max} / ${nth_wafsgrib2}" | bc) export memory_wafsgrib2="80GB" elif [ ${step} = "wafsblending" ]; then export wtime_wafsblending="00:30:00" export npe_wafsblending=1 - export npe_node_wafsblending=$npe_wafsblending export nth_wafsblending=1 + export npe_node_wafsblending=$(echo "${npe_node_max} / ${nth_wafsblending}" | bc) export memory_wafsblending="1GB" elif [ ${step} = "wafsgrib20p25" ]; then export wtime_wafsgrib20p25="00:30:00" export npe_wafsgrib20p25=11 - export npe_node_wafsgrib20p25=$npe_wafsgrib20p25 export nth_wafsgrib20p25=1 + export npe_node_wafsgrib20p25=$(echo "${npe_node_max} / ${nth_wafsgrib20p25}" | bc) export memory_wafsgrib20p25="80GB" elif [ ${step} = "wafsblending0p25" ]; then export wtime_wafsblending0p25="00:30:00" export npe_wafsblending0p25=1 - export npe_node_wafsblending0p25=$npe_wafsblending0p25 export nth_wafsblending0p25=1 + export npe_node_wafsblending0p25=$(echo "${npe_node_max} / ${nth_wafsblending0p25}" | bc) export memory_wafsblending0p25="15GB" elif [ ${step} = "vrfy" ]; then @@ -621,8 +619,8 @@ elif [ ${step} = "ediag" ]; then export wtime_ediag="00:06:00" export npe_ediag=48 export nth_ediag=1 - export npe_node_ediag=${npe_node_max} - export memory_ediag="28GB" + export npe_node_ediag=$(echo "${npe_node_max} / ${nth_ediag}" | bc) + export memory_ediag="24GB" elif [ ${step} = "eupd" ]; then @@ -673,12 +671,11 @@ elif [ ${step} = "esfc" ]; then export wtime_esfc="00:06:00" export npe_esfc=80 - export npe_node_esfc=${npe_esfc} - if [[ "$npe_node_esfc" -gt "$npe_node_max" ]]; then export npe_node_esfc=$npe_node_max ; fi export nth_esfc=1 + export npe_node_esfc=$(echo "${npe_node_max} / ${nth_esfc}" | bc) export nth_cycle=${nth_esfc} - export npe_node_cycle=$(echo "$npe_node_max / $nth_cycle" | bc) - export memory_esfc="80GB" + export npe_node_cycle=$(echo "${npe_node_max} / ${nth_cycle}" | bc) + export memory_esfc="60GB" elif [ ${step} = "efcs" ]; then @@ -725,7 +722,7 @@ elif [ ${step} = "awips" ]; then export npe_awips=1 export npe_node_awips=1 export nth_awips=1 - export memory_awips="10GB" + export memory_awips="3GB" elif [ ${step} = "gempak" ]; then @@ -735,8 +732,8 @@ elif [ ${step} = "gempak" ]; then export npe_node_gempak=2 export npe_node_gempak_gfs=28 export nth_gempak=1 - export memory_gempak="20GB" - export memory_gempak_gfs="200GB" + export memory_gempak="4GB" + export memory_gempak_gfs="2GB" else From f09d487720660564b09e531d58614dfb46174924 Mon Sep 17 00:00:00 2001 From: Kate Friedman Date: Wed, 19 Oct 2022 14:56:24 -0400 Subject: [PATCH 35/42] Increase wafs blending memory to 15GB Refs #419 --- parm/config/config.resources | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/config.resources b/parm/config/config.resources index 4358a3816c..f90cd4d477 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -473,7 +473,7 @@ elif [ ${step} = "wafsblending" ]; then export npe_wafsblending=1 export nth_wafsblending=1 export npe_node_wafsblending=$(echo "${npe_node_max} / ${nth_wafsblending}" | bc) - export memory_wafsblending="1GB" + export memory_wafsblending="15GB" elif [ ${step} = "wafsgrib20p25" ]; then From b021d9dffd3ff30471065d1a2cfb59d86c3b744e Mon Sep 17 00:00:00 2001 From: Kate Friedman Date: Thu, 20 Oct 2022 09:38:57 -0400 Subject: [PATCH 36/42] Increase esfc memory back up to 80GB - Received an OOM kill when testing 60GB. Increased back up to 80GB and the job completed. - This is the only job still encountering a "Cgroup mem" warning however. Further investigation is needed. Refs #419 --- parm/config/config.resources | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/config.resources b/parm/config/config.resources index f90cd4d477..17b9131e49 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -680,7 +680,7 @@ elif [ ${step} = "esfc" ]; then export npe_node_esfc=$(echo "${npe_node_max} / ${nth_esfc}" | bc) export nth_cycle=${nth_esfc} export npe_node_cycle=$(echo "${npe_node_max} / ${nth_cycle}" | bc) - export memory_esfc="60GB" + export memory_esfc="80GB" elif [ ${step} = "efcs" ]; then From 0325c13519bf2be362e928504968f9a3f9afcb95 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Thu, 20 Oct 2022 13:53:20 +0000 Subject: [PATCH 37/42] Address SC2250 linter warnings Refs #419 --- parm/config/config.fv3 | 42 ++++++++++++++++++------------------ parm/config/config.resources | 20 ++++++++--------- 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/parm/config/config.fv3 b/parm/config/config.fv3 index 07fc468bfe..27d24e9edb 100755 --- a/parm/config/config.fv3 +++ b/parm/config/config.fv3 @@ -20,30 +20,30 @@ case_in=$1 echo "BEGIN: config.fv3" -if [[ "$machine" = "WCOSS2" ]]; then +if [[ "${machine}" = "WCOSS2" ]]; then export npe_node_max=128 -elif [[ "$machine" = "JET" ]]; then - if [[ "$PARTITION_BATCH" = "xjet" ]]; then +elif [[ "${machine}" = "JET" ]]; then + if [[ "${PARTITION_BATCH}" = "xjet" ]]; then export npe_node_max=24 - elif [[ "$PARTITION_BATCH" = "vjet" || "$PARTITION_BATCH" = "sjet" ]]; then + elif [[ "${PARTITION_BATCH}" = "vjet" || "${PARTITION_BATCH}" = "sjet" ]]; then export npe_node_max=16 - elif [[ "$PARTITION_BATCH" = "kjet" ]]; then + elif [[ "${PARTITION_BATCH}" = "kjet" ]]; then export npe_node_max=40 fi -elif [[ "$machine" = "HERA" ]]; then +elif [[ "${machine}" = "HERA" ]]; then export npe_node_max=40 -elif [[ "$machine" = "S4" ]]; then - if [[ "$PARTITION_BATCH" = "s4" ]]; then +elif [[ "${machine}" = "S4" ]]; then + if [[ "${PARTITION_BATCH}" = "s4" ]]; then export npe_node_max=32 - elif [[ "$PARTITION_BATCH" = "ivy" ]]; then + elif [[ "${PARTITION_BATCH}" = "ivy" ]]; then export npe_node_max=20 fi -elif [[ "$machine" = "ORION" ]]; then +elif [[ "${machine}" = "ORION" ]]; then export npe_node_max=40 fi # (Standard) Model resolution dependent variables -case $case_in in +case ${case_in} in "C48") export DELTIM=450 export layout_x=3 @@ -55,10 +55,10 @@ case $case_in in export cdmbgwd="0.071,2.1,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export WRITE_GROUP=1 export WRTTASK_PER_GROUP=64 - if [[ "$WRTTASK_PER_GROUP" -gt "$npe_node_max" ]]; then export WRTTASK_PER_GROUP=$npe_node_max ; fi + if [[ "${WRTTASK_PER_GROUP}" -gt "${npe_node_max}" ]]; then export WRTTASK_PER_GROUP=${npe_node_max} ; fi export WRITE_GROUP_GFS=1 export WRTTASK_PER_GROUP_GFS=64 - if [[ "$WRTTASK_PER_GROUP_GFS" -gt "$npe_node_max" ]]; then export WRTTASK_PER_GROUP_GFS=$npe_node_max ; fi + if [[ "${WRTTASK_PER_GROUP_GFS}" -gt "${npe_node_max}" ]]; then export WRTTASK_PER_GROUP_GFS=${npe_node_max} ; fi export WRTIOBUF="1M" ;; "C96") @@ -72,10 +72,10 @@ case $case_in in export cdmbgwd="0.14,1.8,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export WRITE_GROUP=1 export WRTTASK_PER_GROUP=64 - if [[ "$WRTTASK_PER_GROUP" -gt "$npe_node_max" ]]; then export WRTTASK_PER_GROUP=$npe_node_max ; fi + if [[ "${WRTTASK_PER_GROUP}" -gt "${npe_node_max}" ]]; then export WRTTASK_PER_GROUP=${npe_node_max} ; fi export WRITE_GROUP_GFS=1 export WRTTASK_PER_GROUP_GFS=64 - if [[ "$WRTTASK_PER_GROUP_GFS" -gt "$npe_node_max" ]]; then export WRTTASK_PER_GROUP_GFS=$npe_node_max ; fi + if [[ "${WRTTASK_PER_GROUP_GFS}" -gt "${npe_node_max}" ]]; then export WRTTASK_PER_GROUP_GFS=${npe_node_max} ; fi export WRTIOBUF="4M" export n_split=6 ;; @@ -90,10 +90,10 @@ case $case_in in export cdmbgwd="0.23,1.5,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export WRITE_GROUP=1 export WRTTASK_PER_GROUP=64 - if [[ "$WRTTASK_PER_GROUP" -gt "$npe_node_max" ]]; then export WRTTASK_PER_GROUP=$npe_node_max ; fi + if [[ "${WRTTASK_PER_GROUP}" -gt "${npe_node_max}" ]]; then export WRTTASK_PER_GROUP=${npe_node_max} ; fi export WRITE_GROUP_GFS=2 export WRTTASK_PER_GROUP_GFS=64 - if [[ "$WRTTASK_PER_GROUP_GFS" -gt "$npe_node_max" ]]; then export WRTTASK_PER_GROUP_GFS=$npe_node_max ; fi + if [[ "${WRTTASK_PER_GROUP_GFS}" -gt "${npe_node_max}" ]]; then export WRTTASK_PER_GROUP_GFS=${npe_node_max} ; fi export WRTIOBUF="8M" ;; "C384") @@ -107,10 +107,10 @@ case $case_in in export cdmbgwd="1.1,0.72,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export WRITE_GROUP=2 export WRTTASK_PER_GROUP=64 - if [[ "$WRTTASK_PER_GROUP" -gt "$npe_node_max" ]]; then export WRTTASK_PER_GROUP=$npe_node_max ; fi + if [[ "${WRTTASK_PER_GROUP}" -gt "${npe_node_max}" ]]; then export WRTTASK_PER_GROUP=${npe_node_max} ; fi export WRITE_GROUP_GFS=${WRITE_GROUP_GFS:-2} export WRTTASK_PER_GROUP_GFS=${WRTTASK_PER_GROUP_GFS:-64} - if [[ "$WRTTASK_PER_GROUP_GFS" -gt "$npe_node_max" ]]; then export WRTTASK_PER_GROUP_GFS=$npe_node_max ; fi + if [[ "${WRTTASK_PER_GROUP_GFS}" -gt "${npe_node_max}" ]]; then export WRTTASK_PER_GROUP_GFS=${npe_node_max} ; fi export WRTIOBUF=${WRTIOBUF:-"16M"} ;; "C768") @@ -124,10 +124,10 @@ case $case_in in export cdmbgwd="4.0,0.15,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export WRITE_GROUP=2 export WRTTASK_PER_GROUP=64 - if [[ "$WRTTASK_PER_GROUP" -gt "$npe_node_max" ]]; then export WRTTASK_PER_GROUP=$npe_node_max ; fi + if [[ "${WRTTASK_PER_GROUP}" -gt "${npe_node_max}" ]]; then export WRTTASK_PER_GROUP=${npe_node_max} ; fi export WRITE_GROUP_GFS=4 export WRTTASK_PER_GROUP_GFS=64 - if [[ "$WRTTASK_PER_GROUP_GFS" -gt "$npe_node_max" ]]; then export WRTTASK_PER_GROUP_GFS=$npe_node_max ; fi + if [[ "${WRTTASK_PER_GROUP_GFS}" -gt "${npe_node_max}" ]]; then export WRTTASK_PER_GROUP_GFS=${npe_node_max} ; fi export WRTIOBUF="32M" ;; "C1152") diff --git a/parm/config/config.resources b/parm/config/config.resources index 17b9131e49..7ee4d5305c 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -54,7 +54,7 @@ if [ ${step} = "prep" ]; then export npe_prep=4 export npe_node_prep=2 export nth_prep=1 - if [[ "$machine" = "WCOSS2" ]]; then + if [[ "${machine}" = "WCOSS2" ]]; then export is_exclusive=True else export memory_prep="40G" @@ -87,7 +87,7 @@ elif [ ${step} = "waveprep" ]; then export npe_node_waveprep=$(echo "${npe_node_max} / ${nth_waveprep}" | bc) export npe_node_waveprep_gfs=$(echo "${npe_node_max} / ${nth_waveprep_gfs}" | bc) export NTASKS=${npe_waveprep} - export NTASKS_gfs=$npe_waveprep_gfs + export NTASKS_gfs=${npe_waveprep_gfs} export memory_waveprep="100GB" export memory_waveprep_gfs="150GB" @@ -235,11 +235,11 @@ elif [ ${step} = "anal" ]; then export nth_anal=5 export npe_anal_gfs=825 export nth_anal_gfs=5 - if [[ "$machine" = "WCOSS2" ]]; then + if [[ "${machine}" = "WCOSS2" ]]; then export nth_anal=8 export nth_anal_gfs=8 fi - if [ $CASE = "C384" ]; then + if [ ${CASE} = "C384" ]; then export npe_anal=160 export npe_anal_gfs=160 export nth_anal=10 @@ -447,7 +447,7 @@ elif [ ${step} = "wafs" ]; then export wtime_wafs="00:30:00" export npe_wafs=1 - export npe_node_wafs=$npe_wafs + export npe_node_wafs=${npe_wafs} export nth_wafs=1 export memory_wafs="1GB" @@ -522,7 +522,7 @@ elif [ ${step} = "echgres" ]; then export npe_echgres=3 export nth_echgres=${npe_node_max} export npe_node_echgres=1 - if [[ "$machine" = "WCOSS2" ]]; then + if [[ "${machine}" = "WCOSS2" ]]; then export memory_echgres="200GB" fi @@ -633,17 +633,17 @@ elif [ ${step} = "eupd" ]; then if [ ${CASE} = "C768" ]; then export npe_eupd=480 export nth_eupd=6 - if [[ "$machine" = "WCOSS2" ]]; then + if [[ "${machine}" = "WCOSS2" ]]; then export npe_eupd=315 export nth_eupd=14 fi elif [ ${CASE} = "C384" ]; then export npe_eupd=270 export nth_eupd=2 - if [[ "$machine" = "WCOSS2" ]]; then + if [[ "${machine}" = "WCOSS2" ]]; then export npe_eupd=315 export nth_eupd=14 - elif [[ "$machine" = "HERA" ]]; then + elif [[ "${machine}" = "HERA" ]]; then export nth_eupd=8 fi if [[ ${machine} = "S4" ]]; then @@ -698,7 +698,7 @@ elif [ ${step} = "epos" ]; then export wtime_epos="00:15:00" export npe_epos=80 export nth_epos=4 - if [[ "$machine" == "HERA" ]]; then + if [[ "${machine}" == "HERA" ]]; then export nth_epos=6 fi export npe_node_epos=$(echo "${npe_node_max} / ${nth_epos}" | bc) From 0675b4ac12b043ad74e8acecc57c2729e1dac619 Mon Sep 17 00:00:00 2001 From: Kate Friedman Date: Thu, 20 Oct 2022 15:26:42 -0400 Subject: [PATCH 38/42] Add partition_service to WCOSS2 hosts file Matches updates made to other hosts files recently. Refs #419 --- workflow/hosts/wcoss2.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/workflow/hosts/wcoss2.yaml b/workflow/hosts/wcoss2.yaml index a729620dde..4417b22ab2 100644 --- a/workflow/hosts/wcoss2.yaml +++ b/workflow/hosts/wcoss2.yaml @@ -12,6 +12,7 @@ scheduler: pbspro queue: 'dev' queue_service: 'dev_transfer' partition_batch: '' +partition_service: '' chgrp_rstprod: 'YES' chgrp_cmd: 'chgrp rstprod' hpssarch: 'YES' From c2b8cdf31aac0da8d2634ec577a69cb5599140b2 Mon Sep 17 00:00:00 2001 From: Kate Friedman Date: Fri, 21 Oct 2022 10:39:11 -0400 Subject: [PATCH 39/42] Increase esfc memory to 80GB Refs #419 --- ecf/scripts/enkfgdas/analysis/recenter/jenkfgdas_sfc.ecf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecf/scripts/enkfgdas/analysis/recenter/jenkfgdas_sfc.ecf b/ecf/scripts/enkfgdas/analysis/recenter/jenkfgdas_sfc.ecf index 6928765c1f..39d4ec2e8d 100755 --- a/ecf/scripts/enkfgdas/analysis/recenter/jenkfgdas_sfc.ecf +++ b/ecf/scripts/enkfgdas/analysis/recenter/jenkfgdas_sfc.ecf @@ -4,7 +4,7 @@ #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:06:00 -#PBS -l select=1:mpiprocs=80:ompthreads=1:ncpus=80:mem=60GB +#PBS -l select=1:mpiprocs=80:ompthreads=1:ncpus=80:mem=80GB #PBS -l place=vscatter #PBS -l debug=true From 5a6d06ea5753c2c4137096e5391ce4f601a5c0ac Mon Sep 17 00:00:00 2001 From: Kate Friedman Date: Mon, 24 Oct 2022 10:44:38 -0400 Subject: [PATCH 40/42] Update build_ufs.sh to force S2SW on WCOSS2 - Aerosols not yet supported on WCOSS2 so can't build with S2SWA app there. - Force UFS to build with the S2SW app on WCOSS2 for now. Refs #419 --- sorc/build_ufs.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/sorc/build_ufs.sh b/sorc/build_ufs.sh index df1378ca66..21779cdefc 100755 --- a/sorc/build_ufs.sh +++ b/sorc/build_ufs.sh @@ -7,6 +7,13 @@ cwd=$(pwd) APP="S2SWA" CCPP_SUITES="FV3_GFS_v16,FV3_GFS_v16_ugwpv1,FV3_GFS_v17_p8,FV3_GFS_v16_coupled_nsstNoahmpUGWPv1,FV3_GFS_v17_coupled_p8" +export RT_COMPILER="intel" +source $cwd/ufs_model.fd/tests/detect_machine.sh + +if [[ ${MACHINE_ID} =~ WCOSS2.* ]]; then + APP='S2SW' +fi + while getopts ":da:v" option; do case "${option}" in d) BUILD_TYPE="Debug";; @@ -23,8 +30,6 @@ done cd $cwd/ufs_model.fd -export RT_COMPILER="intel" -source $cwd/ufs_model.fd/tests/detect_machine.sh MAKE_OPT="-DAPP=${APP} -DCCPP_SUITES=${CCPP_SUITES}" [[ ${BUILD_TYPE:-"Release"} = "DEBUG" ]] && MAKE_OPT+=" -DDEBUG=ON" COMPILE_NR=0 From 436ad51a60a750e61101eda9dadb625e589a3fe9 Mon Sep 17 00:00:00 2001 From: Kate Friedman Date: Mon, 24 Oct 2022 10:56:12 -0400 Subject: [PATCH 41/42] Change WCOSS2 to lowercase wcoss2 in build_ufs.sh Refs #419 --- sorc/build_ufs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/build_ufs.sh b/sorc/build_ufs.sh index 21779cdefc..7c8265b3d8 100755 --- a/sorc/build_ufs.sh +++ b/sorc/build_ufs.sh @@ -10,7 +10,7 @@ CCPP_SUITES="FV3_GFS_v16,FV3_GFS_v16_ugwpv1,FV3_GFS_v17_p8,FV3_GFS_v16_coupled_n export RT_COMPILER="intel" source $cwd/ufs_model.fd/tests/detect_machine.sh -if [[ ${MACHINE_ID} =~ WCOSS2.* ]]; then +if [[ ${MACHINE_ID} =~ wcoss2.* ]]; then APP='S2SW' fi From a0c450f6c4ce079047a70b11dd545e7750a595b3 Mon Sep 17 00:00:00 2001 From: Kate Friedman Date: Mon, 24 Oct 2022 11:14:42 -0400 Subject: [PATCH 42/42] Set hpssarch to NO by default on WCOSS2 - Update WCOSS2 hosts yaml file to set `hpssarch` to "NO" by default. - Limited bandwidth between WCOSS2 to HPSS so don't want users archiving to HPSS unless necessary. Refs #419 --- workflow/hosts/wcoss2.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/hosts/wcoss2.yaml b/workflow/hosts/wcoss2.yaml index 4417b22ab2..592fa3db79 100644 --- a/workflow/hosts/wcoss2.yaml +++ b/workflow/hosts/wcoss2.yaml @@ -15,7 +15,7 @@ partition_batch: '' partition_service: '' chgrp_rstprod: 'YES' chgrp_cmd: 'chgrp rstprod' -hpssarch: 'YES' +hpssarch: 'NO' localarch: 'NO' atardir: '/NCEPDEV/$HPSS_PROJECT/1year/$USER/$machine/scratch/$PSLOT' make_nsstbufr: 'NO'