From ca5c977acb165d75490f083d90295144d73fe150 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Thu, 17 Nov 2022 08:20:38 -0600 Subject: [PATCH 1/3] Correct wave_mpmd setting in JGLOBAL_WAVE_INIT - When the mpmd variable in the R&D env files was renamed to mpmd_opt the wave_mpmd setting in JGLOBAL_WAVE_INIT was not updated to match and thus broke the job when tested. - Update the wave_mpmd setting in JGLOBAL_WAVE_INIT to use the wave_mpmd setting defined in the env files instead of the old mpmd variable. Refs #1060 --- jobs/JGLOBAL_WAVE_INIT | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/JGLOBAL_WAVE_INIT b/jobs/JGLOBAL_WAVE_INIT index 52fbc80110..c111330ca9 100755 --- a/jobs/JGLOBAL_WAVE_INIT +++ b/jobs/JGLOBAL_WAVE_INIT @@ -64,7 +64,7 @@ fi # Set mpi serial command export wavempexec=${wavempexec:-"mpirun -n"} -export wave_mpmd=${mpmd:-"cfp"} +export wave_mpmd=${wave_mpmd:-"cfp"} # Execute the Script $HOMEgfs/scripts/exgfs_wave_init.sh From 13466962c5010686ce9b54501ac4c1c3c9fdddf9 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Thu, 17 Nov 2022 08:32:09 -0600 Subject: [PATCH 2/3] Update Jet and S4 env files for mpmd_opt - Make matching changes to Jet and S4 env files to set mpmd_opt and use it in launcher commands in place of prior mpmd variable. Refs #1060 --- env/JET.env | 3 ++- env/S4.env | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/env/JET.env b/env/JET.env index 21321b23c9..a5d3c7d1d2 100755 --- a/env/JET.env +++ b/env/JET.env @@ -20,6 +20,7 @@ elif [[ "$PARTITION_BATCH" = "vjet" ]]; then export npe_node_max=16 fi export launcher="srun -l --export=ALL" +export mpmd_opt="--multi-prog" # Configure STACK export OMP_STACKSIZE=2048000 @@ -43,7 +44,7 @@ elif [ $step = "waveinit" -o $step = "waveprep" -o $step = "wavepostsbs" -o $ste if [ $step = "waveprep" ]; then export MP_PULSE=0 ; fi export wavempexec=${launcher} - export wave_mpmd=${mpmd} + export wave_mpmd=${mpmd_opt} elif [ $step = "atmanalrun" ]; then diff --git a/env/S4.env b/env/S4.env index a3faabaecf..16b7df0825 100755 --- a/env/S4.env +++ b/env/S4.env @@ -21,6 +21,7 @@ elif [[ ${PARTITION_BATCH} = "ivy" ]]; then export npe_node_max=20 fi export launcher="srun -l --export=ALL" +export mpmd_opt="--multi-prog" # Configure MPI environment export OMP_STACKSIZE=2048000 @@ -45,11 +46,10 @@ if [[ ${step} = "prep" || ${step} = "prepbufr" ]]; then elif [[ ${step} = "waveinit" || ${step} = "waveprep" || ${step} = "wavepostsbs" || ${step} = "wavepostbndpnt" || ${step} = "wavepostbndpntbll" || ${step} = "wavepostpnt" ]]; then - export mpmd="--multi-prog" export CFP_MP="YES" if [[ ${step} = "waveprep" ]]; then export MP_PULSE=0 ; fi export wavempexec=${launcher} - export wave_mpmd=${mpmd} + export wave_mpmd=${mpmd_opt} elif [[ ${step} = "atmanalrun" ]]; then From 0a21c85b3f7bc5d71585aae91aa286c576b9b6ce Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Thu, 17 Nov 2022 08:34:23 -0600 Subject: [PATCH 3/3] Rename mpmd to mpmd_opt in WCOSS2 env file Refs #1060 --- env/WCOSS2.env | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/env/WCOSS2.env b/env/WCOSS2.env index 71f967181c..eb9969c429 100755 --- a/env/WCOSS2.env +++ b/env/WCOSS2.env @@ -17,7 +17,7 @@ step=$1 # WCOSS2 information export launcher="mpiexec -l" -export mpmd="--cpu-bind verbose,core cfp" +export mpmd_opt="--cpu-bind verbose,core cfp" export npe_node_max=128 @@ -37,13 +37,13 @@ elif [ $step = "waveinit" -o $step = "waveprep" -o $step = "wavepostsbs" -o $ste if [ $step = "waveprep" -a $CDUMP = "gfs" ]; then NTASKS=$NTASKS_gfs ; fi export wavempexec="$launcher -np" - export wave_mpmd=${mpmd} + export wave_mpmd=${mpmd_opt} elif [ $step = "atmanalrun" ]; then export CFP_MP=${CFP_MP:-"YES"} export USE_CFP=${USE_CFP:-"YES"} - export APRUNCFP="$launcher -np \$ncmd $mpmd" + export APRUNCFP="$launcher -np \$ncmd ${mpmd_opt}" nth_max=$(($npe_node_max / $npe_node_atmanalrun)) @@ -55,7 +55,7 @@ elif [ $step = "atmensanalrun" ]; then export CFP_MP=${CFP_MP:-"YES"} export USE_CFP=${USE_CFP:-"YES"} - export APRUNCFP="$launcher -np \$ncmd $mpmd" + export APRUNCFP="$launcher -np \$ncmd ${mpmd_opt}" nth_max=$(($npe_node_max / $npe_node_atmensanalrun)) @@ -65,7 +65,7 @@ elif [ $step = "atmensanalrun" ]; then elif [ $step = "aeroanlrun" ]; then - export APRUNCFP="$launcher -np \$ncmd $mpmd" + export APRUNCFP="$launcher -np \$ncmd ${mpmd_opt}" nth_max=$(($npe_node_max / $npe_node_aeroanlrun)) @@ -108,7 +108,7 @@ elif [ $step = "anal" -o $step = "analcalc" ]; then export CFP_MP=${CFP_MP:-"NO"} export USE_CFP=${USE_CFP:-"YES"} - export APRUNCFP="$launcher -np \$ncmd $mpmd" + export APRUNCFP="$launcher -np \$ncmd ${mpmd_opt}" elif [ $step = "sfcanl" ]; then @@ -138,7 +138,7 @@ elif [ $step = "gldas" ]; then # periods being processed. export USE_CFP=${USE_CFP:-"YES"} npe_gldas_data_proc=$(($gldas_spinup_hours + 12)) - export APRUN_GLDAS_DATA_PROC="$launcher -np $npe_gldas_data_proc $mpmd" + export APRUN_GLDAS_DATA_PROC="$launcher -np $npe_gldas_data_proc ${mpmd_opt}" elif [ $step = "eobs" ]; then @@ -154,7 +154,7 @@ elif [ $step = "eobs" ]; then export CFP_MP=${CFP_MP:-"NO"} export USE_CFP=${USE_CFP:-"YES"} - export APRUNCFP="$launcher -np \$ncmd $mpmd" + export APRUNCFP="$launcher -np \$ncmd ${mpmd_opt}" elif [ $step = "eupd" ]; then @@ -171,7 +171,7 @@ elif [ $step = "eupd" ]; then export CFP_MP=${CFP_MP:-"NO"} export USE_CFP=${USE_CFP:-"YES"} - export APRUNCFP="$launcher -np \$ncmd $mpmd" + export APRUNCFP="$launcher -np \$ncmd ${mpmd_opt}" elif [ $step = "fcst" ]; then @@ -231,7 +231,7 @@ elif [ $step = "post" ]; then export NTHREADS_DWN=${nth_dwn:-1} [[ $NTHREADS_DWN -gt $nth_max ]] && export NTHREADS_DWN=$nth_max - export APRUN_DWN="$launcher -np $npe_dwn $mpmd" + export APRUN_DWN="$launcher -np $npe_dwn ${mpmd_opt}" elif [ $step = "ecen" ]; then @@ -290,7 +290,7 @@ elif [ $step = "postsnd" ]; then export NTHREADS_POSTSNDCFP=${nth_postsndcfp:-1} [[ $NTHREADS_POSTSNDCFP -gt $nth_max ]] && export NTHREADS_POSTSNDCFP=$nth_max - export APRUN_POSTSNDCFP="$launcher -np $npe_postsndcfp $mpmd" + export APRUN_POSTSNDCFP="$launcher -np $npe_postsndcfp ${mpmd_opt}" elif [ $step = "awips" ]; then @@ -298,7 +298,7 @@ elif [ $step = "awips" ]; then export NTHREADS_AWIPS=${nth_awips:-2} [[ $NTHREADS_AWIPS -gt $nth_max ]] && export NTHREADS_AWIPS=$nth_max - export APRUN_AWIPSCFP="$launcher -np $npe_awips $mpmd" + export APRUN_AWIPSCFP="$launcher -np $npe_awips ${mpmd_opt}" elif [ $step = "gempak" ]; then @@ -306,7 +306,7 @@ elif [ $step = "gempak" ]; then export NTHREADS_GEMPAK=${nth_gempak:-1} [[ $NTHREADS_GEMPAK -gt $nth_max ]] && export NTHREADS_GEMPAK=$nth_max - export APRUN_GEMPAKCFP="$launcher -np $npe_gempak $mpmd" + export APRUN_GEMPAKCFP="$launcher -np $npe_gempak ${mpmd_opt}" elif [ $step = "waveawipsbulls" ]; then