Skip to content

Commit

Permalink
Update R&D launcher commands for tasks and multi-prog (#1112)
Browse files Browse the repository at this point in the history 
* Update multi-prog in HERA.env and ORION.env
* Update launcher commands in HERA.env and ORION.env
* Adjust C96 & C48 eobs resources in config.resources

Refs #1060
  • Loading branch information
@KateFriedman-NOAA
KateFriedman-NOAA authored Nov 15, 2022
1 parent f64f9e5 commit 53706b7
Show file tree
Hide file tree
Showing 3 changed files with 58 additions and 60 deletions.
54 changes: 27 additions & 27 deletions env/HERA.env
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ step=$1

export npe_node_max=40
export launcher="srun -l --export=ALL"
export mpmd_opt="--multi-prog"

# Configure MPI environment
#export I_MPI_ADJUST_ALLREDUCE=5
Expand Down Expand Up @@ -47,17 +48,16 @@ if [ $step = "prep" -o $step = "prepbufr" ]; then

elif [ $step = "waveinit" -o $step = "waveprep" -o $step = "wavepostsbs" -o $step = "wavepostbndpnt" -o $step = "wavepostbndpntbll" -o $step = "wavepostpnt" ]; then

export mpmd="--multi-prog"
export CFP_MP="YES"
if [ $step = "waveprep" ]; then export MP_PULSE=0 ; fi
export wavempexec=${launcher}
export wave_mpmd=${mpmd}
export wave_mpmd=${mpmd_opt}

elif [ $step = "atmanalrun" ]; then

export CFP_MP=${CFP_MP:-"YES"}
export USE_CFP=${USE_CFP:-"YES"}
export APRUNCFP="$launcher -n \$ncmd --multi-prog"
export APRUNCFP="$launcher -n \$ncmd ${mpmd_opt}"

nth_max=$(($npe_node_max / $npe_node_atmanalrun))

Expand All @@ -69,7 +69,7 @@ elif [ $step = "atmensanalrun" ]; then

export CFP_MP=${CFP_MP:-"YES"}
export USE_CFP=${USE_CFP:-"YES"}
export APRUNCFP="$launcher -n \$ncmd --multi-prog"
export APRUNCFP="$launcher -n \$ncmd ${mpmd_opt}"

nth_max=$(($npe_node_max / $npe_node_atmensanalrun))

Expand All @@ -79,7 +79,7 @@ elif [ $step = "atmensanalrun" ]; then

elif [ $step = "aeroanlrun" ]; then

export APRUNCFP="$launcher -n \$ncmd --multi-prog"
export APRUNCFP="$launcher -n \$ncmd ${mpmd_opt}"

nth_max=$(($npe_node_max / $npe_node_aeroanlrun))

Expand All @@ -94,17 +94,17 @@ elif [ $step = "anal" -o $step = "analcalc" ]; then

export CFP_MP=${CFP_MP:-"YES"}
export USE_CFP=${USE_CFP:-"YES"}
export APRUNCFP="$launcher -n \$ncmd --multi-prog"
export APRUNCFP="$launcher -n \$ncmd ${mpmd_opt}"

nth_max=$(($npe_node_max / $npe_node_anal))

export NTHREADS_GSI=${nth_anal:-$nth_max}
[[ $NTHREADS_GSI -gt $nth_max ]] && export NTHREADS_GSI=$nth_max
export APRUN_GSI="$launcher"
export APRUN_GSI="$launcher -n ${npe_gsi:-$npe_anal}"

export NTHREADS_CALCINC=${nth_calcinc:-1}
[[ $NTHREADS_CALCINC -gt $nth_max ]] && export NTHREADS_CALCINC=$nth_max
export APRUN_CALCINC="$launcher"
export APRUN_CALCINC="$launcher \$ncmd"

export NTHREADS_CYCLE=${nth_cycle:-12}
[[ $NTHREADS_CYCLE -gt $npe_node_max ]] && export NTHREADS_CYCLE=$npe_node_max
Expand Down Expand Up @@ -143,7 +143,7 @@ elif [ $step = "gldas" ]; then
# periods being processed.

npe_gldas_data_proc=$(($gldas_spinup_hours + 12))
export APRUN_GLDAS_DATA_PROC="$launcher -n $npe_gldas_data_proc --multi-prog"
export APRUN_GLDAS_DATA_PROC="$launcher -n $npe_gldas_data_proc ${mpmd_opt}"

elif [ $step = "eobs" ]; then

Expand All @@ -154,23 +154,23 @@ elif [ $step = "eobs" ]; then

export NTHREADS_GSI=${nth_eobs:-$nth_max}
[[ $NTHREADS_GSI -gt $nth_max ]] && export NTHREADS_GSI=$nth_max
export APRUN_GSI="$launcher"
export APRUN_GSI="$launcher -n ${npe_gsi:-$npe_eobs}"

export CFP_MP=${CFP_MP:-"YES"}
export USE_CFP=${USE_CFP:-"YES"}
export APRUNCFP="$launcher -n \$ncmd --multi-prog"
export APRUNCFP="$launcher -n \$ncmd ${mpmd_opt}"

elif [ $step = "eupd" ]; then

nth_max=$(($npe_node_max / $npe_node_eupd))

export NTHREADS_ENKF=${nth_eupd:-$nth_max}
[[ $NTHREADS_ENKF -gt $nth_max ]] && export NTHREADS_ENKF=$nth_max
export APRUN_ENKF="$launcher"
export APRUN_ENKF="$launcher -n ${npe_enkf:-$npe_eupd}"

export CFP_MP=${CFP_MP:-"YES"}
export USE_CFP=${USE_CFP:-"YES"}
export APRUNCFP="$launcher -n \$ncmd --multi-prog"
export APRUNCFP="$launcher -n \$ncmd ${mpmd_opt}"

elif [ $step = "fcst" ]; then

Expand All @@ -190,11 +190,11 @@ elif [ $step = "fcst" ]; then

export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1}
[[ $NTHREADS_REGRID_NEMSIO -gt $nth_max ]] && export NTHREADS_REGRID_NEMSIO=$nth_max
export APRUN_REGRID_NEMSIO="$launcher"
export APRUN_REGRID_NEMSIO="$launcher -n ${LEVS}"

export NTHREADS_REMAP=${nth_remap:-2}
[[ $NTHREADS_REMAP -gt $nth_max ]] && export NTHREADS_REMAP=$nth_max
export APRUN_REMAP="$launcher"
export APRUN_REMAP="$launcher -n ${npe_remap:-$npe_fcst}"
export I_MPI_DAPL_UD="enable"

elif [ $step = "efcs" ]; then
Expand All @@ -208,35 +208,35 @@ elif [ $step = "efcs" ]; then

export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1}
[[ $NTHREADS_REGRID_NEMSIO -gt $nth_max ]] && export NTHREADS_REGRID_NEMSIO=$nth_max
export APRUN_REGRID_NEMSIO="$launcher $LEVS"
export APRUN_REGRID_NEMSIO="$launcher -n ${LEVS}"

elif [ $step = "post" ]; then

nth_max=$(($npe_node_max / $npe_node_post))

export NTHREADS_NP=${nth_np:-1}
[[ $NTHREADS_NP -gt $nth_max ]] && export NTHREADS_NP=$nth_max
export APRUN_NP="$launcher"
export APRUN_NP="$launcher -n ${npe_post}"

export NTHREADS_DWN=${nth_dwn:-1}
[[ $NTHREADS_DWN -gt $nth_max ]] && export NTHREADS_DWN=$nth_max
export APRUN_DWN="$launcher"
export APRUN_DWN="$launcher -n ${npe_dwn}"

elif [ $step = "ecen" ]; then

nth_max=$(($npe_node_max / $npe_node_ecen))

export NTHREADS_ECEN=${nth_ecen:-$nth_max}
[[ $NTHREADS_ECEN -gt $nth_max ]] && export NTHREADS_ECEN=$nth_max
export APRUN_ECEN="$launcher"
export APRUN_ECEN="$launcher -n ${npe_ecen}"

export NTHREADS_CHGRES=${nth_chgres:-12}
[[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max
export APRUN_CHGRES="time"

export NTHREADS_CALCINC=${nth_calcinc:-1}
[[ $NTHREADS_CALCINC -gt $nth_max ]] && export NTHREADS_CALCINC=$nth_max
export APRUN_CALCINC="$launcher"
export APRUN_CALCINC="$launcher -n ${npe_ecen}"

elif [ $step = "esfc" ]; then

Expand All @@ -256,37 +256,37 @@ elif [ $step = "epos" ]; then

export NTHREADS_EPOS=${nth_epos:-$nth_max}
[[ $NTHREADS_EPOS -gt $nth_max ]] && export NTHREADS_EPOS=$nth_max
export APRUN_EPOS="$launcher"
export APRUN_EPOS="$launcher -n $npe_epos"

elif [ $step = "init" ]; then

export APRUN="$launcher"
export APRUN="$launcher -n ${npe_init}"

elif [ $step = "postsnd" ]; then

nth_max=$(($npe_node_max / $npe_node_postsnd))

export NTHREADS_POSTSND=${nth_postsnd:-1}
[[ $NTHREADS_POSTSND -gt $nth_max ]] && export NTHREADS_POSTSND=$nth_max
export APRUN_POSTSND="$launcher"
export APRUN_POSTSND="$launcher -n ${npe_postsnd}"

export NTHREADS_POSTSNDCFP=${nth_postsndcfp:-1}
[[ $NTHREADS_POSTSNDCFP -gt $nth_max ]] && export NTHREADS_POSTSNDCFP=$nth_max
export APRUN_POSTSNDCFP="$launcher"
export APRUN_POSTSNDCFP="$launcher -n ${npe_postsnd} ${mpmd_opt}"

elif [ $step = "awips" ]; then

nth_max=$(($npe_node_max / $npe_node_awips))

export NTHREADS_AWIPS=${nth_awips:-2}
[[ $NTHREADS_AWIPS -gt $nth_max ]] && export NTHREADS_AWIPS=$nth_max
export APRUN_AWIPSCFP="$launcher -n $npe_awips --multi-prog"
export APRUN_AWIPSCFP="$launcher -n $npe_awips ${mpmd_opt}"

elif [ $step = "gempak" ]; then

nth_max=$(($npe_node_max / $npe_node_gempak))

export NTHREADS_GEMPAK=${nth_gempak:-1}
[[ $NTHREADS_GEMPAK -gt $nth_max ]] && export NTHREADS_GEMPAK=$nth_max
export APRUN="$launcher -n $npe_gempak --multi-prog"
[[ $NTHREADS_GEMPAK -gt ${nth_max} ]] && export NTHREADS_GEMPAK=${nth_max}
export APRUN="${launcher} -n ${npe_gempak} ${mpmd_opt}"
fi
Loading

0 comments on commit 53706b7

Please sign in to comment.