Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

MPMD variable updates and fix #1124

Merged
merged 4 commits into from
Nov 17, 2022
+18 −17
Merged

MPMD variable updates and fix #1124

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
ca5c977
Correct wave_mpmd setting in JGLOBAL_WAVE_INIT
KateFriedman-NOAA Nov 17, 2022
1346696
Update Jet and S4 env files for mpmd_opt
KateFriedman-NOAA Nov 17, 2022
0a21c85
Rename mpmd to mpmd_opt in WCOSS2 env file
KateFriedman-NOAA Nov 17, 2022
613898f
Merge branch 'NOAA-EMC:develop' into feature/wave-mpmd-fix
KateFriedman-NOAA Nov 17, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion env/JET.env
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ elif [[ "$PARTITION_BATCH" = "vjet" ]]; then
export npe_node_max=16
fi
export launcher="srun -l --export=ALL"
export mpmd_opt="--multi-prog"

# Configure STACK
export OMP_STACKSIZE=2048000
Expand All @@ -43,7 +44,7 @@ elif [ $step = "waveinit" -o $step = "waveprep" -o $step = "wavepostsbs" -o $ste

if [ $step = "waveprep" ]; then export MP_PULSE=0 ; fi
export wavempexec=${launcher}
export wave_mpmd=${mpmd}
export wave_mpmd=${mpmd_opt}

elif [ $step = "atmanalrun" ]; then

Expand Down
4 changes: 2 additions & 2 deletions env/S4.env
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ elif [[ ${PARTITION_BATCH} = "ivy" ]]; then
export npe_node_max=20
fi
export launcher="srun -l --export=ALL"
export mpmd_opt="--multi-prog"

# Configure MPI environment
export OMP_STACKSIZE=2048000
Expand All @@ -45,11 +46,10 @@ if [[ ${step} = "prep" || ${step} = "prepbufr" ]]; then

elif [[ ${step} = "waveinit" || ${step} = "waveprep" || ${step} = "wavepostsbs" || ${step} = "wavepostbndpnt" || ${step} = "wavepostbndpntbll" || ${step} = "wavepostpnt" ]]; then

export mpmd="--multi-prog"
export CFP_MP="YES"
if [[ ${step} = "waveprep" ]]; then export MP_PULSE=0 ; fi
export wavempexec=${launcher}
export wave_mpmd=${mpmd}
export wave_mpmd=${mpmd_opt}

elif [[ ${step} = "atmanalrun" ]]; then

Expand Down
26 changes: 13 additions & 13 deletions env/WCOSS2.env
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ step=$1

# WCOSS2 information
export launcher="mpiexec -l"
export mpmd="--cpu-bind verbose,core cfp"
export mpmd_opt="--cpu-bind verbose,core cfp"

export npe_node_max=128

Expand All @@ -37,13 +37,13 @@ elif [ $step = "waveinit" -o $step = "waveprep" -o $step = "wavepostsbs" -o $ste

if [ $step = "waveprep" -a $CDUMP = "gfs" ]; then NTASKS=$NTASKS_gfs ; fi
export wavempexec="$launcher -np"
export wave_mpmd=${mpmd}
export wave_mpmd=${mpmd_opt}

elif [ $step = "atmanalrun" ]; then

export CFP_MP=${CFP_MP:-"YES"}
export USE_CFP=${USE_CFP:-"YES"}
export APRUNCFP="$launcher -np \$ncmd $mpmd"
export APRUNCFP="$launcher -np \$ncmd ${mpmd_opt}"

nth_max=$(($npe_node_max / $npe_node_atmanalrun))

Expand All @@ -55,7 +55,7 @@ elif [ $step = "atmensanalrun" ]; then

export CFP_MP=${CFP_MP:-"YES"}
export USE_CFP=${USE_CFP:-"YES"}
export APRUNCFP="$launcher -np \$ncmd $mpmd"
export APRUNCFP="$launcher -np \$ncmd ${mpmd_opt}"

nth_max=$(($npe_node_max / $npe_node_atmensanalrun))

Expand All @@ -65,7 +65,7 @@ elif [ $step = "atmensanalrun" ]; then

elif [ $step = "aeroanlrun" ]; then

export APRUNCFP="$launcher -np \$ncmd $mpmd"
export APRUNCFP="$launcher -np \$ncmd ${mpmd_opt}"

nth_max=$(($npe_node_max / $npe_node_aeroanlrun))

Expand Down Expand Up @@ -108,7 +108,7 @@ elif [ $step = "anal" -o $step = "analcalc" ]; then

export CFP_MP=${CFP_MP:-"NO"}
export USE_CFP=${USE_CFP:-"YES"}
export APRUNCFP="$launcher -np \$ncmd $mpmd"
export APRUNCFP="$launcher -np \$ncmd ${mpmd_opt}"

elif [ $step = "sfcanl" ]; then

Expand Down Expand Up @@ -138,7 +138,7 @@ elif [ $step = "gldas" ]; then
# periods being processed.
export USE_CFP=${USE_CFP:-"YES"}
npe_gldas_data_proc=$(($gldas_spinup_hours + 12))
export APRUN_GLDAS_DATA_PROC="$launcher -np $npe_gldas_data_proc $mpmd"
export APRUN_GLDAS_DATA_PROC="$launcher -np $npe_gldas_data_proc ${mpmd_opt}"

elif [ $step = "eobs" ]; then

Expand All @@ -154,7 +154,7 @@ elif [ $step = "eobs" ]; then

export CFP_MP=${CFP_MP:-"NO"}
export USE_CFP=${USE_CFP:-"YES"}
export APRUNCFP="$launcher -np \$ncmd $mpmd"
export APRUNCFP="$launcher -np \$ncmd ${mpmd_opt}"

elif [ $step = "eupd" ]; then

Expand All @@ -171,7 +171,7 @@ elif [ $step = "eupd" ]; then

export CFP_MP=${CFP_MP:-"NO"}
export USE_CFP=${USE_CFP:-"YES"}
export APRUNCFP="$launcher -np \$ncmd $mpmd"
export APRUNCFP="$launcher -np \$ncmd ${mpmd_opt}"

elif [ $step = "fcst" ]; then

Expand Down Expand Up @@ -231,7 +231,7 @@ elif [ $step = "post" ]; then

export NTHREADS_DWN=${nth_dwn:-1}
[[ $NTHREADS_DWN -gt $nth_max ]] && export NTHREADS_DWN=$nth_max
export APRUN_DWN="$launcher -np $npe_dwn $mpmd"
export APRUN_DWN="$launcher -np $npe_dwn ${mpmd_opt}"

elif [ $step = "ecen" ]; then

Expand Down Expand Up @@ -290,23 +290,23 @@ elif [ $step = "postsnd" ]; then

export NTHREADS_POSTSNDCFP=${nth_postsndcfp:-1}
[[ $NTHREADS_POSTSNDCFP -gt $nth_max ]] && export NTHREADS_POSTSNDCFP=$nth_max
export APRUN_POSTSNDCFP="$launcher -np $npe_postsndcfp $mpmd"
export APRUN_POSTSNDCFP="$launcher -np $npe_postsndcfp ${mpmd_opt}"

elif [ $step = "awips" ]; then

nth_max=$(($npe_node_max / $npe_node_awips))

export NTHREADS_AWIPS=${nth_awips:-2}
[[ $NTHREADS_AWIPS -gt $nth_max ]] && export NTHREADS_AWIPS=$nth_max
export APRUN_AWIPSCFP="$launcher -np $npe_awips $mpmd"
export APRUN_AWIPSCFP="$launcher -np $npe_awips ${mpmd_opt}"

elif [ $step = "gempak" ]; then

nth_max=$(($npe_node_max / $npe_node_gempak))

export NTHREADS_GEMPAK=${nth_gempak:-1}
[[ $NTHREADS_GEMPAK -gt $nth_max ]] && export NTHREADS_GEMPAK=$nth_max
export APRUN_GEMPAKCFP="$launcher -np $npe_gempak $mpmd"
export APRUN_GEMPAKCFP="$launcher -np $npe_gempak ${mpmd_opt}"

elif [ $step = "waveawipsbulls" ]; then

Expand Down
2 changes: 1 addition & 1 deletion jobs/JGLOBAL_WAVE_INIT
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ fi

# Set mpi serial command
export wavempexec=${wavempexec:-"mpirun -n"}
export wave_mpmd=${mpmd:-"cfp"}
export wave_mpmd=${wave_mpmd:-"cfp"}

# Execute the Script
$HOMEgfs/scripts/exgfs_wave_init.sh
Expand Down