From 2325c873089a8eecab2d906fcfa922ec80932fc6 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Tue, 16 Jan 2024 19:23:21 +0000 Subject: [PATCH 1/3] Add POSTAMBLE_CMD to postamble function in preamble - Will invoke postamble command if set. - Included comments and instructions. Refs #1145 --- ush/preamble.sh | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/ush/preamble.sh b/ush/preamble.sh index be64684aa8..d10d7541c8 100644 --- a/ush/preamble.sh +++ b/ush/preamble.sh @@ -16,6 +16,8 @@ # TRACE (YES/NO): Whether to echo every command (set -x) [default: "YES"] # STRICT (YES/NO): Whether to exit immediately on error or undefined variable # (set -eu) [default: "YES"] +# POSTAMBLE_CMD (empty/set): A command to run at the end of the job +# [default: empty] # ####### set +x @@ -70,6 +72,24 @@ postamble() { start_time="${2}" rc="${3}" + # Execute postamble command + # + # Commands can be added to the postamble by appending them to $POSTAMBLE_CMD: + # POSTAMBLE_CMD="new_thing; ${POSTAMBLE_CMD:-}" # (before existing commands) + # POSTAMBLE_CMD="${POSTAMBLE_CMD:-}; new_thing" # (after existing commands) + # + # Always use this form so previous POSTAMBLE_CMD are not overwritten. This should + # only be used for commands that execute conditionally (i.e. on certain machines + # or jobs). Global changes should just be added to this function. + # These commands will be called when EACH SCRIPT terminates, so be mindful. Please + # consult with global-workflow CMs about permanent changes to $POSTAMBLE_CMD or + # this postamble function. + # + + if [[ -v 'POSTAMBLE_CMD' ]]; then + ${POSTAMBLE_CMD} + fi + # Calculate the elapsed time end_time=$(date +%s) end_time_human=$(date -d@"${end_time}" -u +%H:%M:%S) From 6caaa2882d409c00b4db57c9ee12c6fbc1f4b9d0 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Tue, 16 Jan 2024 19:25:13 +0000 Subject: [PATCH 2/3] Add POSTAMBLE_CMD to HERA.env - Set initial value of POSTAMBLE_CMD to 'report-mem'. Refs #1145 --- env/HERA.env | 2 ++ 1 file changed, 2 insertions(+) diff --git a/env/HERA.env b/env/HERA.env index 284d54820e..e06abf9b68 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -18,6 +18,8 @@ export npe_node_max=40 export launcher="srun -l --export=ALL" export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out" +export POSTAMBLE_CMD='report-mem' + # Configure MPI environment #export I_MPI_ADJUST_ALLREDUCE=5 #export MPI_BUFS_PER_PROC=2048 From 57e273c590cae33b9f0ae47a700132a727f929cc Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Wed, 17 Jan 2024 18:38:28 +0000 Subject: [PATCH 3/3] Comment out POSTAMBLE_CMD in HERA.env Disable POSTAMBLE_CMD in HERA.env for now. Leave it as an example. Refs #1145 --- env/HERA.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/env/HERA.env b/env/HERA.env index e06abf9b68..fb156645f8 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -18,7 +18,7 @@ export npe_node_max=40 export launcher="srun -l --export=ALL" export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out" -export POSTAMBLE_CMD='report-mem' +#export POSTAMBLE_CMD='report-mem' # Configure MPI environment #export I_MPI_ADJUST_ALLREDUCE=5