From dfc1aefa1409c3fd24af4f5adf2a82fb57f9652a Mon Sep 17 00:00:00 2001 From: EdwardSnyder-NOAA <96196752+EdwardSnyder-NOAA@users.noreply.github.com> Date: Thu, 24 Aug 2023 14:12:22 -0500 Subject: [PATCH] [develop] Update wrapper scripts (#883) The wrapper scripts are a set of scripts that run the tasks from a SRW experiment. They are used in a situation where rocoto is not installed or available to use. These set of wrapper scripts have become outdated and don't work in its current state. This PR was created to address this issue. --------- Co-authored-by: Edward Snyder Co-authored-by: Eddie Snyder --- .cicd/Jenkinsfile | 2 +- .cicd/scripts/qsub_srw_ftest.sh | 15 +++++++ .cicd/scripts/sbatch_srw_ftest.sh | 16 ++++++++ .cicd/scripts/srw_ftest.sh | 16 ++++---- .cicd/scripts/wrapper_srw_ftest.sh | 66 ++++++++++++++++++++++++++++++ docs/UsersGuide/source/RunSRW.rst | 8 ---- ush/wrappers/run_fcst.sh | 4 +- ush/wrappers/run_get_ics.sh | 4 +- ush/wrappers/run_get_lbcs.sh | 4 +- ush/wrappers/run_make_grid.sh | 2 +- ush/wrappers/run_make_ics.sh | 5 ++- ush/wrappers/run_make_lbcs.sh | 8 +++- ush/wrappers/run_make_orog.sh | 2 +- ush/wrappers/run_make_sfc_climo.sh | 2 +- ush/wrappers/run_post.sh | 3 +- 15 files changed, 131 insertions(+), 26 deletions(-) create mode 100644 .cicd/scripts/qsub_srw_ftest.sh create mode 100644 .cicd/scripts/sbatch_srw_ftest.sh create mode 100755 .cicd/scripts/wrapper_srw_ftest.sh diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile index 0730e16acd..33c2ece22d 100644 --- a/.cicd/Jenkinsfile +++ b/.cicd/Jenkinsfile @@ -151,7 +151,7 @@ pipeline { stage('Functional WorkflowTaskTests') { steps { echo "Running simple workflow script task tests on ${env.SRW_PLATFORM} (using ${env.WORKSPACE})" - sh 'bash --login "${WORKSPACE}/.cicd/scripts/srw_ftest.sh"' + sh 'bash --login "${WORKSPACE}/.cicd/scripts/wrapper_srw_ftest.sh"' } } diff --git a/.cicd/scripts/qsub_srw_ftest.sh b/.cicd/scripts/qsub_srw_ftest.sh new file mode 100644 index 0000000000..2a669fe61a --- /dev/null +++ b/.cicd/scripts/qsub_srw_ftest.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# +# This is a pbs job card for the srw_ftest.sh script +# and is called by the wrapper_srw_ftest.sh script +# +#PBS -N srw_ftest_run +#PBS -A ${SRW_PROJECT} +#PBS -q main +#PBS -l select=1:ncpus=24:mpiprocs=24:ompthreads=1 +#PBS -l walltime=00:30:00 +#PBS -V +#PBS -o log_wrap.%j.log +#PBS -e err_wrap.%j.err + +bash ${WORKSPACE}/.cicd/scripts/srw_ftest.sh diff --git a/.cicd/scripts/sbatch_srw_ftest.sh b/.cicd/scripts/sbatch_srw_ftest.sh new file mode 100644 index 0000000000..1655148068 --- /dev/null +++ b/.cicd/scripts/sbatch_srw_ftest.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# +# This is a sbatch job card for the srw_ftest.sh script +# and is called by the srw_ftest_wrapper.sh script +# +#SBATCH --job-name=wrapper_test +#SBATCH --account=${SRW_PROJECT} +#SBATCH --qos=batch +#SBATCH --nodes=1 +#SBATCH --tasks-per-node=24 +#SBATCH --cpus-per-task=1 +#SBATCH -t 00:30:00 +#SBATCH -o log_wrap.%j.log +#SBATCH -e err_wrap.%j.err + +bash ${WORKSPACE}/.cicd/scripts/srw_ftest.sh diff --git a/.cicd/scripts/srw_ftest.sh b/.cicd/scripts/srw_ftest.sh index 28c67c32d4..52c0cc45f5 100755 --- a/.cicd/scripts/srw_ftest.sh +++ b/.cicd/scripts/srw_ftest.sh @@ -16,10 +16,10 @@ # SRW_COMPILER= # # Optional: -[[ -n ${ACCOUNT} ]] || ACCOUNT="no_account" +[[ -n ${SRW_PROJECT} ]] || SRW_PROJECT="no_account" [[ -n ${BRANCH} ]] || BRANCH="develop" [[ -n ${TASKS} ]] || TASKS="" -[[ -n ${TASK_DEPTH} ]] || TASK_DEPTH=4 +[[ -n ${TASK_DEPTH} ]] || TASK_DEPTH=9 [[ -n ${FORGIVE_CONDA} ]] || FORGIVE_CONDA=true set -e -u -x @@ -30,6 +30,7 @@ script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd) declare workspace if [[ -n "${WORKSPACE}" ]]; then workspace="${WORKSPACE}" + cd $workspace else workspace="$(cd -- "${script_dir}/../.." && pwd)" fi @@ -58,8 +59,8 @@ pwd echo "BRANCH=${BRANCH}" # Set the ACCOUNT to use for this PLATFORM ... -sed "s|^ ACCOUNT: \"\"| ACCOUNT: \"${ACCOUNT}\"|1" -i ush/config_defaults.yaml -sed "s|hera|${platform,,}|1" ush/config.community.yaml | sed "s|an_account|${ACCOUNT}|1" > ush/config.yaml +sed "s|^ ACCOUNT: \"\"| ACCOUNT: \"${SRW_PROJECT}\"|1" -i ush/config_defaults.yaml +sed "s|hera|${platform,,}|1" ush/config.community.yaml | sed "s|an_account|${SRW_PROJECT}|1" > ush/config.yaml # Set directory paths ... export EXPTDIR=${workspace}/expt_dirs/test_community @@ -67,6 +68,9 @@ echo "EXPTDIR=${EXPTDIR}" sed "s|^workflow:|workflow:\n EXPT_BASEDIR: ${workspace}/expt_dirs|1" -i ush/config.yaml sed "s|^workflow:|workflow:\n EXEC_SUBDIR: ${workspace}/install_${SRW_COMPILER}/exec|1" -i ush/config.yaml +# Decrease forecast length since we are running all the steps +sed "s|^ FCST_LEN_HRS: 12| FCST_LEN_HRS: 6|g" -i ush/config.yaml + # DATA_LOCATION differs on each platform ... find it. export DATA_LOCATION=$(grep TEST_EXTRN_MDL_SOURCE_BASEDIR ${workspace}/ush/machine/${platform,,}.yaml | awk '{printf "%s", $2}') echo "DATA_LOCATION=${DATA_LOCATION}" @@ -107,10 +111,8 @@ cp ${workspace}/ush/wrappers/* . # Set parameters that the task scripts require ... export JOBSdir=${workspace}/jobs export USHdir=${workspace}/ush -export PDY=20190615 -export cyc=18 -export subcyc=0 export OMP_NUM_THREADS=1 +export nprocs=24 [[ -n ${TASKS} ]] || TASKS=( run_make_grid diff --git a/.cicd/scripts/wrapper_srw_ftest.sh b/.cicd/scripts/wrapper_srw_ftest.sh new file mode 100755 index 0000000000..14552d78e1 --- /dev/null +++ b/.cicd/scripts/wrapper_srw_ftest.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash + +############################ +# +# This is a wrapper script for the srw_ftest.sh script. It was created so we can +# test all of the wrapper script tasks within the Jenkins pipeline. It determines which +# workflow manager is installed on the platform and calls the apprioprate job card: +# sbatch: sbatch_srw_ftest.sh +# pbs: qsub_srw_ftest.sh +########################### + +# Set workflow cmd +declare workflow_cmd +declare arg_1 +if [[ "${SRW_PLATFORM}" == cheyenne ]] || [[ "${SRW_PLATFORM}" == derecho ]]; then + workflow_cmd=qsub + arg_1="" + check_job="qstat -u ${USER} -r ${job_id}" +else + workflow_cmd=sbatch + arg_1="--parsable" + check_job="squeue -u ${USER} -j ${job_id} --noheader" +fi + +# Customize wrapper scripts +if [[ "${SRW_PLATFORM}" == gaea ]]; then + sed -i '15i #SBATCH --clusters=c4' ${WORKSPACE}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh + sed -i 's|qos=batch|qos=windfall|g' ${WORKSPACE}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh +fi + +# Call job card and return job_id +echo "Running: ${workflow_cmd} -A ${SRW_PROJECT} ${arg_1} ${WORKSPACE}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh" +job_id=$(${workflow_cmd} -A ${SRW_PROJECT} ${arg_1} ${WORKSPACE}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh) + +echo "Waiting ten seconds for node to initialize" +sleep 10 + +# Check for job and exit when done +while true +do + job_id_info=$($check_job) + if [ ! -z "$job_id_info" ]; then + echo "Job is still running. Check again in two minutes" + sleep 120 + else + echo "Job has completed." + + # Return exit code and check for results file first + results_file="${WORKSPACE}/functional_test_results_${SRW_PLATFORM}_${SRW_COMPILER}.txt" + if [ ! -f "$results_file" ]; then + echo "Missing results file! \nexit 1" + exit 1 + fi + + # Set exit code to number of failures + set +e + failures=$(grep ": FAIL" ${results_file} | wc -l) + if [[ $failures -ne 0 ]]; then + failures=1 + fi + + set -e + echo "exit ${failures}" + exit ${failures} + fi +done diff --git a/docs/UsersGuide/source/RunSRW.rst b/docs/UsersGuide/source/RunSRW.rst index 4d8f78d7d4..b1cac7058c 100644 --- a/docs/UsersGuide/source/RunSRW.rst +++ b/docs/UsersGuide/source/RunSRW.rst @@ -1173,14 +1173,6 @@ The regional workflow can be run using standalone shell scripts in cases where t export EXPTDIR=`pwd` setenv EXPTDIR `pwd` -#. Set the ``PDY`` and ``cyc`` environment variables. ``PDY`` refers to the first 8 characters (YYYYMMDD) of the ``DATE_FIRST_CYCL`` variable defined in the ``config.yaml``. ``cyc`` refers to the last two digits of ``DATE_FIRST_CYCL`` (HH) defined in ``config.yaml``. For example, if the ``config.yaml`` file defines ``DATE_FIRST_CYCL: '2019061518'``, the user should run: - - .. code-block:: console - - export PDY=20190615 && export cyc=18 - - before running the wrapper scripts. - #. Copy the wrapper scripts from the ``ush`` directory into the experiment directory. Each workflow task has a wrapper script that sets environment variables and runs the job script. .. code-block:: console diff --git a/ush/wrappers/run_fcst.sh b/ush/wrappers/run_fcst.sh index a5e589197d..93cc009b58 100755 --- a/ush/wrappers/run_fcst.sh +++ b/ush/wrappers/run_fcst.sh @@ -1,9 +1,11 @@ #!/bin/sh export GLOBAL_VAR_DEFNS_FP="${EXPTDIR}/var_defns.sh" -set -x +set -xa source ${GLOBAL_VAR_DEFNS_FP} export CDATE=${DATE_FIRST_CYCL} export CYCLE_DIR=${EXPTDIR}/${CDATE} +export cyc=${DATE_FIRST_CYCL:8:2} +export PDY=${DATE_FIRST_CYCL:0:8} export SLASH_ENSMEM_SUBDIR="" export ENSMEM_INDX="" diff --git a/ush/wrappers/run_get_ics.sh b/ush/wrappers/run_get_ics.sh index b434e6cec1..4cba2a86e5 100755 --- a/ush/wrappers/run_get_ics.sh +++ b/ush/wrappers/run_get_ics.sh @@ -1,9 +1,11 @@ #!/bin/sh export GLOBAL_VAR_DEFNS_FP="${EXPTDIR}/var_defns.sh" -set -x +set -xa source ${GLOBAL_VAR_DEFNS_FP} export CDATE=${DATE_FIRST_CYCL} export CYCLE_DIR=${EXPTDIR}/${CDATE} +export cyc=${DATE_FIRST_CYCL:8:2} +export PDY=${DATE_FIRST_CYCL:0:8} # get the ICS files export ICS_OR_LBCS="ICS" diff --git a/ush/wrappers/run_get_lbcs.sh b/ush/wrappers/run_get_lbcs.sh index 85eb20b9f0..3447d9f224 100755 --- a/ush/wrappers/run_get_lbcs.sh +++ b/ush/wrappers/run_get_lbcs.sh @@ -1,9 +1,11 @@ #!/bin/sh export GLOBAL_VAR_DEFNS_FP="${EXPTDIR}/var_defns.sh" -set -x +set -xa source ${GLOBAL_VAR_DEFNS_FP} export CDATE=${DATE_FIRST_CYCL} export CYCLE_DIR=${EXPTDIR}/${CDATE} +export cyc=${DATE_FIRST_CYCL:8:2} +export PDY=${DATE_FIRST_CYCL:0:8} # get the LBCS files export ICS_OR_LBCS="LBCS" diff --git a/ush/wrappers/run_make_grid.sh b/ush/wrappers/run_make_grid.sh index b3df718162..deff13c0ee 100755 --- a/ush/wrappers/run_make_grid.sh +++ b/ush/wrappers/run_make_grid.sh @@ -1,6 +1,6 @@ #!/bin/sh export GLOBAL_VAR_DEFNS_FP="${EXPTDIR}/var_defns.sh" -set -x +set -xa source ${GLOBAL_VAR_DEFNS_FP} export CDATE=${DATE_FIRST_CYCL} diff --git a/ush/wrappers/run_make_ics.sh b/ush/wrappers/run_make_ics.sh index 7a62571b90..b3e4333d9d 100755 --- a/ush/wrappers/run_make_ics.sh +++ b/ush/wrappers/run_make_ics.sh @@ -1,10 +1,13 @@ #!/bin/sh export GLOBAL_VAR_DEFNS_FP="${EXPTDIR}/var_defns.sh" -set -x +set -xa source ${GLOBAL_VAR_DEFNS_FP} export CDATE=${DATE_FIRST_CYCL} export CYCLE_DIR=${EXPTDIR}/${CDATE} +export cyc=${DATE_FIRST_CYCL:8:2} +export PDY=${DATE_FIRST_CYCL:0:8} export SLASH_ENSMEM_SUBDIR="" +export NWGES_DIR=${NWGES_BASEDIR}/${DATE_FIRST_CYCL:0:8} ${JOBSdir}/JREGIONAL_MAKE_ICS diff --git a/ush/wrappers/run_make_lbcs.sh b/ush/wrappers/run_make_lbcs.sh index f6dc78e0ee..2cd59c47a2 100755 --- a/ush/wrappers/run_make_lbcs.sh +++ b/ush/wrappers/run_make_lbcs.sh @@ -1,10 +1,14 @@ #!/bin/sh export GLOBAL_VAR_DEFNS_FP="${EXPTDIR}/var_defns.sh" -set -x +set -xa source ${GLOBAL_VAR_DEFNS_FP} export CDATE=${DATE_FIRST_CYCL} export CYCLE_DIR=${EXPTDIR}/${CDATE} +export cyc=${DATE_FIRST_CYCL:8:2} +export PDY=${DATE_FIRST_CYCL:0:8} export SLASH_ENSMEM_SUBDIR="" - +export NWGES_DIR=${NWGES_BASEDIR}/${DATE_FIRST_CYCL:0:8} +export bcgrp="00" +export bcgrpnum="1" ${JOBSdir}/JREGIONAL_MAKE_LBCS diff --git a/ush/wrappers/run_make_orog.sh b/ush/wrappers/run_make_orog.sh index c68cf38a03..81bdd5787b 100755 --- a/ush/wrappers/run_make_orog.sh +++ b/ush/wrappers/run_make_orog.sh @@ -1,6 +1,6 @@ #!/bin/sh export GLOBAL_VAR_DEFNS_FP="${EXPTDIR}/var_defns.sh" -set -x +set -xa source ${GLOBAL_VAR_DEFNS_FP} export CDATE=${DATE_FIRST_CYCL} diff --git a/ush/wrappers/run_make_sfc_climo.sh b/ush/wrappers/run_make_sfc_climo.sh index 3cd3025992..455a8fdf45 100755 --- a/ush/wrappers/run_make_sfc_climo.sh +++ b/ush/wrappers/run_make_sfc_climo.sh @@ -1,6 +1,6 @@ #!/bin/sh export GLOBAL_VAR_DEFNS_FP="${EXPTDIR}/var_defns.sh" -set -x +set -xa source ${GLOBAL_VAR_DEFNS_FP} export CDATE=${DATE_FIRST_CYCL} diff --git a/ush/wrappers/run_post.sh b/ush/wrappers/run_post.sh index c1a4f89bf1..4a4a57513e 100755 --- a/ush/wrappers/run_post.sh +++ b/ush/wrappers/run_post.sh @@ -1,10 +1,11 @@ #!/bin/sh export GLOBAL_VAR_DEFNS_FP="${EXPTDIR}/var_defns.sh" -set -x +set -xa source ${GLOBAL_VAR_DEFNS_FP} export CDATE=${DATE_FIRST_CYCL} export CYCLE_DIR=${EXPTDIR}/${CDATE} export cyc=${DATE_FIRST_CYCL:8:2} +export PDY=${DATE_FIRST_CYCL:0:8} export SLASH_ENSMEM_SUBDIR="" export ENSMEM_INDX=""