diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile index 0730e16acd..33c2ece22d 100644 --- a/.cicd/Jenkinsfile +++ b/.cicd/Jenkinsfile @@ -151,7 +151,7 @@ pipeline { stage('Functional WorkflowTaskTests') { steps { echo "Running simple workflow script task tests on ${env.SRW_PLATFORM} (using ${env.WORKSPACE})" - sh 'bash --login "${WORKSPACE}/.cicd/scripts/srw_ftest.sh"' + sh 'bash --login "${WORKSPACE}/.cicd/scripts/wrapper_srw_ftest.sh"' } } diff --git a/.cicd/scripts/qsub_srw_ftest.sh b/.cicd/scripts/qsub_srw_ftest.sh new file mode 100644 index 0000000000..2a669fe61a --- /dev/null +++ b/.cicd/scripts/qsub_srw_ftest.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# +# This is a pbs job card for the srw_ftest.sh script +# and is called by the wrapper_srw_ftest.sh script +# +#PBS -N srw_ftest_run +#PBS -A ${SRW_PROJECT} +#PBS -q main +#PBS -l select=1:ncpus=24:mpiprocs=24:ompthreads=1 +#PBS -l walltime=00:30:00 +#PBS -V +#PBS -o log_wrap.%j.log +#PBS -e err_wrap.%j.err + +bash ${WORKSPACE}/.cicd/scripts/srw_ftest.sh diff --git a/.cicd/scripts/sbatch_srw_ftest.sh b/.cicd/scripts/sbatch_srw_ftest.sh new file mode 100644 index 0000000000..1655148068 --- /dev/null +++ b/.cicd/scripts/sbatch_srw_ftest.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# +# This is a sbatch job card for the srw_ftest.sh script +# and is called by the srw_ftest_wrapper.sh script +# +#SBATCH --job-name=wrapper_test +#SBATCH --account=${SRW_PROJECT} +#SBATCH --qos=batch +#SBATCH --nodes=1 +#SBATCH --tasks-per-node=24 +#SBATCH --cpus-per-task=1 +#SBATCH -t 00:30:00 +#SBATCH -o log_wrap.%j.log +#SBATCH -e err_wrap.%j.err + +bash ${WORKSPACE}/.cicd/scripts/srw_ftest.sh diff --git a/.cicd/scripts/srw_ftest.sh b/.cicd/scripts/srw_ftest.sh index 28c67c32d4..52c0cc45f5 100755 --- a/.cicd/scripts/srw_ftest.sh +++ b/.cicd/scripts/srw_ftest.sh @@ -16,10 +16,10 @@ # SRW_COMPILER= # # Optional: -[[ -n ${ACCOUNT} ]] || ACCOUNT="no_account" +[[ -n ${SRW_PROJECT} ]] || SRW_PROJECT="no_account" [[ -n ${BRANCH} ]] || BRANCH="develop" [[ -n ${TASKS} ]] || TASKS="" -[[ -n ${TASK_DEPTH} ]] || TASK_DEPTH=4 +[[ -n ${TASK_DEPTH} ]] || TASK_DEPTH=9 [[ -n ${FORGIVE_CONDA} ]] || FORGIVE_CONDA=true set -e -u -x @@ -30,6 +30,7 @@ script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd) declare workspace if [[ -n "${WORKSPACE}" ]]; then workspace="${WORKSPACE}" + cd $workspace else workspace="$(cd -- "${script_dir}/../.." && pwd)" fi @@ -58,8 +59,8 @@ pwd echo "BRANCH=${BRANCH}" # Set the ACCOUNT to use for this PLATFORM ... -sed "s|^ ACCOUNT: \"\"| ACCOUNT: \"${ACCOUNT}\"|1" -i ush/config_defaults.yaml -sed "s|hera|${platform,,}|1" ush/config.community.yaml | sed "s|an_account|${ACCOUNT}|1" > ush/config.yaml +sed "s|^ ACCOUNT: \"\"| ACCOUNT: \"${SRW_PROJECT}\"|1" -i ush/config_defaults.yaml +sed "s|hera|${platform,,}|1" ush/config.community.yaml | sed "s|an_account|${SRW_PROJECT}|1" > ush/config.yaml # Set directory paths ... export EXPTDIR=${workspace}/expt_dirs/test_community @@ -67,6 +68,9 @@ echo "EXPTDIR=${EXPTDIR}" sed "s|^workflow:|workflow:\n EXPT_BASEDIR: ${workspace}/expt_dirs|1" -i ush/config.yaml sed "s|^workflow:|workflow:\n EXEC_SUBDIR: ${workspace}/install_${SRW_COMPILER}/exec|1" -i ush/config.yaml +# Decrease forecast length since we are running all the steps +sed "s|^ FCST_LEN_HRS: 12| FCST_LEN_HRS: 6|g" -i ush/config.yaml + # DATA_LOCATION differs on each platform ... find it. export DATA_LOCATION=$(grep TEST_EXTRN_MDL_SOURCE_BASEDIR ${workspace}/ush/machine/${platform,,}.yaml | awk '{printf "%s", $2}') echo "DATA_LOCATION=${DATA_LOCATION}" @@ -107,10 +111,8 @@ cp ${workspace}/ush/wrappers/* . # Set parameters that the task scripts require ... export JOBSdir=${workspace}/jobs export USHdir=${workspace}/ush -export PDY=20190615 -export cyc=18 -export subcyc=0 export OMP_NUM_THREADS=1 +export nprocs=24 [[ -n ${TASKS} ]] || TASKS=( run_make_grid diff --git a/.cicd/scripts/wrapper_srw_ftest.sh b/.cicd/scripts/wrapper_srw_ftest.sh new file mode 100755 index 0000000000..14552d78e1 --- /dev/null +++ b/.cicd/scripts/wrapper_srw_ftest.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash + +############################ +# +# This is a wrapper script for the srw_ftest.sh script. It was created so we can +# test all of the wrapper script tasks within the Jenkins pipeline. It determines which +# workflow manager is installed on the platform and calls the apprioprate job card: +# sbatch: sbatch_srw_ftest.sh +# pbs: qsub_srw_ftest.sh +########################### + +# Set workflow cmd +declare workflow_cmd +declare arg_1 +if [[ "${SRW_PLATFORM}" == cheyenne ]] || [[ "${SRW_PLATFORM}" == derecho ]]; then + workflow_cmd=qsub + arg_1="" + check_job="qstat -u ${USER} -r ${job_id}" +else + workflow_cmd=sbatch + arg_1="--parsable" + check_job="squeue -u ${USER} -j ${job_id} --noheader" +fi + +# Customize wrapper scripts +if [[ "${SRW_PLATFORM}" == gaea ]]; then + sed -i '15i #SBATCH --clusters=c4' ${WORKSPACE}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh + sed -i 's|qos=batch|qos=windfall|g' ${WORKSPACE}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh +fi + +# Call job card and return job_id +echo "Running: ${workflow_cmd} -A ${SRW_PROJECT} ${arg_1} ${WORKSPACE}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh" +job_id=$(${workflow_cmd} -A ${SRW_PROJECT} ${arg_1} ${WORKSPACE}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh) + +echo "Waiting ten seconds for node to initialize" +sleep 10 + +# Check for job and exit when done +while true +do + job_id_info=$($check_job) + if [ ! -z "$job_id_info" ]; then + echo "Job is still running. Check again in two minutes" + sleep 120 + else + echo "Job has completed." + + # Return exit code and check for results file first + results_file="${WORKSPACE}/functional_test_results_${SRW_PLATFORM}_${SRW_COMPILER}.txt" + if [ ! -f "$results_file" ]; then + echo "Missing results file! \nexit 1" + exit 1 + fi + + # Set exit code to number of failures + set +e + failures=$(grep ": FAIL" ${results_file} | wc -l) + if [[ $failures -ne 0 ]]; then + failures=1 + fi + + set -e + echo "exit ${failures}" + exit ${failures} + fi +done diff --git a/docs/UsersGuide/source/RunSRW.rst b/docs/UsersGuide/source/RunSRW.rst index 4d8f78d7d4..b1cac7058c 100644 --- a/docs/UsersGuide/source/RunSRW.rst +++ b/docs/UsersGuide/source/RunSRW.rst @@ -1173,14 +1173,6 @@ The regional workflow can be run using standalone shell scripts in cases where t export EXPTDIR=`pwd` setenv EXPTDIR `pwd` -#. Set the ``PDY`` and ``cyc`` environment variables. ``PDY`` refers to the first 8 characters (YYYYMMDD) of the ``DATE_FIRST_CYCL`` variable defined in the ``config.yaml``. ``cyc`` refers to the last two digits of ``DATE_FIRST_CYCL`` (HH) defined in ``config.yaml``. For example, if the ``config.yaml`` file defines ``DATE_FIRST_CYCL: '2019061518'``, the user should run: - - .. code-block:: console - - export PDY=20190615 && export cyc=18 - - before running the wrapper scripts. - #. Copy the wrapper scripts from the ``ush`` directory into the experiment directory. Each workflow task has a wrapper script that sets environment variables and runs the job script. .. code-block:: console diff --git a/ush/wrappers/run_fcst.sh b/ush/wrappers/run_fcst.sh index a5e589197d..93cc009b58 100755 --- a/ush/wrappers/run_fcst.sh +++ b/ush/wrappers/run_fcst.sh @@ -1,9 +1,11 @@ #!/bin/sh export GLOBAL_VAR_DEFNS_FP="${EXPTDIR}/var_defns.sh" -set -x +set -xa source ${GLOBAL_VAR_DEFNS_FP} export CDATE=${DATE_FIRST_CYCL} export CYCLE_DIR=${EXPTDIR}/${CDATE} +export cyc=${DATE_FIRST_CYCL:8:2} +export PDY=${DATE_FIRST_CYCL:0:8} export SLASH_ENSMEM_SUBDIR="" export ENSMEM_INDX="" diff --git a/ush/wrappers/run_get_ics.sh b/ush/wrappers/run_get_ics.sh index b434e6cec1..4cba2a86e5 100755 --- a/ush/wrappers/run_get_ics.sh +++ b/ush/wrappers/run_get_ics.sh @@ -1,9 +1,11 @@ #!/bin/sh export GLOBAL_VAR_DEFNS_FP="${EXPTDIR}/var_defns.sh" -set -x +set -xa source ${GLOBAL_VAR_DEFNS_FP} export CDATE=${DATE_FIRST_CYCL} export CYCLE_DIR=${EXPTDIR}/${CDATE} +export cyc=${DATE_FIRST_CYCL:8:2} +export PDY=${DATE_FIRST_CYCL:0:8} # get the ICS files export ICS_OR_LBCS="ICS" diff --git a/ush/wrappers/run_get_lbcs.sh b/ush/wrappers/run_get_lbcs.sh index 85eb20b9f0..3447d9f224 100755 --- a/ush/wrappers/run_get_lbcs.sh +++ b/ush/wrappers/run_get_lbcs.sh @@ -1,9 +1,11 @@ #!/bin/sh export GLOBAL_VAR_DEFNS_FP="${EXPTDIR}/var_defns.sh" -set -x +set -xa source ${GLOBAL_VAR_DEFNS_FP} export CDATE=${DATE_FIRST_CYCL} export CYCLE_DIR=${EXPTDIR}/${CDATE} +export cyc=${DATE_FIRST_CYCL:8:2} +export PDY=${DATE_FIRST_CYCL:0:8} # get the LBCS files export ICS_OR_LBCS="LBCS" diff --git a/ush/wrappers/run_make_grid.sh b/ush/wrappers/run_make_grid.sh index b3df718162..deff13c0ee 100755 --- a/ush/wrappers/run_make_grid.sh +++ b/ush/wrappers/run_make_grid.sh @@ -1,6 +1,6 @@ #!/bin/sh export GLOBAL_VAR_DEFNS_FP="${EXPTDIR}/var_defns.sh" -set -x +set -xa source ${GLOBAL_VAR_DEFNS_FP} export CDATE=${DATE_FIRST_CYCL} diff --git a/ush/wrappers/run_make_ics.sh b/ush/wrappers/run_make_ics.sh index 7a62571b90..b3e4333d9d 100755 --- a/ush/wrappers/run_make_ics.sh +++ b/ush/wrappers/run_make_ics.sh @@ -1,10 +1,13 @@ #!/bin/sh export GLOBAL_VAR_DEFNS_FP="${EXPTDIR}/var_defns.sh" -set -x +set -xa source ${GLOBAL_VAR_DEFNS_FP} export CDATE=${DATE_FIRST_CYCL} export CYCLE_DIR=${EXPTDIR}/${CDATE} +export cyc=${DATE_FIRST_CYCL:8:2} +export PDY=${DATE_FIRST_CYCL:0:8} export SLASH_ENSMEM_SUBDIR="" +export NWGES_DIR=${NWGES_BASEDIR}/${DATE_FIRST_CYCL:0:8} ${JOBSdir}/JREGIONAL_MAKE_ICS diff --git a/ush/wrappers/run_make_lbcs.sh b/ush/wrappers/run_make_lbcs.sh index f6dc78e0ee..2cd59c47a2 100755 --- a/ush/wrappers/run_make_lbcs.sh +++ b/ush/wrappers/run_make_lbcs.sh @@ -1,10 +1,14 @@ #!/bin/sh export GLOBAL_VAR_DEFNS_FP="${EXPTDIR}/var_defns.sh" -set -x +set -xa source ${GLOBAL_VAR_DEFNS_FP} export CDATE=${DATE_FIRST_CYCL} export CYCLE_DIR=${EXPTDIR}/${CDATE} +export cyc=${DATE_FIRST_CYCL:8:2} +export PDY=${DATE_FIRST_CYCL:0:8} export SLASH_ENSMEM_SUBDIR="" - +export NWGES_DIR=${NWGES_BASEDIR}/${DATE_FIRST_CYCL:0:8} +export bcgrp="00" +export bcgrpnum="1" ${JOBSdir}/JREGIONAL_MAKE_LBCS diff --git a/ush/wrappers/run_make_orog.sh b/ush/wrappers/run_make_orog.sh index c68cf38a03..81bdd5787b 100755 --- a/ush/wrappers/run_make_orog.sh +++ b/ush/wrappers/run_make_orog.sh @@ -1,6 +1,6 @@ #!/bin/sh export GLOBAL_VAR_DEFNS_FP="${EXPTDIR}/var_defns.sh" -set -x +set -xa source ${GLOBAL_VAR_DEFNS_FP} export CDATE=${DATE_FIRST_CYCL} diff --git a/ush/wrappers/run_make_sfc_climo.sh b/ush/wrappers/run_make_sfc_climo.sh index 3cd3025992..455a8fdf45 100755 --- a/ush/wrappers/run_make_sfc_climo.sh +++ b/ush/wrappers/run_make_sfc_climo.sh @@ -1,6 +1,6 @@ #!/bin/sh export GLOBAL_VAR_DEFNS_FP="${EXPTDIR}/var_defns.sh" -set -x +set -xa source ${GLOBAL_VAR_DEFNS_FP} export CDATE=${DATE_FIRST_CYCL} diff --git a/ush/wrappers/run_post.sh b/ush/wrappers/run_post.sh index c1a4f89bf1..4a4a57513e 100755 --- a/ush/wrappers/run_post.sh +++ b/ush/wrappers/run_post.sh @@ -1,10 +1,11 @@ #!/bin/sh export GLOBAL_VAR_DEFNS_FP="${EXPTDIR}/var_defns.sh" -set -x +set -xa source ${GLOBAL_VAR_DEFNS_FP} export CDATE=${DATE_FIRST_CYCL} export CYCLE_DIR=${EXPTDIR}/${CDATE} export cyc=${DATE_FIRST_CYCL:8:2} +export PDY=${DATE_FIRST_CYCL:0:8} export SLASH_ENSMEM_SUBDIR="" export ENSMEM_INDX=""