[develop] Update wrapper scripts (#883)

The wrapper scripts are a set of scripts that run the tasks from a SRW experiment. They are used in a situation where rocoto is not installed or available to use. These set of wrapper scripts have become outdated and don't work in its current state. This PR was created to address this issue.

---------

Co-authored-by: Edward Snyder <Edward.Snyder@noaa.com>
Co-authored-by: Eddie Snyder <esnyder@derecho2.hsn.de.hpc.ucar.edu>

.cicd/Jenkinsfile

@@ -151,7 +151,7 @@ pipeline {
                     stage('Functional WorkflowTaskTests') {
                         steps {
                             echo "Running simple workflow script task tests on ${env.SRW_PLATFORM} (using ${env.WORKSPACE})"
-                            sh 'bash --login "${WORKSPACE}/.cicd/scripts/srw_ftest.sh"'
+                            sh 'bash --login "${WORKSPACE}/.cicd/scripts/wrapper_srw_ftest.sh"'
                         }
                     }
 

.cicd/scripts/qsub_srw_ftest.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+#
+# This is a pbs job card for the srw_ftest.sh script
+# and is called by the wrapper_srw_ftest.sh script
+# 
+#PBS -N srw_ftest_run
+#PBS -A ${SRW_PROJECT}
+#PBS -q main
+#PBS -l select=1:ncpus=24:mpiprocs=24:ompthreads=1
+#PBS -l walltime=00:30:00
+#PBS -V
+#PBS -o log_wrap.%j.log
+#PBS -e err_wrap.%j.err 
+
+bash ${WORKSPACE}/.cicd/scripts/srw_ftest.sh

.cicd/scripts/sbatch_srw_ftest.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+#
+# This is a sbatch job card for the srw_ftest.sh script
+# and is called by the srw_ftest_wrapper.sh script
+# 
+#SBATCH --job-name=wrapper_test
+#SBATCH --account=${SRW_PROJECT}
+#SBATCH --qos=batch
+#SBATCH --nodes=1
+#SBATCH --tasks-per-node=24
+#SBATCH --cpus-per-task=1
+#SBATCH -t 00:30:00
+#SBATCH -o log_wrap.%j.log
+#SBATCH -e err_wrap.%j.err 
+
+bash ${WORKSPACE}/.cicd/scripts/srw_ftest.sh

.cicd/scripts/srw_ftest.sh

@@ -16,10 +16,10 @@
 #    SRW_COMPILER=<intel|gnu>
 #
 # Optional:
-[[ -n ${ACCOUNT} ]] || ACCOUNT="no_account"
+[[ -n ${SRW_PROJECT} ]] || SRW_PROJECT="no_account"
 [[ -n ${BRANCH} ]] || BRANCH="develop"
 [[ -n ${TASKS} ]] || TASKS=""
-[[ -n ${TASK_DEPTH} ]] || TASK_DEPTH=4
+[[ -n ${TASK_DEPTH} ]] || TASK_DEPTH=9
 [[ -n ${FORGIVE_CONDA} ]] || FORGIVE_CONDA=true
 set -e -u -x
 
@@ -30,6 +30,7 @@ script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)
 declare workspace
 if [[ -n "${WORKSPACE}" ]]; then
     workspace="${WORKSPACE}"
+    cd $workspace
 else
     workspace="$(cd -- "${script_dir}/../.." && pwd)"
 fi
@@ -58,15 +59,18 @@ pwd
 echo "BRANCH=${BRANCH}"
 
 # Set the ACCOUNT to use for this PLATFORM ...
-sed "s|^  ACCOUNT: \"\"|  ACCOUNT: \"${ACCOUNT}\"|1" -i ush/config_defaults.yaml
-sed "s|hera|${platform,,}|1" ush/config.community.yaml | sed "s|an_account|${ACCOUNT}|1" > ush/config.yaml
+sed "s|^  ACCOUNT: \"\"|  ACCOUNT: \"${SRW_PROJECT}\"|1" -i ush/config_defaults.yaml
+sed "s|hera|${platform,,}|1" ush/config.community.yaml | sed "s|an_account|${SRW_PROJECT}|1" > ush/config.yaml
 
 # Set directory paths ...
 export EXPTDIR=${workspace}/expt_dirs/test_community
 echo "EXPTDIR=${EXPTDIR}"
 sed "s|^workflow:|workflow:\n  EXPT_BASEDIR: ${workspace}/expt_dirs|1" -i ush/config.yaml
 sed "s|^workflow:|workflow:\n  EXEC_SUBDIR: ${workspace}/install_${SRW_COMPILER}/exec|1" -i ush/config.yaml
 
+# Decrease forecast length since we are running all the steps
+sed "s|^  FCST_LEN_HRS: 12|  FCST_LEN_HRS: 6|g" -i ush/config.yaml
+
 # DATA_LOCATION differs on each platform ... find it.
 export DATA_LOCATION=$(grep TEST_EXTRN_MDL_SOURCE_BASEDIR ${workspace}/ush/machine/${platform,,}.yaml | awk '{printf "%s", $2}')
 echo "DATA_LOCATION=${DATA_LOCATION}"
@@ -107,10 +111,8 @@ cp ${workspace}/ush/wrappers/* .
 # Set parameters that the task scripts require ...
 export JOBSdir=${workspace}/jobs
 export USHdir=${workspace}/ush
-export PDY=20190615
-export cyc=18
-export subcyc=0
 export OMP_NUM_THREADS=1
+export nprocs=24
 
 [[ -n ${TASKS} ]] || TASKS=(
                 run_make_grid

.cicd/scripts/wrapper_srw_ftest.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+
+############################
+#
+# This is a wrapper script for the srw_ftest.sh script. It was created so we can 
+# test all of the wrapper script tasks within the Jenkins pipeline. It determines which 
+# workflow manager is installed on the platform and calls the apprioprate job card:
+#    sbatch: sbatch_srw_ftest.sh
+#    pbs: qsub_srw_ftest.sh
+###########################
+
+# Set workflow cmd
+declare workflow_cmd
+declare arg_1
+if [[ "${SRW_PLATFORM}" == cheyenne ]] || [[ "${SRW_PLATFORM}" == derecho ]]; then
+    workflow_cmd=qsub
+    arg_1=""
+    check_job="qstat -u ${USER} -r ${job_id}"
+else
+    workflow_cmd=sbatch
+    arg_1="--parsable"
+    check_job="squeue -u ${USER} -j ${job_id} --noheader"
+fi
+
+# Customize wrapper scripts
+if [[ "${SRW_PLATFORM}" == gaea ]]; then
+    sed -i '15i #SBATCH --clusters=c4' ${WORKSPACE}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh
+    sed -i 's|qos=batch|qos=windfall|g' ${WORKSPACE}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh
+fi
+
+# Call job card and return job_id
+echo "Running: ${workflow_cmd} -A ${SRW_PROJECT} ${arg_1} ${WORKSPACE}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh"
+job_id=$(${workflow_cmd} -A ${SRW_PROJECT} ${arg_1} ${WORKSPACE}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh)
+
+echo "Waiting ten seconds for node to initialize"
+sleep 10
+
+# Check for job and exit when done
+while true
+do
+    job_id_info=$($check_job)
+    if [ ! -z "$job_id_info" ]; then
+        echo "Job is still running. Check again in two minutes"
+        sleep 120
+    else
+        echo "Job has completed."
+
+        # Return exit code and check for results file first
+        results_file="${WORKSPACE}/functional_test_results_${SRW_PLATFORM}_${SRW_COMPILER}.txt"
+        if [ ! -f "$results_file" ]; then
+            echo "Missing results file! \nexit 1"
+            exit 1
+        fi
+
+        # Set exit code to number of failures
+        set +e
+        failures=$(grep ": FAIL" ${results_file} | wc -l)
+        if [[ $failures -ne 0 ]]; then
+            failures=1
+        fi
+
+        set -e
+        echo "exit ${failures}"
+        exit ${failures}
+    fi
+done

docs/UsersGuide/source/RunSRW.rst

@@ -1173,14 +1173,6 @@ The regional workflow can be run using standalone shell scripts in cases where t
       export EXPTDIR=`pwd`
       setenv EXPTDIR `pwd`
 
-#. Set the ``PDY`` and ``cyc`` environment variables. ``PDY`` refers to the first 8 characters (YYYYMMDD) of the ``DATE_FIRST_CYCL`` variable defined in the ``config.yaml``. ``cyc`` refers to the last two digits of ``DATE_FIRST_CYCL`` (HH) defined in ``config.yaml``. For example, if the ``config.yaml`` file defines ``DATE_FIRST_CYCL: '2019061518'``, the user should run:
-
-   .. code-block:: console 
-      
-      export PDY=20190615 && export cyc=18 
-   
-   before running the wrapper scripts.
-
 #. Copy the wrapper scripts from the ``ush`` directory into the experiment directory. Each workflow task has a wrapper script that sets environment variables and runs the job script.
 
    .. code-block:: console

ush/wrappers/run_fcst.sh

@@ -1,9 +1,11 @@
 #!/bin/sh
 export GLOBAL_VAR_DEFNS_FP="${EXPTDIR}/var_defns.sh"
-set -x
+set -xa
 source ${GLOBAL_VAR_DEFNS_FP}
 export CDATE=${DATE_FIRST_CYCL}
 export CYCLE_DIR=${EXPTDIR}/${CDATE}
+export cyc=${DATE_FIRST_CYCL:8:2}
+export PDY=${DATE_FIRST_CYCL:0:8}
 export SLASH_ENSMEM_SUBDIR=""
 export ENSMEM_INDX=""
 

ush/wrappers/run_get_ics.sh

@@ -1,9 +1,11 @@
 #!/bin/sh
 export GLOBAL_VAR_DEFNS_FP="${EXPTDIR}/var_defns.sh"
-set -x
+set -xa
 source ${GLOBAL_VAR_DEFNS_FP}
 export CDATE=${DATE_FIRST_CYCL}
 export CYCLE_DIR=${EXPTDIR}/${CDATE}
+export cyc=${DATE_FIRST_CYCL:8:2}
+export PDY=${DATE_FIRST_CYCL:0:8}
 
 # get the ICS files
 export ICS_OR_LBCS="ICS"

ush/wrappers/run_get_lbcs.sh

@@ -1,9 +1,11 @@
 #!/bin/sh
 export GLOBAL_VAR_DEFNS_FP="${EXPTDIR}/var_defns.sh"
-set -x
+set -xa
 source ${GLOBAL_VAR_DEFNS_FP}
 export CDATE=${DATE_FIRST_CYCL}
 export CYCLE_DIR=${EXPTDIR}/${CDATE}
+export cyc=${DATE_FIRST_CYCL:8:2}
+export PDY=${DATE_FIRST_CYCL:0:8}
 
 # get the LBCS files
 export ICS_OR_LBCS="LBCS"

ush/wrappers/run_make_grid.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 export GLOBAL_VAR_DEFNS_FP="${EXPTDIR}/var_defns.sh"
-set -x
+set -xa
 source ${GLOBAL_VAR_DEFNS_FP}
 
 export CDATE=${DATE_FIRST_CYCL}

ush/wrappers/run_make_ics.sh

@@ -1,10 +1,13 @@
 #!/bin/sh
 export GLOBAL_VAR_DEFNS_FP="${EXPTDIR}/var_defns.sh"
-set -x
+set -xa
 source ${GLOBAL_VAR_DEFNS_FP}
 export CDATE=${DATE_FIRST_CYCL}
 export CYCLE_DIR=${EXPTDIR}/${CDATE}
+export cyc=${DATE_FIRST_CYCL:8:2}
+export PDY=${DATE_FIRST_CYCL:0:8}
 export SLASH_ENSMEM_SUBDIR=""
+export NWGES_DIR=${NWGES_BASEDIR}/${DATE_FIRST_CYCL:0:8}
 
 ${JOBSdir}/JREGIONAL_MAKE_ICS
 

ush/wrappers/run_make_lbcs.sh

@@ -1,10 +1,14 @@
 #!/bin/sh
 export GLOBAL_VAR_DEFNS_FP="${EXPTDIR}/var_defns.sh"
-set -x
+set -xa
 source ${GLOBAL_VAR_DEFNS_FP}
 export CDATE=${DATE_FIRST_CYCL}
 export CYCLE_DIR=${EXPTDIR}/${CDATE}
+export cyc=${DATE_FIRST_CYCL:8:2}
+export PDY=${DATE_FIRST_CYCL:0:8}
 export SLASH_ENSMEM_SUBDIR=""
-
+export NWGES_DIR=${NWGES_BASEDIR}/${DATE_FIRST_CYCL:0:8}
+export bcgrp="00"
+export bcgrpnum="1"
 ${JOBSdir}/JREGIONAL_MAKE_LBCS
 

ush/wrappers/run_make_orog.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 export GLOBAL_VAR_DEFNS_FP="${EXPTDIR}/var_defns.sh"
-set -x
+set -xa
 source ${GLOBAL_VAR_DEFNS_FP}
 
 export CDATE=${DATE_FIRST_CYCL}

ush/wrappers/run_make_sfc_climo.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 export GLOBAL_VAR_DEFNS_FP="${EXPTDIR}/var_defns.sh"
-set -x
+set -xa
 source ${GLOBAL_VAR_DEFNS_FP}
 
 export CDATE=${DATE_FIRST_CYCL}

ush/wrappers/run_post.sh

@@ -1,10 +1,11 @@
 #!/bin/sh
 export GLOBAL_VAR_DEFNS_FP="${EXPTDIR}/var_defns.sh"
-set -x
+set -xa
 source ${GLOBAL_VAR_DEFNS_FP}
 export CDATE=${DATE_FIRST_CYCL}
 export CYCLE_DIR=${EXPTDIR}/${CDATE}
 export cyc=${DATE_FIRST_CYCL:8:2}
+export PDY=${DATE_FIRST_CYCL:0:8}
 export SLASH_ENSMEM_SUBDIR=""
 export ENSMEM_INDX=""