Merge remote-tracking branch 'upstream/develop' into feature/use_jcb_atm

* upstream/develop:
  Update gfs_utils for Gaea (NOAA-EMC#2556)
  Updated GEMPAK version and APRUN launcher. (NOAA-EMC#2555)
  Utilize scale-dependent localization for atmospheric analysis (NOAA-EMC#2542)
  Remove implicit symlink names (NOAA-EMC#2527)
  Fixes sea ice archiving (NOAA-EMC#2541)
  Link ensemble analysis increment files to COMROOT for warm_start (NOAA-EMC#2553)
  Launch Multiple Platforms to Jenkins with polling (NOAA-EMC#2548)
  Turn C48mx500_3DVarAOWCDA back on  (NOAA-EMC#2543)
  Add option to link different orog/ugwd fix files for global nest (NOAA-EMC#2532)
  Retire AWIPS GRIB1 products (NOAA-EMC#2547)
  Add CADS use flexibility (NOAA-EMC#2540)
  Hot fix for bash CI on WCOSS2 (NOAA-EMC#2536)
  Fix comment indentation (NOAA-EMC#2526)

ci/Jenkinsfile

@@ -21,31 +21,60 @@ pipeline {
              // Each Jenknis Node is connected to said machine via an JAVA agent via an ssh tunnel
              // no op 2
 
-        stage('Get Machine') {
+        stage('1. Get Machine') {
             agent { label 'built-in' }
             steps {
                 script {
-                    machine = 'none'
-                    for (label in pullRequest.labels) {
-                        echo "Label: ${label}"
-                        if ((label.matches('CI-Hera-Ready'))) {
-                            machine = 'hera'
-                        } else if ((label.matches('CI-Orion-Ready'))) {
-                            machine = 'orion'
-                        } else if ((label.matches('CI-Hercules-Ready'))) {
-                            machine = 'hercules'
+
+                    def causes = currentBuild.rawBuild.getCauses()
+                    def isSpawnedFromAnotherJob = causes.any { cause ->
+                        cause instanceof hudson.model.Cause.UpstreamCause
+                    }
+
+                    def run_nodes = []
+                    if (isSpawnedFromAnotherJob) {
+                        echo "machine being set to value passed to this spawned job"
+                        echo "passed machine: ${params.machine}"
+                        machine = params.machine
+                    } else {
+                        echo "This is parent job so getting list of nodes matching labels:"
+                        for (label in pullRequest.labels) {
+                            if (label.matches("CI-(.*?)-Ready")) {
+                                def Machine_name = label.split('-')[1].toString()
+                                jenkins.model.Jenkins.get().computers.each { c ->
+                                    if (c.node.selfLabel.name == "${Machine_name}-EMC") {
+                                        run_nodes.add(c.node.selfLabel.name)
+                                    }
+                                }
+                            }
                         }
-                    } // createing a second machine varible with first letter capital
-                      // because the first letter of the machine name is captitalized in the GitHub labels
-                    Machine = machine[0].toUpperCase() + machine.substring(1)
+                        // Spawning all the jobs on the nodes matching the labels
+                        if (run_nodes.size() > 1) {
+                            run_nodes.init().each { node ->
+                                def machine_name = node.split('-')[0].toLowerCase()
+                                echo "Spawning job on node: ${node} with machine name: ${machine_name}"
+                                build job: "/global-workflow/EMC-Global-Pipeline/PR-${env.CHANGE_ID}", parameters: [
+                                    string(name: 'machine', value: machine_name),
+                                    string(name: 'Node', value: node) ],
+                                    wait: false
+                            }
+                            machine = run_nodes.last().split('-')[0].toLowerCase()
+                            echo "Running parent job: ${machine}"
+                        } else {
+                            machine = run_nodes[0].split('-')[0].toLowerCase()
+                            echo "Running only the parent job: ${machine}"
+                        }
+                    }
                 }
             }
         }
 
-        stage('Get Common Workspace') {
+        stage('2. Get Common Workspace') {
             agent { label "${machine}-emc" }
             steps {
                 script {
+                    Machine = machine[0].toUpperCase() + machine.substring(1)
+                    echo "Getting Common Workspace for ${Machine}"
                     ws("${custom_workspace[machine]}/${env.CHANGE_ID}") {
                         properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
                         HOME = "${WORKSPACE}"
@@ -57,7 +86,7 @@ pipeline {
             }
         }
 
-        stage('Build System') {
+        stage('3. Build System') {
             matrix {
                 agent { label "${machine}-emc" }
                 //options {
@@ -141,7 +170,7 @@ pipeline {
             }
         }
 
-        stage('Run Tests') {
+        stage('4. Run Tests') {
             failFast false 
             matrix {
                 agent { label "${machine}-emc" }
@@ -216,7 +245,7 @@ pipeline {
                                         STATUS = 'Failed'
                                         try {
                                             sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" --add-label "CI-${Machine}-${STATUS}" """, returnStatus: true)
-                                            sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${Case} **FAILED** on ${Machine}\nin\\`${HOME}/RUNTESTS/${pslot}\\`" """)
+                                            sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${Case} **FAILED** on ${Machine} in\n\\`${HOME}/RUNTESTS/${pslot}\\`" """)
                                         } catch (Exception e) {
                                             echo "Failed to update label from Running to ${STATUS}: ${e.getMessage()}"
                                         }
@@ -229,7 +258,7 @@ pipeline {
                 }
             }
         }
-        stage( 'FINALIZE' ) {
+        stage( '5. FINALIZE' ) {
             when { 
                 expression {
                    STATUS == 'Passed'

ci/cases/pr/C48mx500_3DVarAOWCDA.yaml

@@ -20,5 +20,4 @@ arguments:
 skip_ci_on_hosts:
   - wcoss2
   - orion
-  - hera
   - hercules

ci/scripts/clone-build_ci.sh

@@ -48,7 +48,7 @@ git clone "${REPO_URL}"
 cd global-workflow || exit 1
 
 # checkout pull request
-"${GH}" pr checkout "${PR}" --repo "${REPO_URL}" --recurse-submodules -j 4
+"${GH}" pr checkout "${PR}" --repo "${REPO_URL}" --recurse-submodules
 HOMEgfs="${PWD}"
 source "${HOMEgfs}/ush/detect_machine.sh"
 

ci/scripts/utils/rocotostat.py

@@ -85,8 +85,6 @@ def rocoto_statcount():
     rocoto_status = rocoto_statcount()
 
     if rocoto_status['CYCLES_TOTAL'] == rocoto_status['CYCLES_DONE']:
-        if not args.export:
-            print(f"All {rocoto_status['CYCLES_TOTAL']} Cycles are Done")
         rocoto_state = 'DONE'
     elif rocoto_status['DEAD'] > 0:
         error_return = rocoto_status['FAIL'] + rocoto_status['DEAD']
@@ -95,11 +93,11 @@ def rocoto_statcount():
         error_return = rocoto_status['UNKNOWN']
         rocoto_state = 'UNKNOWN'
     elif rocoto_status['RUNNING'] + rocoto_status['SUBMITTING'] + rocoto_status['QUEUED'] == 0:
-    #
-    #  TODO for now a STALLED state will be just a warning as it can
-    #  produce a false negative if there is a timestamp on a file dependency.
-    #
-    #   error_return = -3
+        #
+        #  TODO for now a STALLED state will be just a warning as it can
+        #  produce a false negative if there is a timestamp on a file dependency.
+        #
+        #   error_return = -3
         rocoto_state = 'STALLED'
     else:
         rocoto_state = 'RUNNING'

env/HERA.env

@@ -288,15 +288,13 @@ elif [[ "${step}" = "gempak" ]]; then
     export CFP_MP="YES"
 
     if [[ ${CDUMP} == "gfs" ]]; then
-        npe_gempak=${npe_gempak_gfs}
         npe_node_gempak=${npe_node_gempak_gfs}
     fi
 
     nth_max=$((npe_node_max / npe_node_gempak))
 
     export NTHREADS_GEMPAK=${nth_gempak:-1}
     [[ ${NTHREADS_GEMPAK} -gt ${nth_max} ]] && export NTHREADS_GEMPAK=${nth_max}
-    export APRUN="${launcher} -n ${npe_gempak} ${mpmd_opt}"
 
 
 elif [[ "${step}" = "fit2obs" ]]; then

gempak/ush/gdas_ecmwf_meta_ver.sh

@@ -24,7 +24,7 @@ fi
 
 export COMIN="gdas.${PDY}${cyc}"
 if [[ ! -L ${COMIN} ]]; then
-    ln -sf "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
+    ${NLN} "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
 fi
 vergrid="F-GDAS | ${PDY:2}/0600"
 fcsthr="0600f006"
@@ -49,7 +49,7 @@ for area in ${areas}; do
         sdatenum=$(date --utc +%y%m%d -d "${PDY} ${cyc2} - ${fhr} hours")
 
         if [[ ! -L "ecmwf.20${sdatenum}" ]]; then
-            ln -sf "${COMINecmwf}/ecmwf.20${sdatenum}/gempak" "ecmwf.20${sdatenum}"
+            ${NLN} "${COMINecmwf}/ecmwf.20${sdatenum}/gempak" "ecmwf.20${sdatenum}"
         fi
         gdfile="ecmwf.20${sdatenum}/ecmwf_glob_20${sdatenum}12"
 

gempak/ush/gdas_meta_loop.sh

@@ -13,7 +13,7 @@ device="nc | gdasloop.meta"
 #
 export COMIN="${RUN}.${PDY}${cyc}"
 if [[ ! -L "${COMIN}" ]]; then
-    ln -sf "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
+    ${NLN} "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
 fi
 
 if [[ "${envir}" == "para" ]] ; then
@@ -41,7 +41,7 @@ for (( fhr=24; fhr<=144; fhr+=24 )); do
         YMD=${day} HH=${cyc} GRID=1p00 declare_from_tmpl "COM_ATMOS_GEMPAK_1p00_past:COM_ATMOS_GEMPAK_TMPL"
         export COMIN="${RUN}.${day}${cycle}"
         if [[ ! -L "${COMIN}" ]]; then
-            ln -sf "${COM_ATMOS_GEMPAK_1p00_past}" "${COMIN}"
+            ${NLN} "${COM_ATMOS_GEMPAK_1p00_past}" "${COMIN}"
         fi
         gdfile="${COMIN}/gdas_1p00_${day}${cycle}f000"
 

gempak/ush/gdas_meta_na.sh

@@ -13,7 +13,7 @@ device="nc | gdas.meta"
 #
 export COMIN="${RUN}.${PDY}${cyc}"
 if [[ ! -L "${COMIN}" ]]; then
-    ln -sf "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
+    ${NLN} "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
 fi
 
 if [[ "${envir}" == "para" ]] ; then

gempak/ush/gdas_ukmet_meta_ver.sh

@@ -23,7 +23,7 @@ cp "${HOMEgfs}/gempak/fix/datatype.tbl" datatype.tbl
 # SET CURRENT CYCLE AS THE VERIFICATION GRIDDED FILE.
 export COMIN="gdas.${PDY}${cyc}"
 if [[ ! -L ${COMIN} ]]; then
-    ln -sf "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
+    ${NLN} "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
 fi
 vergrid="F-GDAS | ${PDY:2}/0600"
 fcsthr="0600f006"
@@ -54,7 +54,7 @@ for area in ${areas}; do
         cyclenum=${stime:6}
 
         if [[ ! -L "ukmet.20${sdatenum}" ]]; then
-            ln -sf "${COMINukmet}/ukmet.20${sdatenum}/gempak" "ukmet.20${sdatenum}"
+            ${NLN} "${COMINukmet}/ukmet.20${sdatenum}/gempak" "ukmet.20${sdatenum}"
         fi
         gdfile="ukmet.20${sdatenum}/ukmet_20${sdatenum}${cyclenum}${dgdattim}"
 

gempak/ush/gfs_meta_ak.sh

@@ -20,7 +20,7 @@ device="nc | gfs.meta.ak"
 #
 export COMIN="${RUN}.${PDY}${cyc}"
 if [[ ! -L ${COMIN} ]]; then
-    ln -sf "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
+    ${NLN} "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
 fi
 
 fend=F216

gempak/ush/gfs_meta_bwx.sh

@@ -21,7 +21,7 @@ device="nc | ${metaname}"
 #
 export COMIN="${RUN}.${PDY}${cyc}"
 if [[ ! -L ${COMIN} ]]; then
-    ln -sf "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
+    ${NLN} "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
 fi
 
 fend=F180

gempak/ush/gfs_meta_comp.sh

@@ -29,14 +29,14 @@ for cycle in $(seq -f "%02g" -s ' ' 0  "${STEP_GFS}" "${cyc}"); do
     for file_in in "${gempak_dir}/gfs_1p00_${PDY}${cycle}f"*; do
         file_out="${COMIN}/$(basename "${file_in}")"
         if [[ ! -L "${file_out}" ]]; then
-            ln -sf "${file_in}" "${file_out}"
+            ${NLN} "${file_in}" "${file_out}"
         fi
     done
 done
 
 export HPCNAM="nam.${PDY}"
 if [[ ! -L ${HPCNAM} ]]; then
-    ln -sf "${COMINnam}/nam.${PDY}/gempak" "${HPCNAM}"
+    ${NLN} "${COMINnam}/nam.${PDY}/gempak" "${HPCNAM}"
 fi
 
 #
@@ -100,7 +100,7 @@ for gareas in US NP; do
         HPCGFS="${RUN}.${init_time}"
         if [[ ! -L ${HPCGFS} ]]; then
             YMD="${init_PDY}" HH="${init_cyc}" GRID="1p00" declare_from_tmpl source_dir:COM_ATMOS_GEMPAK_TMPL
-            ln -sf "${source_dir}" "${HPCGFS}"
+            ${NLN} "${source_dir}" "${HPCGFS}"
         fi
 
         if [[ ${init_PDY} == "${PDY}" ]]; then
@@ -230,7 +230,7 @@ EOF
         ukmet_cyc=${ukmet_date:8:2}
         export HPCUKMET=ukmet.${ukmet_PDY}
         if [[ ! -L "${HPCUKMET}" ]]; then
-            ln -sf "${COMINukmet}/ukmet.${ukmet_PDY}/gempak" "${HPCUKMET}"
+            ${NLN} "${COMINukmet}/ukmet.${ukmet_PDY}/gempak" "${HPCUKMET}"
         fi
         grid2="F-UKMETHPC | ${ukmet_PDY:2}/${ukmet_date}"
 

gempak/ush/gfs_meta_crb.sh

@@ -23,7 +23,7 @@ device="nc | ${metaname}"
 #
 export COMIN="${RUN}.${PDY}${cyc}"
 if [[ ! -L ${COMIN} ]]; then
-    ln -sf "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
+    ${NLN} "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
 fi
 
 # DEFINE YESTERDAY
@@ -254,13 +254,13 @@ if [[ ${cyc} == 00 ]] ; then
     HPCECMWF_m1=ecmwf.${PDY}
     export HPCUKMET=ukmet.${PDYm1}
     if [[ ! -L "${HPCECMWF}" ]]; then
-        ln -sf "${COMINecmwf}ecmwf.${PDY}/gempak" "${HPCECMWF}"
+        ${NLN} "${COMINecmwf}ecmwf.${PDY}/gempak" "${HPCECMWF}"
     fi
     if [[ ! -L "${HPCECMWF_m1}" ]]; then
-        ln -sf "${COMINecmwf}ecmwf.${PDYm1}/gempak" "${HPCECMWF_m1}"
+        ${NLN} "${COMINecmwf}ecmwf.${PDYm1}/gempak" "${HPCECMWF_m1}"
     fi
     if [[ ! -L "${HPCUKMET}" ]]; then
-        ln -sf "${COMINukmet}/ukmet.${PDYm1}/gempak" "${HPCUKMET}"
+        ${NLN} "${COMINukmet}/ukmet.${PDYm1}/gempak" "${HPCUKMET}"
     fi
 
     grid1="F-${MDL} | ${PDY:2}/${cyc}00"

gempak/ush/gfs_meta_hi.sh

@@ -17,7 +17,7 @@ device="nc | mrfhi.meta"
 #
 export COMIN="${RUN}.${PDY}${cyc}"
 if [[ ! -L ${COMIN} ]]; then
-    ln -sf "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
+    ${NLN} "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
 fi
 
 if [[ "${envir}" = "prod" ]] ; then