Move Fit2Obs to stand-alone job (NOAA-EMC#1456)

This PR moves the Fit2Obs invocation out of the vrfy job and into its own dedicated `fit2obs` job in the gdas suite. This new dedicated job uses the latest Fit2Obs tag `wflow.1.0`.

The Fit2Obs requires a type of spin-up. The job looks back `VBACKUP_FITS` hrs and needs available inputs for that lookback cycle in the `ROTDIR`. The `jobs/JGDAS_FIT2OBS` script will first check that `xdate` (`CDATE` - `VBACKUP_FITS`) > `SDATE` and then if met, will check that the needed inputs exist.
- If `xdate>SDATE` is not yet satisfied, the job will exit 0 with "Too early for FIT2OBS to run. Exiting.". The conditional is greater-than and not greater-than-or-equal since the first half cycle generally does not have some of the needed inputs (e.g. prepbufr). Thus the first half cycle is not included in the valid lookback cycles. This avoids erroneous job failures for the first cycle to run the Fit2Obs package. Additional logic could be introduced to include the half cycle if all available inputs are available.
- If any of the needed inputs are missing the job will abort with "FATAL ERROR: FILE MISSING: ${file}". 

This spin-up means that the first cycles will run the job but exit 0 immediately. The 6th cycle (if `VBACKUP_FITS=24`) will be the first cycle to run the Fit2Obs package and produce output in the online archive.

Changes:

1. Remove fit2obs variables and settings from `config.vrfy` and into newly created `config.fit2obs` for `fit2obs` job.
2. Remove fit2obs submission/invocation from `jobs/rocoto/vrfy.sh`.
3. Create new `fit2obs` job scripts: `jobs/rocoto/fit2obs.sh` and `jobs/JGDAS_FIT2OBS`
4. Add new `fit2obs` job to setup scripts: `workflow/applications.py` and `workflow/rocoto/workflow_tasks.py`
5. Add new `fit2obs` job to all env files.
6. Add new `fit2obs` job into `config.resources` (use 1 node on WCOSS2 and 3 nodes elsewhere).
7. Add `export DO_FIT2OBS="YES"` to `config.base.emc.dyn`.

Resolves NOAA-EMC#1405
Resolves NOAA-EMC#1232

env/HERA.env

@@ -133,6 +133,7 @@ elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
     export APRUN_GAUSFCANL="${launcher} -n ${npe_gausfcanl}"
 
 elif [[ "${step}" = "sfcanl" ]]; then
+
     nth_max=$((npe_node_max / npe_node_sfcanl))
 
     export NTHREADS_CYCLE=${nth_sfcanl:-14}
@@ -291,4 +292,14 @@ elif [[ "${step}" = "gempak" ]]; then
     export NTHREADS_GEMPAK=${nth_gempak:-1}
     [[ ${NTHREADS_GEMPAK} -gt ${nth_max} ]] && export NTHREADS_GEMPAK=${nth_max}
     export APRUN="${launcher} -n ${npe_gempak} ${mpmd_opt}"
+
+
+elif [[ "${step}" = "fit2obs" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_fit2obs))
+
+    export NTHREADS_FIT2OBS=${nth_fit2obs:-1}
+    [[ ${NTHREADS_FIT2OBS} -gt ${nth_max} ]] && export NTHREADS_FIT2OBS=${nth_max}
+    export MPIRUN="${launcher} -n ${npe_fit2obs}"
+
 fi

env/JET.env

@@ -212,4 +212,12 @@ elif [[ "${step}" = "gempak" ]]; then
 
      echo "WARNING: ${step} is not enabled on ${machine}!"
 
+elif [[ "${step}" = "fit2obs" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_fit2obs))
+
+    export NTHREADS_FIT2OBS=${nth_fit2obs:-1}
+    [[ ${NTHREADS_FIT2OBS} -gt ${nth_max} ]] && export NTHREADS_FIT2OBS=${nth_max}
+    export MPIRUN="${launcher} -n ${npe_fit2obs}"
+
 fi

env/ORION.env

@@ -290,4 +290,13 @@ elif [[ "${step}" = "gempak" ]]; then
     export NTHREADS_GEMPAK=${nth_gempak:-1}
     [[ ${NTHREADS_GEMPAK} -gt ${nth_max} ]] && export NTHREADS_GEMPAK=${nth_max}
     export APRUN="${launcher} -n ${npe_gempak} ${mpmd_opt}"
+
+elif [[ "${step}" = "fit2obs" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_fit2obs))
+
+    export NTHREADS_FIT2OBS=${nth_fit2obs:-1}
+    [[ ${NTHREADS_FIT2OBS} -gt ${nth_max} ]] && export NTHREADS_FIT2OBS=${nth_max}
+    export MPIRUN="${launcher} -n ${npe_fit2obs}"
+
 fi

env/S4.env

@@ -260,4 +260,13 @@ elif [[ "${step}" = "awips" ]]; then
 elif [[ "${step}" = "gempak" ]]; then
 
      echo "WARNING: ${step} is not enabled on S4!"
+
+elif [[ "${step}" = "fit2obs" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_fit2obs))
+
+    export NTHREADS_FIT2OBS=${nth_fit2obs:-1}
+    [[ ${NTHREADS_FIT2OBS} -gt ${nth_max} ]] && export NTHREADS_FIT2OBS=${nth_max}
+    export MPIRUN="${launcher} -n ${npe_fit2obs}"
+
 fi

env/WCOSS2.env

@@ -290,6 +290,14 @@ elif [[ "${step}" = "gempak" ]]; then
     [[ ${NTHREADS_GEMPAK} -gt ${nth_max} ]] && export NTHREADS_GEMPAK=${nth_max}
     export APRUN_GEMPAKCFP="${launcher} -np ${npe_gempak} ${mpmd_opt}"
 
+elif [[ "${step}" = "fit2obs" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_fit2obs))
+
+    export NTHREADS_FIT2OBS=${nth_fit2obs:-1}
+    [[ ${NTHREADS_FIT2OBS} -gt ${nth_max} ]] && export NTHREADS_FIT2OBS=${nth_max}
+    export MPIRUN="${launcher} -np ${npe_fit2obs}"
+
 elif [[ "${step}" = "waveawipsbulls" ]]; then
 
     unset PERL5LIB

jobs/JGDAS_FIT2OBS

@@ -0,0 +1,88 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "fit2obs" -c "base fit2obs"
+
+
+##############################################
+# Set variables used in the script
+##############################################
+
+export CDUMP=${RUN/enkf}
+
+# Ignore spelling warning; nothing is misspelled
+# shellcheck disable=SC2153
+CDATE=$(${NDATE} -"${VBACKUP_FITS}" "${PDY}${cyc}") # set CDATE to lookback cycle for use in fit2obs package
+export CDATE
+vday=${CDATE:0:8}
+vcyc=${CDATE:8:2}
+
+export COM_INA=${ROTDIR}/gdas.${vday}/${vcyc}/atmos
+# We want to defer variable expansion, so ignore warning about single quotes
+# shellcheck disable=SC2016
+export COM_INF='$ROTDIR/vrfyarch/gfs.$fdy/$fzz'
+export COM_PRP=${ROTDIR}/gdas.${vday}/${vcyc}/obs
+
+export PRPI=${COM_PRP}/${RUN}.t${vcyc}z.prepbufr
+export sig1=${COM_INA}/${RUN}.t${vcyc}z.atmanl.nc
+export sfc1=${COM_INA}/${RUN}.t${vcyc}z.atmanl.nc
+export CNVS=${COM_INA}/${RUN}.t${vcyc}z.cnvstat
+
+export OUTPUT_FILETYPE=${OUTPUT_FILETYPE:-netcdf}
+
+export FIT_DIR=${ARCDIR}/fits
+[[ ! -d "${FIT_DIR}" ]] && mkdir -p "${FIT_DIR}"
+export HORZ_DIR=${ARCDIR}/horiz
+[[ ! -d "${HORZ_DIR}" ]] && mkdir -p "${HORZ_DIR}"
+export COMLOX=${DATA}/fitx
+[[ ! -d "${COMLOX}" ]] && mkdir -p "${COMLOX}"
+
+echo "echo err_chk">"${DATA}"/err_chk; chmod 755 "${DATA}"/err_chk
+echo "echo postmsg">"${DATA}"/postmsg; chmod 755 "${DATA}"/postmsg
+
+##############################################
+# Check spinup and available inputs
+##############################################
+
+# Ignore spelling warning; nothing is misspelled
+# shellcheck disable=SC2153
+if [[ ${CDATE} -gt ${SDATE} ]]; then
+  for file in ${PRPI} ${sig1} ${sfc1} ${CNVS}; do
+    if [[ ! -f "${file}" ]]; then
+      echo "FATAL ERROR: FILE MISSING: ${file}"
+      exit 1
+    fi
+  done
+
+  ##############################################
+  # RUN FIT2OBS VERIFICATION
+  ##############################################
+
+  "${fitdir}/batrun/excfs_gdas_vrfyfits.sh.ecf"
+  status=$?
+  [[ ${status} -ne 0 ]] && exit "${status}"
+
+  ##############################################
+  # End JOB SPECIFIC work
+  ##############################################
+
+  ##############################################
+  # Final processing
+  ##############################################
+  if [[ -e "${pgmout}" ]] ; then
+    cat "${pgmout}"
+  fi
+
+else
+
+  echo "Too early for FIT2OBS to run. Exiting."
+
+fi
+
+##########################################
+# Remove the Temporary working directory
+##########################################
+cd "${DATAROOT}" || (echo "FATAL ERROR: ${DATAROOT} does not exist. ABORT!"; exit 1)
+[[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}"
+
+exit 0

jobs/rocoto/fit2obs.sh

@@ -0,0 +1,23 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+###############################################################
+echo
+echo "=============== START TO SOURCE FV3GFS WORKFLOW MODULES ==============="
+. "${HOMEgfs}/ush/load_fv3gfs_modules.sh"
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+export job="fit2obs"
+export jobid="${job}.$$"
+
+###############################################################
+echo
+echo "=============== START TO RUN FIT2OBS ==============="
+# Execute the JJOB
+"${HOMEgfs}/jobs/JGDAS_FIT2OBS"
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+exit 0

jobs/rocoto/vrfy.sh

@@ -57,27 +57,6 @@ if [ ${RUNMOS} = "YES" -a ${CDUMP} = "gfs" ]; then
 fi
 
 
-###############################################################
-echo
-echo "=============== START TO RUN FIT2OBS VERIFICATION ==============="
-if [ ${VRFYFITS} = "YES" -a ${CDUMP} = ${CDFNL} -a ${CDATE} != ${SDATE} ]; then
-
-    export CDUMPFCST=${VDUMP}
-    export TMPDIR="${RUNDIR}/${CDATE}/${CDUMP}"
-    [[ ! -d ${TMPDIR} ]] && mkdir -p ${TMPDIR}
-
-    xdate=$(${NDATE} -${VBACKUP_FITS} ${CDATE})
-
-    export RUN_ENVIR_SAVE=${RUN_ENVIR}
-    export RUN_ENVIR="netcdf"
-
-    ${PREPQFITSH} ${PSLOT} ${xdate} ${ROTDIR} ${ARCDIR} ${TMPDIR}
-
-    export RUN_ENVIR=${RUN_ENVIR_SAVE}
-
-fi
-
-
 ###############################################################
 echo
 echo "=============== START TO RUN RADMON DATA EXTRACTION ==============="

parm/config/config.base.emc.dyn

@@ -379,6 +379,7 @@ export binary_diag=".false."
 
 # Verification options
 export DO_METP="YES"         # Run METPLUS jobs - set METPLUS settings in config.metp
+export DO_FIT2OBS="YES"      # Run fit to observations package
 
 # Archiving options
 export HPSSARCH="@HPSSARCH@"        # save data to HPSS archive

parm/config/config.fit2obs

@@ -0,0 +1,30 @@
+#! /usr/bin/env bash
+
+########## config.fit2obs ##########
+# Fit to Observations
+
+echo "BEGIN: config.fit2obs"
+
+# Get task specific resources
+. "${EXPDIR}/config.resources" fit2obs
+
+export fit_ver="wflow.1.0"
+export fitdir="${BASE_GIT}/Fit2Obs/${fit_ver}"
+
+export HOMEcfs=${fitdir}
+export EXECcfs=${HOMEcfs}/exec
+export USHcfs=${HOMEcfs}/ush
+
+export PRVT=${HOMEgfs}/fix/gsi/prepobs_errtable.global
+export HYBLEVS=${HOMEgfs}/fix/am/global_hyblev.l${LEVS}.txt
+
+export VBACKUP_FITS=24
+export OUTPUT_FILETYPE="netcdf"
+export CONVNETC="YES"
+export ACPROFit="YES"
+
+if [[ ${netcdf_diag:-".false."} = ".true." ]]; then
+    export CONVNETC="YES"
+fi
+
+echo "END: config.fit2obs"

parm/config/config.resources

@@ -12,7 +12,7 @@ if [ $# -ne 1 ]; then
     echo "atmanalprep atmanalrun atmanalpost"
     echo "atmensanalprep atmensanalrun atmensanalpost"
     echo "aeroanlinit aeroanlrun aeroanlfinal"
-    echo "anal sfcanl analcalc analdiag gldas fcst post vrfy metp arch echgres"
+    echo "anal sfcanl analcalc analdiag gldas fcst post vrfy fit2obs metp arch echgres"
     echo "eobs ediag eomg eupd ecen esfc efcs epos earc"
     echo "init_chem mom6ic ocnpost"
     echo "waveinit waveprep wavepostsbs wavepostbndpnt wavepostbndpntbll wavepostpnt"
@@ -623,6 +623,14 @@ elif [ ${step} = "vrfy" ]; then
     fi
     export is_exclusive=True
 
+elif [[ "${step}" = "fit2obs" ]]; then
+
+    export wtime_fit2obs="00:20:00"
+    export npe_fit2obs=3
+    export nth_fit2obs=1
+    export npe_node_fit2obs=1
+    if [[ ${machine} == "WCOSS2" ]]; then export npe_node_fit2obs=3 ; fi
+
 elif [ ${step} = "metp" ]; then
 
     export nth_metp=1

parm/config/config.vrfy

@@ -8,12 +8,8 @@ echo "BEGIN: config.vrfy"
 # Get task specific resources
 . $EXPDIR/config.resources vrfy
 
-export VDUMP="gfs"                # Verifying dump
-export CDUMPFCST="gdas"           # Fit-to-obs with GDAS/GFS prepbufr
 export CDFNL="gdas"               # Scores verification against GDAS/GFS analysis
-
 export MKPGB4PRCP="YES"           # Make 0.25-deg pgb files in ARCDIR for precip verification
-export VRFYFITS="YES"             # Fit to observations
 export VRFYRAD="YES"              # Radiance data assimilation monitoring
 export VRFYOZN="YES"              # Ozone data assimilation monitoring
 export VRFYMINMON="YES"           # GSI minimization monitoring
@@ -22,38 +18,6 @@ export VRFYGENESIS="YES"          # Cyclone genesis verification
 export VRFYFSU="NO"               # Cyclone genesis verification (FSU)
 export RUNMOS="NO"                # whether to run entire MOS package
 
-#-------------------------------------------------
-# Fit to Observations
-#-------------------------------------------------
-
-if [ $VRFYFITS = "YES" ]; then
-
-    export fit_ver="newm.1.3"
-    export fitdir="$BASE_GIT/verif/global/Fit2Obs/${fit_ver}/batrun"
-    export PRVT=$HOMEgfs/fix/gsi/prepobs_errtable.global
-    export HYBLEVS=$HOMEgfs/fix/am/global_hyblev.l${LEVS}.txt
-    export CUE2RUN=$QUEUE
-
-    export VBACKUP_FITS=24
-
-    export CONVNETC="NO"
-    if [ ${netcdf_diag:-".false."} = ".true." ]; then
-        export CONVNETC="YES"
-    fi
-
-    if [ $machine = "HERA" ]; then
-        export PREPQFITSH="$fitdir/subfits_hera_slurm"
-    elif [ $machine = "S4" ]; then
-        export PREPQFITSH="$fitdir/subfits_s4_slurm"
-    elif [ $machine = "ORION" ]; then
-        export PREPQFITSH="$fitdir/subfits_orion_netcdf"
-    else
-        echo "Fit2Obs NOT supported on this machine"
-    fi
-
-fi
-
-
 #----------------------------------------------------------
 # Minimization, Radiance and Ozone Monitoring
 #----------------------------------------------------------

workflow/applications.py

@@ -105,6 +105,7 @@ def __init__(self, conf: Configuration) -> None:
         self.do_awips = _base.get('DO_AWIPS', False)
         self.do_wafs = _base.get('WAFSF', False)
         self.do_vrfy = _base.get('DO_VRFY', True)
+        self.do_fit2obs = _base.get('DO_FIT2OBS', True)
         self.do_metp = _base.get('DO_METP', False)
         self.do_jedivar = _base.get('DO_JEDIVAR', False)
         self.do_jediens = _base.get('DO_JEDIENS', False)
@@ -186,7 +187,7 @@ def _cycled_configs(self):
         if self.do_ocean:
             configs += ['ocnpost']
 
-        configs += ['sfcanl', 'analcalc', 'fcst', 'post', 'vrfy', 'arch']
+        configs += ['sfcanl', 'analcalc', 'fcst', 'post', 'vrfy', 'fit2obs', 'arch']
 
         if self.do_gldas:
             configs += ['gldas']
@@ -401,6 +402,9 @@ def _get_cycled_task_names(self):
                 gdas_tasks += wave_bndpnt_tasks
             gdas_tasks += wave_post_tasks
 
+        if self.do_fit2obs:
+            gdas_tasks += ['fit2obs']
+
         gdas_tasks += gdas_gfs_common_cleanup_tasks
 
         # Collect "gfs" cycle tasks

workflow/rocoto/workflow_tasks.py

@@ -993,6 +993,20 @@ def vrfy(self):
 
         return task
 
+    def fit2obs(self):
+        deps = []
+        dep_dict = {'type': 'metatask', 'name': f'{self.cdump}post'}
+        deps.append(rocoto.add_dependency(dep_dict))
+        dependencies = rocoto.create_dependency(dep=deps)
+
+        cycledef = 'gdas_half,gdas' if self.cdump in ['gdas'] else self.cdump
+
+        resources = self.get_resource('fit2obs')
+        task = create_wf_task('fit2obs', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies,
+                              cycledef=cycledef)
+
+        return task
+
     def metp(self):
         deps = []
         dep_dict = {'type': 'metatask', 'name': f'{self.cdump}post'}
@@ -1020,6 +1034,9 @@ def arch(self):
         if self.app_config.do_vrfy:
             dep_dict = {'type': 'task', 'name': f'{self.cdump}vrfy'}
             deps.append(rocoto.add_dependency(dep_dict))
+        if self.app_config.do_fit2obs and self.cdump in ['gdas']:
+            dep_dict = {'type': 'task', 'name': f'{self.cdump}fit2obs'}
+            deps.append(rocoto.add_dependency(dep_dict))
         if self.app_config.do_metp and self.cdump in ['gfs']:
             dep_dict = {'type': 'metatask', 'name': f'{self.cdump}metp'}
             deps.append(rocoto.add_dependency(dep_dict))