Add new task to post-process marine DA (#1485)

The work in this PR is only meant to bring us closer to a viable WCDA system. The refactoring of the marine DA to the new standard introduced by @aerorahul and used by @RussTreadon-NOAA and @CoryMartin-NOAA will be addressed after this [Epic](NOAA-EMC/GDASApp#416) is resolved. 

### Motivation and context
This work adds a separate j-job ```JGDAS_GLOBAL_OCEAN_ANALYSIS_CHKPT ``` that calls a script that will be in the GDASApp for the time being (PR to come once this is merged) and does the following:
- prepares the `SOCA` increment for `MOM6` IAU
- recursively apply the `SOCA2CICE` change of variable. A mapping from the 2D seaice analysis variable to the CICE6 dynamical and thermodynamic variables.
- merge the `Tref` increment from the `NSST` analysis with the `SOCA` increment   

### Summary of the change
- HPC environment: the new j-job runs a `JEDI` executable twice and one python script. All are serial jobs but the JEDI exec need to be called as an MPI job with 1 pe. 
- `jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_CHKPT`, that script point to a ex-script that is not in the GDASApp develop yet.
- addition of the option to merge the Tref NSST increment with the MOM6 increment. This is triggered with the `DO_MERGENSST` switch
- The new j-job dependency was added, with the option to wait for the surface analysis file `sfcanl.nc` if `do_mergensst` is true.

Refs: #1480.
Fixes NOAA-EMC/GDASApp/#418

env/HERA.env

@@ -96,6 +96,16 @@ elif [[ "${step}" = "ocnanalrun" ]]; then
     [[ ${NTHREADS_OCNANAL} -gt ${nth_max} ]] && export NTHREADS_OCNANAL=${nth_max}
     export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalrun}"
 
+elif [[ "${step}" = "ocnanalchkpt" ]]; then
+
+    export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
+
+    nth_max=$((npe_node_max / npe_node_ocnanalchkpt))
+
+    export NTHREADS_OCNANAL=${nth_ocnanalchkpt:-${nth_max}}
+    [[ ${NTHREADS_OCNANAL} -gt ${nth_max} ]] && export NTHREADS_OCNANAL=${nth_max}
+    export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalchkpt}"
+
 elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
 
     export MKL_NUM_THREADS=4

env/ORION.env

@@ -98,6 +98,16 @@ elif [[ "${step}" = "ocnanalrun" ]]; then
     [[ ${NTHREADS_OCNANAL} -gt ${nth_max} ]] && export NTHREADS_OCNANAL=${nth_max}
     export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalrun}"
 
+elif [[ "${step}" = "ocnanalchkpt" ]]; then
+
+    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
+
+    nth_max=$((npe_node_max / npe_node_ocnanalchkpt))
+
+    export NTHREADS_OCNANAL=${nth_ocnanalchkpt:-${nth_max}}
+    [[ ${NTHREADS_OCNANAL} -gt ${nth_max} ]] && export NTHREADS_OCNANAL=${nth_max}
+    export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalchkpt}"
+
 elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
 
     export MKL_NUM_THREADS=4

jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_CHKPT

@@ -0,0 +1,52 @@
+#!/bin/bash
+export STRICT="NO"
+source "${HOMEgfs}/ush/preamble.sh"
+export WIPE_DATA="NO"
+export DATA="${DATAROOT}/${RUN}ocnanal_${cyc}"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "ocnanalchkpt" -c "base ocnanal ocnanalchkpt"
+
+
+##############################################
+# Begin JOB SPECIFIC work
+##############################################
+
+# Ignore possible spelling error (nothing is misspelled)
+# shellcheck disable=SC2153
+GDATE=$(date +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours")
+export GDATE
+export gPDY=${GDATE:0:8}
+export gcyc=${GDATE:8:2}
+export GDUMP=${GDUMP:-"gdas"}
+
+export GPREFIX="${GDUMP}.t${gcyc}z."
+# Ignore possible spelling error (nothing is misspelled)
+# shellcheck disable=SC2153
+export APREFIX="${CDUMP}.t${cyc}z."
+
+export COMOUT=${COMOUT:-${ROTDIR}/${CDUMP}.${PDY}/${cyc}/ocean}
+
+###############################################################
+# Run relevant script
+
+EXSCRIPT=${GDASPREPPY:-${HOMEgfs}/sorc/gdas.cd/scripts/exgdas_global_marine_analysis_chkpt.sh}
+${EXSCRIPT}
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+##############################################
+# End JOB SPECIFIC work
+##############################################
+
+##############################################
+# Final processing
+##############################################
+if [[ -e "${pgmout}" ]] ; then
+  cat "${pgmout}"
+fi
+
+##########################################
+# Do not remove the Temporary working directory (do this in POST)
+##########################################
+cd "${DATAROOT}" || exit 1
+
+exit 0

parm/config/config.base.emc.dyn

@@ -311,6 +311,7 @@ export imp_physics=@IMP_PHYSICS@
 export DO_JEDIVAR="NO"
 export DO_JEDIENS="NO"
 export DO_JEDIOCNVAR="NO"
+export DO_MERGENSST="NO"
 
 # Hybrid related
 export DOHYBVAR="@DOHYBVAR@"

parm/config/config.ocnanal

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 ########## config.ocnanal ##########
-# configuration common to all atm analysis tasks
+# configuration common to all ocean analysis tasks
 
 echo "BEGIN: config.ocnanal"
 
@@ -15,7 +15,7 @@ export SOCA_VARS=tocn,socn,ssh
 export SABER_BLOCKS_YAML=@SABER_BLOCKS_YAML@
 export SOCA_NINNER=@SOCA_NINNER@
 export CASE_ANL=@CASE_ANL@
-export DOMAIN_STACK_SIZE=2000000
+export DOMAIN_STACK_SIZE=116640000 #TODO: Make the stack size reolution dependent 
 export JEDI_BIN=${HOMEgfs}/sorc/gdas.cd/build/bin
 
 # R2D2

parm/config/config.ocnanalchkpt

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+########## config.ocnanalchkpt ##########
+# Ocn Analysis specific
+
+echo "BEGIN: config.ocnanalchkpt"
+
+# Get task specific resources
+. "${EXPDIR}/config.resources" ocnanalchkpt
+
+echo "END: config.ocnanalchkpt"

parm/config/config.resources

@@ -19,6 +19,8 @@ if [[ $# -ne 1 ]]; then
     echo "wavegempak waveawipsbulls waveawipsgridded"
     echo "postsnd awips gempak"
     echo "wafs wafsgrib2 wafsblending wafsgrib20p25 wafsblending0p25 wafsgcip"
+    echo "ocnanalprep ocnanalbmat ocnanalrun ocnanalchkpt ocnanalpost"
+
     exit 1
 
 fi
@@ -327,6 +329,25 @@ elif [[ "${step}" = "ocnanalrun" ]]; then
     npe_node_ocnanalrun=$(echo "${npe_node_max} / ${nth_ocnanalrun}" | bc)
     export npe_node_ocnanalrun
 
+elif [[ "${step}" = "ocnanalchkpt" ]]; then
+
+   export wtime_ocnanalchkpt="00:10:00"
+   export npe_ocnanalchkpt=1
+   export nth_ocnanalchkpt=1
+   npe_node_ocnanalchkpt=$(echo "${npe_node_max} / ${nth_ocnanalchkpt}" | bc)
+   export npe_node_ocnanalchkpt
+   case ${CASE} in
+      C384)
+        export memory_ocnanalchkpt="128GB"
+        ;;
+      C48)
+        export memory_ocnanalchkpt="32GB"
+        ;;
+      *)
+          echo "FATAL: Resolution not supported'"
+          exit 1
+    esac
+
 elif [[ "${step}" = "ocnanalpost" ]]; then
 
     export wtime_ocnanalpost="00:30:00"

ush/forecast_postdet.sh

@@ -1001,7 +1001,12 @@ CICE_postdet() {
 
   # Copy/link CICE IC to DATA
   if [[ "${warm_start}" = ".true." ]]; then
-    $NLN "${ROTDIR}/${CDUMP}.${gPDY}/${gcyc}/ice/RESTART/${PDY}.${cyc}0000.cice_model.res.nc" "${DATA}/cice_model.res.nc"
+    cice_ana="${ROTDIR}/${CDUMP}.${PDY}/${cyc}/ice/RESTART/${PDY}.${cyc}0000.cice_model_anl.res.nc"
+    if [[ -e ${cice_ana} ]]; then
+      ${NLN} "${cice_ana}" "${DATA}/cice_model.res.nc"
+    else
+      ${NLN} "${ROTDIR}/${CDUMP}.${gPDY}/${gcyc}/ice/RESTART/${PDY}.${cyc}0000.cice_model.res.nc" "${DATA}/cice_model.res.nc"
+    fi
   else # cold start are typically SIS2 restarts obtained from somewhere else e.g. CPC
     $NLN "${ROTDIR}/${CDUMP}.${PDY}/${cyc}/ice/RESTART/${PDY}.${cyc}0000.cice_model.res.nc" "${DATA}/cice_model.res.nc"
   fi

workflow/applications.py

@@ -110,6 +110,7 @@ def __init__(self, conf: Configuration) -> None:
         self.do_jediatmvar = _base.get('DO_JEDIVAR', False)
         self.do_jediens = _base.get('DO_JEDIENS', False)
         self.do_jediocnvar = _base.get('DO_JEDIOCNVAR', False)
+        self.do_mergensst = _base.get('DO_MERGENSST', False)
 
         self.do_hpssarch = _base.get('HPSSARCH', False)
 
@@ -183,7 +184,7 @@ def _cycled_configs(self):
             configs += ['anal', 'analdiag']
 
         if self.do_jediocnvar:
-            configs += ['ocnanalprep', 'ocnanalbmat', 'ocnanalrun', 'ocnanalpost']
+            configs += ['ocnanalprep', 'ocnanalbmat', 'ocnanalrun', 'ocnanalchkpt', 'ocnanalpost']
         if self.do_ocean:
             configs += ['ocnpost']
 
@@ -360,7 +361,8 @@ def _get_cycled_task_names(self):
             gdas_gfs_common_tasks_before_fcst += ['anal']
 
         if self.do_jediocnvar:
-            gdas_gfs_common_tasks_before_fcst += ['ocnanalprep', 'ocnanalbmat', 'ocnanalrun', 'ocnanalpost']
+            gdas_gfs_common_tasks_before_fcst += ['ocnanalprep', 'ocnanalbmat', 'ocnanalrun',
+                                                  'ocnanalchkpt', 'ocnanalpost']
 
         gdas_gfs_common_tasks_before_fcst += ['sfcanl', 'analcalc']
 

workflow/rocoto/workflow_tasks.py

@@ -13,7 +13,7 @@ class Tasks:
     VALID_TASKS = ['aerosol_init', 'coupled_ic', 'getic', 'init',
                    'prep', 'anal', 'sfcanl', 'analcalc', 'analdiag', 'gldas', 'arch',
                    'atmanlinit', 'atmanlrun', 'atmanlfinal',
-                   'ocnanalprep', 'ocnanalbmat', 'ocnanalrun', 'ocnanalpost',
+                   'ocnanalprep', 'ocnanalbmat', 'ocnanalrun', 'ocnanalchkpt', 'ocnanalpost',
                    'earc', 'ecen', 'echgres', 'ediag', 'efcs',
                    'eobs', 'eomg', 'epos', 'esfc', 'eupd',
                    'atmensanalprep', 'atmensanalrun', 'atmensanalpost',
@@ -533,11 +533,31 @@ def ocnanalrun(self):
 
         return task
 
-    def ocnanalpost(self):
+    def ocnanalchkpt(self):
 
         deps = []
         dep_dict = {'type': 'task', 'name': f'{self.cdump}ocnanalrun'}
         deps.append(rocoto.add_dependency(dep_dict))
+        if self.app_config.do_mergensst:
+            data = f'&ROTDIR;/{self.cdump}.@Y@m@d/@H/atmos/{self.cdump}.t@Hz.sfcanl.nc'
+            dep_dict = {'type': 'data', 'data': data}
+            deps.append(rocoto.add_dependency(dep_dict))
+        dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
+
+        resources = self.get_resource('ocnanalchkpt')
+        task = create_wf_task('ocnanalchkpt',
+                              resources,
+                              cdump=self.cdump,
+                              envar=self.envars,
+                              dependency=dependencies)
+
+        return task
+
+    def ocnanalpost(self):
+
+        deps = []
+        dep_dict = {'type': 'task', 'name': f'{self.cdump}ocnanalchkpt'}
+        deps.append(rocoto.add_dependency(dep_dict))
         dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
 
         resources = self.get_resource('ocnanalpost')