Issue #5 - Sync merge branch 'feature/gfsv16b' into port2orion

* feature/gfsv16b: Update config.resources Issue NOAA-EMC#85: update DA checkout to point at release/gfsda.v16.0.0 Issue NOAA-EMC#85: (1) rename "chgresfcst" as "echgres", (2) add chgres variables to env Issue NOAA-EMC#85: add cfp option to chgresfcst; enable threads with chgresfcst add checkout.sh to test on Dell Commit changes from debugging addition of chgresfcst on hera First draft to add chgresfcst to rocoto workflow
NOAA-GSL · Jun 22, 2020 · df8dc52 · df8dc52
2 parents ff46607 + df89cc8
commit df8dc52
Show file tree

Hide file tree

Showing 10 changed files with 90 additions and 20 deletions.
diff --git a/env/HERA.env b/env/HERA.env
@@ -74,9 +74,15 @@ elif [ $step = "anal" ]; then
 
     export NTHREADS_GAUSFCANL=1
     npe_gausfcanl=${npe_gausfcanl:-1}
-
     export APRUN_GAUSFCANL="$launcher -n $npe_gausfcanl"
 
+    export NTHREADS_CHGRES=${nth_echgres:-1}
+    [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max
+    export APRUN_CHGRES=""
+
+    export USE_CFP=${USE_CFP:-"YES"}
+    export APRUNCFP="$launcher"
+
 elif [ $step = "gldas" ]; then
 
     nth_max=$(($npe_node_max / $npe_node_gldas))

diff --git a/env/JET.env b/env/JET.env
@@ -58,6 +58,10 @@ elif [ $step = "anal" ]; then
     npe_gausfcanl=${npe_gausfcanl:-1}
     export APRUN_GAUSFCANL="$launcher $npe_gausfcanl"
 
+    export NTHREADS_CHGRES=${nth_echgres:-1}
+    [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max
+    export APRUN_CHGRES=""
+
 elif [ $step = "gldas" ]; then
 
     nth_max=$(($npe_node_max / $npe_node_gldas))

diff --git a/env/WCOSS_C.env b/env/WCOSS_C.env
@@ -66,6 +66,10 @@ elif [ $step = "anal" ]; then
     [[ $npe_node_gausfcanl -gt $npe_gausfcanl ]] && npe_node_gausfcanl=$npe_gausfcanl
     export APRUN_GAUSFCANL="$launcher -j 1 -n $npe_gausfcanl -N $npe_node_gausfcanl -d $NTHREADS_GAUSFCANL -cc depth"
 
+    export NTHREADS_CHGRES=${nth_echgres:-1}
+    [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max
+    export APRUN_CHGRES="$launcher -j 1 -n 1 -N 1 -d $NTHREADS_CHGRES -cc depth"
+
 elif [ $step = "gldas" ]; then
 
     nth_max=$(($npe_node_max / $npe_node_gldas))

diff --git a/env/WCOSS_DELL_P3.env b/env/WCOSS_DELL_P3.env
@@ -66,6 +66,10 @@ elif [ $step = "anal" ]; then
     npe_gausfcanl=${npe_gausfcanl:-1}
     export APRUN_GAUSFCANL="$launcher $npe_gausfcanl"
 
+    export NTHREADS_CHGRES=${nth_echgres:-14}
+    [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max
+    export APRUN_CHGRES=""
+
     if [ ${USE_CFP:-"NO"} = "YES" ]; then
         export APRUNCFP="$launcher \$ncmd $mpmd"
     fi

diff --git a/jobs/rocoto/echgres.sh b/jobs/rocoto/echgres.sh
@@ -0,0 +1,13 @@
+#!/bin/ksh -x
+
+###############################################################
+# Source FV3GFS workflow modules
+. $HOMEgfs/ush/load_fv3gfs_modules.sh
+status=$?
+[[ $status -ne 0 ]] && exit $status
+
+###############################################################
+# Execute the JJOB
+$HOMEgfs/jobs/JGDAS_ENKF_CHGRES
+status=$?
+exit $status
diff --git a/parm/config/config.echgres b/parm/config/config.echgres
@@ -0,0 +1,13 @@
+#!/bin/ksh -x
+
+########## config.echgres ##########
+# regrid full-res forecast for use in ensemble-res analysis generation
+
+echo "BEGIN: config.echgres"
+
+# Get task specific resources
+. $EXPDIR/config.resources echgres
+
+export CHGRESFCSTSH=$HOMEgfs/scripts/exglobal_enkf_chgres_fv3gfs.sh.ecf
+
+echo "END: config.echgres"
diff --git a/parm/config/config.resources b/parm/config/config.resources
@@ -8,7 +8,7 @@ if [ $# -ne 1 ]; then
 
     echo "Must specify an input task argument to set resource variables!"
     echo "argument can be any one of the following:"
-    echo "anal analcalc analdiag gldas fcst post vrfy metp arch"
+    echo "anal analcalc analdiag gldas fcst post vrfy metp arch echgres"
     echo "eobs ediag eomg eupd ecen esfc efcs epos earc"
     echo "waveinit waveprep wavepostsbs wavegempaksbs waveawipssbs"
     echo "wavepost waveawips wavestat"
@@ -202,6 +202,13 @@ elif [ $step = "metp" ]; then
             export memory_metp="16384M"
     fi
 
+elif [ $step = "echgres" ]; then
+
+    export wtime_echgres="01:00:00"
+    export npe_echgres=3
+    export nth_echgres=$npe_node_max
+    export npe_node_echgres=1
+
 elif [ $step = "arch" -o $step = "earc" -o $step = "getic" ]; then
 
     eval "export wtime_$step='06:00:00'"

diff --git a/sorc/enkf_chgres_recenter_nc.fd/setup.f90 b/sorc/enkf_chgres_recenter_nc.fd/setup.f90
@@ -23,15 +23,17 @@ subroutine program_setup
  implicit none
 
  integer                           :: istat
+ character(len=500)                :: filenamelist
 
  namelist /chgres_setup/ i_output, j_output, input_file, output_file, &
                       terrain_file, vcoord_file, cld_amt
 
  cld_amt = .false. ! default option
 
  print*
- print*,"OPEN SETUP NAMELIST."
- open(43, file="./chgres_nc_gauss.nml", iostat=istat)
+ call getarg(1,filenamelist)
+ print*,"OPEN SETUP NAMELIST ",trim(filenamelist)
+ open(43, file=filenamelist, iostat=istat)
  if (istat /= 0) then
    print*,"FATAL ERROR OPENING NAMELIST FILE. ISTAT IS: ",istat
    stop 

diff --git a/sorc/link_fv3gfs.sh b/sorc/link_fv3gfs.sh
@@ -116,6 +116,7 @@ cd ${pwd}/../jobs               ||exit 8
     $LINK ../sorc/gsi.fd/jobs/JGDAS_ENKF_SURFACE         .    
     $LINK ../sorc/gsi.fd/jobs/JGDAS_ENKF_FCST            .
     $LINK ../sorc/gsi.fd/jobs/JGDAS_ENKF_POST            .
+    $LINK ../sorc/gsi.fd/jobs/JGDAS_ENKF_CHGRES          .
 cd ${pwd}/../scripts            ||exit 8
     $LINK ../sorc/gsi.fd/scripts/exglobal_analysis_fv3gfs.sh.ecf           .
     $LINK ../sorc/gsi.fd/scripts/exglobal_analcalc_fv3gfs.sh.ecf           .
@@ -127,6 +128,7 @@ cd ${pwd}/../scripts            ||exit 8
     $LINK ../sorc/gsi.fd/scripts/exglobal_enkf_surface_fv3gfs.sh.ecf       .
     $LINK ../sorc/gsi.fd/scripts/exglobal_enkf_fcst_fv3gfs.sh.ecf          .
     $LINK ../sorc/gsi.fd/scripts/exglobal_enkf_post_fv3gfs.sh.ecf          .
+    $LINK ../sorc/gsi.fd/scripts/exglobal_enkf_chgres_fv3gfs.sh.ecf        .
 cd ${pwd}/../fix                ||exit 8
     [[ -d fix_gsi ]] && rm -rf fix_gsi
     $LINK ../sorc/gsi.fd/fix  fix_gsi

diff --git a/ush/rocoto/setup_workflow.py b/ush/rocoto/setup_workflow.py
@@ -54,7 +54,7 @@ def main():
     #wav_steps_awips = ['waveawipssbs', 'waveawips']
 # From gfsv16b latest
 #    gfs_steps = ['prep', 'anal', 'gldas', 'fcst', 'postsnd', 'post', 'awips', 'gempak', 'vrfy', 'metp', 'arch']
-    hyb_steps = ['eobs', 'ediag', 'eomg', 'eupd', 'ecen', 'esfc', 'efcs', 'epos', 'earc']
+    hyb_steps = ['eobs', 'ediag', 'eomg', 'eupd', 'ecen', 'esfc', 'efcs', 'echgres', 'epos', 'earc']
 
     steps = gfs_steps + hyb_steps if _base.get('DOHYBVAR', 'NO') == 'YES' else gfs_steps
     steps = steps + metp_steps if _base.get('DO_METP', 'NO') == 'YES' else steps
@@ -312,9 +312,9 @@ def get_hyb_resources(dict_configs):
 
     # These tasks can be run in either or both cycles
     if lobsdiag_forenkf in ['.T.', '.TRUE.']:
-        tasks1 = ['eobs', 'ediag', 'eupd']
+        tasks1 = ['eobs', 'ediag', 'eupd', 'echgres']
     else:
-        tasks1 = ['eobs', 'eomg', 'eupd']
+        tasks1 = ['eobs', 'eomg', 'eupd', 'echgres']
 
     if eupd_cyc in ['BOTH']:
         cdumps = ['gfs', 'gdas']
@@ -478,18 +478,14 @@ def get_gdasgfs_tasks(dict_configs, cdump='gdas'):
     dep_dict = {'type': 'data', 'data': data}
     deps1.append(rocoto.add_dependency(dep_dict))
     dep_dict = {'type': 'task', 'name': '%sanal' % cdump}
-    deps1.append(rocoto.add_dependency(dep_dict))
-    dependencies1 = rocoto.create_dependency(dep_condition='or', dep=deps1)
-
-    if cdump in ['gdas']:
-        deps2 = []
-        deps2 = dependencies1
-        dep_dict = {'type': 'cycleexist', 'offset': '-06:00:00'}
-        deps2.append(rocoto.add_dependency(dep_dict))
-        dependencies2 = rocoto.create_dependency(dep_condition='and', dep=deps2)
-        task = wfu.create_wf_task('analcalc', cdump=cdump, envar=envars, dependency=dependencies2)
+    deps.append(rocoto.add_dependency(dep_dict))
+    if dohybvar in ['y', 'Y', 'yes', 'YES'] and cdump == 'gdas':
+        dep_dict = {'type': 'task', 'name': '%sechgres' % 'gdas', 'offset': '-06:00:00'}
+        deps.append(rocoto.add_dependency(dep_dict))
+        dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
     else:
-        task = wfu.create_wf_task('analcalc', cdump=cdump, envar=envars, dependency=dependencies1)
+        dependencies = rocoto.create_dependency(dep_condition='or', dep=deps)
+    task = wfu.create_wf_task('analcalc', cdump=cdump, envar=envars, dependency=dependencies)
 
     dict_tasks['%sanalcalc' % cdump] = task
 
@@ -888,7 +884,6 @@ def get_hyb_tasks(dict_configs, cycledef='enkf'):
 
     dict_tasks['%sesfc' % cdump] = task
 
-
     # efmn, efcs
     deps1 = []
     dep_dict = {'type': 'metatask', 'name': '%secmn' % cdump}
@@ -909,6 +904,17 @@ def get_hyb_tasks(dict_configs, cycledef='enkf'):
 
     dict_tasks['%sefmn' % cdump] = task
 
+    # echgres
+    deps1 = []
+    dep_dict = {'type': 'task', 'name': '%sfcst' % cdump}
+    deps1.append(rocoto.add_dependency(dep_dict))
+    dep_dict = {'type': 'metatask', 'name': '%sefmn' % cdump}
+    deps1.append(rocoto.add_dependency(dep_dict))
+    dependencies1 = rocoto.create_dependency(dep_condition='and', dep=deps1)
+    task = wfu.create_wf_task('echgres', cdump=cdump, envar=envars1, dependency=dependencies1, cycledef=cycledef)
+
+    dict_tasks['%sechgres' % cdump] = task
+
     # epmn, epos
     deps = []
     dep_dict = {'type': 'metatask', 'name': '%sefmn' % cdump}
@@ -1131,7 +1137,16 @@ def create_xml(dict_configs):
         dict_hyb_tasks = get_hyb_tasks(dict_configs)
 
         # Removes <memory>&MEMORY_JOB_DUMP</memory> post mortem from hyb tasks
-        hyp_tasks = {'gdaseobs':'gdaseobs', 'gdasediag':'gdasediag', 'gdaseomg':'gdaseomn', 'gdaseupd':'gdaseupd','gdasecen':'gdasecmn','gdasesfc':'gdasesfc','gdasefcs':'gdasefmn','gdasepos':'gdasepmn','gdasearc':'gdaseamn'}
+        hyp_tasks = {'gdaseobs':'gdaseobs',
+                     'gdasediag':'gdasediag',
+                     'gdaseomg':'gdaseomn',
+                     'gdaseupd':'gdaseupd',
+                     'gdasecen':'gdasecmn',
+                     'gdasesfc':'gdasesfc',
+                     'gdasefcs':'gdasefmn',
+                     'gdasepos':'gdasepmn',
+                     'gdasearc':'gdaseamn',
+                     'gdasechgres':'gdasechgres'}
         for each_task, each_resource_string in dict_hyb_resources.iteritems():
             #print each_task,hyp_tasks[each_task]
             #print dict_hyb_tasks[hyp_tasks[each_task]]