From fc3066c2b7a5edd9f0d510b88f8542b07b8a8589 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Wed, 10 Jun 2020 14:19:07 +0000 Subject: [PATCH 1/7] First draft to add chgresfcst to rocoto workflow --- parm/config/config.chgresfcst | 13 +++++++++++++ parm/config/config.resources | 11 ++++++++++- sorc/link_fv3gfs.sh | 2 ++ ush/rocoto/setup_workflow.py | 21 ++++++++++++++++++--- 4 files changed, 43 insertions(+), 4 deletions(-) create mode 100755 parm/config/config.chgresfcst diff --git a/parm/config/config.chgresfcst b/parm/config/config.chgresfcst new file mode 100755 index 0000000000..2f43f6e176 --- /dev/null +++ b/parm/config/config.chgresfcst @@ -0,0 +1,13 @@ +#!/bin/ksh -x + +########## config.chgresfcst ########## +# regrid full-res forecast for use in ensemble-res analysis generation + +echo "BEGIN: config.chgresfcst" + +# Get task specific resources +. $EXPDIR/config.resources chgresfcst + +export CHGRESFCSTSH=$HOMEgsi/scripts/exglobal_chgresfcst_fv3gfs.sh.ecf + +echo "END: config.chgresfcst" diff --git a/parm/config/config.resources b/parm/config/config.resources index 1f24fe11b6..8382e552f9 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -8,7 +8,7 @@ if [ $# -ne 1 ]; then echo "Must specify an input task argument to set resource variables!" echo "argument can be any one of the following:" - echo "anal analcalc analdiag gldas fcst post vrfy metp arch" + echo "anal analcalc analdiag gldas fcst post vrfy metp arch chgresfcst" echo "eobs ediag eomg eupd ecen esfc efcs epos earc" echo "waveinit waveprep wavepostsbs wavegempaksbs waveawipssbs" echo "wavepost waveawips wavestat" @@ -200,6 +200,15 @@ elif [ $step = "metp" ]; then export memory_metp="16384M" fi +elif [ $step = "chgresfcst" ]; then + + export wtime_chgresfcst="01:00:00" + export npe_chgresfcst=24 + export nth_chgresfcst=1 + export npe_node_chgresfcst=$npe_node_max + if [[ "$machine" = "WCOSS_DELL_P3" ]]; then export npe_chgresfcst=24 ; fi + if [[ "$machine" == "WCOSS_C" ]]; then export memory_chgresfcst="3072M"; fi + elif [ $step = "arch" -o $step = "earc" -o $step = "getic" ]; then eval "export wtime_$step='06:00:00'" diff --git a/sorc/link_fv3gfs.sh b/sorc/link_fv3gfs.sh index fafcd34d02..57ceda4563 100755 --- a/sorc/link_fv3gfs.sh +++ b/sorc/link_fv3gfs.sh @@ -106,6 +106,7 @@ cd ${pwd}/../jobs ||exit 8 $LINK ../sorc/gsi.fd/jobs/JGLOBAL_ANALYSIS . $LINK ../sorc/gsi.fd/jobs/JGLOBAL_ANALCALC . $LINK ../sorc/gsi.fd/jobs/JGLOBAL_ANALDIAG . + $LINK ../sorc/gsi.fd/jobs/JGLOBAL_CHGRESFCST . $LINK ../sorc/gsi.fd/jobs/JGLOBAL_ENKF_SELECT_OBS . $LINK ../sorc/gsi.fd/jobs/JGLOBAL_ENKF_ANALDIAG . $LINK ../sorc/gsi.fd/jobs/JGLOBAL_ENKF_INNOVATE_OBS . @@ -118,6 +119,7 @@ cd ${pwd}/../scripts ||exit 8 $LINK ../sorc/gsi.fd/scripts/exglobal_analysis_fv3gfs.sh.ecf . $LINK ../sorc/gsi.fd/scripts/exglobal_analcalc_fv3gfs.sh.ecf . $LINK ../sorc/gsi.fd/scripts/exglobal_analdiag_fv3gfs.sh.ecf . + $LINK ../sorc/gsi.fd/scripts/exglobal_chgresfcst_fv3gfs.sh.ecf . $LINK ../sorc/gsi.fd/scripts/exglobal_innovate_obs_fv3gfs.sh.ecf . $LINK ../sorc/gsi.fd/scripts/exglobal_enkf_innovate_obs_fv3gfs.sh.ecf . $LINK ../sorc/gsi.fd/scripts/exglobal_enkf_update_fv3gfs.sh.ecf . diff --git a/ush/rocoto/setup_workflow.py b/ush/rocoto/setup_workflow.py index c3cc41e46c..98dc64fabe 100755 --- a/ush/rocoto/setup_workflow.py +++ b/ush/rocoto/setup_workflow.py @@ -54,7 +54,7 @@ def main(): #wav_steps_awips = ['waveawipssbs', 'waveawips'] # From gfsv16b latest # gfs_steps = ['prep', 'anal', 'gldas', 'fcst', 'postsnd', 'post', 'awips', 'gempak', 'vrfy', 'metp', 'arch'] - hyb_steps = ['eobs', 'ediag', 'eomg', 'eupd', 'ecen', 'esfc', 'efcs', 'epos', 'earc'] + hyb_steps = ['eobs', 'ediag', 'eomg', 'eupd', 'ecen', 'esfc', 'efcs', 'epos', 'earc', 'chgresfcst'] steps = gfs_steps + hyb_steps if _base.get('DOHYBVAR', 'NO') == 'YES' else gfs_steps steps = steps + metp_steps if _base.get('DO_METP', 'NO') == 'YES' else steps @@ -342,7 +342,7 @@ def get_hyb_resources(dict_configs): # These tasks are always run as part of the GDAS cycle cdump = 'gdas' - tasks2 = ['ecen', 'esfc', 'efcs', 'epos', 'earc'] + tasks2 = ['ecen', 'esfc', 'efcs', 'epos', 'earc', 'chgresfcst'] for task in tasks2: cfg = dict_configs[task] @@ -473,7 +473,12 @@ def get_gdasgfs_tasks(dict_configs, cdump='gdas'): deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': '%sanal' % cdump} deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) + if dohybvar in ['y', 'Y', 'yes', 'YES']: + dep_dict = {'type': 'task', 'name': '%schgresfcst' % 'gdas', 'offset': '-06:00:00'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + else: + dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) task = wfu.create_wf_task('analcalc', cdump=cdump, envar=envars, dependency=dependencies) dict_tasks['%sanalcalc' % cdump] = task @@ -855,6 +860,16 @@ def get_hyb_tasks(dict_configs, cycledef='enkf'): dict_tasks['%sesfc' % cdump] = task + # chgresfcst + deps1 = [] + dep_dict = {'type': 'task', 'name': '%sfcst' % cdump} + deps1.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'metatask', 'name': '%sefmn' % cdump} + deps1.append(rocoto.add_dependency(dep_dict)) + dependencies1 = rocoto.create_dependency(dep_condition='and', dep=deps2) + task = wfu.create_wf_task('chgresfcst', cdump=cdump, envar=envars1, dependency=dependencies1, cycledef=cycledef) + + dict_tasks['%schgresfcst ' % cdump] = task # efmn, efcs deps1 = [] From 201609b2d43acd13a08bf1d5ab2251db90a11d32 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Wed, 10 Jun 2020 20:51:51 +0000 Subject: [PATCH 2/7] Commit changes from debugging addition of chgresfcst on hera --- jobs/rocoto/chgresfcst.sh | 13 +++++++++++++ parm/config/config.resources | 1 - ush/rocoto/setup_workflow.py | 23 ++++++++++++++++------- 3 files changed, 29 insertions(+), 8 deletions(-) create mode 100755 jobs/rocoto/chgresfcst.sh diff --git a/jobs/rocoto/chgresfcst.sh b/jobs/rocoto/chgresfcst.sh new file mode 100755 index 0000000000..6372dbf83c --- /dev/null +++ b/jobs/rocoto/chgresfcst.sh @@ -0,0 +1,13 @@ +#!/bin/ksh -x + +############################################################### +# Source FV3GFS workflow modules +. $HOMEgfs/ush/load_fv3gfs_modules.sh +status=$? +[[ $status -ne 0 ]] && exit $status + +############################################################### +# Execute the JJOB +$HOMEgfs/jobs/JGLOBAL_CHGRESFCST +status=$? +exit $status diff --git a/parm/config/config.resources b/parm/config/config.resources index 8382e552f9..a01bf32e36 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -207,7 +207,6 @@ elif [ $step = "chgresfcst" ]; then export nth_chgresfcst=1 export npe_node_chgresfcst=$npe_node_max if [[ "$machine" = "WCOSS_DELL_P3" ]]; then export npe_chgresfcst=24 ; fi - if [[ "$machine" == "WCOSS_C" ]]; then export memory_chgresfcst="3072M"; fi elif [ $step = "arch" -o $step = "earc" -o $step = "getic" ]; then diff --git a/ush/rocoto/setup_workflow.py b/ush/rocoto/setup_workflow.py index 98dc64fabe..e67f7f9ec4 100755 --- a/ush/rocoto/setup_workflow.py +++ b/ush/rocoto/setup_workflow.py @@ -308,9 +308,9 @@ def get_hyb_resources(dict_configs): # These tasks can be run in either or both cycles if lobsdiag_forenkf in ['.T.', '.TRUE.']: - tasks1 = ['eobs', 'ediag', 'eupd'] + tasks1 = ['eobs', 'ediag', 'eupd', 'chgresfcst'] else: - tasks1 = ['eobs', 'eomg', 'eupd'] + tasks1 = ['eobs', 'eomg', 'eupd', 'chgresfcst'] if eupd_cyc in ['BOTH']: cdumps = ['gfs', 'gdas'] @@ -342,7 +342,7 @@ def get_hyb_resources(dict_configs): # These tasks are always run as part of the GDAS cycle cdump = 'gdas' - tasks2 = ['ecen', 'esfc', 'efcs', 'epos', 'earc', 'chgresfcst'] + tasks2 = ['ecen', 'esfc', 'efcs', 'epos', 'earc'] for task in tasks2: cfg = dict_configs[task] @@ -473,7 +473,7 @@ def get_gdasgfs_tasks(dict_configs, cdump='gdas'): deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': '%sanal' % cdump} deps.append(rocoto.add_dependency(dep_dict)) - if dohybvar in ['y', 'Y', 'yes', 'YES']: + if dohybvar in ['y', 'Y', 'yes', 'YES'] and cdump == 'gdas': dep_dict = {'type': 'task', 'name': '%schgresfcst' % 'gdas', 'offset': '-06:00:00'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -866,10 +866,10 @@ def get_hyb_tasks(dict_configs, cycledef='enkf'): deps1.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'metatask', 'name': '%sefmn' % cdump} deps1.append(rocoto.add_dependency(dep_dict)) - dependencies1 = rocoto.create_dependency(dep_condition='and', dep=deps2) + dependencies1 = rocoto.create_dependency(dep_condition='and', dep=deps1) task = wfu.create_wf_task('chgresfcst', cdump=cdump, envar=envars1, dependency=dependencies1, cycledef=cycledef) - dict_tasks['%schgresfcst ' % cdump] = task + dict_tasks['%schgresfcst' % cdump] = task # efmn, efcs deps1 = [] @@ -1113,7 +1113,16 @@ def create_xml(dict_configs): dict_hyb_tasks = get_hyb_tasks(dict_configs) # Removes &MEMORY_JOB_DUMP post mortem from hyb tasks - hyp_tasks = {'gdaseobs':'gdaseobs', 'gdasediag':'gdasediag', 'gdaseomg':'gdaseomn', 'gdaseupd':'gdaseupd','gdasecen':'gdasecmn','gdasesfc':'gdasesfc','gdasefcs':'gdasefmn','gdasepos':'gdasepmn','gdasearc':'gdaseamn'} + hyp_tasks = {'gdaseobs':'gdaseobs', + 'gdasediag':'gdasediag', + 'gdaseomg':'gdaseomn', + 'gdaseupd':'gdaseupd', + 'gdasecen':'gdasecmn', + 'gdasesfc':'gdasesfc', + 'gdasefcs':'gdasefmn', + 'gdasepos':'gdasepmn', + 'gdasearc':'gdaseamn', + 'gdaschgresfcst':'gdaschgresfcst'} for each_task, each_resource_string in dict_hyb_resources.iteritems(): #print each_task,hyp_tasks[each_task] #print dict_hyb_tasks[hyp_tasks[each_task]] From c330e60197c38acb724cfdf4a30a20417a6618b4 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Wed, 10 Jun 2020 21:17:00 +0000 Subject: [PATCH 3/7] add checkout.sh to test on Dell --- sorc/checkout.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sorc/checkout.sh b/sorc/checkout.sh index e067fa7d04..9a5675fb29 100755 --- a/sorc/checkout.sh +++ b/sorc/checkout.sh @@ -19,9 +19,11 @@ fi echo gsi checkout ... if [[ ! -d gsi.fd ]] ; then rm -f ${topdir}/checkout-gsi.log - git clone --recursive https://github.com/NOAA-EMC/GSI.git gsi.fd >> ${topdir}/checkout-gsi.log 2>&1 + #git clone --recursive https://github.com/NOAA-EMC/GSI.git gsi.fd >> ${topdir}/checkout-gsi.log 2>&1 + git clone --recursive https://github.com/CoryMartin-NOAA/GSI.git gsi.fd >> ${topdir}/checkout-gsi.log 2>&1 cd gsi.fd - git checkout release/gfsda.v16.0.0 + #git checkout release/gfsda.v16.0.0 + git checkout feature/gdaschgres git submodule update cd ${topdir} else From 15d5bed4119b067905f5a4bf47656557f2211883 Mon Sep 17 00:00:00 2001 From: "russ.treadon" Date: Mon, 15 Jun 2020 20:47:47 +0000 Subject: [PATCH 4/7] Issue #85: add cfp option to chgresfcst; enable threads with chgresfcst --- env/WCOSS_DELL_P3.env | 4 ++++ parm/config/config.resources | 6 +++--- sorc/enkf_chgres_recenter_nc.fd/setup.f90 | 6 ++++-- ush/rocoto/setup_workflow.py | 24 +++++++++++------------ 4 files changed, 23 insertions(+), 17 deletions(-) diff --git a/env/WCOSS_DELL_P3.env b/env/WCOSS_DELL_P3.env index 484b5ce73c..06e68653d3 100755 --- a/env/WCOSS_DELL_P3.env +++ b/env/WCOSS_DELL_P3.env @@ -66,6 +66,10 @@ elif [ $step = "anal" ]; then npe_gausfcanl=${npe_gausfcanl:-1} export APRUN_GAUSFCANL="$launcher $npe_gausfcanl" + export NTHREADS_CHGRES=${nth_chgresfcst:-14} + [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max + export APRUN_CHGRES="" + if [ ${USE_CFP:-"NO"} = "YES" ]; then export APRUNCFP="$launcher \$ncmd $mpmd" fi diff --git a/parm/config/config.resources b/parm/config/config.resources index a01bf32e36..fba9802223 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -203,10 +203,10 @@ elif [ $step = "metp" ]; then elif [ $step = "chgresfcst" ]; then export wtime_chgresfcst="01:00:00" - export npe_chgresfcst=24 + export npe_chgresfcst=3 export nth_chgresfcst=1 - export npe_node_chgresfcst=$npe_node_max - if [[ "$machine" = "WCOSS_DELL_P3" ]]; then export npe_chgresfcst=24 ; fi + export npe_node_chgresfcst=1 + if [[ "$machine" = "WCOSS_DELL_P3" ]]; then export nth_chgresfcst=28 ; fi elif [ $step = "arch" -o $step = "earc" -o $step = "getic" ]; then diff --git a/sorc/enkf_chgres_recenter_nc.fd/setup.f90 b/sorc/enkf_chgres_recenter_nc.fd/setup.f90 index 4bb1a0253c..1ea5b98a82 100644 --- a/sorc/enkf_chgres_recenter_nc.fd/setup.f90 +++ b/sorc/enkf_chgres_recenter_nc.fd/setup.f90 @@ -23,6 +23,7 @@ subroutine program_setup implicit none integer :: istat + character(len=500) :: filenamelist namelist /chgres_setup/ i_output, j_output, input_file, output_file, & terrain_file, vcoord_file, cld_amt @@ -30,8 +31,9 @@ subroutine program_setup cld_amt = .false. ! default option print* - print*,"OPEN SETUP NAMELIST." - open(43, file="./chgres_nc_gauss.nml", iostat=istat) + call getarg(1,filenamelist) + print*,"OPEN SETUP NAMELIST ",trim(filenamelist) + open(43, file=filenamelist, iostat=istat) if (istat /= 0) then print*,"FATAL ERROR OPENING NAMELIST FILE. ISTAT IS: ",istat stop diff --git a/ush/rocoto/setup_workflow.py b/ush/rocoto/setup_workflow.py index e67f7f9ec4..b2a3617821 100755 --- a/ush/rocoto/setup_workflow.py +++ b/ush/rocoto/setup_workflow.py @@ -54,7 +54,7 @@ def main(): #wav_steps_awips = ['waveawipssbs', 'waveawips'] # From gfsv16b latest # gfs_steps = ['prep', 'anal', 'gldas', 'fcst', 'postsnd', 'post', 'awips', 'gempak', 'vrfy', 'metp', 'arch'] - hyb_steps = ['eobs', 'ediag', 'eomg', 'eupd', 'ecen', 'esfc', 'efcs', 'epos', 'earc', 'chgresfcst'] + hyb_steps = ['eobs', 'ediag', 'eomg', 'eupd', 'ecen', 'esfc', 'efcs', 'chgresfcst', 'epos', 'earc'] steps = gfs_steps + hyb_steps if _base.get('DOHYBVAR', 'NO') == 'YES' else gfs_steps steps = steps + metp_steps if _base.get('DO_METP', 'NO') == 'YES' else steps @@ -860,17 +860,6 @@ def get_hyb_tasks(dict_configs, cycledef='enkf'): dict_tasks['%sesfc' % cdump] = task - # chgresfcst - deps1 = [] - dep_dict = {'type': 'task', 'name': '%sfcst' % cdump} - deps1.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': '%sefmn' % cdump} - deps1.append(rocoto.add_dependency(dep_dict)) - dependencies1 = rocoto.create_dependency(dep_condition='and', dep=deps1) - task = wfu.create_wf_task('chgresfcst', cdump=cdump, envar=envars1, dependency=dependencies1, cycledef=cycledef) - - dict_tasks['%schgresfcst' % cdump] = task - # efmn, efcs deps1 = [] dep_dict = {'type': 'metatask', 'name': '%secmn' % cdump} @@ -891,6 +880,17 @@ def get_hyb_tasks(dict_configs, cycledef='enkf'): dict_tasks['%sefmn' % cdump] = task + # chgresfcst + deps1 = [] + dep_dict = {'type': 'task', 'name': '%sfcst' % cdump} + deps1.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'metatask', 'name': '%sefmn' % cdump} + deps1.append(rocoto.add_dependency(dep_dict)) + dependencies1 = rocoto.create_dependency(dep_condition='and', dep=deps1) + task = wfu.create_wf_task('chgresfcst', cdump=cdump, envar=envars1, dependency=dependencies1, cycledef=cycledef) + + dict_tasks['%schgresfcst' % cdump] = task + # epmn, epos deps = [] dep_dict = {'type': 'metatask', 'name': '%sefmn' % cdump} From 71d714119960feed1f0807708cdf85677ab81d73 Mon Sep 17 00:00:00 2001 From: "russ.treadon" Date: Tue, 16 Jun 2020 20:11:33 +0000 Subject: [PATCH 5/7] Issue #85: (1) rename "chgresfcst" as "echgres", (2) add chgres variables to env --- env/HERA.env | 5 ++++- env/JET.env | 4 ++++ env/WCOSS_C.env | 4 ++++ env/WCOSS_DELL_P3.env | 2 +- jobs/rocoto/{chgresfcst.sh => echgres.sh} | 2 +- parm/config/config.chgresfcst | 13 ------------- parm/config/config.echgres | 13 +++++++++++++ parm/config/config.resources | 14 +++++++------- sorc/link_fv3gfs.sh | 4 ++-- ush/rocoto/setup_workflow.py | 16 ++++++++-------- 10 files changed, 44 insertions(+), 33 deletions(-) rename jobs/rocoto/{chgresfcst.sh => echgres.sh} (90%) delete mode 100755 parm/config/config.chgresfcst create mode 100755 parm/config/config.echgres diff --git a/env/HERA.env b/env/HERA.env index dce975a383..bd7e34ca31 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -67,9 +67,12 @@ elif [ $step = "anal" ]; then export NTHREADS_GAUSFCANL=1 npe_gausfcanl=${npe_gausfcanl:-1} - export APRUN_GAUSFCANL="$launcher -n $npe_gausfcanl" + export NTHREADS_CHGRES=${nth_echgres:-1} + [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max + export APRUN_CHGRES="" + export USE_CFP=${USE_CFP:-"YES"} export APRUNCFP="$launcher" diff --git a/env/JET.env b/env/JET.env index c9ab6c3c7f..a63222e3b4 100755 --- a/env/JET.env +++ b/env/JET.env @@ -58,6 +58,10 @@ elif [ $step = "anal" ]; then npe_gausfcanl=${npe_gausfcanl:-1} export APRUN_GAUSFCANL="$launcher $npe_gausfcanl" + export NTHREADS_CHGRES=${nth_echgres:-1} + [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max + export APRUN_CHGRES="" + elif [ $step = "gldas" ]; then nth_max=$(($npe_node_max / $npe_node_gldas)) diff --git a/env/WCOSS_C.env b/env/WCOSS_C.env index b6a40b977d..8be3f66fe3 100755 --- a/env/WCOSS_C.env +++ b/env/WCOSS_C.env @@ -66,6 +66,10 @@ elif [ $step = "anal" ]; then [[ $npe_node_gausfcanl -gt $npe_gausfcanl ]] && npe_node_gausfcanl=$npe_gausfcanl export APRUN_GAUSFCANL="$launcher -j 1 -n $npe_gausfcanl -N $npe_node_gausfcanl -d $NTHREADS_GAUSFCANL -cc depth" + export NTHREADS_CHGRES=${nth_echgres:-1} + [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max + export APRUN_CHGRES="$launcher -j 1 -n 1 -N 1 -d $NTHREADS_CHGRES -cc depth" + elif [ $step = "gldas" ]; then nth_max=$(($npe_node_max / $npe_node_gldas)) diff --git a/env/WCOSS_DELL_P3.env b/env/WCOSS_DELL_P3.env index 06e68653d3..c12ec6a327 100755 --- a/env/WCOSS_DELL_P3.env +++ b/env/WCOSS_DELL_P3.env @@ -66,7 +66,7 @@ elif [ $step = "anal" ]; then npe_gausfcanl=${npe_gausfcanl:-1} export APRUN_GAUSFCANL="$launcher $npe_gausfcanl" - export NTHREADS_CHGRES=${nth_chgresfcst:-14} + export NTHREADS_CHGRES=${nth_echgres:-14} [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max export APRUN_CHGRES="" diff --git a/jobs/rocoto/chgresfcst.sh b/jobs/rocoto/echgres.sh similarity index 90% rename from jobs/rocoto/chgresfcst.sh rename to jobs/rocoto/echgres.sh index 6372dbf83c..5278f58539 100755 --- a/jobs/rocoto/chgresfcst.sh +++ b/jobs/rocoto/echgres.sh @@ -8,6 +8,6 @@ status=$? ############################################################### # Execute the JJOB -$HOMEgfs/jobs/JGLOBAL_CHGRESFCST +$HOMEgfs/jobs/JGDAS_ENKF_CHGRES status=$? exit $status diff --git a/parm/config/config.chgresfcst b/parm/config/config.chgresfcst deleted file mode 100755 index 2f43f6e176..0000000000 --- a/parm/config/config.chgresfcst +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/ksh -x - -########## config.chgresfcst ########## -# regrid full-res forecast for use in ensemble-res analysis generation - -echo "BEGIN: config.chgresfcst" - -# Get task specific resources -. $EXPDIR/config.resources chgresfcst - -export CHGRESFCSTSH=$HOMEgsi/scripts/exglobal_chgresfcst_fv3gfs.sh.ecf - -echo "END: config.chgresfcst" diff --git a/parm/config/config.echgres b/parm/config/config.echgres new file mode 100755 index 0000000000..c7b2061545 --- /dev/null +++ b/parm/config/config.echgres @@ -0,0 +1,13 @@ +#!/bin/ksh -x + +########## config.echgres ########## +# regrid full-res forecast for use in ensemble-res analysis generation + +echo "BEGIN: config.echgres" + +# Get task specific resources +. $EXPDIR/config.resources echgres + +export CHGRESFCSTSH=$HOMEgfs/scripts/exglobal_enkf_chgres_fv3gfs.sh.ecf + +echo "END: config.echgres" diff --git a/parm/config/config.resources b/parm/config/config.resources index fba9802223..16b224f4c2 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -8,7 +8,7 @@ if [ $# -ne 1 ]; then echo "Must specify an input task argument to set resource variables!" echo "argument can be any one of the following:" - echo "anal analcalc analdiag gldas fcst post vrfy metp arch chgresfcst" + echo "anal analcalc analdiag gldas fcst post vrfy metp arch echgres" echo "eobs ediag eomg eupd ecen esfc efcs epos earc" echo "waveinit waveprep wavepostsbs wavegempaksbs waveawipssbs" echo "wavepost waveawips wavestat" @@ -200,13 +200,13 @@ elif [ $step = "metp" ]; then export memory_metp="16384M" fi -elif [ $step = "chgresfcst" ]; then +elif [ $step = "echgres" ]; then - export wtime_chgresfcst="01:00:00" - export npe_chgresfcst=3 - export nth_chgresfcst=1 - export npe_node_chgresfcst=1 - if [[ "$machine" = "WCOSS_DELL_P3" ]]; then export nth_chgresfcst=28 ; fi + export wtime_echgres="01:00:00" + export npe_echgres=3 + export nth_echgres=1 + export npe_node_echgres=1 + if [[ "$machine" = "WCOSS_DELL_P3" ]]; then export nth_echgres=28 ; fi elif [ $step = "arch" -o $step = "earc" -o $step = "getic" ]; then diff --git a/sorc/link_fv3gfs.sh b/sorc/link_fv3gfs.sh index 57ceda4563..74e5635423 100755 --- a/sorc/link_fv3gfs.sh +++ b/sorc/link_fv3gfs.sh @@ -106,7 +106,6 @@ cd ${pwd}/../jobs ||exit 8 $LINK ../sorc/gsi.fd/jobs/JGLOBAL_ANALYSIS . $LINK ../sorc/gsi.fd/jobs/JGLOBAL_ANALCALC . $LINK ../sorc/gsi.fd/jobs/JGLOBAL_ANALDIAG . - $LINK ../sorc/gsi.fd/jobs/JGLOBAL_CHGRESFCST . $LINK ../sorc/gsi.fd/jobs/JGLOBAL_ENKF_SELECT_OBS . $LINK ../sorc/gsi.fd/jobs/JGLOBAL_ENKF_ANALDIAG . $LINK ../sorc/gsi.fd/jobs/JGLOBAL_ENKF_INNOVATE_OBS . @@ -115,11 +114,11 @@ cd ${pwd}/../jobs ||exit 8 $LINK ../sorc/gsi.fd/jobs/JGDAS_ENKF_SURFACE . $LINK ../sorc/gsi.fd/jobs/JGDAS_ENKF_FCST . $LINK ../sorc/gsi.fd/jobs/JGDAS_ENKF_POST . + $LINK ../sorc/gsi.fd/jobs/JGDAS_ENKF_CHGRES . cd ${pwd}/../scripts ||exit 8 $LINK ../sorc/gsi.fd/scripts/exglobal_analysis_fv3gfs.sh.ecf . $LINK ../sorc/gsi.fd/scripts/exglobal_analcalc_fv3gfs.sh.ecf . $LINK ../sorc/gsi.fd/scripts/exglobal_analdiag_fv3gfs.sh.ecf . - $LINK ../sorc/gsi.fd/scripts/exglobal_chgresfcst_fv3gfs.sh.ecf . $LINK ../sorc/gsi.fd/scripts/exglobal_innovate_obs_fv3gfs.sh.ecf . $LINK ../sorc/gsi.fd/scripts/exglobal_enkf_innovate_obs_fv3gfs.sh.ecf . $LINK ../sorc/gsi.fd/scripts/exglobal_enkf_update_fv3gfs.sh.ecf . @@ -127,6 +126,7 @@ cd ${pwd}/../scripts ||exit 8 $LINK ../sorc/gsi.fd/scripts/exglobal_enkf_surface_fv3gfs.sh.ecf . $LINK ../sorc/gsi.fd/scripts/exglobal_enkf_fcst_fv3gfs.sh.ecf . $LINK ../sorc/gsi.fd/scripts/exglobal_enkf_post_fv3gfs.sh.ecf . + $LINK ../sorc/gsi.fd/scripts/exglobal_enkf_chgres_fv3gfs.sh.ecf . cd ${pwd}/../fix ||exit 8 [[ -d fix_gsi ]] && rm -rf fix_gsi $LINK ../sorc/gsi.fd/fix fix_gsi diff --git a/ush/rocoto/setup_workflow.py b/ush/rocoto/setup_workflow.py index b2a3617821..b0559e4bed 100755 --- a/ush/rocoto/setup_workflow.py +++ b/ush/rocoto/setup_workflow.py @@ -54,7 +54,7 @@ def main(): #wav_steps_awips = ['waveawipssbs', 'waveawips'] # From gfsv16b latest # gfs_steps = ['prep', 'anal', 'gldas', 'fcst', 'postsnd', 'post', 'awips', 'gempak', 'vrfy', 'metp', 'arch'] - hyb_steps = ['eobs', 'ediag', 'eomg', 'eupd', 'ecen', 'esfc', 'efcs', 'chgresfcst', 'epos', 'earc'] + hyb_steps = ['eobs', 'ediag', 'eomg', 'eupd', 'ecen', 'esfc', 'efcs', 'echgres', 'epos', 'earc'] steps = gfs_steps + hyb_steps if _base.get('DOHYBVAR', 'NO') == 'YES' else gfs_steps steps = steps + metp_steps if _base.get('DO_METP', 'NO') == 'YES' else steps @@ -308,9 +308,9 @@ def get_hyb_resources(dict_configs): # These tasks can be run in either or both cycles if lobsdiag_forenkf in ['.T.', '.TRUE.']: - tasks1 = ['eobs', 'ediag', 'eupd', 'chgresfcst'] + tasks1 = ['eobs', 'ediag', 'eupd', 'echgres'] else: - tasks1 = ['eobs', 'eomg', 'eupd', 'chgresfcst'] + tasks1 = ['eobs', 'eomg', 'eupd', 'echgres'] if eupd_cyc in ['BOTH']: cdumps = ['gfs', 'gdas'] @@ -474,7 +474,7 @@ def get_gdasgfs_tasks(dict_configs, cdump='gdas'): dep_dict = {'type': 'task', 'name': '%sanal' % cdump} deps.append(rocoto.add_dependency(dep_dict)) if dohybvar in ['y', 'Y', 'yes', 'YES'] and cdump == 'gdas': - dep_dict = {'type': 'task', 'name': '%schgresfcst' % 'gdas', 'offset': '-06:00:00'} + dep_dict = {'type': 'task', 'name': '%sechgres' % 'gdas', 'offset': '-06:00:00'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) else: @@ -880,16 +880,16 @@ def get_hyb_tasks(dict_configs, cycledef='enkf'): dict_tasks['%sefmn' % cdump] = task - # chgresfcst + # echgres deps1 = [] dep_dict = {'type': 'task', 'name': '%sfcst' % cdump} deps1.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'metatask', 'name': '%sefmn' % cdump} deps1.append(rocoto.add_dependency(dep_dict)) dependencies1 = rocoto.create_dependency(dep_condition='and', dep=deps1) - task = wfu.create_wf_task('chgresfcst', cdump=cdump, envar=envars1, dependency=dependencies1, cycledef=cycledef) + task = wfu.create_wf_task('echgres', cdump=cdump, envar=envars1, dependency=dependencies1, cycledef=cycledef) - dict_tasks['%schgresfcst' % cdump] = task + dict_tasks['%sechgres' % cdump] = task # epmn, epos deps = [] @@ -1122,7 +1122,7 @@ def create_xml(dict_configs): 'gdasefcs':'gdasefmn', 'gdasepos':'gdasepmn', 'gdasearc':'gdaseamn', - 'gdaschgresfcst':'gdaschgresfcst'} + 'gdasechgres':'gdasechgres'} for each_task, each_resource_string in dict_hyb_resources.iteritems(): #print each_task,hyp_tasks[each_task] #print dict_hyb_tasks[hyp_tasks[each_task]] From 965ff4203f132c6b032398a7a56494329714247c Mon Sep 17 00:00:00 2001 From: "russ.treadon" Date: Thu, 18 Jun 2020 20:02:23 +0000 Subject: [PATCH 6/7] Issue #85: update DA checkout to point at release/gfsda.v16.0.0 --- sorc/checkout.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/sorc/checkout.sh b/sorc/checkout.sh index 9a5675fb29..e067fa7d04 100755 --- a/sorc/checkout.sh +++ b/sorc/checkout.sh @@ -19,11 +19,9 @@ fi echo gsi checkout ... if [[ ! -d gsi.fd ]] ; then rm -f ${topdir}/checkout-gsi.log - #git clone --recursive https://github.com/NOAA-EMC/GSI.git gsi.fd >> ${topdir}/checkout-gsi.log 2>&1 - git clone --recursive https://github.com/CoryMartin-NOAA/GSI.git gsi.fd >> ${topdir}/checkout-gsi.log 2>&1 + git clone --recursive https://github.com/NOAA-EMC/GSI.git gsi.fd >> ${topdir}/checkout-gsi.log 2>&1 cd gsi.fd - #git checkout release/gfsda.v16.0.0 - git checkout feature/gdaschgres + git checkout release/gfsda.v16.0.0 git submodule update cd ${topdir} else From df89cc800d3c479c132a5e679a2562af91b32f62 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA <26926959+RussTreadon-NOAA@users.noreply.github.com> Date: Fri, 19 Jun 2020 18:40:26 -0400 Subject: [PATCH 7/7] Update config.resources Generalize setting of echgres threads to be maximum permitted on given platform instead of explicitly setting echgres threads on every platform. --- parm/config/config.resources | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/parm/config/config.resources b/parm/config/config.resources index 16b224f4c2..9af606b5e2 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -204,9 +204,8 @@ elif [ $step = "echgres" ]; then export wtime_echgres="01:00:00" export npe_echgres=3 - export nth_echgres=1 + export nth_echgres=$npe_node_max export npe_node_echgres=1 - if [[ "$machine" = "WCOSS_DELL_P3" ]]; then export nth_echgres=28 ; fi elif [ $step = "arch" -o $step = "earc" -o $step = "getic" ]; then