updates for global-nest on Gaea C6

.gitignore

@@ -41,6 +41,7 @@ fix/ugwd
 fix/ugwd_nest
 fix/verif
 fix/wave
+fix/nest
 
 # Ignore parm file symlinks
 #--------------------------

parm/config/gefs/config.base

@@ -13,6 +13,7 @@ export RUN_ENVIR="emc"
 
 # Account, queue, etc.
 export ACCOUNT="@ACCOUNT@"
+export ACCOUNT_SERVICE="@ACCOUNT_SERVICE@"
 export QUEUE="@QUEUE@"
 export QUEUE_SERVICE="@QUEUE_SERVICE@"
 export PARTITION_BATCH="@PARTITION_BATCH@"

parm/config/gfs/config.base

@@ -13,6 +13,7 @@ export RUN_ENVIR="emc"
 
 # Account, queue, etc.
 export ACCOUNT="@ACCOUNT@"
+export ACCOUNT_SERVICE="@ACCOUNT_SERVICE@"
 export QUEUE="@QUEUE@"
 export QUEUE_SERVICE="@QUEUE_SERVICE@"
 export PARTITION_BATCH="@PARTITION_BATCH@"
@@ -192,6 +193,7 @@ if [[ "${DO_NEST:-NO}" == "YES" ]] ; then
   export NEST_OUTPUT_GRID="regional_latlon"
   export FIXugwd="${FIXgfs}/ugwd_nest"
   export FIXorog="${FIXgfs}/orog_nest"
+  export FIXgfs="${HOMEgfs}/fix/nest"
 else
   export ntiles=6
   export FIXugwd="${FIXgfs}/ugwd"

parm/config/gfs/config.ufs

@@ -112,6 +112,19 @@ case "${fv3_res}" in
         export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=1
         ;;
     "C96")
+        export nthreads_fv3=1
+        export nthreads_fv3_gfs=1
+        export nthreads_ufs=1
+        export nthreads_ufs_gfs=1
+        export xr_cnvcld=.false. # Do not pass conv. clouds to Xu-Randall cloud fraction
+        export cdmbgwd="0.14,1.8,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
+        export cdmbgwd_gsl="20.0,2.5,1.0,1.0"   # settings for GSL drag suite
+        export knob_ugwp_tauamp=3.0e-3      # setting for UGWPv1 non-stationary GWD
+        export k_split=1
+        export n_split=4
+        export tau=8.0
+        export rf_cutoff=100.0
+        export fv_sg_adj=1800
         if [[ "${DO_NEST:-NO}" == "YES" ]] ; then
           export DELTIM=450
           export layout_x_gfs=4
@@ -157,6 +170,18 @@ case "${fv3_res}" in
         fi
         ;;
     "C192")
+        export nthreads_fv3=1
+        export nthreads_fv3_gfs=2
+        export nthreads_ufs=1
+        export nthreads_ufs_gfs=2
+        export cdmbgwd="0.23,1.5,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
+        export cdmbgwd_gsl="10.0,3.5,1.0,1.0"   # settings for GSL drag suite
+        export knob_ugwp_tauamp=1.5e-3      # setting for UGWPv1 non-stationary GWD
+        export k_split=2
+        export n_split=4
+        export tau=6.0
+        export rf_cutoff=100.0
+        export fv_sg_adj=1800
         if [[ "${DO_NEST:-NO}" == "YES" ]] ; then
           export DELTIM=225
           export layout_x_gfs=5
@@ -201,6 +226,18 @@ case "${fv3_res}" in
         fi
         ;;
     "C384")
+        export nthreads_fv3=2
+        export nthreads_fv3_gfs=2
+        export nthreads_ufs=2
+        export nthreads_ufs_gfs=2
+        export cdmbgwd="1.1,0.72,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
+        export cdmbgwd_gsl="5.0,5.0,1.0,1.0"   # settings for GSL drag suite
+        export knob_ugwp_tauamp=0.8e-3      # setting for UGWPv1 non-stationary GWD
+        export k_split=2
+        export n_split=4
+        export tau=4.0
+        export rf_cutoff=100.0
+        export fv_sg_adj=900
         if [[ "${DO_NEST:-NO}" == "YES" ]] ; then
           export DELTIM=150
           export layout_x_gdas=8
@@ -220,7 +257,7 @@ case "${fv3_res}" in
           export WRITE_GROUP_GDAS=2
           export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GDAS=20
           export WRITE_GROUP_GFS=2
-          export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=20
+          export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=30
         else
           export DELTIM=300
           export layout_x_gdas=8
@@ -247,6 +284,16 @@ case "${fv3_res}" in
         fi
         ;;
     "C768")
+        export nthreads_ufs=4
+        export nthreads_ufs_gfs=4
+        export cdmbgwd="4.0,0.15,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
+        export cdmbgwd_gsl="2.5,7.5,1.0,1.0"   # settings for GSL drag suite
+        export knob_ugwp_tauamp=0.5e-3      # setting for UGWPv1 non-stationary GWD
+        export k_split=2
+        export n_split=4
+        export tau=3.0
+        export rf_cutoff=100.0
+        export fv_sg_adj=450
         if [[ "${DO_NEST:-NO}" == "YES" ]] ; then
           export DELTIM=75
           export layout_x_gdas=16

parm/ufs/model_configure_nest.IN

@@ -0,0 +1,53 @@
+start_year:              @[SYEAR]
+start_month:             @[SMONTH]
+start_day:               @[SDAY]
+start_hour:              @[SHOUR]
+start_minute:            0
+start_second:            0
+nhours_fcst:             @[FHMAX]
+fhrot:                   @[FHROT]
+
+dt_atmos:                @[DT_ATMOS]
+restart_interval:        @[RESTART_INTERVAL]
+
+quilting:                @[QUILTING]
+quilting_restart:        @[QUILTING_RESTART]
+write_groups:            @[WRITE_GROUP]
+write_tasks_per_group:   @[WRTTASK_PER_GROUP]
+itasks:                  @[ITASKS]
+output_history:          @[OUTPUT_HISTORY]
+history_file_on_native_grid: @[HISTORY_FILE_ON_NATIVE_GRID]
+write_dopost:            @[WRITE_DOPOST]
+write_nsflip:            @[WRITE_NSFLIP]
+num_files:               @[NUM_FILES]
+filename_base:           @[FILENAME_BASE]
+output_grid:             @[OUTPUT_GRID]
+output_file:             @[OUTPUT_FILE]
+zstandard_level:         @[ZSTANDARD_LEVEL]
+ideflate:                @[IDEFLATE]
+quantize_mode:           'quantize_bitround'
+quantize_nsd:            @[QUANTIZE_NSD]
+ichunk2d:                @[ICHUNK2D]
+jchunk2d:                @[JCHUNK2D]
+ichunk3d:                @[ICHUNK3D]
+jchunk3d:                @[JCHUNK3D]
+kchunk3d:                @[KCHUNK3D]
+imo:                     @[IMO]
+jmo:                     @[JMO]
+output_fh:               @[OUTPUT_FH]
+iau_offset:              @[IAU_OFFSET]
+
+# Output settings for the nest.
+<output_grid_02>
+output_grid:             @[NEST_OUTPUT_GRID]
+imo:                     @[NEST_IMO]
+jmo:                     @[NEST_JMO]
+cen_lon:                 @[TARGET_LON]
+cen_lat:                 @[TARGET_LAT]
+lon1:                    @[NEST_LON1]
+lat1:                    @[NEST_LAT1]
+lon2:                    @[NEST_LON2]
+lat2:                    @[NEST_LAT2]
+dlon:                    @[NEST_DLON]
+dlat:                    @[NEST_DLAT]
+</output_grid_02>

sorc/link_workflow.sh

@@ -10,7 +10,7 @@ function usage() {
 Builds all of the global-workflow components by calling the individual build
   scripts in sequence.
 
-Usage: ${BASH_SOURCE[0]} [-h][-o][--nest]
+Usage: ${BASH_SOURCE[0]} [-h][-o]
   -h:
     Print this help message and exit
   -o:
@@ -23,7 +23,7 @@ RUN_ENVIR="emc"
 
 # Reset option counter in case this script is sourced
 OPTIND=1
-while getopts ":ho-:" option; do
+while getopts ":ho" option; do
   case "${option}" in
   h) usage ;;
   o)
@@ -99,6 +99,7 @@ done
 # Link fix directories
 if [[ -n "${FIX_DIR}" ]]; then
   if [[ ! -d "${HOMEgfs}/fix" ]]; then mkdir "${HOMEgfs}/fix" || exit 1; fi
+  if [[ ! -d "${HOMEgfs}/fix/nest" ]]; then mkdir "${HOMEgfs}/fix/nest" || exit 1; fi
 fi
 cd "${HOMEgfs}/fix" || exit 1
 for dir in aer \

ush/forecast_predet.sh

@@ -48,6 +48,10 @@ FV3_coldstarts(){
       fv3_input_files+=("${tt}.tile${nn}.nc")
     done
   done
+  if [[ "${DO_NEST:-NO}" == "YES" ]] ; then
+    fv3_input_files+=("gfs_data.nest02.tile7.nc")
+    fv3_input_files+=("sfc_data.nest02.tile7.nc")
+  fi
   # Create a comma separated string from array using IFS
   IFS=, echo "${fv3_input_files[*]}"
 }

ush/parsing_namelists_FV3_nest.sh

@@ -136,7 +136,7 @@ cat > "${nml_file}" <<EOF
   npy = ${npy_here}
   ntiles = ${ntiles_here}
   ${nested_here:+nested = ${nested_here}}
-  ${twowaynest_here:+twowaynest = ${twowaynest_herei}}
+  ${twowaynest_here:+twowaynest = ${twowaynest_here}}
   ${nestupdate_here:+nestupdate = ${nestupdate_here}}
   npz = ${npz}
   dz_min =  ${dz_min:-"6"}

workflow/hosts/container.yaml

@@ -8,6 +8,7 @@ PTMP: '/home/${USER}'
 NOSCRUB: $HOMEDIR
 SCHEDULER: none
 ACCOUNT: ''
+ACCOUNT_SERVICE: ''
 QUEUE: ''
 QUEUE_SERVICE: ''
 PARTITION_BATCH: ''

workflow/hosts/hera.yaml

@@ -4,11 +4,12 @@ BASE_DATA: '/scratch1/NCEPDEV/global/glopara/data'
 BASE_IC: '/scratch1/NCEPDEV/global/glopara/data/ICSDIR'
 PACKAGEROOT: '/scratch1/NCEPDEV/global/glopara/nwpara'
 COMINsyn: '/scratch1/NCEPDEV/global/glopara/com/gfs/prod/syndat'
-HOMEDIR: '/scratch1/NCEPDEV/global/${USER}'
-STMP: '/scratch1/NCEPDEV/stmp2/${USER}'
-PTMP: '/scratch1/NCEPDEV/stmp4/${USER}'
+HOMEDIR: '/scratch2/BMC/wrfruc/Guoqing.Ge/ufs-ar/noscrub'
+STMP: '/scratch2/BMC/wrfruc/Guoqing.Ge/ufs-ar/stmp'
+PTMP: '/scratch2/BMC/wrfruc/Guoqing.Ge/ufs-ar/ptmp'
 NOSCRUB: $HOMEDIR
-ACCOUNT: fv3-cpu
+ACCOUNT: wrfruc
+ACCOUNT_SERVICE: wrfruc
 SCHEDULER: slurm
 QUEUE: batch
 QUEUE_SERVICE: batch
@@ -19,9 +20,9 @@ CHGRP_RSTPROD: 'YES'
 CLUSTERS: ''
 CHGRP_CMD: 'chgrp rstprod'
 HPSSARCH: 'YES'
-HPSS_PROJECT: emc-global
+HPSS_PROJECT: wrfruc
 LOCALARCH: 'NO'
-ATARDIR: '/NCEPDEV/${HPSS_PROJECT}/1year/${USER}/${machine}/scratch/${PSLOT}'
+ATARDIR: '/ESRL/BMC/wrfruc/5year/ufsar/${USER}/${machine}/${PSLOT}'
 MAKE_NSSTBUFR: 'NO'
 MAKE_ACFTBUFR: 'NO'
 DO_AWIPS: 'NO'

workflow/hosts/hercules.yaml

@@ -10,6 +10,7 @@ PTMP: '/work/noaa/stmp/${USER}/HERCULES'
 NOSCRUB: $HOMEDIR
 SCHEDULER: slurm
 ACCOUNT: fv3-cpu
+ACCOUNT_SERVICE: fv3-cpu
 QUEUE: batch
 QUEUE_SERVICE: batch
 PARTITION_BATCH: hercules

workflow/hosts/jet.yaml

@@ -1,31 +1,30 @@
-BASE_GIT: '/lfs5/HFIP/hfv3gfs/glopara/git'
-DMPDIR: '/lfs5/HFIP/hfv3gfs/glopara/dump'
-BASE_DATA: '/lfs5/HFIP/hfv3gfs/glopara/data'
-BASE_IC: '/mnt/lfs5/HFIP/hfv3gfs/glopara/data/ICSDIR'
-PACKAGEROOT: '/lfs5/HFIP/hfv3gfs/glopara/nwpara'
-COMINsyn: '/lfs5/HFIP/hfv3gfs/glopara/com/gfs/prod/syndat'
-HOMEDIR: '/lfs5/HFIP/hfv3gfs/${USER}'
-STMP: '/lfs5/HFIP/hfv3gfs/${USER}/stmp'
-PTMP: '/lfs5/HFIP/hfv3gfs/${USER}/ptmp'
+BASE_GIT: '/lfs4/HFIP/hfv3gfs/glopara/git'
+DMPDIR: '/lfs4/BMC/nrtrr/gge/ufs-ar/dump/6hourly'
+BASE_CPLIC: '/mnt/lfs4/HFIP/hfv3gfs/glopara/data/ICSDIR/prototype_ICs'
+PACKAGEROOT: '/lfs4/HFIP/hfv3gfs/glopara/nwpara'
+COMINsyn: '/lfs4/HFIP/hfv3gfs/glopara/com/gfs/prod/syndat'
+HOMEDIR: '/lfs4/BMC/nrtrr/gge/ufs-ar/noscrub'
+STMP: '/lfs4/BMC/nrtrr/gge/ufs-ar/stmp'
+PTMP: '/lfs4/BMC/nrtrr/gge/ufs-ar/ptmp'
 NOSCRUB: $HOMEDIR
-ACCOUNT: hfv3gfs
+ACCOUNT: rtwrfruc
+ACCOUNT_SERVICE: nrtrr
 SCHEDULER: slurm
-QUEUE: batch
+QUEUE: rth
 QUEUE_SERVICE: batch
 PARTITION_BATCH: kjet
 PARTITION_SERVICE: service
-RESERVATION: ''
+RESERVATION: rrfsens
 CHGRP_RSTPROD: 'YES'
 CLUSTERS: ''
 CHGRP_CMD: 'chgrp rstprod'
 HPSSARCH: 'YES'
-HPSS_PROJECT: emc-global
+HPSS_PROJECT: wrfruc
 LOCALARCH: 'NO'
-ATARDIR: '/NCEPDEV/${HPSS_PROJECT}/1year/${USER}/${machine}/scratch/${PSLOT}'
+ATARDIR: '/ESRL/BMC/wrfruc/5year/ufsar/${USER}/${machine}/${PSLOT}'
 MAKE_NSSTBUFR: 'NO'
 MAKE_ACFTBUFR: 'NO'
-SUPPORTED_RESOLUTIONS: ['C384', 'C192', 'C96', 'C48']
-COMINecmwf: /mnt/lfs5/HFIP/hfv3gfs/glopara/data/external_gempak/ecmwf
-COMINnam: /mnt/lfs5/HFIP/hfv3gfs/glopara/data/external_gempak/nam
-COMINukmet: /mnt/lfs5/HFIP/hfv3gfs/glopara/data/external_gempak/ukmet
-AERO_INPUTS_DIR: /lfs5/HFIP/hfv3gfs/glopara/data/gocart_emissions
+SUPPORTED_RESOLUTIONS: ['C768', 'C384', 'C192', 'C96', 'C48']
+COMINecmwf: /mnt/lfs4/HFIP/hfv3gfs/glopara/data/external_gempak/ecmwf
+COMINnam: /mnt/lfs4/HFIP/hfv3gfs/glopara/data/external_gempak/nam
+COMINukmet: /mnt/lfs4/HFIP/hfv3gfs/glopara/data/external_gempak/ukmet

workflow/hosts/orion.yaml

@@ -10,6 +10,7 @@ PTMP: '/work/noaa/stmp/${USER}/ORION'
 NOSCRUB: $HOMEDIR
 SCHEDULER: slurm
 ACCOUNT: fv3-cpu
+ACCOUNT_SERVICE: fv3-cpu
 QUEUE: batch
 QUEUE_SERVICE: batch
 PARTITION_BATCH: orion

workflow/hosts/s4.yaml

@@ -9,6 +9,7 @@ STMP: '/scratch/users/${USER}'
 PTMP: '/scratch/users/${USER}'
 NOSCRUB: ${HOMEDIR}
 ACCOUNT: star
+ACCOUNT_SERVICE: star
 SCHEDULER: slurm
 QUEUE: s4
 QUEUE_SERVICE: serial

workflow/hosts/wcoss2.yaml

@@ -9,6 +9,7 @@ STMP: '/lfs/h2/emc/stmp/${USER}'
 PTMP: '/lfs/h2/emc/ptmp/${USER}'
 NOSCRUB: $HOMEDIR
 ACCOUNT: 'GFS-DEV'
+ACCOUNT_SERVICE: 'GFS-DEV'
 SCHEDULER: pbspro
 QUEUE: 'dev'
 QUEUE_SERVICE: 'dev_transfer'

workflow/rocoto/tasks.py

@@ -310,7 +310,7 @@ def get_resource(self, task_name):
 
         task_config = self._configs[task_name]
 
-        account = task_config['ACCOUNT']
+        account = task_config['ACCOUNT_SERVICE'] if task_name in Tasks.SERVICE_TASKS else task_config['ACCOUNT']
 
         walltime = task_config[f'walltime']
         ntasks = task_config[f'ntasks']