Changes to RRFS 3- and 13-km domains, setup.sh script bug fixes, make…

…_ics task modification, and tweaks to stochastic physics namelist settings (#721)

* Modify RRFS North America 3- and 13-km domain configuration and WE2E test.

* Change sotyp_from_climo to "true" based on operational RAP grib2 files.

* Update for changes to stochastic physics namelist options.

* Check for DO_ENSEMBLE="TRUE" when running ensemble verification and turn of VX when running in NCO mode.

* Revert to 3-km domain.

* Remove commented-out GFDL grid for the RRFS_NA_13km domain

* Add RRFS_NA_13km WE2E test

* Changes to comments.

scripts/exregional_make_ics.sh

@@ -437,7 +437,7 @@ case "${EXTRN_MDL_NAME_ICS}" in
 #
   geogrid_file_input_grid="${FIXgsm}/geo_em.d01.nc_RAPX"
   vgtyp_from_climo=True
-  sotyp_from_climo=False
+  sotyp_from_climo=True
   vgfrc_from_climo=True
   minmax_vgfrc_from_climo=True
   lai_from_climo=True

tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.sh

@@ -0,0 +1,54 @@
+#
+# TEST PURPOSE/DESCRIPTION:
+# ------------------------
+#
+# This test is to ensure that the workflow running in community mode 
+# completes successfully on the RRFS_NA_13km grid using the RRFS_v1beta
+# physics suite with ICs and LBCs derived from the FV3GFS.
+#
+# Note that this test also sets various resource parameters for several
+# of the rocoto tasks in order to more efficiently run the code on this
+# (very large) grid.
+#
+
+RUN_ENVIR="community"
+PREEXISTING_DIR_METHOD="rename"
+
+PREDEF_GRID_NAME="RRFS_NA_13km"
+CCPP_PHYS_SUITE="FV3_RRFS_v1beta"
+
+EXTRN_MDL_NAME_ICS="FV3GFS"
+EXTRN_MDL_NAME_LBCS="FV3GFS"
+USE_USER_STAGED_EXTRN_FILES="TRUE"
+
+DATE_FIRST_CYCL="20190701"
+DATE_LAST_CYCL="20190701"
+CYCL_HRS=( "00" )
+
+FCST_LEN_HRS="6"
+LBC_SPEC_INTVL_HRS="6"
+
+#########################################################################
+# The following code/namelist/workflow setting changes are necessary to #
+# run/optimize end-to-end experiments using the 3-km NA grid            #
+#########################################################################
+
+# The model should be built in 32-bit mode (64-bit will result in much
+# longer run times.
+
+# Use k_split=2 and n_split=5, the previous namelist values (k_split=4
+# and n_split=5) will result in significantly longer run times.
+
+NNODES_MAKE_ICS="12"
+NNODES_MAKE_LBCS="12"
+PPN_MAKE_ICS="4"
+PPN_MAKE_LBCS="4"
+WTIME_MAKE_LBCS="01:00:00"
+
+NNODES_RUN_POST="6"
+PPN_RUN_POST="12"
+
+OMP_STACKSIZE_MAKE_ICS="2048m"
+OMP_STACKSIZE_RUN_FCST="2048m"
+
+###############################################################################

tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1alpha.sh

@@ -45,7 +45,7 @@ PPN_MAKE_ICS="4"
 PPN_MAKE_LBCS="4"
 WTIME_MAKE_LBCS="01:00:00"
 
-NNODES_RUN_POST="6"
+NNODES_RUN_POST="8"
 PPN_RUN_POST="12"
 
 OMP_STACKSIZE_MAKE_ICS="2048m"

ush/generate_FV3LAM_wflow.sh

@@ -856,8 +856,9 @@ settings="$settings
 fi
 #
 # Add the relevant tendency-based stochastic physics namelist variables to
-# "settings" when running with SPPT, SHUM, or SKEB turned on. Otherwise 
-# only include an empty "nam_stochy" stanza.
+# "settings" when running with SPPT, SHUM, or SKEB turned on. If running 
+# with SPP or LSM SPP, set the "new_lscale" variable.  Otherwise only 
+# include an empty "nam_stochy" stanza.
 #
 settings="$settings
 'nam_stochy': {"
@@ -881,8 +882,7 @@ if [ "${DO_SHUM}" = "TRUE" ]; then
     'shum': ${SHUM_MAG},
     'shum_lscale': ${SHUM_LSCALE},
     'shum_tau': ${SHUM_TSCALE},
-    'shumint': ${SHUM_INT},
-    'use_zmtnblck': ${USE_ZMTNBLCK},"
+    'shumint': ${SHUM_INT},"
 fi
 
 if [ "${DO_SKEB}" = "TRUE" ]; then
@@ -894,8 +894,12 @@ if [ "${DO_SKEB}" = "TRUE" ]; then
     'skebnorm': ${SKEBNORM},
     'skeb_tau': ${SKEB_TSCALE},
     'skebint': ${SKEB_INT},
-    'skeb_vdof': ${SKEB_VDOF},
-    'use_zmtnblck': ${USE_ZMTNBLCK},"
+    'skeb_vdof': ${SKEB_VDOF},"
+fi
+
+if [ "${DO_SPP}" = "TRUE" ] || [ "${DO_LSM_SPP}" = "TRUE" ]; then
+    settings="$settings
+    'new_lscale': ${NEW_LSCALE},"
 fi
 settings="$settings
   }"

ush/set_predef_grid_params.sh

@@ -1121,60 +1121,16 @@ case ${PREDEF_GRID_NAME} in
 #
 "RRFS_NA_13km")
 
-#  if [ "${GRID_GEN_METHOD}" = "GFDLgrid" ]; then
-#
-#    GFDLgrid_LON_T6_CTR="-106.0"
-#    GFDLgrid_LAT_T6_CTR="54.0"
-#    GFDLgrid_STRETCH_FAC="0.63"
-#    GFDLgrid_RES="384"
-#    GFDLgrid_REFINE_RATIO="3"
-#
-#    num_margin_cells_T6_left="10"
-#    GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G=$(( num_margin_cells_T6_left + 1 ))
-#
-#    num_margin_cells_T6_right="10"
-#    GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G=$(( GFDLgrid_RES - num_margin_cells_T6_right ))
-#
-#    num_margin_cells_T6_bottom="10"
-#    GFDLgrid_JSTART_OF_RGNL_DOM_ON_T6G=$(( num_margin_cells_T6_bottom + 1 ))
-#
-#    num_margin_cells_T6_top="10"
-#    GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G=$(( GFDLgrid_RES - num_margin_cells_T6_top ))
-#
-#    GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES="FALSE"
-#
-#    DT_ATMOS="50"
-#
-#    LAYOUT_X="14"
-#    LAYOUT_Y="14"
-#    BLOCKSIZE="26"
-#
-#    if [ "$QUILTING" = "TRUE" ]; then
-#      WRTCMP_write_groups="1"
-#      WRTCMP_write_tasks_per_group="14"
-#      WRTCMP_output_grid="rotated_latlon"
-#      WRTCMP_cen_lon="${GFDLgrid_LON_T6_CTR}"
-#      WRTCMP_cen_lat="${GFDLgrid_LAT_T6_CTR}"
-#      WRTCMP_lon_lwr_left="-57.9926"
-#      WRTCMP_lat_lwr_left="-50.74344"
-#      WRTCMP_lon_upr_rght="57.99249"
-#      WRTCMP_lat_upr_rght="50.74344"
-#      WRTCMP_dlon="0.1218331"
-#      WRTCMP_dlat="0.121833"
-#    fi
-#
-#  elif [ "${GRID_GEN_METHOD}" = "ESGgrid" ]; then
-
   GRID_GEN_METHOD="ESGgrid"
 
-  ESGgrid_LON_CTR="-106.0"
-  ESGgrid_LAT_CTR="54.0"
+  ESGgrid_LON_CTR="-112.5"
+  ESGgrid_LAT_CTR="55.0"
 
   ESGgrid_DELX="13000.0"
   ESGgrid_DELY="13000.0"
 
-  ESGgrid_NX="960"
-  ESGgrid_NY="960"
+  ESGgrid_NX="912"
+  ESGgrid_NY="623"
 
   ESGgrid_PAZI="0.0"
 
@@ -1190,12 +1146,12 @@ case ${PREDEF_GRID_NAME} in
     WRTCMP_write_groups="1"
     WRTCMP_write_tasks_per_group="16"
     WRTCMP_output_grid="rotated_latlon"
-    WRTCMP_cen_lon="${ESGgrid_LON_CTR}"
-    WRTCMP_cen_lat="${ESGgrid_LAT_CTR}"
-    WRTCMP_lon_lwr_left="-55.82538869"
-    WRTCMP_lat_lwr_left="-48.57685654"
-    WRTCMP_lon_upr_rght="55.82538869"
-    WRTCMP_lat_upr_rght="48.57685654"
+    WRTCMP_cen_lon="-113.0" #"${ESGgrid_LON_CTR}"
+    WRTCMP_cen_lat="55.0" #"${ESGgrid_LAT_CTR}"
+    WRTCMP_lon_lwr_left="-61.0"
+    WRTCMP_lat_lwr_left="-37.0"
+    WRTCMP_lon_upr_rght="61.0"
+    WRTCMP_lat_upr_rght="37.0"
     WRTCMP_dlon=$( printf "%.9f" $( bc -l <<< "(${ESGgrid_DELX}/${radius_Earth})*${degs_per_radian}" ) )
     WRTCMP_dlat=$( printf "%.9f" $( bc -l <<< "(${ESGgrid_DELY}/${radius_Earth})*${degs_per_radian}" ) )
   fi
@@ -1212,35 +1168,35 @@ case ${PREDEF_GRID_NAME} in
 
   GRID_GEN_METHOD="ESGgrid"
 
-  ESGgrid_LON_CTR=-107.5
-  ESGgrid_LAT_CTR=51.5
+  ESGgrid_LON_CTR=-112.5
+  ESGgrid_LAT_CTR=55.0
 
   ESGgrid_DELX="3000.0"
   ESGgrid_DELY="3000.0"
 
-  ESGgrid_NX=3640
-  ESGgrid_NY=2520
+  ESGgrid_NX=3950
+  ESGgrid_NY=2700
 
-  ESGgrid_PAZI="-13.0"
+  ESGgrid_PAZI="0.0"
 
   ESGgrid_WIDE_HALO_WIDTH=6
 
   DT_ATMOS="${DT_ATMOS:-36}"
 
-  LAYOUT_X="${LAYOUT_X:-18}"   # 40 - EMC operational configuration
-  LAYOUT_Y="${LAYOUT_Y:-36}"   # 45 - EMC operational configuration
+  LAYOUT_X="${LAYOUT_X:-20}"   # 40 - EMC operational configuration
+  LAYOUT_Y="${LAYOUT_Y:-35}"   # 45 - EMC operational configuration
   BLOCKSIZE="${BLOCKSIZE:-28}"
 
   if [ "$QUILTING" = "TRUE" ]; then
     WRTCMP_write_groups="1"
     WRTCMP_write_tasks_per_group="144"
     WRTCMP_output_grid="rotated_latlon"
-    WRTCMP_cen_lon="-112.0" #${ESGgrid_LON_CTR}"
-    WRTCMP_cen_lat="48.0" #${ESGgrid_LAT_CTR}"
-    WRTCMP_lon_lwr_left="-51.0"
-    WRTCMP_lat_lwr_left="-33.0"
-    WRTCMP_lon_upr_rght="51.0"
-    WRTCMP_lat_upr_rght="33.0"
+    WRTCMP_cen_lon="-113.0" #"${ESGgrid_LON_CTR}"
+    WRTCMP_cen_lat="55.0" #"${ESGgrid_LAT_CTR}"
+    WRTCMP_lon_lwr_left="-61.0"
+    WRTCMP_lat_lwr_left="-37.0"
+    WRTCMP_lon_upr_rght="61.0"
+    WRTCMP_lat_upr_rght="37.0"
     WRTCMP_dlon="0.025" #$( printf "%.9f" $( bc -l <<< "(${ESGgrid_DELX}/${radius_Earth})*${degs_per_radian}" ) )
     WRTCMP_dlat="0.025" #$( printf "%.9f" $( bc -l <<< "(${ESGgrid_DELY}/${radius_Earth})*${degs_per_radian}" ) )
   fi

ush/setup.sh

@@ -1404,6 +1404,21 @@ fi
 #
 #-----------------------------------------------------------------------
 #
+# Make sure that DO_ENSEMBLE is set to TRUE when running ensemble vx.
+#
+#-----------------------------------------------------------------------
+#
+if [ "${DO_ENSEMBLE}" = "FALSE" ] && [ "${RUN_TASK_VX_ENSGRID}" = "TRUE" -o \
+   "${RUN_TASK_VX_ENSPOINT}" = "TRUE" ]; then
+  print_err_msg_exit "\
+Ensemble verification can not be run unless running in ensemble mode:
+   DO_ENSEMBLE = \"${DO_ENSEMBLE}\"
+   RUN_TASK_VX_ENSGRID = \"${RUN_TASK_VX_ENSGRID}\"
+   RUN_TASK_VX_ENSPOINT = \"${RUN_TASK_VX_ENSPOINT}\""
+fi
+#
+#-----------------------------------------------------------------------
+#
 # Set the full path to the forecast model executable.
 #
 #-----------------------------------------------------------------------
@@ -1587,8 +1602,7 @@ one above.  Reset values are:
 
   fi
 
-  if [ "${RUN_TASK_VX_GRIDSTAT}" = "TRUE" ] || \
-     [ "${RUN_TASK_VX_GRIDSTAT}" = "FALSE" ]; then
+  if [ "${RUN_TASK_VX_GRIDSTAT}" = "TRUE" ]; then
 
     msg="
 When RUN_ENVIR is set to \"nco\", it is assumed that the verification
@@ -1607,8 +1621,7 @@ Reset value is:"
 
   fi
 
-  if [ "${RUN_TASK_VX_POINTSTAT}" = "TRUE" ] || \
-     [ "${RUN_TASK_VX_POINTSTAT}" = "FALSE" ]; then
+  if [ "${RUN_TASK_VX_POINTSTAT}" = "TRUE" ]; then
 
     msg="
 When RUN_ENVIR is set to \"nco\", it is assumed that the verification
@@ -1627,8 +1640,7 @@ Reset value is:"
 
   fi
 
-  if [ "${RUN_TASK_VX_ENSGRID}" = "TRUE" ] || \
-     [ "${RUN_TASK_VX_ENSGRID}" = "FALSE" ]; then
+  if [ "${RUN_TASK_VX_ENSGRID}" = "TRUE" ]; then
 
     msg="
 When RUN_ENVIR is set to \"nco\", it is assumed that the verification
@@ -2022,7 +2034,6 @@ SUB_HOURLY_POST is NOT available with Inline Post yet."
   fi
 fi
 
-
 check_var_valid_value "QUILTING" "valid_vals_QUILTING"
 QUILTING=$(boolify $QUILTING)