[develop] Split EnsembleStat vx tasks into GenEnsProd and EnsembleStat

jobs/JREGIONAL_CHECK_POST_OUTPUT

@@ -0,0 +1,99 @@
+#!/bin/bash
+
+#
+#-----------------------------------------------------------------------
+#
+#
+#
+#-----------------------------------------------------------------------
+#
+
+#
+#-----------------------------------------------------------------------
+#
+# Source the variable definitions file and the bash utility functions.
+#
+#-----------------------------------------------------------------------
+#
+. $USHdir/source_util_funcs.sh
+source_config_for_task "" ${GLOBAL_VAR_DEFNS_FP}
+. $USHdir/job_preamble.sh
+#
+#-----------------------------------------------------------------------
+#
+# Save current shell options (in a global array).  Then set new options
+# for this script/function.
+#
+#-----------------------------------------------------------------------
+#
+{ save_shell_opts; . $USHdir/preamble.sh; } > /dev/null 2>&1
+#
+#-----------------------------------------------------------------------
+#
+# Get the full path to the file in which this script/function is located 
+# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in
+# which the file is located (scrfunc_dir).
+#
+#-----------------------------------------------------------------------
+#
+scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" )
+scrfunc_fn=$( basename "${scrfunc_fp}" )
+scrfunc_dir=$( dirname "${scrfunc_fp}" )
+#
+#-----------------------------------------------------------------------
+#
+# Print message indicating entry into script.
+#
+#-----------------------------------------------------------------------
+#
+print_info_msg "
+========================================================================
+Entering script:  \"${scrfunc_fn}\"
+In directory:     \"${scrfunc_dir}\"
+
+This is the J-job script for the task that checks that all the post-
+processed output files in fact exist and are at least a certain age.
+These files may have been generated by UPP as part of the current SRW
+App workflow, or they may be user-staged. 
+========================================================================"
+#
+#-----------------------------------------------------------------------
+#
+# Call the ex-script for this J-job and pass to it the necessary varia-
+# bles. 
+#
+#-----------------------------------------------------------------------
+#
+$SCRIPTSdir/exregional_check_post_output.sh || \
+print_err_msg_exit "\
+Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed."
+#
+#-----------------------------------------------------------------------
+#
+# Create a flag file to make rocoto aware that the make_grid task has 
+# successfully completed (so that other tasks that depend on it can be
+# launched).  
+#
+#-----------------------------------------------------------------------
+#
+ensmem_name="mem${ENSMEM_INDX}"
+cycle_dir="$EXPTDIR/$CDATE"
+mkdir_vrfy -p "${cycle_dir}"
+touch "${cycle_dir}/post_files_exist_${ensmem_name}.txt"
+#
+#-----------------------------------------------------------------------
+#
+# Run job postamble.
+#
+#-----------------------------------------------------------------------
+#
+job_postamble
+#
+#-----------------------------------------------------------------------
+#
+# Restore the shell options saved at the beginning of this script/func-
+# tion.
+#
+#-----------------------------------------------------------------------
+#
+{ restore_shell_opts; } > /dev/null 2>&1

parm/metplus/EnsembleStat_APCP01h.conf

@@ -2,7 +2,7 @@
 
 [config]
 
-# List of applications to run - only GridStat for this case
+# List of applications (tools) to run.
 PROCESS_LIST = EnsembleStat
 
 # time looping - options are INIT, VALID, RETRO, and REALTIME
@@ -54,19 +54,23 @@ METPLUS_CONF = {ENSEMBLE_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}}
 #
 # Location of MET configuration file to pass to EnsembleStat.
 #
+# References PARM_BASE, which is the location of the parm directory
+# corresponding to the ush directory of the run_metplus.py script that
+# is called or the value of the environment variable METPLUS_PARM_BASE
+# if set.
+#
 ENSEMBLE_STAT_CONFIG_FILE = {PARM_BASE}/met_config/EnsembleStatConfig_wrapped
 #
-# Used in the MET config file for:  model, output_prefix
+# Name to identify model (forecast) data in output.
 #
 MODEL = {{vx_fcst_model_name}}
 
 ENSEMBLE_STAT_DESC = NA
-
-# Name to identify observation data in output
+#
+# Name to identify observation data in output.
+#
 OBTYPE = {{obtype}}
 
-#ENSEMBLE_STAT_DESC =
-
 # The METplus EnsembleStat logging level
 # 0 quiet to 5 loud, Verbosity setting for METplus EnsembleStat output, 2 is default.
 # This takes precendence over the general LOG_MET_VERBOSITY set in metplus_logging.conf
@@ -89,8 +93,6 @@ ENSEMBLE_STAT_ENS_THRESH = 0.05
 # ens.vld_thresh value in the MET config file
 ENSEMBLE_STAT_ENS_VLD_THRESH = 1.0
 
-ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_{FIELDNAME_IN_MET_FILEDIR_NAMES}_{OBTYPE}
-
 FIELDNAME_IN_MET_FILEDIR_NAMES = {{fieldname_in_met_filedir_names}}
 
 # ENSEMBLE_STAT_MET_OBS_ERR_TABLE is not required.
@@ -144,7 +146,7 @@ ENSEMBLE_STAT_ENS_PHIST_BIN_SIZE = 0.05
 
 ENSEMBLE_STAT_CLIMO_CDF_BINS = 1
 ENSEMBLE_STAT_CLIMO_CDF_CENTER_BINS = False
-ENSEMBLE_STAT_CLIMO_CDF_WRITE_BINS = True
+ENSEMBLE_STAT_CLIMO_CDF_WRITE_BINS = False
 
 ENSEMBLE_STAT_MASK_GRID =
 
@@ -164,36 +166,22 @@ ENSEMBLE_STAT_OUTPUT_FLAG_ORANK = STAT
 ENSEMBLE_STAT_OUTPUT_FLAG_SSVAR = STAT
 ENSEMBLE_STAT_OUTPUT_FLAG_RELP = STAT
 
-ENSEMBLE_STAT_ENSEMBLE_FLAG_LATLON = TRUE
-ENSEMBLE_STAT_ENSEMBLE_FLAG_MEAN = TRUE
-ENSEMBLE_STAT_ENSEMBLE_FLAG_STDEV = TRUE
+ENSEMBLE_STAT_ENSEMBLE_FLAG_LATLON = FALSE
+ENSEMBLE_STAT_ENSEMBLE_FLAG_MEAN = FALSE
+ENSEMBLE_STAT_ENSEMBLE_FLAG_STDEV = FALSE
 ENSEMBLE_STAT_ENSEMBLE_FLAG_MINUS = FALSE
 ENSEMBLE_STAT_ENSEMBLE_FLAG_PLUS = FALSE
 ENSEMBLE_STAT_ENSEMBLE_FLAG_MIN = FALSE
 ENSEMBLE_STAT_ENSEMBLE_FLAG_MAX = FALSE
 ENSEMBLE_STAT_ENSEMBLE_FLAG_RANGE = FALSE
-ENSEMBLE_STAT_ENSEMBLE_FLAG_VLD_COUNT = TRUE
-ENSEMBLE_STAT_ENSEMBLE_FLAG_FREQUENCY = TRUE
+ENSEMBLE_STAT_ENSEMBLE_FLAG_VLD_COUNT = FALSE
+ENSEMBLE_STAT_ENSEMBLE_FLAG_FREQUENCY = FALSE
 ENSEMBLE_STAT_ENSEMBLE_FLAG_NEP = FALSE
 ENSEMBLE_STAT_ENSEMBLE_FLAG_NMEP = FALSE
-ENSEMBLE_STAT_ENSEMBLE_FLAG_RANK = TRUE
+ENSEMBLE_STAT_ENSEMBLE_FLAG_RANK = FALSE
 ENSEMBLE_STAT_ENSEMBLE_FLAG_WEIGHT = FALSE
 #
-# Ensemble variables and levels as specified in the ens field dictionary
-# of the MET configuration file.  Specify as ENS_VARn_NAME, ENS_VARn_LEVELS,
-# (optional) ENS_VARn_OPTION.
-#
-# ENS_VARn_THRESH must be included here in order for the netCDF file
-# generated by EnsembleStat to include the ensemble probability (aka
-# ensembe frequency) fields corresponding to the specified thresholds.
-# These must be included because the corresonding GridStat task for
-# ensemble probabilities expects to read them in as input.
-#
-ENS_VAR1_NAME = {{fieldname_in_fcst_input}}
-ENS_VAR1_LEVELS = A{ACCUM_HH}
-ENS_VAR1_THRESH = {{field_thresholds}}
-#
-# Get the variable ACCUM_HH from the environment.
+# Precipitation accumulation interval in 2-digit hours.
 #
 ACCUM_HH = {{accum_hh}}
 #
@@ -216,7 +204,6 @@ OBS_VAR1_OPTIONS = {FCST_VAR1_OPTIONS}
 # configuration file.
 #
 OUTPUT_BASE = {{output_base}}
-OUTPUT_DIR = {{output_dir}}
 #
 # Point observation input directory for EnsembleStat.
 #
@@ -242,31 +229,37 @@ ENSEMBLE_STAT_CLIMO_STDEV_INPUT_DIR =
 #
 # Output directory for EnsembleStat.
 #
-ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_DIR}
+ENSEMBLE_STAT_OUTPUT_DIR = {{output_dir}}
 #
 # Directory for staging data.
 #
 STAGING_DIR = {{staging_dir}}
 
+# End of [dir] section and start of [filename_templates] section.
 [filename_templates]
 #
-# FCST_ENSEMBLE_STAT_INPUT_TEMPLATE  - comma separated list of ensemble members
-# or a single line, - filename wildcard characters may be used, ? or *.
+# Template for point observation input to EnsembleStat relative to
+# OBS_ENSEMBLE_STAT_POINT_INPUT_DIR.
 #
-# The template FCST_ENSEMBLE_STAT_INPUT_TEMPLATE is relative to
-# FCST_ENSEMBLE_STAT_INPUT_DIR.
+OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE =
 #
-FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}}
+# Template for gridded observation input to EnsembleStat relative to
+# OBS_ENSEMBLE_STAT_GRID_INPUT_DIR.
 #
-# Template for point observations relative to OBS_ENSEMBLE_STAT_POINT_INPUT_DIR.
+OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = {{obs_input_fn_template}}
 #
-OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE =
+# Template for forecast input to EnsembleStat relative to 
+# FCST_ENSEMBLE_STAT_INPUT_DIR.
 #
-# Template for gridded observations relative to OBS_ENSEMBLE_STAT_GRID_INPUT_DIR.
+# Note that this can be a comma separated list of ensemble members
+# or a single line, - filename wildcard characters may be used, ? or *.
 #
-OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = {{obs_input_fn_template}}
-
-ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly
+FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}}
+#
+# Template for output from EnsembleStat relative to ENSEMBLE_STAT_OUTPUT_DIR.
+#
+ENSEMBLE_STAT_OUTPUT_TEMPLATE =
+ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_{FIELDNAME_IN_MET_FILEDIR_NAMES}_{OBTYPE}_{lead?fmt=%H%M%S}L
 #
 # Template for climatology input to EnsembleStat relative to
 # ENSEMBLE_STAT_CLIMO_MEAN_INPUT_DIR.  Not used in this example.
@@ -278,7 +271,7 @@ ENSEMBLE_STAT_CLIMO_MEAN_INPUT_TEMPLATE =
 #
 ENSEMBLE_STAT_CLIMO_STDEV_INPUT_TEMPLATE =
 #
-# The template ENSEMBLE_STAT_OUTPUT_TEMPLATE is relative to
-# ENSEMBLE_STAT_OUTPUT_DIR.
+# Variable used to specify one or more verification mask files for
+# EnsembleStat.  Not used for this example.
 #
-ENSEMBLE_STAT_OUTPUT_TEMPLATE =
+ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly