From a46a5af50a64f42f3ebf9231f740efc3c0aed67c Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Thu, 23 Mar 2023 09:36:30 -0600 Subject: [PATCH 01/43] Cleanup of EnsembleStat configuration files. --- parm/metplus/EnsembleStat_APCP01h.conf | 19 ++++++++-------- parm/metplus/EnsembleStat_APCPgt01h.conf | 28 ++++++++++++------------ parm/metplus/EnsembleStat_REFC.conf | 17 +++++++------- parm/metplus/EnsembleStat_RETOP.conf | 17 +++++++------- parm/metplus/EnsembleStat_SFC.conf | 17 +++++++------- parm/metplus/EnsembleStat_UPA.conf | 17 +++++++------- 6 files changed, 55 insertions(+), 60 deletions(-) diff --git a/parm/metplus/EnsembleStat_APCP01h.conf b/parm/metplus/EnsembleStat_APCP01h.conf index 4f78827e08..0d285d33f5 100644 --- a/parm/metplus/EnsembleStat_APCP01h.conf +++ b/parm/metplus/EnsembleStat_APCP01h.conf @@ -2,7 +2,7 @@ [config] -# List of applications to run - only GridStat for this case +# List of applications (tools) to run. PROCESS_LIST = EnsembleStat # time looping - options are INIT, VALID, RETRO, and REALTIME @@ -61,12 +61,11 @@ ENSEMBLE_STAT_CONFIG_FILE = {PARM_BASE}/met_config/EnsembleStatConfig_wrapped MODEL = {{vx_fcst_model_name}} ENSEMBLE_STAT_DESC = NA - +# # Name to identify observation data in output +# OBTYPE = {{obtype}} -#ENSEMBLE_STAT_DESC = - # The METplus EnsembleStat logging level # 0 quiet to 5 loud, Verbosity setting for METplus EnsembleStat output, 2 is default. # This takes precendence over the general LOG_MET_VERBOSITY set in metplus_logging.conf @@ -193,7 +192,7 @@ ENS_VAR1_NAME = {{fieldname_in_fcst_input}} ENS_VAR1_LEVELS = A{ACCUM_HH} ENS_VAR1_THRESH = {{field_thresholds}} # -# Get the variable ACCUM_HH from the environment. +# Precipitation accumulation interval in 2-digit hours. # ACCUM_HH = {{accum_hh}} # @@ -265,6 +264,11 @@ OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE = # Template for gridded observations relative to OBS_ENSEMBLE_STAT_GRID_INPUT_DIR. # OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = {{obs_input_fn_template}} +# +# The template ENSEMBLE_STAT_OUTPUT_TEMPLATE is relative to +# ENSEMBLE_STAT_OUTPUT_DIR. +# +ENSEMBLE_STAT_OUTPUT_TEMPLATE = ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly # @@ -277,8 +281,3 @@ ENSEMBLE_STAT_CLIMO_MEAN_INPUT_TEMPLATE = # ENSEMBLE_STAT_CLIMO_STDEV_INPUT_DIR. Not used in this example. # ENSEMBLE_STAT_CLIMO_STDEV_INPUT_TEMPLATE = -# -# The template ENSEMBLE_STAT_OUTPUT_TEMPLATE is relative to -# ENSEMBLE_STAT_OUTPUT_DIR. -# -ENSEMBLE_STAT_OUTPUT_TEMPLATE = diff --git a/parm/metplus/EnsembleStat_APCPgt01h.conf b/parm/metplus/EnsembleStat_APCPgt01h.conf index e8b64b6763..beac8a7164 100644 --- a/parm/metplus/EnsembleStat_APCPgt01h.conf +++ b/parm/metplus/EnsembleStat_APCPgt01h.conf @@ -2,7 +2,7 @@ [config] -# List of applications to run - only GridStat for this case +# List of applications (tools) to run. PROCESS_LIST = EnsembleStat # time looping - options are INIT, VALID, RETRO, and REALTIME @@ -60,18 +60,18 @@ ENSEMBLE_STAT_CONFIG_FILE = {PARM_BASE}/met_config/EnsembleStatConfig_wrapped # MODEL = {{vx_fcst_model_name}} -FCST_NATIVE_DATA_TYPE = GRIB - ENSEMBLE_STAT_DESC = NA - +# # Name to identify observation data in output +# OBTYPE = {{obtype}} -OBS_NATIVE_DATA_TYPE = GRIB - -# Forecast data description variables +# +# Forecast data description variables. +# FCST_IS_PROB = false -#ENSEMBLE_STAT_DESC = +FCST_NATIVE_DATA_TYPE = GRIB +OBS_NATIVE_DATA_TYPE = GRIB # The METplus EnsembleStat logging level # 0 quiet to 5 loud, Verbosity setting for METplus EnsembleStat output, 2 is default. @@ -199,7 +199,7 @@ ENS_VAR1_NAME = {{fieldname_in_met_output}} ENS_VAR1_LEVELS = A{ACCUM_HH} ENS_VAR1_THRESH = {{field_thresholds}} # -# Get the variable ACCUM_HH from the environment. +# Precipitation accumulation interval in 2-digit hours. # ACCUM_HH = {{accum_hh}} # @@ -271,6 +271,11 @@ OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE = # Template for gridded observations relative to OBS_ENSEMBLE_STAT_GRID_INPUT_DIR. # OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = {{obs_input_fn_template}} +# +# The template ENSEMBLE_STAT_OUTPUT_TEMPLATE is relative to +# ENSEMBLE_STAT_OUTPUT_DIR. +# +ENSEMBLE_STAT_OUTPUT_TEMPLATE = ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly # @@ -283,8 +288,3 @@ ENSEMBLE_STAT_CLIMO_MEAN_INPUT_TEMPLATE = # ENSEMBLE_STAT_CLIMO_STDEV_INPUT_DIR. Not used in this example. # ENSEMBLE_STAT_CLIMO_STDEV_INPUT_TEMPLATE = -# -# The template ENSEMBLE_STAT_OUTPUT_TEMPLATE is relative to -# ENSEMBLE_STAT_OUTPUT_DIR. -# -ENSEMBLE_STAT_OUTPUT_TEMPLATE = diff --git a/parm/metplus/EnsembleStat_REFC.conf b/parm/metplus/EnsembleStat_REFC.conf index 5379540b76..1772431b69 100644 --- a/parm/metplus/EnsembleStat_REFC.conf +++ b/parm/metplus/EnsembleStat_REFC.conf @@ -2,7 +2,7 @@ [config] -# List of applications to run - only GridStat for this case +# List of applications (tools) to run. PROCESS_LIST = EnsembleStat # time looping - options are INIT, VALID, RETRO, and REALTIME @@ -61,12 +61,11 @@ ENSEMBLE_STAT_CONFIG_FILE = {PARM_BASE}/met_config/EnsembleStatConfig_wrapped MODEL = {{vx_fcst_model_name}} ENSEMBLE_STAT_DESC = NA - +# # Name to identify observation data in output +# OBTYPE = {{obtype}} -#ENSEMBLE_STAT_DESC = - # The METplus EnsembleStat logging level # 0 quiet to 5 loud, Verbosity setting for METplus EnsembleStat output, 2 is default. # This takes precendence over the general LOG_MET_VERBOSITY set in metplus_logging.conf @@ -263,6 +262,11 @@ OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE = # Template for gridded observations relative to OBS_ENSEMBLE_STAT_GRID_INPUT_DIR. # OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = {{obs_input_fn_template}} +# +# The template ENSEMBLE_STAT_OUTPUT_TEMPLATE is relative to +# ENSEMBLE_STAT_OUTPUT_DIR. +# +ENSEMBLE_STAT_OUTPUT_TEMPLATE = ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly # @@ -275,8 +279,3 @@ ENSEMBLE_STAT_CLIMO_MEAN_INPUT_TEMPLATE = # ENSEMBLE_STAT_CLIMO_STDEV_INPUT_DIR. Not used in this example. # ENSEMBLE_STAT_CLIMO_STDEV_INPUT_TEMPLATE = -# -# The template ENSEMBLE_STAT_OUTPUT_TEMPLATE is relative to -# ENSEMBLE_STAT_OUTPUT_DIR. -# -ENSEMBLE_STAT_OUTPUT_TEMPLATE = diff --git a/parm/metplus/EnsembleStat_RETOP.conf b/parm/metplus/EnsembleStat_RETOP.conf index 46286e1632..a4ab12189d 100644 --- a/parm/metplus/EnsembleStat_RETOP.conf +++ b/parm/metplus/EnsembleStat_RETOP.conf @@ -2,7 +2,7 @@ [config] -# List of applications to run - only GridStat for this case +# List of applications (tools) to run. PROCESS_LIST = EnsembleStat # time looping - options are INIT, VALID, RETRO, and REALTIME @@ -61,12 +61,11 @@ ENSEMBLE_STAT_CONFIG_FILE = {PARM_BASE}/met_config/EnsembleStatConfig_wrapped MODEL = {{vx_fcst_model_name}} ENSEMBLE_STAT_DESC = NA - +# # Name to identify observation data in output +# OBTYPE = {{obtype}} -#ENSEMBLE_STAT_DESC = - # The METplus EnsembleStat logging level # 0 quiet to 5 loud, Verbosity setting for METplus EnsembleStat output, 2 is default. # This takes precendence over the general LOG_MET_VERBOSITY set in metplus_logging.conf @@ -267,6 +266,11 @@ OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE = # Template for gridded observations relative to OBS_ENSEMBLE_STAT_GRID_INPUT_DIR. # OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = {{obs_input_fn_template}} +# +# The template ENSEMBLE_STAT_OUTPUT_TEMPLATE is relative to +# ENSEMBLE_STAT_OUTPUT_DIR. +# +ENSEMBLE_STAT_OUTPUT_TEMPLATE = ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly # @@ -279,8 +283,3 @@ ENSEMBLE_STAT_CLIMO_MEAN_INPUT_TEMPLATE = # ENSEMBLE_STAT_CLIMO_STDEV_INPUT_DIR. Not used in this example. # ENSEMBLE_STAT_CLIMO_STDEV_INPUT_TEMPLATE = -# -# The template ENSEMBLE_STAT_OUTPUT_TEMPLATE is relative to -# ENSEMBLE_STAT_OUTPUT_DIR. -# -ENSEMBLE_STAT_OUTPUT_TEMPLATE = diff --git a/parm/metplus/EnsembleStat_SFC.conf b/parm/metplus/EnsembleStat_SFC.conf index 1cd1495ca6..9b0ae70b88 100644 --- a/parm/metplus/EnsembleStat_SFC.conf +++ b/parm/metplus/EnsembleStat_SFC.conf @@ -2,7 +2,7 @@ [config] -# List of applications to run - only GridStat for this case +# List of applications (tools) to run. PROCESS_LIST = EnsembleStat # time looping - options are INIT, VALID, RETRO, and REALTIME @@ -61,12 +61,11 @@ ENSEMBLE_STAT_CONFIG_FILE = {PARM_BASE}/met_config/EnsembleStatConfig_wrapped MODEL = {{vx_fcst_model_name}} ENSEMBLE_STAT_DESC = NA - +# # Name to identify observation data in output +# OBTYPE = {{obtype}} -#ENSEMBLE_STAT_DESC = - # The METplus EnsembleStat logging level # 0 quiet to 5 loud, Verbosity setting for METplus EnsembleStat output, 2 is default. # This takes precendence over the general LOG_MET_VERBOSITY set in metplus_logging.conf @@ -330,6 +329,11 @@ OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE = {{obs_input_fn_template}} # Template for gridded observations relative to OBS_ENSEMBLE_STAT_GRID_INPUT_DIR. # OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = +# +# The template ENSEMBLE_STAT_OUTPUT_TEMPLATE is relative to +# ENSEMBLE_STAT_OUTPUT_DIR. +# +ENSEMBLE_STAT_OUTPUT_TEMPLATE = ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly # @@ -342,8 +346,3 @@ ENSEMBLE_STAT_CLIMO_MEAN_INPUT_TEMPLATE = # ENSEMBLE_STAT_CLIMO_STDEV_INPUT_DIR. Not used in this example. # ENSEMBLE_STAT_CLIMO_STDEV_INPUT_TEMPLATE = -# -# The template ENSEMBLE_STAT_OUTPUT_TEMPLATE is relative to -# ENSEMBLE_STAT_OUTPUT_DIR. -# -ENSEMBLE_STAT_OUTPUT_TEMPLATE = diff --git a/parm/metplus/EnsembleStat_UPA.conf b/parm/metplus/EnsembleStat_UPA.conf index 9a60c5ae0f..51b34fb353 100644 --- a/parm/metplus/EnsembleStat_UPA.conf +++ b/parm/metplus/EnsembleStat_UPA.conf @@ -2,7 +2,7 @@ [config] -# List of applications to run - only GridStat for this case +# List of applications (tools) to run. PROCESS_LIST = EnsembleStat # time looping - options are INIT, VALID, RETRO, and REALTIME @@ -61,12 +61,11 @@ ENSEMBLE_STAT_CONFIG_FILE = {PARM_BASE}/met_config/EnsembleStatConfig_wrapped MODEL = {{vx_fcst_model_name}} ENSEMBLE_STAT_DESC = NA - +# # Name to identify observation data in output +# OBTYPE = {{obtype}} -#ENSEMBLE_STAT_DESC = - # The METplus EnsembleStat logging level # 0 quiet to 5 loud, Verbosity setting for METplus EnsembleStat output, 2 is default. # This takes precendence over the general LOG_MET_VERBOSITY set in metplus_logging.conf @@ -393,6 +392,11 @@ OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE = {{obs_input_fn_template}} # Template for gridded observations relative to OBS_ENSEMBLE_STAT_GRID_INPUT_DIR. # OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = +# +# The template ENSEMBLE_STAT_OUTPUT_TEMPLATE is relative to +# ENSEMBLE_STAT_OUTPUT_DIR. +# +ENSEMBLE_STAT_OUTPUT_TEMPLATE = ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly # @@ -405,8 +409,3 @@ ENSEMBLE_STAT_CLIMO_MEAN_INPUT_TEMPLATE = # ENSEMBLE_STAT_CLIMO_STDEV_INPUT_DIR. Not used in this example. # ENSEMBLE_STAT_CLIMO_STDEV_INPUT_TEMPLATE = -# -# The template ENSEMBLE_STAT_OUTPUT_TEMPLATE is relative to -# ENSEMBLE_STAT_OUTPUT_DIR. -# -ENSEMBLE_STAT_OUTPUT_TEMPLATE = From f77a8a6e2f5de2caa621a97e02926aabdc572f61 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Thu, 23 Mar 2023 09:47:09 -0600 Subject: [PATCH 02/43] Clean up comments in METplus config files. --- parm/metplus/GridStat_APCP01h.conf | 4 +--- parm/metplus/GridStat_APCPgt01h.conf | 4 +--- parm/metplus/GridStat_REFC.conf | 4 +--- parm/metplus/GridStat_RETOP.conf | 4 +--- parm/metplus/GridStat_ensmean_APCP01h.conf | 4 +--- parm/metplus/GridStat_ensmean_APCPgt01h.conf | 4 +--- parm/metplus/GridStat_ensmean_REFC.conf | 4 +--- parm/metplus/GridStat_ensmean_RETOP.conf | 4 +--- parm/metplus/GridStat_ensprob_APCP01h.conf | 4 +--- parm/metplus/GridStat_ensprob_APCP03h.conf | 4 +--- parm/metplus/GridStat_ensprob_APCP06h.conf | 4 +--- parm/metplus/GridStat_ensprob_APCP24h.conf | 4 +--- parm/metplus/GridStat_ensprob_REFC.conf | 4 +--- parm/metplus/GridStat_ensprob_RETOP.conf | 4 +--- parm/metplus/Pb2nc_obs.conf | 2 +- parm/metplus/PcpCombine_fcst.conf | 4 +--- parm/metplus/PcpCombine_obs.conf | 4 +--- parm/metplus/PointStat_SFC.conf | 2 +- parm/metplus/PointStat_UPA.conf | 2 +- parm/metplus/PointStat_ensmean_SFC.conf | 2 +- parm/metplus/PointStat_ensmean_UPA.conf | 2 +- parm/metplus/PointStat_ensprob_SFC.conf | 2 +- parm/metplus/PointStat_ensprob_UPA.conf | 2 +- 23 files changed, 23 insertions(+), 55 deletions(-) diff --git a/parm/metplus/GridStat_APCP01h.conf b/parm/metplus/GridStat_APCP01h.conf index d8a686ab00..e58f26e062 100644 --- a/parm/metplus/GridStat_APCP01h.conf +++ b/parm/metplus/GridStat_APCP01h.conf @@ -1,10 +1,8 @@ # GridStat METplus Configuration -# section heading for [config] variables - all items below this line and -# before the next section heading correspond to the [config] section [config] -# List of applications to run - only GridStat for this case +# List of applications (tools) to run. PROCESS_LIST = GridStat # time looping - options are INIT, VALID, RETRO, and REALTIME diff --git a/parm/metplus/GridStat_APCPgt01h.conf b/parm/metplus/GridStat_APCPgt01h.conf index 6d8968796d..9543f62b0e 100644 --- a/parm/metplus/GridStat_APCPgt01h.conf +++ b/parm/metplus/GridStat_APCPgt01h.conf @@ -1,10 +1,8 @@ # GridStat METplus Configuration -# section heading for [config] variables - all items below this line and -# before the next section heading correspond to the [config] section [config] -# List of applications to run - only GridStat for this case +# List of applications (tools) to run. PROCESS_LIST = GridStat # time looping - options are INIT, VALID, RETRO, and REALTIME diff --git a/parm/metplus/GridStat_REFC.conf b/parm/metplus/GridStat_REFC.conf index 9fd7e2f86c..7c80444bb0 100644 --- a/parm/metplus/GridStat_REFC.conf +++ b/parm/metplus/GridStat_REFC.conf @@ -1,10 +1,8 @@ # GridStat METplus Configuration -# section heading for [config] variables - all items below this line and -# before the next section heading correspond to the [config] section [config] -# List of applications to run - only GridStat for this case +# List of applications (tools) to run. PROCESS_LIST = GridStat # time looping - options are INIT, VALID, RETRO, and REALTIME diff --git a/parm/metplus/GridStat_RETOP.conf b/parm/metplus/GridStat_RETOP.conf index fde17ded5d..8cdebca0dc 100644 --- a/parm/metplus/GridStat_RETOP.conf +++ b/parm/metplus/GridStat_RETOP.conf @@ -1,10 +1,8 @@ # GridStat METplus Configuration -# section heading for [config] variables - all items below this line and -# before the next section heading correspond to the [config] section [config] -# List of applications to run - only GridStat for this case +# List of applications (tools) to run. PROCESS_LIST = GridStat # time looping - options are INIT, VALID, RETRO, and REALTIME diff --git a/parm/metplus/GridStat_ensmean_APCP01h.conf b/parm/metplus/GridStat_ensmean_APCP01h.conf index 5e215f5395..b7816d3338 100644 --- a/parm/metplus/GridStat_ensmean_APCP01h.conf +++ b/parm/metplus/GridStat_ensmean_APCP01h.conf @@ -1,10 +1,8 @@ # Ensemble mean GridStat METplus Configuration -# section heading for [config] variables - all items below this line and -# before the next section heading correspond to the [config] section [config] -# List of applications to run - only GridStat for this case +# List of applications (tools) to run. PROCESS_LIST = GridStat # time looping - options are INIT, VALID, RETRO, and REALTIME diff --git a/parm/metplus/GridStat_ensmean_APCPgt01h.conf b/parm/metplus/GridStat_ensmean_APCPgt01h.conf index 09dd54d68c..47bcf4d795 100644 --- a/parm/metplus/GridStat_ensmean_APCPgt01h.conf +++ b/parm/metplus/GridStat_ensmean_APCPgt01h.conf @@ -1,10 +1,8 @@ # Ensemble mean GridStat METplus Configuration -# section heading for [config] variables - all items below this line and -# before the next section heading correspond to the [config] section [config] -# List of applications to run - only GridStat for this case +# List of applications (tools) to run. PROCESS_LIST = GridStat # time looping - options are INIT, VALID, RETRO, and REALTIME diff --git a/parm/metplus/GridStat_ensmean_REFC.conf b/parm/metplus/GridStat_ensmean_REFC.conf index 17a1996211..ffa2d05730 100644 --- a/parm/metplus/GridStat_ensmean_REFC.conf +++ b/parm/metplus/GridStat_ensmean_REFC.conf @@ -1,10 +1,8 @@ # Ensemble mean GridStat METplus Configuration -# section heading for [config] variables - all items below this line and -# before the next section heading correspond to the [config] section [config] -# List of applications to run - only GridStat for this case +# List of applications (tools) to run. PROCESS_LIST = GridStat # time looping - options are INIT, VALID, RETRO, and REALTIME diff --git a/parm/metplus/GridStat_ensmean_RETOP.conf b/parm/metplus/GridStat_ensmean_RETOP.conf index 36c6282671..1caeccbe94 100644 --- a/parm/metplus/GridStat_ensmean_RETOP.conf +++ b/parm/metplus/GridStat_ensmean_RETOP.conf @@ -1,10 +1,8 @@ # Ensemble mean GridStat METplus Configuration -# section heading for [config] variables - all items below this line and -# before the next section heading correspond to the [config] section [config] -# List of applications to run - only GridStat for this case +# List of applications (tools) to run. PROCESS_LIST = GridStat # time looping - options are INIT, VALID, RETRO, and REALTIME diff --git a/parm/metplus/GridStat_ensprob_APCP01h.conf b/parm/metplus/GridStat_ensprob_APCP01h.conf index 728a80275c..534cab0e7a 100644 --- a/parm/metplus/GridStat_ensprob_APCP01h.conf +++ b/parm/metplus/GridStat_ensprob_APCP01h.conf @@ -1,10 +1,8 @@ # Ensemble probabilistic GridStat METplus Configuration -# section heading for [config] variables - all items below this line and -# before the next section heading correspond to the [config] section [config] -# List of applications to run - only GridStat for this case +# List of applications (tools) to run. PROCESS_LIST = GridStat # time looping - options are INIT, VALID, RETRO, and REALTIME diff --git a/parm/metplus/GridStat_ensprob_APCP03h.conf b/parm/metplus/GridStat_ensprob_APCP03h.conf index 3b3f9a3e20..c9a442b8bf 100644 --- a/parm/metplus/GridStat_ensprob_APCP03h.conf +++ b/parm/metplus/GridStat_ensprob_APCP03h.conf @@ -1,10 +1,8 @@ # Ensemble probabilistic GridStat METplus Configuration -# section heading for [config] variables - all items below this line and -# before the next section heading correspond to the [config] section [config] -# List of applications to run - only GridStat for this case +# List of applications (tools) to run. PROCESS_LIST = GridStat # time looping - options are INIT, VALID, RETRO, and REALTIME diff --git a/parm/metplus/GridStat_ensprob_APCP06h.conf b/parm/metplus/GridStat_ensprob_APCP06h.conf index d9856518c6..37211c3ebf 100644 --- a/parm/metplus/GridStat_ensprob_APCP06h.conf +++ b/parm/metplus/GridStat_ensprob_APCP06h.conf @@ -1,10 +1,8 @@ # Ensemble probabilistic GridStat METplus Configuration -# section heading for [config] variables - all items below this line and -# before the next section heading correspond to the [config] section [config] -# List of applications to run - only GridStat for this case +# List of applications (tools) to run. PROCESS_LIST = GridStat # time looping - options are INIT, VALID, RETRO, and REALTIME diff --git a/parm/metplus/GridStat_ensprob_APCP24h.conf b/parm/metplus/GridStat_ensprob_APCP24h.conf index 4c82f06829..53fa2fa918 100644 --- a/parm/metplus/GridStat_ensprob_APCP24h.conf +++ b/parm/metplus/GridStat_ensprob_APCP24h.conf @@ -1,10 +1,8 @@ # Ensemble probabilistic GridStat METplus Configuration -# section heading for [config] variables - all items below this line and -# before the next section heading correspond to the [config] section [config] -# List of applications to run - only GridStat for this case +# List of applications (tools) to run. PROCESS_LIST = GridStat # time looping - options are INIT, VALID, RETRO, and REALTIME diff --git a/parm/metplus/GridStat_ensprob_REFC.conf b/parm/metplus/GridStat_ensprob_REFC.conf index 2348b46832..1f9334a622 100644 --- a/parm/metplus/GridStat_ensprob_REFC.conf +++ b/parm/metplus/GridStat_ensprob_REFC.conf @@ -1,10 +1,8 @@ # Ensemble probabilistic GridStat METplus Configuration -# section heading for [config] variables - all items below this line and -# before the next section heading correspond to the [config] section [config] -# List of applications to run - only GridStat for this case +# List of applications (tools) to run. PROCESS_LIST = GridStat # time looping - options are INIT, VALID, RETRO, and REALTIME diff --git a/parm/metplus/GridStat_ensprob_RETOP.conf b/parm/metplus/GridStat_ensprob_RETOP.conf index 071d47fdc7..972d611255 100644 --- a/parm/metplus/GridStat_ensprob_RETOP.conf +++ b/parm/metplus/GridStat_ensprob_RETOP.conf @@ -1,10 +1,8 @@ # Ensemble probabilistic GridStat METplus Configuration -# section heading for [config] variables - all items below this line and -# before the next section heading correspond to the [config] section [config] -# List of applications to run - only GridStat for this case +# List of applications (tools) to run. PROCESS_LIST = GridStat # time looping - options are INIT, VALID, RETRO, and REALTIME diff --git a/parm/metplus/Pb2nc_obs.conf b/parm/metplus/Pb2nc_obs.conf index 263ad864ed..d24a66ab22 100644 --- a/parm/metplus/Pb2nc_obs.conf +++ b/parm/metplus/Pb2nc_obs.conf @@ -1,6 +1,6 @@ [config] -# List of applications to run. +# List of applications (tools) to run. PROCESS_LIST = PB2NC # time looping - options are INIT, VALID, RETRO, and REALTIME diff --git a/parm/metplus/PcpCombine_fcst.conf b/parm/metplus/PcpCombine_fcst.conf index 1ab2b22425..17a70625fb 100644 --- a/parm/metplus/PcpCombine_fcst.conf +++ b/parm/metplus/PcpCombine_fcst.conf @@ -1,10 +1,8 @@ # PcpCombine METplus Configuration for Forecasts -# section heading for [config] variables - all items below this line and -# before the next section heading correspond to the [config] section [config] -# List of applications to run - only PcpCombine for this case +# List of applications (tools) to run. PROCESS_LIST = PcpCombine # time looping - options are INIT, VALID, RETRO, and REALTIME diff --git a/parm/metplus/PcpCombine_obs.conf b/parm/metplus/PcpCombine_obs.conf index 8b9202d081..f61bc015bd 100644 --- a/parm/metplus/PcpCombine_obs.conf +++ b/parm/metplus/PcpCombine_obs.conf @@ -1,10 +1,8 @@ # PcpCombine METplus Configuration for Observations -# section heading for [config] variables - all items below this line and -# before the next section heading correspond to the [config] section [config] -# List of applications to run - only PcpCombine for this case +# List of applications (tools) to run. PROCESS_LIST = PcpCombine # time looping - options are INIT, VALID, RETRO, and REALTIME diff --git a/parm/metplus/PointStat_SFC.conf b/parm/metplus/PointStat_SFC.conf index 2d5fe0a2e4..d4255ebe36 100644 --- a/parm/metplus/PointStat_SFC.conf +++ b/parm/metplus/PointStat_SFC.conf @@ -2,7 +2,7 @@ [config] -# List of applications to run - only PointStat for this case +# List of applications (tools) to run. PROCESS_LIST = PointStat # time looping - options are INIT, VALID, RETRO, and REALTIME diff --git a/parm/metplus/PointStat_UPA.conf b/parm/metplus/PointStat_UPA.conf index 1d20b16fd8..0fa7d296a5 100644 --- a/parm/metplus/PointStat_UPA.conf +++ b/parm/metplus/PointStat_UPA.conf @@ -2,7 +2,7 @@ [config] -# List of applications to run - only PointStat for this case +# List of applications (tools) to run. PROCESS_LIST = PointStat # time looping - options are INIT, VALID, RETRO, and REALTIME diff --git a/parm/metplus/PointStat_ensmean_SFC.conf b/parm/metplus/PointStat_ensmean_SFC.conf index 9771081456..11c95db98d 100644 --- a/parm/metplus/PointStat_ensmean_SFC.conf +++ b/parm/metplus/PointStat_ensmean_SFC.conf @@ -2,7 +2,7 @@ [config] -# List of applications to run - only PointStat for this case +# List of applications (tools) to run. PROCESS_LIST = PointStat # time looping - options are INIT, VALID, RETRO, and REALTIME diff --git a/parm/metplus/PointStat_ensmean_UPA.conf b/parm/metplus/PointStat_ensmean_UPA.conf index 3fb3f1b790..7eb0c7a0cf 100644 --- a/parm/metplus/PointStat_ensmean_UPA.conf +++ b/parm/metplus/PointStat_ensmean_UPA.conf @@ -2,7 +2,7 @@ [config] -# List of applications to run - only PointStat for this case +# List of applications (tools) to run. PROCESS_LIST = PointStat # time looping - options are INIT, VALID, RETRO, and REALTIME diff --git a/parm/metplus/PointStat_ensprob_SFC.conf b/parm/metplus/PointStat_ensprob_SFC.conf index 10937c2ec7..0df0a36e3b 100644 --- a/parm/metplus/PointStat_ensprob_SFC.conf +++ b/parm/metplus/PointStat_ensprob_SFC.conf @@ -2,7 +2,7 @@ [config] -# List of applications to run - only PointStat for this case +# List of applications (tools) to run. PROCESS_LIST = PointStat # time looping - options are INIT, VALID, RETRO, and REALTIME diff --git a/parm/metplus/PointStat_ensprob_UPA.conf b/parm/metplus/PointStat_ensprob_UPA.conf index ea6e380422..06b7d7f563 100644 --- a/parm/metplus/PointStat_ensprob_UPA.conf +++ b/parm/metplus/PointStat_ensprob_UPA.conf @@ -2,7 +2,7 @@ [config] -# List of applications to run - only PointStat for this case +# List of applications (tools) to run. PROCESS_LIST = PointStat # time looping - options are INIT, VALID, RETRO, and REALTIME From 4298da05993159af57fc623002db2f13398fa0d9 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Thu, 23 Mar 2023 13:25:52 -0600 Subject: [PATCH 03/43] Standardize METplus conf files as much as possible. --- parm/metplus/EnsembleStat_APCP01h.conf | 42 ++++++++++------ parm/metplus/EnsembleStat_APCPgt01h.conf | 42 ++++++++++------ parm/metplus/EnsembleStat_REFC.conf | 42 ++++++++++------ parm/metplus/EnsembleStat_RETOP.conf | 42 ++++++++++------ parm/metplus/EnsembleStat_SFC.conf | 42 ++++++++++------ parm/metplus/EnsembleStat_UPA.conf | 42 ++++++++++------ parm/metplus/GridStat_APCP01h.conf | 20 +++++--- parm/metplus/GridStat_APCPgt01h.conf | 18 +++---- parm/metplus/GridStat_REFC.conf | 33 ++++++++---- parm/metplus/GridStat_RETOP.conf | 33 ++++++++---- parm/metplus/GridStat_ensmean_APCP01h.conf | 36 ++++++++----- parm/metplus/GridStat_ensmean_APCPgt01h.conf | 26 +++++++--- parm/metplus/GridStat_ensmean_REFC.conf | 35 ++++++++----- parm/metplus/GridStat_ensmean_RETOP.conf | 31 +++++++----- parm/metplus/GridStat_ensprob_APCP01h.conf | 17 +++++-- parm/metplus/GridStat_ensprob_APCP03h.conf | 24 ++++++--- parm/metplus/GridStat_ensprob_APCP06h.conf | 24 ++++++--- parm/metplus/GridStat_ensprob_APCP24h.conf | 24 ++++++--- parm/metplus/GridStat_ensprob_REFC.conf | 31 +++++++----- parm/metplus/GridStat_ensprob_RETOP.conf | 33 ++++++++---- parm/metplus/Pb2nc_obs.conf | 53 ++++++++++++-------- parm/metplus/PcpCombine_fcst.conf | 34 ++++++------- parm/metplus/PcpCombine_obs.conf | 34 ++++++------- parm/metplus/PointStat_SFC.conf | 22 ++++---- parm/metplus/PointStat_UPA.conf | 22 ++++---- parm/metplus/PointStat_ensmean_SFC.conf | 19 ++++--- parm/metplus/PointStat_ensmean_UPA.conf | 19 ++++--- parm/metplus/PointStat_ensprob_SFC.conf | 19 ++++--- parm/metplus/PointStat_ensprob_UPA.conf | 19 ++++--- 29 files changed, 537 insertions(+), 341 deletions(-) diff --git a/parm/metplus/EnsembleStat_APCP01h.conf b/parm/metplus/EnsembleStat_APCP01h.conf index 0d285d33f5..845089044c 100644 --- a/parm/metplus/EnsembleStat_APCP01h.conf +++ b/parm/metplus/EnsembleStat_APCP01h.conf @@ -32,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} -# + # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -41,7 +41,6 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run -# LOOP_ORDER = times # # Specify the name of the METplus log file. @@ -54,15 +53,20 @@ METPLUS_CONF = {ENSEMBLE_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} # # Location of MET configuration file to pass to EnsembleStat. # +# References PARM_BASE, which is the location of the parm directory +# corresponding to the ush directory of the run_metplus.py script that +# is called or the value of the environment variable METPLUS_PARM_BASE +# if set. +# ENSEMBLE_STAT_CONFIG_FILE = {PARM_BASE}/met_config/EnsembleStatConfig_wrapped # -# Used in the MET config file for: model, output_prefix +# Name to identify model (forecast) data in output. # MODEL = {{vx_fcst_model_name}} ENSEMBLE_STAT_DESC = NA # -# Name to identify observation data in output +# Name to identify observation data in output. # OBTYPE = {{obtype}} @@ -247,30 +251,31 @@ ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_DIR} # STAGING_DIR = {{staging_dir}} +# End of [dir] section and start of [filename_templates] section. [filename_templates] # -# FCST_ENSEMBLE_STAT_INPUT_TEMPLATE - comma separated list of ensemble members -# or a single line, - filename wildcard characters may be used, ? or *. +# Template for point observation input to EnsembleStat relative to +# OBS_ENSEMBLE_STAT_POINT_INPUT_DIR. # -# The template FCST_ENSEMBLE_STAT_INPUT_TEMPLATE is relative to -# FCST_ENSEMBLE_STAT_INPUT_DIR. +OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE = # -FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}} +# Template for gridded observation input to EnsembleStat relative to +# OBS_ENSEMBLE_STAT_GRID_INPUT_DIR. # -# Template for point observations relative to OBS_ENSEMBLE_STAT_POINT_INPUT_DIR. +OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = {{obs_input_fn_template}} # -OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE = +# Template for forecast input to EnsembleStat relative to +# FCST_ENSEMBLE_STAT_INPUT_DIR. # -# Template for gridded observations relative to OBS_ENSEMBLE_STAT_GRID_INPUT_DIR. +# Note that this can be a comma separated list of ensemble members +# or a single line, - filename wildcard characters may be used, ? or *. # -OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = {{obs_input_fn_template}} +FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}} # -# The template ENSEMBLE_STAT_OUTPUT_TEMPLATE is relative to +# Template for output from EnsembleStat relative to # ENSEMBLE_STAT_OUTPUT_DIR. # ENSEMBLE_STAT_OUTPUT_TEMPLATE = - -ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly # # Template for climatology input to EnsembleStat relative to # ENSEMBLE_STAT_CLIMO_MEAN_INPUT_DIR. Not used in this example. @@ -281,3 +286,8 @@ ENSEMBLE_STAT_CLIMO_MEAN_INPUT_TEMPLATE = # ENSEMBLE_STAT_CLIMO_STDEV_INPUT_DIR. Not used in this example. # ENSEMBLE_STAT_CLIMO_STDEV_INPUT_TEMPLATE = +# +# Variable used to specify one or more verification mask files for +# EnsembleStat. Not used for this example. +# +ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly diff --git a/parm/metplus/EnsembleStat_APCPgt01h.conf b/parm/metplus/EnsembleStat_APCPgt01h.conf index beac8a7164..dccaecd9b7 100644 --- a/parm/metplus/EnsembleStat_APCPgt01h.conf +++ b/parm/metplus/EnsembleStat_APCPgt01h.conf @@ -32,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} -# + # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -41,7 +41,6 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run -# LOOP_ORDER = times # # Specify the name of the METplus log file. @@ -54,15 +53,20 @@ METPLUS_CONF = {ENSEMBLE_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} # # Location of MET configuration file to pass to EnsembleStat. # +# References PARM_BASE, which is the location of the parm directory +# corresponding to the ush directory of the run_metplus.py script that +# is called or the value of the environment variable METPLUS_PARM_BASE +# if set. +# ENSEMBLE_STAT_CONFIG_FILE = {PARM_BASE}/met_config/EnsembleStatConfig_wrapped # -# Used in the MET config file for: model, output_prefix +# Name to identify model (forecast) data in output. # MODEL = {{vx_fcst_model_name}} ENSEMBLE_STAT_DESC = NA # -# Name to identify observation data in output +# Name to identify observation data in output. # OBTYPE = {{obtype}} # @@ -254,30 +258,31 @@ ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_DIR} # STAGING_DIR = {{staging_dir}} +# End of [dir] section and start of [filename_templates] section. [filename_templates] # -# FCST_ENSEMBLE_STAT_INPUT_TEMPLATE - comma separated list of ensemble members -# or a single line, - filename wildcard characters may be used, ? or *. +# Template for point observation input to EnsembleStat relative to +# OBS_ENSEMBLE_STAT_POINT_INPUT_DIR. # -# The template FCST_ENSEMBLE_STAT_INPUT_TEMPLATE is relative to -# FCST_ENSEMBLE_STAT_INPUT_DIR. +OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE = # -FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}} +# Template for gridded observation input to EnsembleStat relative to +# OBS_ENSEMBLE_STAT_GRID_INPUT_DIR. # -# Template for point observations relative to OBS_ENSEMBLE_STAT_POINT_INPUT_DIR. +OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = {{obs_input_fn_template}} # -OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE = +# Template for forecast input to EnsembleStat relative to +# FCST_ENSEMBLE_STAT_INPUT_DIR. # -# Template for gridded observations relative to OBS_ENSEMBLE_STAT_GRID_INPUT_DIR. +# Note that this can be a comma separated list of ensemble members +# or a single line, - filename wildcard characters may be used, ? or *. # -OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = {{obs_input_fn_template}} +FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}} # -# The template ENSEMBLE_STAT_OUTPUT_TEMPLATE is relative to +# Template for output from EnsembleStat relative to # ENSEMBLE_STAT_OUTPUT_DIR. # ENSEMBLE_STAT_OUTPUT_TEMPLATE = - -ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly # # Template for climatology input to EnsembleStat relative to # ENSEMBLE_STAT_CLIMO_MEAN_INPUT_DIR. Not used in this example. @@ -288,3 +293,8 @@ ENSEMBLE_STAT_CLIMO_MEAN_INPUT_TEMPLATE = # ENSEMBLE_STAT_CLIMO_STDEV_INPUT_DIR. Not used in this example. # ENSEMBLE_STAT_CLIMO_STDEV_INPUT_TEMPLATE = +# +# Variable used to specify one or more verification mask files for +# EnsembleStat. Not used for this example. +# +ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly diff --git a/parm/metplus/EnsembleStat_REFC.conf b/parm/metplus/EnsembleStat_REFC.conf index 1772431b69..3f52156d97 100644 --- a/parm/metplus/EnsembleStat_REFC.conf +++ b/parm/metplus/EnsembleStat_REFC.conf @@ -32,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} -# + # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -41,7 +41,6 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run -# LOOP_ORDER = times # # Specify the name of the METplus log file. @@ -54,15 +53,20 @@ METPLUS_CONF = {ENSEMBLE_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} # # Location of MET configuration file to pass to EnsembleStat. # +# References PARM_BASE, which is the location of the parm directory +# corresponding to the ush directory of the run_metplus.py script that +# is called or the value of the environment variable METPLUS_PARM_BASE +# if set. +# ENSEMBLE_STAT_CONFIG_FILE = {PARM_BASE}/met_config/EnsembleStatConfig_wrapped # -# Used in the MET config file for: model, output_prefix +# Name to identify model (forecast) data in output. # MODEL = {{vx_fcst_model_name}} ENSEMBLE_STAT_DESC = NA # -# Name to identify observation data in output +# Name to identify observation data in output. # OBTYPE = {{obtype}} @@ -245,30 +249,31 @@ ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_DIR} # STAGING_DIR = {{staging_dir}} +# End of [dir] section and start of [filename_templates] section. [filename_templates] # -# FCST_ENSEMBLE_STAT_INPUT_TEMPLATE - comma separated list of ensemble members -# or a single line, - filename wildcard characters may be used, ? or *. +# Template for point observation input to EnsembleStat relative to +# OBS_ENSEMBLE_STAT_POINT_INPUT_DIR. # -# The template FCST_ENSEMBLE_STAT_INPUT_TEMPLATE is relative to -# FCST_ENSEMBLE_STAT_INPUT_DIR. +OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE = # -FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}} +# Template for gridded observation input to EnsembleStat relative to +# OBS_ENSEMBLE_STAT_GRID_INPUT_DIR. # -# Template for point observations relative to OBS_ENSEMBLE_STAT_POINT_INPUT_DIR. +OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = {{obs_input_fn_template}} # -OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE = +# Template for forecast input to EnsembleStat relative to +# FCST_ENSEMBLE_STAT_INPUT_DIR. # -# Template for gridded observations relative to OBS_ENSEMBLE_STAT_GRID_INPUT_DIR. +# Note that this can be a comma separated list of ensemble members +# or a single line, - filename wildcard characters may be used, ? or *. # -OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = {{obs_input_fn_template}} +FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}} # -# The template ENSEMBLE_STAT_OUTPUT_TEMPLATE is relative to +# Template for output from EnsembleStat relative to # ENSEMBLE_STAT_OUTPUT_DIR. # ENSEMBLE_STAT_OUTPUT_TEMPLATE = - -ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly # # Template for climatology input to EnsembleStat relative to # ENSEMBLE_STAT_CLIMO_MEAN_INPUT_DIR. Not used in this example. @@ -279,3 +284,8 @@ ENSEMBLE_STAT_CLIMO_MEAN_INPUT_TEMPLATE = # ENSEMBLE_STAT_CLIMO_STDEV_INPUT_DIR. Not used in this example. # ENSEMBLE_STAT_CLIMO_STDEV_INPUT_TEMPLATE = +# +# Variable used to specify one or more verification mask files for +# EnsembleStat. Not used for this example. +# +ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly diff --git a/parm/metplus/EnsembleStat_RETOP.conf b/parm/metplus/EnsembleStat_RETOP.conf index a4ab12189d..916ad3d55f 100644 --- a/parm/metplus/EnsembleStat_RETOP.conf +++ b/parm/metplus/EnsembleStat_RETOP.conf @@ -32,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} -# + # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -41,7 +41,6 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run -# LOOP_ORDER = times # # Specify the name of the METplus log file. @@ -54,15 +53,20 @@ METPLUS_CONF = {ENSEMBLE_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} # # Location of MET configuration file to pass to EnsembleStat. # +# References PARM_BASE, which is the location of the parm directory +# corresponding to the ush directory of the run_metplus.py script that +# is called or the value of the environment variable METPLUS_PARM_BASE +# if set. +# ENSEMBLE_STAT_CONFIG_FILE = {PARM_BASE}/met_config/EnsembleStatConfig_wrapped # -# Used in the MET config file for: model, output_prefix +# Name to identify model (forecast) data in output. # MODEL = {{vx_fcst_model_name}} ENSEMBLE_STAT_DESC = NA # -# Name to identify observation data in output +# Name to identify observation data in output. # OBTYPE = {{obtype}} @@ -249,30 +253,31 @@ ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_DIR} # STAGING_DIR = {{staging_dir}} +# End of [dir] section and start of [filename_templates] section. [filename_templates] # -# FCST_ENSEMBLE_STAT_INPUT_TEMPLATE - comma separated list of ensemble members -# or a single line, - filename wildcard characters may be used, ? or *. +# Template for point observation input to EnsembleStat relative to +# OBS_ENSEMBLE_STAT_POINT_INPUT_DIR. # -# The template FCST_ENSEMBLE_STAT_INPUT_TEMPLATE is relative to -# FCST_ENSEMBLE_STAT_INPUT_DIR. +OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE = # -FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}} +# Template for gridded observation input to EnsembleStat relative to +# OBS_ENSEMBLE_STAT_GRID_INPUT_DIR. # -# Template for point observations relative to OBS_ENSEMBLE_STAT_POINT_INPUT_DIR. +OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = {{obs_input_fn_template}} # -OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE = +# Template for forecast input to EnsembleStat relative to +# FCST_ENSEMBLE_STAT_INPUT_DIR. # -# Template for gridded observations relative to OBS_ENSEMBLE_STAT_GRID_INPUT_DIR. +# Note that this can be a comma separated list of ensemble members +# or a single line, - filename wildcard characters may be used, ? or *. # -OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = {{obs_input_fn_template}} +FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}} # -# The template ENSEMBLE_STAT_OUTPUT_TEMPLATE is relative to +# Template for output from EnsembleStat relative to # ENSEMBLE_STAT_OUTPUT_DIR. # ENSEMBLE_STAT_OUTPUT_TEMPLATE = - -ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly # # Template for climatology input to EnsembleStat relative to # ENSEMBLE_STAT_CLIMO_MEAN_INPUT_DIR. Not used in this example. @@ -283,3 +288,8 @@ ENSEMBLE_STAT_CLIMO_MEAN_INPUT_TEMPLATE = # ENSEMBLE_STAT_CLIMO_STDEV_INPUT_DIR. Not used in this example. # ENSEMBLE_STAT_CLIMO_STDEV_INPUT_TEMPLATE = +# +# Variable used to specify one or more verification mask files for +# EnsembleStat. Not used for this example. +# +ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly diff --git a/parm/metplus/EnsembleStat_SFC.conf b/parm/metplus/EnsembleStat_SFC.conf index 9b0ae70b88..1c2460ba5d 100644 --- a/parm/metplus/EnsembleStat_SFC.conf +++ b/parm/metplus/EnsembleStat_SFC.conf @@ -32,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} -# + # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -41,7 +41,6 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run -# LOOP_ORDER = times # # Specify the name of the METplus log file. @@ -54,15 +53,20 @@ METPLUS_CONF = {ENSEMBLE_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} # # Location of MET configuration file to pass to EnsembleStat. # +# References PARM_BASE, which is the location of the parm directory +# corresponding to the ush directory of the run_metplus.py script that +# is called or the value of the environment variable METPLUS_PARM_BASE +# if set. +# ENSEMBLE_STAT_CONFIG_FILE = {PARM_BASE}/met_config/EnsembleStatConfig_wrapped # -# Used in the MET config file for: model, output_prefix +# Name to identify model (forecast) data in output. # MODEL = {{vx_fcst_model_name}} ENSEMBLE_STAT_DESC = NA # -# Name to identify observation data in output +# Name to identify observation data in output. # OBTYPE = {{obtype}} @@ -312,30 +316,31 @@ ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_DIR} # STAGING_DIR = {{staging_dir}} +# End of [dir] section and start of [filename_templates] section. [filename_templates] # -# FCST_ENSEMBLE_STAT_INPUT_TEMPLATE - comma separated list of ensemble members -# or a single line, - filename wildcard characters may be used, ? or *. +# Template for point observation input to EnsembleStat relative to +# OBS_ENSEMBLE_STAT_POINT_INPUT_DIR. # -# The template FCST_ENSEMBLE_STAT_INPUT_TEMPLATE is relative to -# FCST_ENSEMBLE_STAT_INPUT_DIR. +OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE = {{obs_input_fn_template}} # -FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}} +# Template for gridded observation input to EnsembleStat relative to +# OBS_ENSEMBLE_STAT_GRID_INPUT_DIR. # -# Template for point observations relative to OBS_ENSEMBLE_STAT_POINT_INPUT_DIR. +OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = # -OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE = {{obs_input_fn_template}} +# Template for forecast input to EnsembleStat relative to +# FCST_ENSEMBLE_STAT_INPUT_DIR. # -# Template for gridded observations relative to OBS_ENSEMBLE_STAT_GRID_INPUT_DIR. +# Note that this can be a comma separated list of ensemble members +# or a single line, - filename wildcard characters may be used, ? or *. # -OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = +FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}} # -# The template ENSEMBLE_STAT_OUTPUT_TEMPLATE is relative to +# Template for output from EnsembleStat relative to # ENSEMBLE_STAT_OUTPUT_DIR. # ENSEMBLE_STAT_OUTPUT_TEMPLATE = - -ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly # # Template for climatology input to EnsembleStat relative to # ENSEMBLE_STAT_CLIMO_MEAN_INPUT_DIR. Not used in this example. @@ -346,3 +351,8 @@ ENSEMBLE_STAT_CLIMO_MEAN_INPUT_TEMPLATE = # ENSEMBLE_STAT_CLIMO_STDEV_INPUT_DIR. Not used in this example. # ENSEMBLE_STAT_CLIMO_STDEV_INPUT_TEMPLATE = +# +# Variable used to specify one or more verification mask files for +# EnsembleStat. Not used for this example. +# +ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly diff --git a/parm/metplus/EnsembleStat_UPA.conf b/parm/metplus/EnsembleStat_UPA.conf index 51b34fb353..de14c23f7a 100644 --- a/parm/metplus/EnsembleStat_UPA.conf +++ b/parm/metplus/EnsembleStat_UPA.conf @@ -32,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} -# + # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -41,7 +41,6 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run -# LOOP_ORDER = times # # Specify the name of the METplus log file. @@ -54,15 +53,20 @@ METPLUS_CONF = {ENSEMBLE_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} # # Location of MET configuration file to pass to EnsembleStat. # +# References PARM_BASE, which is the location of the parm directory +# corresponding to the ush directory of the run_metplus.py script that +# is called or the value of the environment variable METPLUS_PARM_BASE +# if set. +# ENSEMBLE_STAT_CONFIG_FILE = {PARM_BASE}/met_config/EnsembleStatConfig_wrapped # -# Used in the MET config file for: model, output_prefix +# Name to identify model (forecast) data in output. # MODEL = {{vx_fcst_model_name}} ENSEMBLE_STAT_DESC = NA # -# Name to identify observation data in output +# Name to identify observation data in output. # OBTYPE = {{obtype}} @@ -375,30 +379,31 @@ ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_DIR} # STAGING_DIR = {{staging_dir}} +# End of [dir] section and start of [filename_templates] section. [filename_templates] # -# FCST_ENSEMBLE_STAT_INPUT_TEMPLATE - comma separated list of ensemble members -# or a single line, - filename wildcard characters may be used, ? or *. +# Template for point observation input to EnsembleStat relative to +# OBS_ENSEMBLE_STAT_POINT_INPUT_DIR. # -# The template FCST_ENSEMBLE_STAT_INPUT_TEMPLATE is relative to -# FCST_ENSEMBLE_STAT_INPUT_DIR. +OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE = {{obs_input_fn_template}} # -FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}} +# Template for gridded observation input to EnsembleStat relative to +# OBS_ENSEMBLE_STAT_GRID_INPUT_DIR. # -# Template for point observations relative to OBS_ENSEMBLE_STAT_POINT_INPUT_DIR. +OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = # -OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE = {{obs_input_fn_template}} +# Template for forecast input to EnsembleStat relative to +# FCST_ENSEMBLE_STAT_INPUT_DIR. # -# Template for gridded observations relative to OBS_ENSEMBLE_STAT_GRID_INPUT_DIR. +# Note that this can be a comma separated list of ensemble members +# or a single line, - filename wildcard characters may be used, ? or *. # -OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = +FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}} # -# The template ENSEMBLE_STAT_OUTPUT_TEMPLATE is relative to +# Template for output from EnsembleStat relative to # ENSEMBLE_STAT_OUTPUT_DIR. # ENSEMBLE_STAT_OUTPUT_TEMPLATE = - -ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly # # Template for climatology input to EnsembleStat relative to # ENSEMBLE_STAT_CLIMO_MEAN_INPUT_DIR. Not used in this example. @@ -409,3 +414,8 @@ ENSEMBLE_STAT_CLIMO_MEAN_INPUT_TEMPLATE = # ENSEMBLE_STAT_CLIMO_STDEV_INPUT_DIR. Not used in this example. # ENSEMBLE_STAT_CLIMO_STDEV_INPUT_TEMPLATE = +# +# Variable used to specify one or more verification mask files for +# EnsembleStat. Not used for this example. +# +ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly diff --git a/parm/metplus/GridStat_APCP01h.conf b/parm/metplus/GridStat_APCP01h.conf index e58f26e062..de01b4ad2e 100644 --- a/parm/metplus/GridStat_APCP01h.conf +++ b/parm/metplus/GridStat_APCP01h.conf @@ -45,7 +45,6 @@ LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only #LOG_GRID_STAT_VERBOSITY = 2 - # # Specify the name of the METplus log file. # @@ -57,6 +56,11 @@ METPLUS_CONF = {GRID_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} # # Location of MET configuration file to pass to GridStat. # +# References PARM_BASE, which is the location of the parm directory +# corresponding to the ush directory of the run_metplus.py script that +# is called or the value of the environment variable METPLUS_PARM_BASE +# if set. +# GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped # grid to remap data. Value is set as the 'to_grid' variable in the 'regrid' dictionary @@ -282,19 +286,23 @@ GRID_STAT_CLIMO_STDEV_INPUT_DIR = # OUTPUT_BASE = {{output_base}} GRID_STAT_OUTPUT_DIR = {{output_dir}} - +# +# Directory for staging data. +# STAGING_DIR = {{staging_dir}} # End of [dir] section and start of [filename_templates] section. [filename_templates] # -# Template for forecast input to GridStat relative to FCST_GRID_STAT_INPUT_DIR. +# Template for observation input to GridStat relative to +# OBS_GRID_STAT_INPUT_DIR. # -FCST_GRID_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}} +OBS_GRID_STAT_INPUT_TEMPLATE = {{obs_input_fn_template}} # -# Template for observation input to GridStat relative to OBS_GRID_STAT_INPUT_DIR. +# Template for forecast input to GridStat relative to +# FCST_GRID_STAT_INPUT_DIR. # -OBS_GRID_STAT_INPUT_TEMPLATE = {{obs_input_fn_template}} +FCST_GRID_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}} # # Template for output from GridStat relative to GRID_STAT_OUTPUT_DIR. # diff --git a/parm/metplus/GridStat_APCPgt01h.conf b/parm/metplus/GridStat_APCPgt01h.conf index 9543f62b0e..fb2014e6a5 100644 --- a/parm/metplus/GridStat_APCPgt01h.conf +++ b/parm/metplus/GridStat_APCPgt01h.conf @@ -45,7 +45,6 @@ LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only #LOG_GRID_STAT_VERBOSITY = 2 - # # Specify the name of the METplus log file. # @@ -55,7 +54,12 @@ LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} # METPLUS_CONF = {GRID_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} # -# Location of MET config file to pass to GridStat. +# Location of MET configuration file to pass to GridStat. +# +# References PARM_BASE, which is the location of the parm directory +# corresponding to the ush directory of the run_metplus.py script that +# is called or the value of the environment variable METPLUS_PARM_BASE +# if set. # GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped @@ -141,7 +145,7 @@ FCST_VAR1_LEVELS = A{ACCUM_HH} FCST_VAR1_OPTIONS = set_attr_lead = "{lead?fmt=%H%M%S}"; # # List of thresholds to evaluate for each name/level combination for -# forecast variable 1. +# both obs and forecast variable 1. # BOTH_VAR1_THRESH = {{field_thresholds}} # @@ -269,12 +273,10 @@ GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK = FALSE # # Directory containing observation input to GridStat. # -#OBS_GRID_STAT_INPUT_DIR = {ENV[EXPTDIR]}/metprd/PcpCombine_obs OBS_GRID_STAT_INPUT_DIR = {{obs_input_dir}} # # Directory containing forecast input to GridStat. # -#FCST_GRID_STAT_INPUT_DIR = {OUTPUT_BASE}/metprd/PcpCombine_fcst FCST_GRID_STAT_INPUT_DIR = {{fcst_input_dir}} # # Directory containing climatology mean input to GridStat. Not used in @@ -293,14 +295,11 @@ GRID_STAT_CLIMO_STDEV_INPUT_DIR = # to its default value. But it is not explicitly used elsewhere in this # configuration file. # -#OUTPUT_BASE = {ENV[OUTPUT_BASE]} -#GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} OUTPUT_BASE = {{output_base}} GRID_STAT_OUTPUT_DIR = {{output_dir}} # # Directory for staging data. # -#STAGING_DIR = {OUTPUT_BASE}/stage/APCP_03h STAGING_DIR = {{staging_dir}} # End of [dir] section and start of [filename_templates] section. @@ -309,18 +308,15 @@ STAGING_DIR = {{staging_dir}} # Template for observation input to GridStat relative to # OBS_GRID_STAT_INPUT_DIR. # -#OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hrap.conus.gb2_a{level?fmt=%HH}h.nc OBS_GRID_STAT_INPUT_TEMPLATE = {{obs_input_fn_template}} # # Template for forecast input to GridStat relative to # FCST_GRID_STAT_INPUT_DIR. # -#FCST_GRID_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}_a{level?fmt=%HH}h.nc FCST_GRID_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}} # # Template for output from GridStat relative to GRID_STAT_OUTPUT_DIR. # -#GRID_STAT_OUTPUT_TEMPLATE = metprd/GridStat GRID_STAT_OUTPUT_TEMPLATE = # # Template for climatology input to GridStat relative to diff --git a/parm/metplus/GridStat_REFC.conf b/parm/metplus/GridStat_REFC.conf index 7c80444bb0..22b7230478 100644 --- a/parm/metplus/GridStat_REFC.conf +++ b/parm/metplus/GridStat_REFC.conf @@ -45,7 +45,6 @@ LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only #LOG_GRID_STAT_VERBOSITY = 2 - # # Specify the name of the METplus log file. # @@ -55,7 +54,12 @@ LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} # METPLUS_CONF = {GRID_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} # -# Location of MET config file to pass to GridStat. +# Location of MET configuration file to pass to GridStat. +# +# References PARM_BASE, which is the location of the parm directory +# corresponding to the ush directory of the run_metplus.py script that +# is called or the value of the environment variable METPLUS_PARM_BASE +# if set. # GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped @@ -154,6 +158,7 @@ OBS_VAR1_OPTIONS = censor_thresh = [eq-999, <-20]; censor_val = [-9999, -20]; cn # Time relative to valid time (in seconds) to allow files to be considered # valid. Set both BEGIN and END to 0 to require the exact time in the # filename. Not used in this example. +# #FCST_GRID_STAT_FILE_WINDOW_BEGIN = 0 #FCST_GRID_STAT_FILE_WINDOW_END = 0 OBS_GRID_STAT_FILE_WINDOW_BEGIN = -300 @@ -186,12 +191,14 @@ FCST_IS_PROB = False FCST_GRID_STAT_PROB_THRESH = ==0.1 GRID_STAT_OUTPUT_PREFIX = {MODEL}_{{fieldname_in_met_filedir_names}}_{OBTYPE} - -# Set to true if observation data is probabilistic. -# Only used if configuring forecast data as the 'OBS' input. +# +# Set to true if observation data is probabilistic. Only used if +# configuring forecast data as the 'OBS' input. +# OBS_IS_PROB = false - +# # Only used if OBS_IS_PROB is true - sets probabilistic threshold. +# OBS_GRID_STAT_PROB_THRESH = ==0.1 # Climatology data @@ -289,19 +296,23 @@ GRID_STAT_CLIMO_STDEV_INPUT_DIR = # OUTPUT_BASE = {{output_base}} GRID_STAT_OUTPUT_DIR = {{output_dir}} - +# +# Directory for staging data. +# STAGING_DIR = {{staging_dir}} # End of [dir] section and start of [filename_templates] section. [filename_templates] # -# Template for forecast input to GridStat relative to FCST_GRID_STAT_INPUT_DIR. +# Template for observation input to GridStat relative to +# OBS_GRID_STAT_INPUT_DIR. # -FCST_GRID_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}} +OBS_GRID_STAT_INPUT_TEMPLATE = {{obs_input_fn_template}} # -# Template for observation input to GridStat relative to OBS_GRID_STAT_INPUT_DIR. +# Template for forecast input to GridStat relative to +# FCST_GRID_STAT_INPUT_DIR. # -OBS_GRID_STAT_INPUT_TEMPLATE = {{obs_input_fn_template}} +FCST_GRID_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}} # # Template for output from GridStat relative to GRID_STAT_OUTPUT_DIR. # diff --git a/parm/metplus/GridStat_RETOP.conf b/parm/metplus/GridStat_RETOP.conf index 8cdebca0dc..0ebcf7e1a8 100644 --- a/parm/metplus/GridStat_RETOP.conf +++ b/parm/metplus/GridStat_RETOP.conf @@ -45,7 +45,6 @@ LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only #LOG_GRID_STAT_VERBOSITY = 2 - # # Specify the name of the METplus log file. # @@ -55,7 +54,12 @@ LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} # METPLUS_CONF = {GRID_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} # -# Location of MET config file to pass to GridStat. +# Location of MET configuration file to pass to GridStat. +# +# References PARM_BASE, which is the location of the parm directory +# corresponding to the ush directory of the run_metplus.py script that +# is called or the value of the environment variable METPLUS_PARM_BASE +# if set. # GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped @@ -155,6 +159,7 @@ OBS_VAR1_OPTIONS = convert(x) = x * 3280.84 * 0.001; censor_thresh = [<=-9.84252 # Time relative to valid time (in seconds) to allow files to be considered # valid. Set both BEGIN and END to 0 to require the exact time in the # filename. Not used in this example. +# #FCST_GRID_STAT_FILE_WINDOW_BEGIN = 0 #FCST_GRID_STAT_FILE_WINDOW_END = 0 OBS_GRID_STAT_FILE_WINDOW_BEGIN = -300 @@ -187,12 +192,14 @@ FCST_IS_PROB = False FCST_GRID_STAT_PROB_THRESH = ==0.1 GRID_STAT_OUTPUT_PREFIX = {MODEL}_{{fieldname_in_met_filedir_names}}_{OBTYPE} - -# Set to true if observation data is probabilistic. -# Only used if configuring forecast data as the 'OBS' input. +# +# Set to true if observation data is probabilistic. Only used if +# configuring forecast data as the 'OBS' input. +# OBS_IS_PROB = false - +# # Only used if OBS_IS_PROB is true - sets probabilistic threshold. +# OBS_GRID_STAT_PROB_THRESH = ==0.1 # Climatology data @@ -290,19 +297,23 @@ GRID_STAT_CLIMO_STDEV_INPUT_DIR = # OUTPUT_BASE = {{output_base}} GRID_STAT_OUTPUT_DIR = {{output_dir}} - +# +# Directory for staging data. +# STAGING_DIR = {{staging_dir}} # End of [dir] section and start of [filename_templates] section. [filename_templates] # -# Template for forecast input to GridStat relative to FCST_GRID_STAT_INPUT_DIR. +# Template for observation input to GridStat relative to +# OBS_GRID_STAT_INPUT_DIR. # -FCST_GRID_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}} +OBS_GRID_STAT_INPUT_TEMPLATE = {{obs_input_fn_template}} # -# Template for observation input to GridStat relative to OBS_GRID_STAT_INPUT_DIR. +# Template for forecast input to GridStat relative to +# FCST_GRID_STAT_INPUT_DIR. # -OBS_GRID_STAT_INPUT_TEMPLATE = {{obs_input_fn_template}} +FCST_GRID_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}} # # Template for output from GridStat relative to GRID_STAT_OUTPUT_DIR. # diff --git a/parm/metplus/GridStat_ensmean_APCP01h.conf b/parm/metplus/GridStat_ensmean_APCP01h.conf index b7816d3338..bf01073ad6 100644 --- a/parm/metplus/GridStat_ensmean_APCP01h.conf +++ b/parm/metplus/GridStat_ensmean_APCP01h.conf @@ -45,7 +45,6 @@ LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only #LOG_GRID_STAT_VERBOSITY = 2 - # # Specify the name of the METplus log file. # @@ -55,7 +54,12 @@ LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} # METPLUS_CONF = {GRID_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} # -# Location of MET config file to pass to GridStat. +# Location of MET configuration file to pass to GridStat. +# +# References PARM_BASE, which is the location of the parm directory +# corresponding to the ush directory of the run_metplus.py script that +# is called or the value of the environment variable METPLUS_PARM_BASE +# if set. # GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped @@ -91,8 +95,9 @@ FCST_NATIVE_DATA_TYPE = GRIB # OBTYPE = {{obtype}} OBS_NATIVE_DATA_TYPE = GRIB - -# set the desc value in the GridStat MET config file +# +# Value to enter under the DESC column in the output stat file. +# GRID_STAT_DESC = NA # List of variables to compare in GridStat - FCST_VAR1 variables correspond @@ -102,7 +107,9 @@ GRID_STAT_DESC = NA GRID_STAT_MET_CONFIG_OVERRIDES = cat_thresh = [NA]; cnt_thresh = [NA]; cnt_logic = UNION; wind_thresh = [NA]; wind_logic = UNION; ci_alpha = [0.05]; rank_corr_flag = FALSE; # -# Name of forecast variable 1 +# String to search for in the foreast input files for forecast variable +# 1. +# # Note: # This is the name of the field in the NetCDF file(s) created by MET's # gen_ens_prod tool. This tool reads in the grib2 file(s) (in this case @@ -133,11 +140,12 @@ ACCUM_HH = {{accum_hh}} FCST_VAR1_LEVELS = A{ACCUM_HH} # # List of thresholds to evaluate for each name/level combination for -# observation and forecast variable 1. +# both obs and forecast variable 1. # BOTH_VAR1_THRESH = {{field_thresholds}} # -# Name of observation variable 1. +# String to search for in the observation input files for observation +# variable 1. # # Note: # This is the name of the field in the grib2 observation file. Thus, @@ -288,22 +296,24 @@ GRID_STAT_CLIMO_STDEV_INPUT_DIR = # OUTPUT_BASE = {{output_base}} GRID_STAT_OUTPUT_DIR = {{output_dir}} - +# +# Directory for staging data. +# STAGING_DIR = {{staging_dir}} # End of [dir] section and start of [filename_templates] section. [filename_templates] # -# Template for forecast input to GridStat relative to -# FCST_GRID_STAT_INPUT_DIR. -# -FCST_GRID_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}} -# # Template for observation input to GridStat relative to # OBS_GRID_STAT_INPUT_DIR. # OBS_GRID_STAT_INPUT_TEMPLATE = {{obs_input_fn_template}} # +# Template for forecast input to GridStat relative to +# FCST_GRID_STAT_INPUT_DIR. +# +FCST_GRID_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}} +# # Template for output from GridStat relative to GRID_STAT_OUTPUT_DIR. # GRID_STAT_OUTPUT_TEMPLATE = diff --git a/parm/metplus/GridStat_ensmean_APCPgt01h.conf b/parm/metplus/GridStat_ensmean_APCPgt01h.conf index 47bcf4d795..9b0bc71f4d 100644 --- a/parm/metplus/GridStat_ensmean_APCPgt01h.conf +++ b/parm/metplus/GridStat_ensmean_APCPgt01h.conf @@ -45,7 +45,6 @@ LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only #LOG_GRID_STAT_VERBOSITY = 2 - # # Specify the name of the METplus log file. # @@ -55,7 +54,12 @@ LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} # METPLUS_CONF = {GRID_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} # -# Location of MET config file to pass to GridStat. +# Location of MET configuration file to pass to GridStat. +# +# References PARM_BASE, which is the location of the parm directory +# corresponding to the ush directory of the run_metplus.py script that +# is called or the value of the environment variable METPLUS_PARM_BASE +# if set. # GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped @@ -91,8 +95,9 @@ FCST_NATIVE_DATA_TYPE = GRIB # OBTYPE = {{obtype}} OBS_NATIVE_DATA_TYPE = GRIB - -# set the desc value in the GridStat MET config file +# +# Value to enter under the DESC column in the output stat file. +# GRID_STAT_DESC = NA # List of variables to compare in GridStat - FCST_VAR1 variables correspond @@ -102,7 +107,9 @@ GRID_STAT_DESC = NA GRID_STAT_MET_CONFIG_OVERRIDES = cat_thresh = [NA]; cnt_thresh = [NA]; cnt_logic = UNION; wind_thresh = [NA]; wind_logic = UNION; ci_alpha = [0.05]; rank_corr_flag = FALSE; # -# Name of forecast variable 1 +# String to search for in the foreast input files for forecast variable +# 1. +# # Note: # This is the name of the field in the NetCDF file(s) created by MET's # gen_ens_prod tool. This tool reads in the NetCDF file(s) generated by @@ -129,11 +136,12 @@ ACCUM_HH = {{accum_hh}} FCST_VAR1_LEVELS = A{ACCUM_HH} # # List of thresholds to evaluate for each name/level combination for -# observation and forecast variable 1. +# both obs and forecast variable 1. # BOTH_VAR1_THRESH = {{field_thresholds}} # -# Name of observation variable 1. +# String to search for in the observation input files for observation +# variable 1. # # Note: # This is the name of the field in the NetCDF file created by MET's @@ -284,7 +292,9 @@ GRID_STAT_CLIMO_STDEV_INPUT_DIR = # OUTPUT_BASE = {{output_base}} GRID_STAT_OUTPUT_DIR = {{output_dir}} - +# +# Directory for staging data. +# STAGING_DIR = {{staging_dir}} # End of [dir] section and start of [filename_templates] section. diff --git a/parm/metplus/GridStat_ensmean_REFC.conf b/parm/metplus/GridStat_ensmean_REFC.conf index ffa2d05730..983d827610 100644 --- a/parm/metplus/GridStat_ensmean_REFC.conf +++ b/parm/metplus/GridStat_ensmean_REFC.conf @@ -45,7 +45,6 @@ LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only #LOG_GRID_STAT_VERBOSITY = 2 - # # Specify the name of the METplus log file. # @@ -55,7 +54,12 @@ LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} # METPLUS_CONF = {GRID_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} # -# Location of MET config file to pass to GridStat. +# Location of MET configuration file to pass to GridStat. +# +# References PARM_BASE, which is the location of the parm directory +# corresponding to the ush directory of the run_metplus.py script that +# is called or the value of the environment variable METPLUS_PARM_BASE +# if set. # GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped @@ -88,8 +92,9 @@ MODEL = {{vx_fcst_model_name}}_ensmean # OBTYPE = {{obtype}} OBS_NATIVE_DATA_TYPE = GRIB - -# set the desc value in the GridStat MET config file +# +# Value to enter under the DESC column in the output stat file. +# GRID_STAT_DESC = NA # List of variables to compare in GridStat - FCST_VAR1 variables correspond @@ -99,16 +104,18 @@ GRID_STAT_DESC = NA GRID_STAT_MET_CONFIG_OVERRIDES = cat_thresh = []; cnt_thresh = [NA]; cnt_logic = UNION; wind_thresh = [NA]; wind_logic = UNION; ci_alpha = [0.05]; rank_corr_flag = FALSE; # -# Name of forecast variable 1. +# String to search for in the foreast input files for forecast variable +# 1. +# # Note: # This is the name of the field in the NetCDF file(s) created by MET's # gen_ens_prod tool. This tool reads in the grib2 file(s) (in this case # of forecasts) and outputs NetCDF file(s) in which the array names -# consist of the value of FIELDNAME_IN_MET_OUTPUT plus a suffix that +# consist of the value of fieldname_in_met_output plus a suffix that # specifies additional properties of the data in the array such as the # level, the type of statistic, etc. In this case, this suffix is # "_L0_ENS_MEAN". Thus, below, FCST_VAR1_NAME must be set to the value -# of FIELDNAME_IN_MET_OUTPUT with "_L0_ENS_MEAN" appended to it. +# of fieldname_in_met_output with "_L0_ENS_MEAN" appended to it. # FCST_VAR1_NAME = {{fieldname_in_met_output}}_L0_ENS_MEAN # @@ -119,11 +126,12 @@ FCST_VAR1_LEVELS = L0 #FCST_VAR1_OPTIONS = cnt_thresh = [ >15 ]; cnt_logic = UNION; # # List of thresholds to evaluate for each name/level combination for -# observation and forecast variable 1. +# both obs and forecast variable 1. # BOTH_VAR1_THRESH = {{field_thresholds}} # -# Name of observation variable 1. +# String to search for in the observation input files for observation +# variable 1. # # Note: # This is the name of the field in the grib2 observation file. Thus, @@ -140,7 +148,6 @@ OBS_VAR1_NAME = {{fieldname_in_obs_input}} # same length as FCST_VAR1_LEVELS. # OBS_VAR1_LEVELS = Z500 - OBS_VAR1_OPTIONS = censor_thresh = lt-20; censor_val = -20.0; cnt_thresh = [ >15 ]; cnt_logic = UNION; # # Time relative to valid time (in seconds) to allow files to be considered @@ -177,6 +184,8 @@ FCST_IS_PROB = False # Only used if FCST_IS_PROB is true - sets probabilistic threshold # FCST_GRID_STAT_PROB_THRESH = ==0.1 + +GRID_STAT_OUTPUT_PREFIX = {MODEL}_{{fieldname_in_met_filedir_names}}_{OBTYPE} # # Set to true if observation data is probabilistic. Only used if # configuring forecast data as the 'OBS' input. @@ -187,8 +196,6 @@ OBS_IS_PROB = false # OBS_GRID_STAT_PROB_THRESH = ==0.1 -GRID_STAT_OUTPUT_PREFIX = {MODEL}_{{fieldname_in_met_filedir_names}}_{OBTYPE} - # Climatology data #GRID_STAT_CLIMO_MEAN_FILE_NAME = #GRID_STAT_CLIMO_MEAN_FIELD = @@ -284,7 +291,9 @@ GRID_STAT_CLIMO_STDEV_INPUT_DIR = # OUTPUT_BASE = {{output_base}} GRID_STAT_OUTPUT_DIR = {{output_dir}} - +# +# Directory for staging data. +# STAGING_DIR = {{staging_dir}} # End of [dir] section and start of [filename_templates] section. diff --git a/parm/metplus/GridStat_ensmean_RETOP.conf b/parm/metplus/GridStat_ensmean_RETOP.conf index 1caeccbe94..7cc6f2b580 100644 --- a/parm/metplus/GridStat_ensmean_RETOP.conf +++ b/parm/metplus/GridStat_ensmean_RETOP.conf @@ -45,7 +45,6 @@ LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only #LOG_GRID_STAT_VERBOSITY = 2 - # # Specify the name of the METplus log file. # @@ -55,7 +54,12 @@ LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} # METPLUS_CONF = {GRID_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} # -# Location of MET config file to pass to GridStat. +# Location of MET configuration file to pass to GridStat. +# +# References PARM_BASE, which is the location of the parm directory +# corresponding to the ush directory of the run_metplus.py script that +# is called or the value of the environment variable METPLUS_PARM_BASE +# if set. # GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped @@ -88,8 +92,9 @@ MODEL = {{vx_fcst_model_name}}_ensmean # OBTYPE = {{obtype}} OBS_NATIVE_DATA_TYPE = GRIB - -# set the desc value in the GridStat MET config file +# +# Value to enter under the DESC column in the output stat file. +# GRID_STAT_DESC = NA # List of variables to compare in GridStat - FCST_VAR1 variables correspond @@ -99,7 +104,8 @@ GRID_STAT_DESC = NA GRID_STAT_MET_CONFIG_OVERRIDES = cat_thresh = []; cnt_thresh = [NA]; cnt_logic = UNION; wind_thresh = [NA]; wind_logic = UNION; ci_alpha = [0.05]; rank_corr_flag = FALSE; # -# Name of forecast variable 1. +# String to search for in the foreast input files for forecast variable +# 1. # # Note: # This is the name of the field in the NetCDF file(s) created by MET's @@ -116,15 +122,15 @@ FCST_VAR1_NAME = {{fieldname_in_met_output}}_L0_ENS_MEAN # List of levels to evaluate for forecast variable 1. # FCST_VAR1_LEVELS = L0 - FCST_VAR1_OPTIONS = convert(x) = x * 3.28084 * 0.001; # # List of thresholds to evaluate for each name/level combination for -# observation and forecast variable 1. +# both obs and forecast variable 1. # BOTH_VAR1_THRESH = {{field_thresholds}} # -# Name of observation variable 1. +# String to search for in the observation input files for observation +# variable 1. # # Note: # This is the name of the field in the grib2 observation file. Thus, @@ -141,7 +147,6 @@ OBS_VAR1_NAME = {{fieldname_in_obs_input}} # same length as FCST_VAR1_LEVELS. # OBS_VAR1_LEVELS = Z500 - OBS_VAR1_OPTIONS = censor_thresh = lt-20.0; censor_val = -20.0; cnt_thresh = [ >15 ]; cnt_logic = UNION; convert(x) = x * 3280.84 * 0.001; # # Time relative to valid time (in seconds) to allow files to be considered @@ -178,6 +183,8 @@ FCST_IS_PROB = False # Only used if FCST_IS_PROB is true - sets probabilistic threshold # FCST_GRID_STAT_PROB_THRESH = ==0.1 + +GRID_STAT_OUTPUT_PREFIX = {MODEL}_{{fieldname_in_met_filedir_names}}_{OBTYPE} # # Set to true if observation data is probabilistic. Only used if # configuring forecast data as the 'OBS' input. @@ -188,8 +195,6 @@ OBS_IS_PROB = false # OBS_GRID_STAT_PROB_THRESH = ==0.1 -GRID_STAT_OUTPUT_PREFIX = {MODEL}_{{fieldname_in_met_filedir_names}}_{OBTYPE} - # Climatology data #GRID_STAT_CLIMO_MEAN_FILE_NAME = #GRID_STAT_CLIMO_MEAN_FIELD = @@ -285,7 +290,9 @@ GRID_STAT_CLIMO_STDEV_INPUT_DIR = # OUTPUT_BASE = {{output_base}} GRID_STAT_OUTPUT_DIR = {{output_dir}} - +# +# Directory for staging data. +# STAGING_DIR = {{staging_dir}} # End of [dir] section and start of [filename_templates] section. diff --git a/parm/metplus/GridStat_ensprob_APCP01h.conf b/parm/metplus/GridStat_ensprob_APCP01h.conf index 534cab0e7a..0c31ed33ef 100644 --- a/parm/metplus/GridStat_ensprob_APCP01h.conf +++ b/parm/metplus/GridStat_ensprob_APCP01h.conf @@ -45,7 +45,6 @@ LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only #LOG_GRID_STAT_VERBOSITY = 2 - # # Specify the name of the METplus log file. # @@ -55,7 +54,12 @@ LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} # METPLUS_CONF = {GRID_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} # -# Location of MET config file to pass to GridStat. +# Location of MET configuration file to pass to GridStat. +# +# References PARM_BASE, which is the location of the parm directory +# corresponding to the ush directory of the run_metplus.py script that +# is called or the value of the environment variable METPLUS_PARM_BASE +# if set. # GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped @@ -91,8 +95,9 @@ FCST_NATIVE_DATA_TYPE = GRIB # OBTYPE = {{obtype}} OBS_NATIVE_DATA_TYPE = GRIB - -# set the desc value in the GridStat MET config file +# +# Value to enter under the DESC column in the output stat file. +# GRID_STAT_DESC = NA # List of variables to compare in GridStat - FCST_VAR1 variables correspond @@ -317,7 +322,9 @@ GRID_STAT_CLIMO_STDEV_INPUT_DIR = # OUTPUT_BASE = {{output_base}} GRID_STAT_OUTPUT_DIR = {{output_dir}} - +# +# Directory for staging data. +# STAGING_DIR = {{staging_dir}} # End of [dir] section and start of [filename_templates] section. diff --git a/parm/metplus/GridStat_ensprob_APCP03h.conf b/parm/metplus/GridStat_ensprob_APCP03h.conf index c9a442b8bf..f31ed807aa 100644 --- a/parm/metplus/GridStat_ensprob_APCP03h.conf +++ b/parm/metplus/GridStat_ensprob_APCP03h.conf @@ -45,7 +45,6 @@ LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only #LOG_GRID_STAT_VERBOSITY = 2 - # # Specify the name of the METplus log file. # @@ -55,7 +54,12 @@ LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} # METPLUS_CONF = {GRID_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} # -# Location of MET config file to pass to GridStat. +# Location of MET configuration file to pass to GridStat. +# +# References PARM_BASE, which is the location of the parm directory +# corresponding to the ush directory of the run_metplus.py script that +# is called or the value of the environment variable METPLUS_PARM_BASE +# if set. # GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped @@ -91,8 +95,9 @@ FCST_NATIVE_DATA_TYPE = GRIB # OBTYPE = {{obtype}} OBS_NATIVE_DATA_TYPE = GRIB - -# set the desc value in the GridStat MET config file +# +# Value to enter under the DESC column in the output stat file. +# GRID_STAT_DESC = NA # List of variables to compare in GridStat - FCST_VAR1 variables correspond @@ -108,7 +113,9 @@ FIELDNAME_IN_MET_OUTPUT = {{fieldname_in_met_output}} # Process as probability # -# Name of forecast variable 1 +# String to search for in the foreast input files for forecast variable +# 1. +# # Note: # This is the name of the field in the NetCDF file(s) created by MET's # gen_ens_prod tool. This tool reads in the NetCDF file(s) generated by @@ -132,7 +139,8 @@ FCST_VAR1_NAME = {FIELDNAME_IN_MET_OUTPUT}_A3_ENS_FREQ_gt0.0 FCST_VAR1_LEVELS = A03 FCST_VAR1_THRESH = ==0.1 # -# Name of observation variable 1 +# List of levels to evaluate for observation variable 1. Must be the +# same length as FCST_VAR1_LEVELS. # # Note: # This is the name of the field in the NetCDF file created by MET's @@ -366,7 +374,9 @@ GRID_STAT_CLIMO_STDEV_INPUT_DIR = # OUTPUT_BASE = {{output_base}} GRID_STAT_OUTPUT_DIR = {{output_dir}} - +# +# Directory for staging data. +# STAGING_DIR = {{staging_dir}} # End of [dir] section and start of [filename_templates] section. diff --git a/parm/metplus/GridStat_ensprob_APCP06h.conf b/parm/metplus/GridStat_ensprob_APCP06h.conf index 37211c3ebf..dda11cff7c 100644 --- a/parm/metplus/GridStat_ensprob_APCP06h.conf +++ b/parm/metplus/GridStat_ensprob_APCP06h.conf @@ -45,7 +45,6 @@ LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only #LOG_GRID_STAT_VERBOSITY = 2 - # # Specify the name of the METplus log file. # @@ -55,7 +54,12 @@ LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} # METPLUS_CONF = {GRID_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} # -# Location of MET config file to pass to GridStat. +# Location of MET configuration file to pass to GridStat. +# +# References PARM_BASE, which is the location of the parm directory +# corresponding to the ush directory of the run_metplus.py script that +# is called or the value of the environment variable METPLUS_PARM_BASE +# if set. # GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped @@ -91,8 +95,9 @@ FCST_NATIVE_DATA_TYPE = GRIB # OBTYPE = {{obtype}} OBS_NATIVE_DATA_TYPE = GRIB - -# set the desc value in the GridStat MET config file +# +# Value to enter under the DESC column in the output stat file. +# GRID_STAT_DESC = NA # List of variables to compare in GridStat - FCST_VAR1 variables correspond @@ -108,7 +113,9 @@ FIELDNAME_IN_MET_OUTPUT = {{fieldname_in_met_output}} # Process as probability # -# Name of forecast variable 1 +# String to search for in the foreast input files for forecast variable +# 1. +# # Note: # This is the name of the field in the NetCDF file(s) created by MET's # gen_ens_prod tool. This tool reads in the NetCDF file(s) generated by @@ -132,7 +139,8 @@ FCST_VAR1_NAME = {FIELDNAME_IN_MET_OUTPUT}_A6_ENS_FREQ_gt0.0 FCST_VAR1_LEVELS = A06 FCST_VAR1_THRESH = ==0.1 # -# Name of observation variable 1 +# List of levels to evaluate for observation variable 1. Must be the +# same length as FCST_VAR1_LEVELS. # # Note: # This is the name of the field in the NetCDF file created by MET's @@ -366,7 +374,9 @@ GRID_STAT_CLIMO_STDEV_INPUT_DIR = # OUTPUT_BASE = {{output_base}} GRID_STAT_OUTPUT_DIR = {{output_dir}} - +# +# Directory for staging data. +# STAGING_DIR = {{staging_dir}} # End of [dir] section and start of [filename_templates] section. diff --git a/parm/metplus/GridStat_ensprob_APCP24h.conf b/parm/metplus/GridStat_ensprob_APCP24h.conf index 53fa2fa918..4bf135305d 100644 --- a/parm/metplus/GridStat_ensprob_APCP24h.conf +++ b/parm/metplus/GridStat_ensprob_APCP24h.conf @@ -45,7 +45,6 @@ LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only #LOG_GRID_STAT_VERBOSITY = 2 - # # Specify the name of the METplus log file. # @@ -55,7 +54,12 @@ LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} # METPLUS_CONF = {GRID_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} # -# Location of MET config file to pass to GridStat. +# Location of MET configuration file to pass to GridStat. +# +# References PARM_BASE, which is the location of the parm directory +# corresponding to the ush directory of the run_metplus.py script that +# is called or the value of the environment variable METPLUS_PARM_BASE +# if set. # GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped @@ -91,8 +95,9 @@ FCST_NATIVE_DATA_TYPE = GRIB # OBTYPE = {{obtype}} OBS_NATIVE_DATA_TYPE = GRIB - -# set the desc value in the GridStat MET config file +# +# Value to enter under the DESC column in the output stat file. +# GRID_STAT_DESC = NA # List of variables to compare in GridStat - FCST_VAR1 variables correspond @@ -108,7 +113,9 @@ FIELDNAME_IN_MET_OUTPUT = {{fieldname_in_met_output}} # Process as probability # -# Name of forecast variable 1 +# String to search for in the foreast input files for forecast variable +# 1. +# # Note: # This is the name of the field in the NetCDF file(s) created by MET's # gen_ens_prod tool. This tool reads in the NetCDF file(s) generated by @@ -132,7 +139,8 @@ FCST_VAR1_NAME = {FIELDNAME_IN_MET_OUTPUT}_A24_ENS_FREQ_gt0.0 FCST_VAR1_LEVELS = A24 FCST_VAR1_THRESH = ==0.1 # -# Name of observation variable 1 +# List of levels to evaluate for observation variable 1. Must be the +# same length as FCST_VAR1_LEVELS. # # Note: # This is the name of the field in the NetCDF file created by MET's @@ -366,7 +374,9 @@ GRID_STAT_CLIMO_STDEV_INPUT_DIR = # OUTPUT_BASE = {{output_base}} GRID_STAT_OUTPUT_DIR = {{output_dir}} - +# +# Directory for staging data. +# STAGING_DIR = {{staging_dir}} # End of [dir] section and start of [filename_templates] section. diff --git a/parm/metplus/GridStat_ensprob_REFC.conf b/parm/metplus/GridStat_ensprob_REFC.conf index 1f9334a622..fa020df40f 100644 --- a/parm/metplus/GridStat_ensprob_REFC.conf +++ b/parm/metplus/GridStat_ensprob_REFC.conf @@ -45,7 +45,6 @@ LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only #LOG_GRID_STAT_VERBOSITY = 2 - # # Specify the name of the METplus log file. # @@ -55,7 +54,12 @@ LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} # METPLUS_CONF = {GRID_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} # -# Location of MET config file to pass to GridStat. +# Location of MET configuration file to pass to GridStat. +# +# References PARM_BASE, which is the location of the parm directory +# corresponding to the ush directory of the run_metplus.py script that +# is called or the value of the environment variable METPLUS_PARM_BASE +# if set. # GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped @@ -90,8 +94,9 @@ MODEL = {{vx_fcst_model_name}}_ensprob # OBTYPE = {{obtype}} OBS_NATIVE_DATA_TYPE = GRIB - -# set the desc value in the GridStat MET config file +# +# Value to enter under the DESC column in the output stat file. +# GRID_STAT_DESC = NA # List of variables to compare in GridStat - FCST_VAR1 variables correspond @@ -105,7 +110,9 @@ GRID_STAT_MET_CONFIG_OVERRIDES = cat_thresh = []; cnt_thresh = [NA]; cnt_logic = # Process as probability # -# Name of forecast variable 1 +# String to search for in the foreast input files for forecast variable +# 1. +# # Note: # This is the name of the field in the NetCDF file(s) created by MET's # gen_ens_prod tool. This tool reads in the grib2 file(s) (in this case @@ -120,10 +127,7 @@ GRID_STAT_MET_CONFIG_OVERRIDES = cat_thresh = []; cnt_thresh = [NA]; cnt_logic = # #FCST_VAR1_NAME = REFC_L0_ENS_FREQ_ge20 FCST_VAR1_NAME = {{fieldname_in_met_output}}_L0_ENS_FREQ_ge20 - -# List of levels to evaluate for forecast variable 1 FCST_VAR1_LEVELS = L0 - #FCST_VAR1_OPTIONS = cnt_thresh = [ >15 ]; cnt_logic = UNION; # # List of thresholds to evaluate for each name/level combination for @@ -133,7 +137,8 @@ FCST_VAR1_THRESH = ==0.1 #FCST_GRID_STAT_FILE_TYPE = # -# Name of observation variable 1. +# String to search for in the observation input files for observation +# variable 1. # # Note: # This is the name of the field in the grib2 observation file. Thus, @@ -272,6 +277,8 @@ FCST_PROB_IN_GRIB_PDS = False # Only used if FCST_IS_PROB is true - sets probabilistic threshold # FCST_GRID_STAT_PROB_THRESH = ==0.1 + +GRID_STAT_OUTPUT_PREFIX = {MODEL}_{{fieldname_in_met_filedir_names}}_{OBTYPE} # # Set to true if observation data is probabilistic. Only used if # configuring forecast data as the 'OBS' input. @@ -282,8 +289,6 @@ OBS_IS_PROB = false # OBS_GRID_STAT_PROB_THRESH = ==0.1 -GRID_STAT_OUTPUT_PREFIX = {MODEL}_{{fieldname_in_met_filedir_names}}_{OBTYPE} - # Climatology data #GRID_STAT_CLIMO_MEAN_FILE_NAME = #GRID_STAT_CLIMO_MEAN_FIELD = @@ -377,7 +382,9 @@ GRID_STAT_CLIMO_STDEV_INPUT_DIR = # OUTPUT_BASE = {{output_base}} GRID_STAT_OUTPUT_DIR = {{output_dir}} - +# +# Directory for staging data. +# STAGING_DIR = {{staging_dir}} # End of [dir] section and start of [filename_templates] section. diff --git a/parm/metplus/GridStat_ensprob_RETOP.conf b/parm/metplus/GridStat_ensprob_RETOP.conf index 972d611255..ea060c122e 100644 --- a/parm/metplus/GridStat_ensprob_RETOP.conf +++ b/parm/metplus/GridStat_ensprob_RETOP.conf @@ -45,7 +45,6 @@ LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only #LOG_GRID_STAT_VERBOSITY = 2 - # # Specify the name of the METplus log file. # @@ -55,7 +54,12 @@ LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} # METPLUS_CONF = {GRID_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} # -# Location of MET config file to pass to GridStat. +# Location of MET configuration file to pass to GridStat. +# +# References PARM_BASE, which is the location of the parm directory +# corresponding to the ush directory of the run_metplus.py script that +# is called or the value of the environment variable METPLUS_PARM_BASE +# if set. # GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped @@ -90,8 +94,9 @@ MODEL = {{vx_fcst_model_name}}_ensprob # OBTYPE = {{obtype}} OBS_NATIVE_DATA_TYPE = GRIB - -# set the desc value in the GridStat MET config file +# +# Value to enter under the DESC column in the output stat file. +# GRID_STAT_DESC = NA # List of variables to compare in GridStat - FCST_VAR1 variables correspond @@ -105,7 +110,8 @@ GRID_STAT_MET_CONFIG_OVERRIDES = cat_thresh = []; cnt_thresh = [NA]; cnt_logic = # Process as probability # -# Name of forecast variable 1. +# String to search for in the foreast input files for forecast variable +# 1. # # Note: # This is the name of the field in the NetCDF file(s) created by MET's @@ -121,12 +127,17 @@ GRID_STAT_MET_CONFIG_OVERRIDES = cat_thresh = []; cnt_thresh = [NA]; cnt_logic = # #FCST_VAR1_NAME = RETOP_L0_ENS_FREQ_ge20 FCST_VAR1_NAME = {{fieldname_in_met_output}}_L0_ENS_FREQ_ge20 -FCST_VAR1_LEVELS = L0 +FCST_VAR1_LEVELS = L0 +# +# List of thresholds to evaluate for each name/level combination for +# forecast variable 1. +# FCST_VAR1_THRESH = ==0.1 #FCST_GRID_STAT_FILE_TYPE = # -# Name of observation variable 1. +# String to search for in the observation input files for observation +# variable 1. # # Note: # This is the name of the field in the grib2 observation file. Thus, @@ -265,6 +276,8 @@ FCST_PROB_IN_GRIB_PDS = False # Only used if FCST_IS_PROB is true - sets probabilistic threshold # FCST_GRID_STAT_PROB_THRESH = ==0.1 + +GRID_STAT_OUTPUT_PREFIX = {MODEL}_{{fieldname_in_met_filedir_names}}_{OBTYPE} # # Set to true if observation data is probabilistic. Only used if # configuring forecast data as the 'OBS' input. @@ -275,8 +288,6 @@ OBS_IS_PROB = false # OBS_GRID_STAT_PROB_THRESH = ==0.1 -GRID_STAT_OUTPUT_PREFIX = {MODEL}_{{fieldname_in_met_filedir_names}}_{OBTYPE} - # Climatology data #GRID_STAT_CLIMO_MEAN_FILE_NAME = #GRID_STAT_CLIMO_MEAN_FIELD = @@ -370,7 +381,9 @@ GRID_STAT_CLIMO_STDEV_INPUT_DIR = # OUTPUT_BASE = {{output_base}} GRID_STAT_OUTPUT_DIR = {{output_dir}} - +# +# Directory for staging data. +# STAGING_DIR = {{staging_dir}} # End of [dir] section and start of [filename_templates] section. diff --git a/parm/metplus/Pb2nc_obs.conf b/parm/metplus/Pb2nc_obs.conf index d24a66ab22..ae2b5e19f0 100644 --- a/parm/metplus/Pb2nc_obs.conf +++ b/parm/metplus/Pb2nc_obs.conf @@ -1,3 +1,5 @@ +# Pb2nc METplus Configuration + [config] # List of applications (tools) to run. @@ -14,7 +16,7 @@ LOOP_BY = INIT # %Y = 4 digit year, %m = 2 digit month, %d = 2 digit day, etc. # see www.strftime.org for more information # %Y%m%d%H expands to YYYYMMDDHH -INIT_TIME_FMT = %Y%m%d%H +INIT_TIME_FMT = %Y%m%d%H # Start time for METplus run - must match INIT_TIME_FMT INIT_BEG = {{cdate}} @@ -22,8 +24,8 @@ INIT_BEG = {{cdate}} # End time for METplus run - must match INIT_TIME_FMT INIT_END = {{cdate}} -# Increment between METplus runs (in seconds if no units are specified) -# Must be >= 60 seconds +# Increment between METplus runs (in seconds if no units are specified). +# Must be >= 60 seconds. INIT_INCREMENT = 3600 # List of forecast leads to process for each run time (init or valid) @@ -39,13 +41,31 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run -LOOP_ORDER = times +LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for PointStat only LOG_POINT_STAT_VERBOSITY = 2 - -# Location of MET config file to pass to PB2NC +# +# Specify the name of the METplus log file. +# +LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} +# +# Specify the location and name of the final METplus conf file. +# +METPLUS_CONF = {PB2NC_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} +# +# Location of MET configuration file to pass to Pb2nc. +# +# References PARM_BASE, which is the location of the parm directory +# corresponding to the ush directory of the run_metplus.py script that +# is called or the value of the environment variable METPLUS_PARM_BASE +# if set. +# PB2NC_CONFIG_FILE = {PARM_BASE}/met_config/PB2NCConfig_wrapped +# +# Name to identify observation data in output. +# +OBTYPE = {{obtype}} OBS_WINDOW_BEGIN = -1799 OBS_WINDOW_END = 1800 @@ -84,41 +104,30 @@ PB2NC_TIME_SUMMARY_BEG = 000000 PB2NC_TIME_SUMMARY_END = 235959 PB2NC_TIME_SUMMARY_VAR_NAMES = PMO,TOB,TDO,UOB,VOB,PWO,TOCC PB2NC_TIME_SUMMARY_TYPES = min, max, range, mean, stdev, median, p80 -# -# Specify the name of the METplus log file. -# -LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} -# -# Specify the location and name of the final METplus configuration file. -# -METPLUS_CONF = {PB2NC_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} - -OBTYPE = NDAS # End of [config] section and start of [dir] section [dir] # -# Directory containing (obs) input to PB2NC. +# Directory containing (observation) input to Pb2nc. # PB2NC_INPUT_DIR = {{obs_input_dir}} # -# Directory in which to write output from PB2NC. -# -PB2NC_OUTPUT_DIR = {{output_dir}} +# Directory in which to write output from Pb2nc. # # OUTPUT_BASE apparently has to be set to something; it cannot be left # to its default value. But it is not explicitly used elsewhere in this # configuration file. # OUTPUT_BASE = {{output_base}} +PB2NC_OUTPUT_DIR = {{output_dir}} # End of [dir] section and start of [filename_templates] section. [filename_templates] # -# Template for prepbufr input to PB2NC relative to PB2NC_INPUT_DIR. +# Template for prepbufr input to Pb2nc relative to PB2NC_INPUT_DIR. # PB2NC_INPUT_TEMPLATE = {{obs_input_fn_template}} # -# Template for output from PB2NC. +# Template for output from Pb2nc relative to PB2NC_OUTPUT_DIR. # PB2NC_OUTPUT_TEMPLATE = {{output_fn_template}} diff --git a/parm/metplus/PcpCombine_fcst.conf b/parm/metplus/PcpCombine_fcst.conf index 17a70625fb..65204c7caa 100644 --- a/parm/metplus/PcpCombine_fcst.conf +++ b/parm/metplus/PcpCombine_fcst.conf @@ -28,9 +28,9 @@ INIT_END = {{cdate}} # Must be >= 60 seconds. INIT_INCREMENT = 3600 -# List of forecast leads to process for each run time (init or valid). -# In hours if units are not specified. -# If unset, defaults to 0 (don't loop through forecast leads). +# List of forecast leads to process for each run time (init or valid) +# In hours if units are not specified +# If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} # Order of loops to process data - Options are times, processes @@ -43,6 +43,14 @@ LEAD_SEQ = {{fhr_list}} # wrappers have been run LOOP_ORDER = times # +# Specify the name of the METplus log file. +# +LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} +# +# Specify the location and name of the final METplus conf file. +# +METPLUS_CONF = {FCST_PCP_COMBINE_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} +# # Run PcpCombine on forecast data but not observation (observation input # files are not provided). # @@ -81,14 +89,6 @@ FCST_PCP_COMBINE_MAX_FORECAST = 2d # Keep initialization time constant. FCST_PCP_COMBINE_CONSTANT_INIT = True -# -# Specify the name of the METplus log file. -# -LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} -# -# Specify the location and name of the final METplus configuration file. -# -METPLUS_CONF = {FCST_PCP_COMBINE_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} FCST_PCP_COMBINE_INPUT_DATATYPE = GRIB FCST_NATIVE_DATA_TYPE = GRIB @@ -102,13 +102,12 @@ FCST_PCP_COMBINE_INPUT_DIR = {{fcst_input_dir}} # # Directory in which to write output from PcpCombine. # -FCST_PCP_COMBINE_OUTPUT_DIR = {{output_dir}} -# # OUTPUT_BASE apparently has to be set to something; it cannot be left # to its default value. But it is not explicitly used elsewhere in this # configuration file. # OUTPUT_BASE = {{output_base}} +FCST_PCP_COMBINE_OUTPUT_DIR = {{output_dir}} # # Directory for staging data. # @@ -117,13 +116,12 @@ STAGING_DIR = {{staging_dir}} # End of [dir] section and start of [filename_templates] section. [filename_templates] # -# The template FCST_PCP_COMBINE_INPUT_TEMPLATE is used to search for -# forecast input files. It is relative to FCST_PCP_COMBINE_INPUT_DIR. +# Template for forecast input to PcpCombine relative to +# FCST_PCP_COMBINE_INPUT_DIR. # FCST_PCP_COMBINE_INPUT_TEMPLATE = {{fcst_input_fn_template}} # -# FCST_PCP_COMBINE_OUTPUT_TEMPLATE is a path/file template that specifies -# the name of output file, including optional subdirectories. It is -# relative to FCST_PCP_COMBINE_OUTPUT_DIR. +# Template for output from PCPCOMBINE relative to +# FCST_PCP_COMBINE_OUTPUT_DIR. # FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {{output_fn_template}} diff --git a/parm/metplus/PcpCombine_obs.conf b/parm/metplus/PcpCombine_obs.conf index f61bc015bd..bc2c4180eb 100644 --- a/parm/metplus/PcpCombine_obs.conf +++ b/parm/metplus/PcpCombine_obs.conf @@ -28,9 +28,9 @@ INIT_END = {{cdate}} # Must be >= 60 seconds. INIT_INCREMENT = 3600 -# List of forecast leads to process for each run time (init or valid). -# In hours if units are not specified. -# If unset, defaults to 0 (don't loop through forecast leads). +# List of forecast leads to process for each run time (init or valid) +# In hours if units are not specified +# If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} # Order of loops to process data - Options are times, processes @@ -43,6 +43,14 @@ LEAD_SEQ = {{fhr_list}} # wrappers have been run LOOP_ORDER = times # +# Specify the name of the METplus log file. +# +LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} +# +# Specify the location and name of the final METplus conf file. +# +METPLUS_CONF = {OBS_PCP_COMBINE_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} +# # Run PcpCombine on observation data but not forecast (forecast input # files are not provided). # @@ -88,14 +96,6 @@ OBS_PCP_COMBINE_OUTPUT_ACCUM = {{accum_hh}} # PCP_COMBINE_SKIP_IF_OUTPUT_EXISTS = True # -# Specify the name of the METplus log file. -# -LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} -# -# Specify the location and name of the final METplus configuration file. -# -METPLUS_CONF = {OBS_PCP_COMBINE_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} -# # Name to identify observation data in output. # OBTYPE = CCPA @@ -110,13 +110,12 @@ OBS_PCP_COMBINE_INPUT_DIR = {{obs_input_dir}} # # Directory in which to write output from PcpCombine. # -OBS_PCP_COMBINE_OUTPUT_DIR = {{output_dir}} -# # OUTPUT_BASE apparently has to be set to something; it cannot be left # to its default value. But it is not explicitly used elsewhere in this # configuration file. # OUTPUT_BASE = {{output_base}} +OBS_PCP_COMBINE_OUTPUT_DIR = {{output_dir}} # # Directory for staging data. # @@ -125,13 +124,12 @@ STAGING_DIR = {{staging_dir}} # End of [dir] section and start of [filename_templates] section. [filename_templates] # -# The template OBS_PCP_COMBINE_INPUT_TEMPLATE is used to search for -# observation input files. It is relative to OBS_PCP_COMBINE_INPUT_DIR. +# Template for observation input to PcpCombine relative to +# OBS_PCP_COMBINE_INPUT_DIR. # OBS_PCP_COMBINE_INPUT_TEMPLATE = {{obs_input_fn_template}} # -# OBS_PCP_COMBINE_OUTPUT_TEMPLATE is a path/file template that specifies -# the name of output file, including optional subdirectories. It is -# relative to OBS_PCP_COMBINE_OUTPUT_DIR. +# Template for output from PcpCombine relative to +# OBS_PCP_COMBINE_OUTPUT_DIR. # OBS_PCP_COMBINE_OUTPUT_TEMPLATE = {{output_fn_template}} diff --git a/parm/metplus/PointStat_SFC.conf b/parm/metplus/PointStat_SFC.conf index d4255ebe36..11de0e85e3 100644 --- a/parm/metplus/PointStat_SFC.conf +++ b/parm/metplus/PointStat_SFC.conf @@ -24,7 +24,7 @@ INIT_BEG = {{cdate}} # End time for METplus run - must match INIT_TIME_FMT INIT_END = {{cdate}} -# Increment between METplus runs (in seconds if no units are specified) +# Increment between METplus runs (in seconds if no units are specified). # Must be >= 60 seconds. INIT_INCREMENT = 3600 @@ -45,8 +45,6 @@ LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for PointStat only LOG_POINT_STAT_VERBOSITY = 2 - -FIELDNAME_IN_MET_FILEDIR_NAMES = {{fieldname_in_met_filedir_names}} # # Specify the name of the METplus log file. # @@ -55,12 +53,14 @@ LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} # Specify the location and name of the final METplus conf file. # METPLUS_CONF = {POINT_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} - -# Location of MET config file to pass to GridStat +# +# Location of MET configuration file to pass to PointStat. +# # References PARM_BASE, which is the location of the parm directory # corresponding to the ush directory of the run_metplus.py script that # is called or the value of the environment variable METPLUS_PARM_BASE # if set. +# POINT_STAT_CONFIG_FILE = {PARM_BASE}/met_config/PointStatConfig_wrapped POINT_STAT_OBS_QUALITY_INC = 0, 1, 2, 3, 9, NA @@ -124,6 +124,10 @@ POINT_STAT_OFFSETS = 0 # MODEL = {{vx_fcst_model_name}}_mem{ENV[ENSMEM_INDX]} # +# Name to identify observation data in output. +# +OBTYPE = {{obtype}} +# # Value to enter under the DESC column in the output stat file. Here, # we store the value of the original lead in this column, i.e. the lead # with zero corresponding to the actual start time of the forecast (which @@ -134,10 +138,6 @@ MODEL = {{vx_fcst_model_name}}_mem{ENV[ENSMEM_INDX]} # Uncomment this line only after upgrading to METplus 5.x. #POINT_STAT_DESC = {lead?fmt=%H%M%S?shift={{time_lag}}} POINT_STAT_DESC = NA -# -# Name to identify observation data in output. -# -OBTYPE = {{obtype}} # Regrid to specified grid. Indicate NONE if no regridding, or the grid id # (e.g. G212) @@ -145,7 +145,7 @@ POINT_STAT_REGRID_TO_GRID = NONE POINT_STAT_REGRID_METHOD = BILIN POINT_STAT_REGRID_WIDTH = 2 -POINT_STAT_OUTPUT_PREFIX = {MODEL}_ADP{FIELDNAME_IN_MET_FILEDIR_NAMES}_{OBTYPE} +POINT_STAT_OUTPUT_PREFIX = {MODEL}_ADP{{fieldname_in_met_filedir_names}}_{OBTYPE} # sets the -obs_valid_beg command line argument (optional) # not used for this example @@ -169,7 +169,7 @@ POINT_STAT_STATION_ID = # Message types, if all message types are to be returned, leave this empty, # otherwise indicate the message types of interest. -POINT_STAT_MESSAGE_TYPE = ADP{FIELDNAME_IN_MET_FILEDIR_NAMES} +POINT_STAT_MESSAGE_TYPE = ADP{{fieldname_in_met_filedir_names}} # set to True to run PointStat once for each name/level combination # set to False to run PointStat once per run time including all fields diff --git a/parm/metplus/PointStat_UPA.conf b/parm/metplus/PointStat_UPA.conf index 0fa7d296a5..c971683d16 100644 --- a/parm/metplus/PointStat_UPA.conf +++ b/parm/metplus/PointStat_UPA.conf @@ -24,7 +24,7 @@ INIT_BEG = {{cdate}} # End time for METplus run - must match INIT_TIME_FMT INIT_END = {{cdate}} -# Increment between METplus runs (in seconds if no units are specified) +# Increment between METplus runs (in seconds if no units are specified). # Must be >= 60 seconds. INIT_INCREMENT = 3600 @@ -45,8 +45,6 @@ LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for PointStat only LOG_POINT_STAT_VERBOSITY = 2 - -FIELDNAME_IN_MET_FILEDIR_NAMES = {{fieldname_in_met_filedir_names}} # # Specify the name of the METplus log file. # @@ -55,12 +53,14 @@ LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} # Specify the location and name of the final METplus conf file. # METPLUS_CONF = {POINT_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} - -# Location of MET config file to pass to GridStat +# +# Location of MET configuration file to pass to PointStat. +# # References PARM_BASE, which is the location of the parm directory # corresponding to the ush directory of the run_metplus.py script that # is called or the value of the environment variable METPLUS_PARM_BASE # if set. +# POINT_STAT_CONFIG_FILE = {PARM_BASE}/met_config/PointStatConfig_wrapped POINT_STAT_OBS_QUALITY_INC = 0, 1, 2, 3, 9, NA @@ -124,6 +124,10 @@ POINT_STAT_OFFSETS = 0 # MODEL = {{vx_fcst_model_name}}_mem{ENV[ENSMEM_INDX]} # +# Name to identify observation data in output. +# +OBTYPE = {{obtype}} +# # Value to enter under the DESC column in the output stat file. Here, # we store the value of the original lead in this column, i.e. the lead # with zero corresponding to the actual start time of the forecast (which @@ -134,10 +138,6 @@ MODEL = {{vx_fcst_model_name}}_mem{ENV[ENSMEM_INDX]} # Uncomment this line only after upgrading to METplus 5.x. #POINT_STAT_DESC = {lead?fmt=%H%M%S?shift={{time_lag}}} POINT_STAT_DESC = NA -# -# Name to identify observation data in output. -# -OBTYPE = {{obtype}} # Regrid to specified grid. Indicate NONE if no regridding, or the grid id # (e.g. G212) @@ -145,7 +145,7 @@ POINT_STAT_REGRID_TO_GRID = NONE POINT_STAT_REGRID_METHOD = BILIN POINT_STAT_REGRID_WIDTH = 2 -POINT_STAT_OUTPUT_PREFIX = {MODEL}_ADP{FIELDNAME_IN_MET_FILEDIR_NAMES}_{OBTYPE} +POINT_STAT_OUTPUT_PREFIX = {MODEL}_ADP{{fieldname_in_met_filedir_names}}_{OBTYPE} # sets the -obs_valid_beg command line argument (optional) # not used for this example @@ -169,7 +169,7 @@ POINT_STAT_STATION_ID = # Message types, if all message types are to be returned, leave this empty, # otherwise indicate the message types of interest. -POINT_STAT_MESSAGE_TYPE = ADP{FIELDNAME_IN_MET_FILEDIR_NAMES} +POINT_STAT_MESSAGE_TYPE = ADP{{fieldname_in_met_filedir_names}} # set to True to run PointStat once for each name/level combination # set to False to run PointStat once per run time including all fields diff --git a/parm/metplus/PointStat_ensmean_SFC.conf b/parm/metplus/PointStat_ensmean_SFC.conf index 11c95db98d..8319c4ef37 100644 --- a/parm/metplus/PointStat_ensmean_SFC.conf +++ b/parm/metplus/PointStat_ensmean_SFC.conf @@ -45,8 +45,6 @@ LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for PointStat only LOG_POINT_STAT_VERBOSITY = 2 - -FIELDNAME_IN_MET_FILEDIR_NAMES = {{fieldname_in_met_filedir_names}} # # Specify the name of the METplus log file. # @@ -56,7 +54,8 @@ LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} # METPLUS_CONF = {POINT_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} # -# Location of MET config file to pass to GridStat +# Location of MET configuration file to pass to PointStat. +# # References PARM_BASE, which is the location of the parm directory # corresponding to the ush directory of the run_metplus.py script that # is called or the value of the environment variable METPLUS_PARM_BASE @@ -124,12 +123,14 @@ POINT_STAT_OFFSETS = 0 # mean. This makes it easier to identify each curve. # MODEL = {{vx_fcst_model_name}}_ensmean - -POINT_STAT_DESC = NA # # Name to identify observation data in output. # OBTYPE = {{obtype}} +# +# Value to enter under the DESC column in the output stat file. +# +POINT_STAT_DESC = NA # Regrid to specified grid. Indicate NONE if no regridding, or the grid id # (e.g. G212) @@ -137,7 +138,7 @@ POINT_STAT_REGRID_TO_GRID = NONE POINT_STAT_REGRID_METHOD = BILIN POINT_STAT_REGRID_WIDTH = 2 -POINT_STAT_OUTPUT_PREFIX = {MODEL}_ADP{FIELDNAME_IN_MET_FILEDIR_NAMES}_{OBTYPE} +POINT_STAT_OUTPUT_PREFIX = {MODEL}_ADP{{fieldname_in_met_filedir_names}}_{OBTYPE} # sets the -obs_valid_beg command line argument (optional) # not used for this example @@ -161,7 +162,7 @@ POINT_STAT_STATION_ID = # Message types, if all message types are to be returned, leave this empty, # otherwise indicate the message types of interest. -POINT_STAT_MESSAGE_TYPE = ADP{FIELDNAME_IN_MET_FILEDIR_NAMES} +POINT_STAT_MESSAGE_TYPE = ADP{{fieldname_in_met_filedir_names}} # set to True to run PointStat once for each name/level combination # set to False to run PointStat once per run time including all fields @@ -228,7 +229,9 @@ POINT_STAT_CLIMO_STDEV_INPUT_DIR = # OUTPUT_BASE = {{output_base}} POINT_STAT_OUTPUT_DIR = {{output_dir}} - +# +# Directory for staging data. +# STAGING_DIR = {{staging_dir}} # End of [dir] section and start of [filename_templates] section. diff --git a/parm/metplus/PointStat_ensmean_UPA.conf b/parm/metplus/PointStat_ensmean_UPA.conf index 7eb0c7a0cf..d872d07152 100644 --- a/parm/metplus/PointStat_ensmean_UPA.conf +++ b/parm/metplus/PointStat_ensmean_UPA.conf @@ -45,8 +45,6 @@ LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for PointStat only LOG_POINT_STAT_VERBOSITY = 2 - -FIELDNAME_IN_MET_FILEDIR_NAMES = {{fieldname_in_met_filedir_names}} # # Specify the name of the METplus log file. # @@ -56,7 +54,8 @@ LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} # METPLUS_CONF = {POINT_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} # -# Location of MET config file to pass to GridStat +# Location of MET configuration file to pass to PointStat. +# # References PARM_BASE, which is the location of the parm directory # corresponding to the ush directory of the run_metplus.py script that # is called or the value of the environment variable METPLUS_PARM_BASE @@ -124,12 +123,14 @@ POINT_STAT_OFFSETS = 0 # mean. This makes it easier to identify each curve. # MODEL = {{vx_fcst_model_name}}_ensmean - -POINT_STAT_DESC = NA # # Name to identify observation data in output. # OBTYPE = {{obtype}} +# +# Value to enter under the DESC column in the output stat file. +# +POINT_STAT_DESC = NA # Regrid to specified grid. Indicate NONE if no regridding, or the grid id # (e.g. G212) @@ -137,7 +138,7 @@ POINT_STAT_REGRID_TO_GRID = NONE POINT_STAT_REGRID_METHOD = BILIN POINT_STAT_REGRID_WIDTH = 2 -POINT_STAT_OUTPUT_PREFIX = {MODEL}_ADP{FIELDNAME_IN_MET_FILEDIR_NAMES}_{OBTYPE} +POINT_STAT_OUTPUT_PREFIX = {MODEL}_ADP{{fieldname_in_met_filedir_names}}_{OBTYPE} # sets the -obs_valid_beg command line argument (optional) # not used for this example @@ -161,7 +162,7 @@ POINT_STAT_STATION_ID = # Message types, if all message types are to be returned, leave this empty, # otherwise indicate the message types of interest. -POINT_STAT_MESSAGE_TYPE = ADP{FIELDNAME_IN_MET_FILEDIR_NAMES} +POINT_STAT_MESSAGE_TYPE = ADP{{fieldname_in_met_filedir_names}} # set to True to run PointStat once for each name/level combination # set to False to run PointStat once per run time including all fields @@ -303,7 +304,9 @@ POINT_STAT_CLIMO_STDEV_INPUT_DIR = # OUTPUT_BASE = {{output_base}} POINT_STAT_OUTPUT_DIR = {{output_dir}} - +# +# Directory for staging data. +# STAGING_DIR = {{staging_dir}} # End of [dir] section and start of [filename_templates] section. diff --git a/parm/metplus/PointStat_ensprob_SFC.conf b/parm/metplus/PointStat_ensprob_SFC.conf index 0df0a36e3b..a2f56c59d6 100644 --- a/parm/metplus/PointStat_ensprob_SFC.conf +++ b/parm/metplus/PointStat_ensprob_SFC.conf @@ -45,8 +45,6 @@ LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for PointStat only LOG_POINT_STAT_VERBOSITY = 2 - -FIELDNAME_IN_MET_FILEDIR_NAMES = {{fieldname_in_met_filedir_names}} # # Specify the name of the METplus log file. # @@ -56,7 +54,8 @@ LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} # METPLUS_CONF = {POINT_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} # -# Location of MET config file to pass to GridStat +# Location of MET configuration file to pass to PointStat. +# # References PARM_BASE, which is the location of the parm directory # corresponding to the ush directory of the run_metplus.py script that # is called or the value of the environment variable METPLUS_PARM_BASE @@ -124,12 +123,14 @@ POINT_STAT_OFFSETS = 0 # This makes it easier to identify each curve. # MODEL = {{vx_fcst_model_name}}_ensprob - -POINT_STAT_DESC = NA # # Name to identify observation data in output. # OBTYPE = {{obtype}} +# +# Value to enter under the DESC column in the output stat file. +# +POINT_STAT_DESC = NA # Regrid to specified grid. Indicate NONE if no regridding, or the grid id # (e.g. G212) @@ -137,7 +138,7 @@ POINT_STAT_REGRID_TO_GRID = NONE POINT_STAT_REGRID_METHOD = BILIN POINT_STAT_REGRID_WIDTH = 2 -POINT_STAT_OUTPUT_PREFIX = {MODEL}_ADP{FIELDNAME_IN_MET_FILEDIR_NAMES}_{OBTYPE} +POINT_STAT_OUTPUT_PREFIX = {MODEL}_ADP{{fieldname_in_met_filedir_names}}_{OBTYPE} # sets the -obs_valid_beg command line argument (optional) # not used for this example @@ -161,7 +162,7 @@ POINT_STAT_STATION_ID = # Message types, if all message types are to be returned, leave this empty, # otherwise indicate the message types of interest. -POINT_STAT_MESSAGE_TYPE = ADP{FIELDNAME_IN_MET_FILEDIR_NAMES} +POINT_STAT_MESSAGE_TYPE = ADP{{fieldname_in_met_filedir_names}} # set to True to run PointStat once for each name/level combination # set to False to run PointStat once per run time including all fields @@ -406,7 +407,9 @@ POINT_STAT_CLIMO_STDEV_INPUT_DIR = # OUTPUT_BASE = {{output_base}} POINT_STAT_OUTPUT_DIR = {{output_dir}} - +# +# Directory for staging data. +# STAGING_DIR = {{staging_dir}} # End of [dir] section and start of [filename_templates] section. diff --git a/parm/metplus/PointStat_ensprob_UPA.conf b/parm/metplus/PointStat_ensprob_UPA.conf index 06b7d7f563..5160202175 100644 --- a/parm/metplus/PointStat_ensprob_UPA.conf +++ b/parm/metplus/PointStat_ensprob_UPA.conf @@ -45,8 +45,6 @@ LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for PointStat only LOG_POINT_STAT_VERBOSITY = 2 - -FIELDNAME_IN_MET_FILEDIR_NAMES = {{fieldname_in_met_filedir_names}} # # Specify the name of the METplus log file. # @@ -56,7 +54,8 @@ LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} # METPLUS_CONF = {POINT_STAT_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} # -# Location of MET config file to pass to GridStat +# Location of MET configuration file to pass to PointStat. +# # References PARM_BASE, which is the location of the parm directory # corresponding to the ush directory of the run_metplus.py script that # is called or the value of the environment variable METPLUS_PARM_BASE @@ -124,12 +123,14 @@ POINT_STAT_OFFSETS = 0 # This makes it easier to identify each curve. # MODEL = {{vx_fcst_model_name}}_ensprob - -POINT_STAT_DESC = NA # # Name to identify observation data in output. # OBTYPE = {{obtype}} +# +# Value to enter under the DESC column in the output stat file. +# +POINT_STAT_DESC = NA # Regrid to specified grid. Indicate NONE if no regridding, or the grid id # (e.g. G212) @@ -137,7 +138,7 @@ POINT_STAT_REGRID_TO_GRID = NONE POINT_STAT_REGRID_METHOD = BILIN POINT_STAT_REGRID_WIDTH = 2 -POINT_STAT_OUTPUT_PREFIX = {MODEL}_ADP{FIELDNAME_IN_MET_FILEDIR_NAMES}_{OBTYPE} +POINT_STAT_OUTPUT_PREFIX = {MODEL}_ADP{{fieldname_in_met_filedir_names}}_{OBTYPE} # sets the -obs_valid_beg command line argument (optional) # not used for this example @@ -161,7 +162,7 @@ POINT_STAT_STATION_ID = # Message types, if all message types are to be returned, leave this empty, # otherwise indicate the message types of interest. -POINT_STAT_MESSAGE_TYPE = ADP{FIELDNAME_IN_MET_FILEDIR_NAMES} +POINT_STAT_MESSAGE_TYPE = ADP{{fieldname_in_met_filedir_names}} # set to True to run PointStat once for each name/level combination # set to False to run PointStat once per run time including all fields @@ -528,7 +529,9 @@ POINT_STAT_CLIMO_STDEV_INPUT_DIR = # OUTPUT_BASE = {{output_base}} POINT_STAT_OUTPUT_DIR = {{output_dir}} - +# +# Directory for staging data. +# STAGING_DIR = {{staging_dir}} # End of [dir] section and start of [filename_templates] section. From 627c94f55ce88ff908780db7c2e442e69d1ad30c Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Thu, 23 Mar 2023 15:11:28 -0600 Subject: [PATCH 04/43] Fix up comments in METplus conf files. --- parm/metplus/EnsembleStat_APCP01h.conf | 3 ++- parm/metplus/EnsembleStat_APCPgt01h.conf | 3 ++- parm/metplus/EnsembleStat_REFC.conf | 3 ++- parm/metplus/EnsembleStat_RETOP.conf | 3 ++- parm/metplus/EnsembleStat_SFC.conf | 3 ++- parm/metplus/EnsembleStat_UPA.conf | 3 ++- parm/metplus/GridStat_APCP01h.conf | 3 ++- parm/metplus/GridStat_APCPgt01h.conf | 3 ++- parm/metplus/GridStat_REFC.conf | 3 ++- parm/metplus/GridStat_RETOP.conf | 3 ++- parm/metplus/GridStat_ensmean_APCP01h.conf | 3 ++- parm/metplus/GridStat_ensmean_APCPgt01h.conf | 3 ++- parm/metplus/GridStat_ensmean_REFC.conf | 3 ++- parm/metplus/GridStat_ensmean_RETOP.conf | 3 ++- parm/metplus/GridStat_ensprob_APCP01h.conf | 3 ++- parm/metplus/GridStat_ensprob_APCP03h.conf | 3 ++- parm/metplus/GridStat_ensprob_APCP06h.conf | 3 ++- parm/metplus/GridStat_ensprob_APCP24h.conf | 3 ++- parm/metplus/GridStat_ensprob_REFC.conf | 3 ++- parm/metplus/GridStat_ensprob_RETOP.conf | 3 ++- parm/metplus/PointStat_SFC.conf | 3 ++- parm/metplus/PointStat_UPA.conf | 3 ++- parm/metplus/PointStat_ensmean_SFC.conf | 3 ++- parm/metplus/PointStat_ensmean_UPA.conf | 3 ++- parm/metplus/PointStat_ensprob_SFC.conf | 3 ++- parm/metplus/PointStat_ensprob_UPA.conf | 3 ++- 26 files changed, 52 insertions(+), 26 deletions(-) diff --git a/parm/metplus/EnsembleStat_APCP01h.conf b/parm/metplus/EnsembleStat_APCP01h.conf index 845089044c..4a4e6cddb4 100644 --- a/parm/metplus/EnsembleStat_APCP01h.conf +++ b/parm/metplus/EnsembleStat_APCP01h.conf @@ -32,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} - +# # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -41,6 +41,7 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run +# LOOP_ORDER = times # # Specify the name of the METplus log file. diff --git a/parm/metplus/EnsembleStat_APCPgt01h.conf b/parm/metplus/EnsembleStat_APCPgt01h.conf index dccaecd9b7..bb008c35b1 100644 --- a/parm/metplus/EnsembleStat_APCPgt01h.conf +++ b/parm/metplus/EnsembleStat_APCPgt01h.conf @@ -32,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} - +# # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -41,6 +41,7 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run +# LOOP_ORDER = times # # Specify the name of the METplus log file. diff --git a/parm/metplus/EnsembleStat_REFC.conf b/parm/metplus/EnsembleStat_REFC.conf index 3f52156d97..16af79ea05 100644 --- a/parm/metplus/EnsembleStat_REFC.conf +++ b/parm/metplus/EnsembleStat_REFC.conf @@ -32,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} - +# # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -41,6 +41,7 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run +# LOOP_ORDER = times # # Specify the name of the METplus log file. diff --git a/parm/metplus/EnsembleStat_RETOP.conf b/parm/metplus/EnsembleStat_RETOP.conf index 916ad3d55f..b479e3784a 100644 --- a/parm/metplus/EnsembleStat_RETOP.conf +++ b/parm/metplus/EnsembleStat_RETOP.conf @@ -32,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} - +# # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -41,6 +41,7 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run +# LOOP_ORDER = times # # Specify the name of the METplus log file. diff --git a/parm/metplus/EnsembleStat_SFC.conf b/parm/metplus/EnsembleStat_SFC.conf index 1c2460ba5d..6a6b7e8e24 100644 --- a/parm/metplus/EnsembleStat_SFC.conf +++ b/parm/metplus/EnsembleStat_SFC.conf @@ -32,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} - +# # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -41,6 +41,7 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run +# LOOP_ORDER = times # # Specify the name of the METplus log file. diff --git a/parm/metplus/EnsembleStat_UPA.conf b/parm/metplus/EnsembleStat_UPA.conf index de14c23f7a..48551fdc2a 100644 --- a/parm/metplus/EnsembleStat_UPA.conf +++ b/parm/metplus/EnsembleStat_UPA.conf @@ -32,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} - +# # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -41,6 +41,7 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run +# LOOP_ORDER = times # # Specify the name of the METplus log file. diff --git a/parm/metplus/GridStat_APCP01h.conf b/parm/metplus/GridStat_APCP01h.conf index de01b4ad2e..9f23de0ea2 100644 --- a/parm/metplus/GridStat_APCP01h.conf +++ b/parm/metplus/GridStat_APCP01h.conf @@ -32,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} - +# # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -41,6 +41,7 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run +# LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only diff --git a/parm/metplus/GridStat_APCPgt01h.conf b/parm/metplus/GridStat_APCPgt01h.conf index fb2014e6a5..464374f965 100644 --- a/parm/metplus/GridStat_APCPgt01h.conf +++ b/parm/metplus/GridStat_APCPgt01h.conf @@ -32,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} - +# # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -41,6 +41,7 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run +# LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only diff --git a/parm/metplus/GridStat_REFC.conf b/parm/metplus/GridStat_REFC.conf index 22b7230478..0bb4a35ecd 100644 --- a/parm/metplus/GridStat_REFC.conf +++ b/parm/metplus/GridStat_REFC.conf @@ -32,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} - +# # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -41,6 +41,7 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run +# LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only diff --git a/parm/metplus/GridStat_RETOP.conf b/parm/metplus/GridStat_RETOP.conf index 0ebcf7e1a8..134d10bee9 100644 --- a/parm/metplus/GridStat_RETOP.conf +++ b/parm/metplus/GridStat_RETOP.conf @@ -32,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} - +# # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -41,6 +41,7 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run +# LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only diff --git a/parm/metplus/GridStat_ensmean_APCP01h.conf b/parm/metplus/GridStat_ensmean_APCP01h.conf index bf01073ad6..a8408f90d9 100644 --- a/parm/metplus/GridStat_ensmean_APCP01h.conf +++ b/parm/metplus/GridStat_ensmean_APCP01h.conf @@ -32,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} - +# # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -41,6 +41,7 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run +# LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only diff --git a/parm/metplus/GridStat_ensmean_APCPgt01h.conf b/parm/metplus/GridStat_ensmean_APCPgt01h.conf index 9b0bc71f4d..8eb87fafca 100644 --- a/parm/metplus/GridStat_ensmean_APCPgt01h.conf +++ b/parm/metplus/GridStat_ensmean_APCPgt01h.conf @@ -32,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} - +# # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -41,6 +41,7 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run +# LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only diff --git a/parm/metplus/GridStat_ensmean_REFC.conf b/parm/metplus/GridStat_ensmean_REFC.conf index 983d827610..bfd4c65990 100644 --- a/parm/metplus/GridStat_ensmean_REFC.conf +++ b/parm/metplus/GridStat_ensmean_REFC.conf @@ -32,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} - +# # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -41,6 +41,7 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run +# LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only diff --git a/parm/metplus/GridStat_ensmean_RETOP.conf b/parm/metplus/GridStat_ensmean_RETOP.conf index 7cc6f2b580..04e337e0d4 100644 --- a/parm/metplus/GridStat_ensmean_RETOP.conf +++ b/parm/metplus/GridStat_ensmean_RETOP.conf @@ -32,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} - +# # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -41,6 +41,7 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run +# LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only diff --git a/parm/metplus/GridStat_ensprob_APCP01h.conf b/parm/metplus/GridStat_ensprob_APCP01h.conf index 0c31ed33ef..289c407d31 100644 --- a/parm/metplus/GridStat_ensprob_APCP01h.conf +++ b/parm/metplus/GridStat_ensprob_APCP01h.conf @@ -32,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} - +# # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -41,6 +41,7 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run +# LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only diff --git a/parm/metplus/GridStat_ensprob_APCP03h.conf b/parm/metplus/GridStat_ensprob_APCP03h.conf index f31ed807aa..145f3e5fa1 100644 --- a/parm/metplus/GridStat_ensprob_APCP03h.conf +++ b/parm/metplus/GridStat_ensprob_APCP03h.conf @@ -32,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} - +# # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -41,6 +41,7 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run +# LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only diff --git a/parm/metplus/GridStat_ensprob_APCP06h.conf b/parm/metplus/GridStat_ensprob_APCP06h.conf index dda11cff7c..f3e2243c15 100644 --- a/parm/metplus/GridStat_ensprob_APCP06h.conf +++ b/parm/metplus/GridStat_ensprob_APCP06h.conf @@ -32,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} - +# # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -41,6 +41,7 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run +# LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only diff --git a/parm/metplus/GridStat_ensprob_APCP24h.conf b/parm/metplus/GridStat_ensprob_APCP24h.conf index 4bf135305d..089b8d3821 100644 --- a/parm/metplus/GridStat_ensprob_APCP24h.conf +++ b/parm/metplus/GridStat_ensprob_APCP24h.conf @@ -32,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} - +# # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -41,6 +41,7 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run +# LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only diff --git a/parm/metplus/GridStat_ensprob_REFC.conf b/parm/metplus/GridStat_ensprob_REFC.conf index fa020df40f..671d351dfb 100644 --- a/parm/metplus/GridStat_ensprob_REFC.conf +++ b/parm/metplus/GridStat_ensprob_REFC.conf @@ -32,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} - +# # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -41,6 +41,7 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run +# LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only diff --git a/parm/metplus/GridStat_ensprob_RETOP.conf b/parm/metplus/GridStat_ensprob_RETOP.conf index ea060c122e..581dd4164e 100644 --- a/parm/metplus/GridStat_ensprob_RETOP.conf +++ b/parm/metplus/GridStat_ensprob_RETOP.conf @@ -32,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} - +# # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -41,6 +41,7 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run +# LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only diff --git a/parm/metplus/PointStat_SFC.conf b/parm/metplus/PointStat_SFC.conf index 11de0e85e3..58c7d9c228 100644 --- a/parm/metplus/PointStat_SFC.conf +++ b/parm/metplus/PointStat_SFC.conf @@ -32,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} - +# # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -41,6 +41,7 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run +# LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for PointStat only diff --git a/parm/metplus/PointStat_UPA.conf b/parm/metplus/PointStat_UPA.conf index c971683d16..06bc948c6d 100644 --- a/parm/metplus/PointStat_UPA.conf +++ b/parm/metplus/PointStat_UPA.conf @@ -32,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} - +# # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -41,6 +41,7 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run +# LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for PointStat only diff --git a/parm/metplus/PointStat_ensmean_SFC.conf b/parm/metplus/PointStat_ensmean_SFC.conf index 8319c4ef37..0d5ee3b696 100644 --- a/parm/metplus/PointStat_ensmean_SFC.conf +++ b/parm/metplus/PointStat_ensmean_SFC.conf @@ -32,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} - +# # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -41,6 +41,7 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run +# LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for PointStat only diff --git a/parm/metplus/PointStat_ensmean_UPA.conf b/parm/metplus/PointStat_ensmean_UPA.conf index d872d07152..1846e0cf6e 100644 --- a/parm/metplus/PointStat_ensmean_UPA.conf +++ b/parm/metplus/PointStat_ensmean_UPA.conf @@ -32,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} - +# # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -41,6 +41,7 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run +# LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for PointStat only diff --git a/parm/metplus/PointStat_ensprob_SFC.conf b/parm/metplus/PointStat_ensprob_SFC.conf index a2f56c59d6..84b3ac9445 100644 --- a/parm/metplus/PointStat_ensprob_SFC.conf +++ b/parm/metplus/PointStat_ensprob_SFC.conf @@ -32,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} - +# # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -41,6 +41,7 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run +# LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for PointStat only diff --git a/parm/metplus/PointStat_ensprob_UPA.conf b/parm/metplus/PointStat_ensprob_UPA.conf index 5160202175..6b4628112f 100644 --- a/parm/metplus/PointStat_ensprob_UPA.conf +++ b/parm/metplus/PointStat_ensprob_UPA.conf @@ -32,7 +32,7 @@ INIT_INCREMENT = 3600 # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {{fhr_list}} - +# # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST # times = run all wrappers in the PROCESS_LIST for a single run time, then @@ -41,6 +41,7 @@ LEAD_SEQ = {{fhr_list}} # processes = run the first wrapper in the PROCESS_LIST for all times # specified, then repeat for the next item in the PROCESS_LIST until all # wrappers have been run +# LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for PointStat only From 6b2e934218f2c933835e2dc80056a9f568a1b04e Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Fri, 24 Mar 2023 01:19:26 -0600 Subject: [PATCH 05/43] Simplificatons to EnsembleStat conf files. --- parm/metplus/EnsembleStat_APCP01h.conf | 3 +-- parm/metplus/EnsembleStat_APCPgt01h.conf | 3 +-- parm/metplus/EnsembleStat_REFC.conf | 3 +-- parm/metplus/EnsembleStat_RETOP.conf | 3 +-- parm/metplus/EnsembleStat_SFC.conf | 3 +-- parm/metplus/EnsembleStat_UPA.conf | 3 +-- 6 files changed, 6 insertions(+), 12 deletions(-) diff --git a/parm/metplus/EnsembleStat_APCP01h.conf b/parm/metplus/EnsembleStat_APCP01h.conf index 4a4e6cddb4..8d2277fe10 100644 --- a/parm/metplus/EnsembleStat_APCP01h.conf +++ b/parm/metplus/EnsembleStat_APCP01h.conf @@ -220,7 +220,6 @@ OBS_VAR1_OPTIONS = {FCST_VAR1_OPTIONS} # configuration file. # OUTPUT_BASE = {{output_base}} -OUTPUT_DIR = {{output_dir}} # # Point observation input directory for EnsembleStat. # @@ -246,7 +245,7 @@ ENSEMBLE_STAT_CLIMO_STDEV_INPUT_DIR = # # Output directory for EnsembleStat. # -ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_DIR} +ENSEMBLE_STAT_OUTPUT_DIR = {{output_dir}} # # Directory for staging data. # diff --git a/parm/metplus/EnsembleStat_APCPgt01h.conf b/parm/metplus/EnsembleStat_APCPgt01h.conf index bb008c35b1..fba99e460f 100644 --- a/parm/metplus/EnsembleStat_APCPgt01h.conf +++ b/parm/metplus/EnsembleStat_APCPgt01h.conf @@ -227,7 +227,6 @@ OBS_VAR1_OPTIONS = {FCST_VAR1_OPTIONS} # configuration file. # OUTPUT_BASE = {{output_base}} -OUTPUT_DIR = {{output_dir}} # # Point observation input directory for EnsembleStat. # @@ -253,7 +252,7 @@ ENSEMBLE_STAT_CLIMO_STDEV_INPUT_DIR = # # Output directory for EnsembleStat. # -ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_DIR} +ENSEMBLE_STAT_OUTPUT_DIR = {{output_dir}} # # Directory for staging data. # diff --git a/parm/metplus/EnsembleStat_REFC.conf b/parm/metplus/EnsembleStat_REFC.conf index 16af79ea05..34ed94929e 100644 --- a/parm/metplus/EnsembleStat_REFC.conf +++ b/parm/metplus/EnsembleStat_REFC.conf @@ -218,7 +218,6 @@ OBS_VAR1_OPTIONS = censor_thresh = lt-20; censor_val = -20.0; cnt_thresh = [ >15 # configuration file. # OUTPUT_BASE = {{output_base}} -OUTPUT_DIR = {{output_dir}} # # Point observation input directory for EnsembleStat. # @@ -244,7 +243,7 @@ ENSEMBLE_STAT_CLIMO_STDEV_INPUT_DIR = # # Output directory for EnsembleStat. # -ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_DIR} +ENSEMBLE_STAT_OUTPUT_DIR = {{output_dir}} # # Directory for staging data. # diff --git a/parm/metplus/EnsembleStat_RETOP.conf b/parm/metplus/EnsembleStat_RETOP.conf index b479e3784a..28094c1cd5 100644 --- a/parm/metplus/EnsembleStat_RETOP.conf +++ b/parm/metplus/EnsembleStat_RETOP.conf @@ -222,7 +222,6 @@ OBS_VAR1_OPTIONS = censor_thresh = lt-20; censor_val = -20.0; cnt_thresh = [ >15 # configuration file. # OUTPUT_BASE = {{output_base}} -OUTPUT_DIR = {{output_dir}} # # Point observation input directory for EnsembleStat. # @@ -248,7 +247,7 @@ ENSEMBLE_STAT_CLIMO_STDEV_INPUT_DIR = # # Output directory for EnsembleStat. # -ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_DIR} +ENSEMBLE_STAT_OUTPUT_DIR = {{output_dir}} # # Directory for staging data. # diff --git a/parm/metplus/EnsembleStat_SFC.conf b/parm/metplus/EnsembleStat_SFC.conf index 6a6b7e8e24..1cb6e4f11e 100644 --- a/parm/metplus/EnsembleStat_SFC.conf +++ b/parm/metplus/EnsembleStat_SFC.conf @@ -285,7 +285,6 @@ OBS_VAR6_THRESH = <152, <305, >=914 # configuration file. # OUTPUT_BASE = {{output_base}} -OUTPUT_DIR = {{output_dir}} # # Point observation input directory for EnsembleStat. # @@ -311,7 +310,7 @@ ENSEMBLE_STAT_CLIMO_STDEV_INPUT_DIR = # # Output directory for EnsembleStat. # -ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_DIR} +ENSEMBLE_STAT_OUTPUT_DIR = {{output_dir}} # # Directory for staging data. # diff --git a/parm/metplus/EnsembleStat_UPA.conf b/parm/metplus/EnsembleStat_UPA.conf index 48551fdc2a..40f11f2d59 100644 --- a/parm/metplus/EnsembleStat_UPA.conf +++ b/parm/metplus/EnsembleStat_UPA.conf @@ -348,7 +348,6 @@ OBS_VAR12_THRESH = <500, <1500, >1500 # configuration file. # OUTPUT_BASE = {{output_base}} -OUTPUT_DIR = {{output_dir}} # # Point observation input directory for EnsembleStat. # @@ -374,7 +373,7 @@ ENSEMBLE_STAT_CLIMO_STDEV_INPUT_DIR = # # Output directory for EnsembleStat. # -ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_DIR} +ENSEMBLE_STAT_OUTPUT_DIR = {{output_dir}} # # Directory for staging data. # From 203020a69722b5e5dce0ee81753b19f396f30c69 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Fri, 24 Mar 2023 02:14:58 -0600 Subject: [PATCH 06/43] Add GenEnsProd tasks to verification workflow. --- parm/metplus/GenEnsProd_APCP01h.conf | 192 ++++++++++++++++++++ parm/metplus/GenEnsProd_APCPgt01h.conf | 205 +++++++++++++++++++++ parm/metplus/GenEnsProd_REFC.conf | 192 ++++++++++++++++++++ parm/metplus/GenEnsProd_RETOP.conf | 194 ++++++++++++++++++++ parm/metplus/GenEnsProd_SFC.conf | 217 ++++++++++++++++++++++ parm/metplus/GenEnsProd_UPA.conf | 237 +++++++++++++++++++++++++ ush/get_met_metplus_tool_name.sh | 8 +- 7 files changed, 1243 insertions(+), 2 deletions(-) create mode 100644 parm/metplus/GenEnsProd_APCP01h.conf create mode 100644 parm/metplus/GenEnsProd_APCPgt01h.conf create mode 100644 parm/metplus/GenEnsProd_REFC.conf create mode 100644 parm/metplus/GenEnsProd_RETOP.conf create mode 100644 parm/metplus/GenEnsProd_SFC.conf create mode 100644 parm/metplus/GenEnsProd_UPA.conf diff --git a/parm/metplus/GenEnsProd_APCP01h.conf b/parm/metplus/GenEnsProd_APCP01h.conf new file mode 100644 index 0000000000..1731306975 --- /dev/null +++ b/parm/metplus/GenEnsProd_APCP01h.conf @@ -0,0 +1,192 @@ +# GenEnsProd METplus Configuration + +[config] + +# List of applications (tools) to run. +PROCESS_LIST = GenEnsProd + +# time looping - options are INIT, VALID, RETRO, and REALTIME +# If set to INIT or RETRO: +# INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set +# If set to VALID or REALTIME: +# VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set +LOOP_BY = INIT + +# Format of INIT_BEG and INIT_END using % items +# %Y = 4 digit year, %m = 2 digit month, %d = 2 digit day, etc. +# see www.strftime.org for more information +# %Y%m%d%H expands to YYYYMMDDHH +INIT_TIME_FMT = %Y%m%d%H + +# Start time for METplus run - must match INIT_TIME_FMT +INIT_BEG = {{cdate}} + +# End time for METplus run - must match INIT_TIME_FMT +INIT_END = {{cdate}} + +# Increment between METplus runs (in seconds if no units are specified). +# Must be >= 60 seconds. +INIT_INCREMENT = 3600 + +# List of forecast leads to process for each run time (init or valid) +# In hours if units are not specified +# If unset, defaults to 0 (don't loop through forecast leads) +LEAD_SEQ = {{fhr_list}} +# +# Order of loops to process data - Options are times, processes +# Not relevant if only one item is in the PROCESS_LIST +# times = run all wrappers in the PROCESS_LIST for a single run time, then +# increment the run time and run all wrappers again until all times have +# been evaluated. +# processes = run the first wrapper in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST until all +# wrappers have been run +# +LOOP_ORDER = times +# +# Specify the name of the METplus log file. +# +LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} +# +# Specify the location and name of the final METplus conf file. +# +METPLUS_CONF = {GEN_ENS_PROD_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} +# +# Name to identify model (forecast) data in output. +# +MODEL = {{vx_fcst_model_name}} + +GEN_ENS_PROD_DESC = NA +# +# Name to identify observation data in output. +# +OBTYPE = {{obtype}} + +# The METplus GenEnsProd logging level +# 0 quiet to 5 loud, Verbosity setting for MET gen_ens_prod output, 2 is default. +# This takes precendence over the general LOG_MET_VERBOSITY set in metplus_logging.conf +#LOG_GEN_ENS_PROD_VERBOSITY = 2 + +### +# File I/O +### + +# +# Forecast model input directory for GenEnsProd. +# +GEN_ENS_PROD_INPUT_DIR = {{fcst_input_dir}} +# +# Template for forecast input to GenEnsProd relative to +# GEN_ENS_PROD_INPUT_DIR. +# +# Note that this can be a comma separated list of ensemble members +# or a single line, - filename wildcard characters may be used, ? or *. +# +GEN_ENS_PROD_INPUT_TEMPLATE = {{fcst_input_fn_template}} + +# GEN_ENS_PROD_CTRL_INPUT_DIR = {INPUT_BASE} +# GEN_ENS_PROD_CTRL_INPUT_TEMPLATE = +# {init?fmt=%Y%m%d%H}/mem1/postprd/{ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 + +# +# Output directory for GenEnsProd. +# +# OUTPUT_BASE apparently has to be set to something; it cannot be left +# to its default value. But it is not explicitly used elsewhere in this +# configuration file. +# +OUTPUT_BASE = {{output_base}} +GEN_ENS_PROD_OUTPUT_DIR = {{output_dir}} +# +# Template for output from GenEnsProd relative to +# GEN_ENS_PROD_OUTPUT_DIR. +# +GEN_ENS_PROD_OUTPUT_TEMPLATE = gen_ens_prod_{MODEL}_{{fieldname_in_met_filedir_names}}_{OBTYPE}_{valid?fmt=%Y%m%d_%H%M%S}V.nc +# +# Directory for staging data. +# +STAGING_DIR = {{staging_dir}} +# +# There are n ensembles but 1 is used as control, so specify n-1 members. +# +GEN_ENS_PROD_N_MEMBERS = {{num_ens_members}} + +### +# Field Info +### + +# Ensemble Variables and levels as specified in the ens field dictionary +# of the MET configuration file. Specify as ENS_VARn_NAME, ENS_VARn_LEVELS, +# (optional) ENS_VARn_OPTION +ENS_VAR1_NAME = {{fieldname_in_fcst_input}} +ENS_VAR1_LEVELS = A{{accum_hh}} +ENS_VAR1_THRESH = {{field_thresholds}} + +### +# GenEnsProd +### + +# GEN_ENS_PROD_REGRID_TO_GRID = NONE +# GEN_ENS_PROD_REGRID_METHOD = NEAREST +# GEN_ENS_PROD_REGRID_WIDTH = 1 +# GEN_ENS_PROD_REGRID_VLD_THRESH = 0.5 +# GEN_ENS_PROD_REGRID_SHAPE = SQUARE + +# GEN_ENS_PROD_CENSOR_THRESH = +# GEN_ENS_PROD_CENSOR_VAL = +# GEN_ENS_PROD_CAT_THRESH = +# GEN_ENS_PROD_NC_VAR_STR = + +# Threshold for ratio of valid files to expected files to allow app to run +GEN_ENS_PROD_ENS_THRESH = 0.05 + +GEN_ENS_PROD_NBRHD_PROB_WIDTH = 27 +GEN_ENS_PROD_NBRHD_PROB_SHAPE = CIRCLE +GEN_ENS_PROD_NBRHD_PROB_VLD_THRESH = 0.0 + +# GEN_ENS_PROD_NMEP_SMOOTH_VLD_THRESH = 0.0 +# GEN_ENS_PROD_NMEP_SMOOTH_SHAPE = CIRCLE +# GEN_ENS_PROD_NMEP_SMOOTH_GAUSSIAN_DX = 81.27 +# GEN_ENS_PROD_NMEP_SMOOTH_GAUSSIAN_RADIUS = 120 +# GEN_ENS_PROD_NMEP_SMOOTH_METHOD = GAUSSIAN +# GEN_ENS_PROD_NMEP_SMOOTH_WIDTH = 1 + +# GEN_ENS_PROD_CLIMO_MEAN_FILE_NAME = +# GEN_ENS_PROD_CLIMO_MEAN_FIELD = +# GEN_ENS_PROD_CLIMO_MEAN_REGRID_METHOD = +# GEN_ENS_PROD_CLIMO_MEAN_REGRID_WIDTH = +# GEN_ENS_PROD_CLIMO_MEAN_REGRID_VLD_THRESH = +# GEN_ENS_PROD_CLIMO_MEAN_REGRID_SHAPE = +# GEN_ENS_PROD_CLIMO_MEAN_TIME_INTERP_METHOD = +# GEN_ENS_PROD_CLIMO_MEAN_MATCH_MONTH = +# GEN_ENS_PROD_CLIMO_MEAN_DAY_INTERVAL = 31 +# GEN_ENS_PROD_CLIMO_MEAN_HOUR_INTERVAL = 6 + +# GEN_ENS_PROD_CLIMO_STDEV_FILE_NAME = +# GEN_ENS_PROD_CLIMO_STDEV_FIELD = +# GEN_ENS_PROD_CLIMO_STDEV_REGRID_METHOD = +# GEN_ENS_PROD_CLIMO_STDEV_REGRID_WIDTH = +# GEN_ENS_PROD_CLIMO_STDEV_REGRID_VLD_THRESH = +# GEN_ENS_PROD_CLIMO_STDEV_REGRID_SHAPE = +# GEN_ENS_PROD_CLIMO_STDEV_TIME_INTERP_METHOD = +# GEN_ENS_PROD_CLIMO_STDEV_MATCH_MONTH = +# GEN_ENS_PROD_CLIMO_STDEV_DAY_INTERVAL = 31 +# GEN_ENS_PROD_CLIMO_STDEV_HOUR_INTERVAL = 6 + +GEN_ENS_PROD_ENSEMBLE_FLAG_LATLON = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_MEAN = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_STDEV = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_MINUS = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_PLUS = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_MIN = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_MAX = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_RANGE = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_VLD_COUNT = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_FREQUENCY = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_NEP = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_NMEP = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_CLIMO = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_CLIMO_CDF = FALSE + +# GEN_ENS_PROD_ENS_MEMBER_IDS = +# GEN_ENS_PROD_CONTROL_ID = diff --git a/parm/metplus/GenEnsProd_APCPgt01h.conf b/parm/metplus/GenEnsProd_APCPgt01h.conf new file mode 100644 index 0000000000..e0df15ed68 --- /dev/null +++ b/parm/metplus/GenEnsProd_APCPgt01h.conf @@ -0,0 +1,205 @@ +# GenEnsProd METplus Configuration + +[config] + +# List of applications (tools) to run. +PROCESS_LIST = GenEnsProd + +# time looping - options are INIT, VALID, RETRO, and REALTIME +# If set to INIT or RETRO: +# INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set +# If set to VALID or REALTIME: +# VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set +LOOP_BY = INIT + +# Format of INIT_BEG and INIT_END using % items +# %Y = 4 digit year, %m = 2 digit month, %d = 2 digit day, etc. +# see www.strftime.org for more information +# %Y%m%d%H expands to YYYYMMDDHH +INIT_TIME_FMT = %Y%m%d%H + +# Start time for METplus run - must match INIT_TIME_FMT +INIT_BEG = {{cdate}} + +# End time for METplus run - must match INIT_TIME_FMT +INIT_END = {{cdate}} + +# Increment between METplus runs (in seconds if no units are specified). +# Must be >= 60 seconds. +INIT_INCREMENT = 3600 + +# List of forecast leads to process for each run time (init or valid) +# In hours if units are not specified +# If unset, defaults to 0 (don't loop through forecast leads) +LEAD_SEQ = {{fhr_list}} +# +# Order of loops to process data - Options are times, processes +# Not relevant if only one item is in the PROCESS_LIST +# times = run all wrappers in the PROCESS_LIST for a single run time, then +# increment the run time and run all wrappers again until all times have +# been evaluated. +# processes = run the first wrapper in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST until all +# wrappers have been run +# +LOOP_ORDER = times +# +# Specify the name of the METplus log file. +# +LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} +# +# Specify the location and name of the final METplus conf file. +# +METPLUS_CONF = {GEN_ENS_PROD_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} +# +# Name to identify model (forecast) data in output. +# +MODEL = {{vx_fcst_model_name}} + +GEN_ENS_PROD_DESC = NA +# +# Name to identify observation data in output. +# +OBTYPE = {{obtype}} + +# The METplus GenEnsProd logging level +# 0 quiet to 5 loud, Verbosity setting for MET gen_ens_prod output, 2 is default. +# This takes precendence over the general LOG_MET_VERBOSITY set in metplus_logging.conf +#LOG_GEN_ENS_PROD_VERBOSITY = 2 + +### +# File I/O +### + +# +# Forecast model input directory for GenEnsProd. +# +GEN_ENS_PROD_INPUT_DIR = {{fcst_input_dir}} +# +# Template for forecast input to GenEnsProd relative to +# GEN_ENS_PROD_INPUT_DIR. +# +# Note that this can be a comma separated list of ensemble members +# or a single line, - filename wildcard characters may be used, ? or *. +# +GEN_ENS_PROD_INPUT_TEMPLATE = {{fcst_input_fn_template}} + +# Forecast data description variables +FCST_IS_PROB = false + +# GEN_ENS_PROD_CTRL_INPUT_DIR = {INPUT_BASE} +# GEN_ENS_PROD_CTRL_INPUT_TEMPLATE = +# {init?fmt=%Y%m%d%H}/mem1/postprd/{ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 + +# +# Output directory for GenEnsProd. +# +# OUTPUT_BASE apparently has to be set to something; it cannot be left +# to its default value. But it is not explicitly used elsewhere in this +# configuration file. +# +OUTPUT_BASE = {{output_base}} +GEN_ENS_PROD_OUTPUT_DIR = {{output_dir}} +# +# Template for output from GenEnsProd relative to +# GEN_ENS_PROD_OUTPUT_DIR. +# +GEN_ENS_PROD_OUTPUT_TEMPLATE = gen_ens_prod_{MODEL}_{{fieldname_in_met_filedir_names}}_{OBTYPE}_{valid?fmt=%Y%m%d_%H%M%S}V.nc +# +# Directory for staging data. +# +STAGING_DIR = {{staging_dir}} +# +# There are n ensembles but 1 is used as control, so specify n-1 members. +# +GEN_ENS_PROD_N_MEMBERS = {{num_ens_members}} + +### +# Field Info +### + +# Ensemble Variables and levels as specified in the ens field dictionary +# of the MET configuration file. Specify as ENS_VARn_NAME, ENS_VARn_LEVELS, +# (optional) ENS_VARn_OPTION +# Note: +# For APCP variables with accumulation periods greater than 1 hour (e.g. +# 03h, 06h, 24h), ENS_VAR1_NAME in the corresponding METplus configuration +# file specifies the name of the field in the NetCDF file(s) generated by +# MET's pcp_combine tool. This tool reads in the grib2 file(s) containing +# 1 hour accumulation data and outputs NetCDF file(s) with the appropriate +# > 1 hour accumulation. These output NetCDF files name their arrays as +# specified by the environment variable fieldname_in_met_output. Thus, +# that is the value we set ENS_VAR1_NAME to here, not to the name of the +# variable in the grib2 file. +ENS_VAR1_NAME = {{fieldname_in_met_output}} +ENS_VAR1_LEVELS = A{{accum_hh}} +ENS_VAR1_THRESH = {{field_thresholds}} + +### +# GenEnsProd +### + +# GEN_ENS_PROD_REGRID_TO_GRID = NONE +# GEN_ENS_PROD_REGRID_METHOD = NEAREST +# GEN_ENS_PROD_REGRID_WIDTH = 1 +# GEN_ENS_PROD_REGRID_VLD_THRESH = 0.5 +# GEN_ENS_PROD_REGRID_SHAPE = SQUARE + +# GEN_ENS_PROD_CENSOR_THRESH = +# GEN_ENS_PROD_CENSOR_VAL = +# GEN_ENS_PROD_CAT_THRESH = +# GEN_ENS_PROD_NC_VAR_STR = + +# Threshold for ratio of valid files to expected files to allow app to run +GEN_ENS_PROD_ENS_THRESH = 0.05 + +GEN_ENS_PROD_NBRHD_PROB_WIDTH = 27 +GEN_ENS_PROD_NBRHD_PROB_SHAPE = CIRCLE +GEN_ENS_PROD_NBRHD_PROB_VLD_THRESH = 0.0 + +# GEN_ENS_PROD_NMEP_SMOOTH_VLD_THRESH = 0.0 +# GEN_ENS_PROD_NMEP_SMOOTH_SHAPE = CIRCLE +# GEN_ENS_PROD_NMEP_SMOOTH_GAUSSIAN_DX = 81.27 +# GEN_ENS_PROD_NMEP_SMOOTH_GAUSSIAN_RADIUS = 120 +# GEN_ENS_PROD_NMEP_SMOOTH_METHOD = GAUSSIAN +# GEN_ENS_PROD_NMEP_SMOOTH_WIDTH = 1 + +# GEN_ENS_PROD_CLIMO_MEAN_FILE_NAME = +# GEN_ENS_PROD_CLIMO_MEAN_FIELD = +# GEN_ENS_PROD_CLIMO_MEAN_REGRID_METHOD = +# GEN_ENS_PROD_CLIMO_MEAN_REGRID_WIDTH = +# GEN_ENS_PROD_CLIMO_MEAN_REGRID_VLD_THRESH = +# GEN_ENS_PROD_CLIMO_MEAN_REGRID_SHAPE = +# GEN_ENS_PROD_CLIMO_MEAN_TIME_INTERP_METHOD = +# GEN_ENS_PROD_CLIMO_MEAN_MATCH_MONTH = +# GEN_ENS_PROD_CLIMO_MEAN_DAY_INTERVAL = 31 +# GEN_ENS_PROD_CLIMO_MEAN_HOUR_INTERVAL = 6 + +# GEN_ENS_PROD_CLIMO_STDEV_FILE_NAME = +# GEN_ENS_PROD_CLIMO_STDEV_FIELD = +# GEN_ENS_PROD_CLIMO_STDEV_REGRID_METHOD = +# GEN_ENS_PROD_CLIMO_STDEV_REGRID_WIDTH = +# GEN_ENS_PROD_CLIMO_STDEV_REGRID_VLD_THRESH = +# GEN_ENS_PROD_CLIMO_STDEV_REGRID_SHAPE = +# GEN_ENS_PROD_CLIMO_STDEV_TIME_INTERP_METHOD = +# GEN_ENS_PROD_CLIMO_STDEV_MATCH_MONTH = +# GEN_ENS_PROD_CLIMO_STDEV_DAY_INTERVAL = 31 +# GEN_ENS_PROD_CLIMO_STDEV_HOUR_INTERVAL = 6 + +GEN_ENS_PROD_ENSEMBLE_FLAG_LATLON = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_MEAN = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_STDEV = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_MINUS = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_PLUS = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_MIN = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_MAX = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_RANGE = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_VLD_COUNT = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_FREQUENCY = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_NEP = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_NMEP = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_CLIMO = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_CLIMO_CDF = FALSE + +# GEN_ENS_PROD_ENS_MEMBER_IDS = +# GEN_ENS_PROD_CONTROL_ID = diff --git a/parm/metplus/GenEnsProd_REFC.conf b/parm/metplus/GenEnsProd_REFC.conf new file mode 100644 index 0000000000..08dea7b571 --- /dev/null +++ b/parm/metplus/GenEnsProd_REFC.conf @@ -0,0 +1,192 @@ +# GenEnsProd METplus Configuration + +[config] + +# List of applications (tools) to run. +PROCESS_LIST = GenEnsProd + +# time looping - options are INIT, VALID, RETRO, and REALTIME +# If set to INIT or RETRO: +# INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set +# If set to VALID or REALTIME: +# VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set +LOOP_BY = INIT + +# Format of INIT_BEG and INIT_END using % items +# %Y = 4 digit year, %m = 2 digit month, %d = 2 digit day, etc. +# see www.strftime.org for more information +# %Y%m%d%H expands to YYYYMMDDHH +INIT_TIME_FMT = %Y%m%d%H + +# Start time for METplus run - must match INIT_TIME_FMT +INIT_BEG = {{cdate}} + +# End time for METplus run - must match INIT_TIME_FMT +INIT_END = {{cdate}} + +# Increment between METplus runs (in seconds if no units are specified). +# Must be >= 60 seconds. +INIT_INCREMENT = 3600 + +# List of forecast leads to process for each run time (init or valid) +# In hours if units are not specified +# If unset, defaults to 0 (don't loop through forecast leads) +LEAD_SEQ = {{fhr_list}} +# +# Order of loops to process data - Options are times, processes +# Not relevant if only one item is in the PROCESS_LIST +# times = run all wrappers in the PROCESS_LIST for a single run time, then +# increment the run time and run all wrappers again until all times have +# been evaluated. +# processes = run the first wrapper in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST until all +# wrappers have been run +# +LOOP_ORDER = times +# +# Specify the name of the METplus log file. +# +LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} +# +# Specify the location and name of the final METplus conf file. +# +METPLUS_CONF = {GEN_ENS_PROD_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} +# +# Name to identify model (forecast) data in output. +# +MODEL = {{vx_fcst_model_name}} + +GEN_ENS_PROD_DESC = NA +# +# Name to identify observation data in output. +# +OBTYPE = {{obtype}} + +# The METplus GenEnsProd logging level +# 0 quiet to 5 loud, Verbosity setting for MET gen_ens_prod output, 2 is default. +# This takes precendence over the general LOG_MET_VERBOSITY set in metplus_logging.conf +#LOG_GEN_ENS_PROD_VERBOSITY = 2 + +### +# File I/O +### + +# +# Forecast model input directory for GenEnsProd. +# +GEN_ENS_PROD_INPUT_DIR = {{fcst_input_dir}} +# +# Template for forecast input to GenEnsProd relative to +# GEN_ENS_PROD_INPUT_DIR. +# +# Note that this can be a comma separated list of ensemble members +# or a single line, - filename wildcard characters may be used, ? or *. +# +GEN_ENS_PROD_INPUT_TEMPLATE = {{fcst_input_fn_template}} + +# GEN_ENS_PROD_CTRL_INPUT_DIR = {INPUT_BASE} +# GEN_ENS_PROD_CTRL_INPUT_TEMPLATE = +# {init?fmt=%Y%m%d%H}/mem1/postprd/{ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 + +# +# Output directory for GenEnsProd. +# +# OUTPUT_BASE apparently has to be set to something; it cannot be left +# to its default value. But it is not explicitly used elsewhere in this +# configuration file. +# +OUTPUT_BASE = {{output_base}} +GEN_ENS_PROD_OUTPUT_DIR = {{output_dir}} +# +# Template for output from GenEnsProd relative to +# GEN_ENS_PROD_OUTPUT_DIR. +# +GEN_ENS_PROD_OUTPUT_TEMPLATE = gen_ens_prod_{MODEL}_{{fieldname_in_met_filedir_names}}_{OBTYPE}_{valid?fmt=%Y%m%d_%H%M%S}V.nc +# +# Directory for staging data. +# +STAGING_DIR = {{staging_dir}} +# +# There are n ensembles but 1 is used as control, so specify n-1 members. +# +GEN_ENS_PROD_N_MEMBERS = {{num_ens_members}} + +### +# Field Info +### + +# Ensemble Variables and levels as specified in the ens field dictionary +# of the MET configuration file. Specify as ENS_VARn_NAME, ENS_VARn_LEVELS, +# (optional) ENS_VARn_OPTION +ENS_VAR1_NAME = {{fieldname_in_fcst_input}} +ENS_VAR1_LEVELS = L0 +ENS_VAR1_THRESH = {{field_thresholds}} + +### +# GenEnsProd +### + +# GEN_ENS_PROD_REGRID_TO_GRID = NONE +# GEN_ENS_PROD_REGRID_METHOD = NEAREST +# GEN_ENS_PROD_REGRID_WIDTH = 1 +# GEN_ENS_PROD_REGRID_VLD_THRESH = 0.5 +# GEN_ENS_PROD_REGRID_SHAPE = SQUARE + +# GEN_ENS_PROD_CENSOR_THRESH = +# GEN_ENS_PROD_CENSOR_VAL = +# GEN_ENS_PROD_CAT_THRESH = +# GEN_ENS_PROD_NC_VAR_STR = + +# Threshold for ratio of valid files to expected files to allow app to run +GEN_ENS_PROD_ENS_THRESH = 0.05 + +GEN_ENS_PROD_NBRHD_PROB_WIDTH = 27 +GEN_ENS_PROD_NBRHD_PROB_SHAPE = CIRCLE +GEN_ENS_PROD_NBRHD_PROB_VLD_THRESH = 0.0 + +# GEN_ENS_PROD_NMEP_SMOOTH_VLD_THRESH = 0.0 +# GEN_ENS_PROD_NMEP_SMOOTH_SHAPE = CIRCLE +# GEN_ENS_PROD_NMEP_SMOOTH_GAUSSIAN_DX = 81.27 +# GEN_ENS_PROD_NMEP_SMOOTH_GAUSSIAN_RADIUS = 120 +# GEN_ENS_PROD_NMEP_SMOOTH_METHOD = GAUSSIAN +# GEN_ENS_PROD_NMEP_SMOOTH_WIDTH = 1 + +# GEN_ENS_PROD_CLIMO_MEAN_FILE_NAME = +# GEN_ENS_PROD_CLIMO_MEAN_FIELD = +# GEN_ENS_PROD_CLIMO_MEAN_REGRID_METHOD = +# GEN_ENS_PROD_CLIMO_MEAN_REGRID_WIDTH = +# GEN_ENS_PROD_CLIMO_MEAN_REGRID_VLD_THRESH = +# GEN_ENS_PROD_CLIMO_MEAN_REGRID_SHAPE = +# GEN_ENS_PROD_CLIMO_MEAN_TIME_INTERP_METHOD = +# GEN_ENS_PROD_CLIMO_MEAN_MATCH_MONTH = +# GEN_ENS_PROD_CLIMO_MEAN_DAY_INTERVAL = 31 +# GEN_ENS_PROD_CLIMO_MEAN_HOUR_INTERVAL = 6 + +# GEN_ENS_PROD_CLIMO_STDEV_FILE_NAME = +# GEN_ENS_PROD_CLIMO_STDEV_FIELD = +# GEN_ENS_PROD_CLIMO_STDEV_REGRID_METHOD = +# GEN_ENS_PROD_CLIMO_STDEV_REGRID_WIDTH = +# GEN_ENS_PROD_CLIMO_STDEV_REGRID_VLD_THRESH = +# GEN_ENS_PROD_CLIMO_STDEV_REGRID_SHAPE = +# GEN_ENS_PROD_CLIMO_STDEV_TIME_INTERP_METHOD = +# GEN_ENS_PROD_CLIMO_STDEV_MATCH_MONTH = +# GEN_ENS_PROD_CLIMO_STDEV_DAY_INTERVAL = 31 +# GEN_ENS_PROD_CLIMO_STDEV_HOUR_INTERVAL = 6 + +GEN_ENS_PROD_ENSEMBLE_FLAG_LATLON = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_MEAN = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_STDEV = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_MINUS = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_PLUS = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_MIN = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_MAX = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_RANGE = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_VLD_COUNT = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_FREQUENCY = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_NEP = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_NMEP = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_CLIMO = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_CLIMO_CDF = FALSE + +# GEN_ENS_PROD_ENS_MEMBER_IDS = +# GEN_ENS_PROD_CONTROL_ID = diff --git a/parm/metplus/GenEnsProd_RETOP.conf b/parm/metplus/GenEnsProd_RETOP.conf new file mode 100644 index 0000000000..205fc3eb75 --- /dev/null +++ b/parm/metplus/GenEnsProd_RETOP.conf @@ -0,0 +1,194 @@ +# GenEnsProd METplus Configuration + +[config] + +# List of applications (tools) to run. +PROCESS_LIST = GenEnsProd + +# time looping - options are INIT, VALID, RETRO, and REALTIME +# If set to INIT or RETRO: +# INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set +# If set to VALID or REALTIME: +# VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set +LOOP_BY = INIT + +# Format of INIT_BEG and INIT_END using % items +# %Y = 4 digit year, %m = 2 digit month, %d = 2 digit day, etc. +# see www.strftime.org for more information +# %Y%m%d%H expands to YYYYMMDDHH +INIT_TIME_FMT = %Y%m%d%H + +# Start time for METplus run - must match INIT_TIME_FMT +INIT_BEG = {{cdate}} + +# End time for METplus run - must match INIT_TIME_FMT +INIT_END = {{cdate}} + +# Increment between METplus runs (in seconds if no units are specified). +# Must be >= 60 seconds. +INIT_INCREMENT = 3600 + +# List of forecast leads to process for each run time (init or valid) +# In hours if units are not specified +# If unset, defaults to 0 (don't loop through forecast leads) +LEAD_SEQ = {{fhr_list}} +# +# Order of loops to process data - Options are times, processes +# Not relevant if only one item is in the PROCESS_LIST +# times = run all wrappers in the PROCESS_LIST for a single run time, then +# increment the run time and run all wrappers again until all times have +# been evaluated. +# processes = run the first wrapper in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST until all +# wrappers have been run +# +LOOP_ORDER = times +# +# Specify the name of the METplus log file. +# +LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} +# +# Specify the location and name of the final METplus conf file. +# +METPLUS_CONF = {GEN_ENS_PROD_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} +# +# Name to identify model (forecast) data in output. +# +MODEL = {{vx_fcst_model_name}} + +GEN_ENS_PROD_DESC = NA +# +# Name to identify observation data in output. +# +OBTYPE = {{obtype}} + +# The METplus GenEnsProd logging level +# 0 quiet to 5 loud, Verbosity setting for MET gen_ens_prod output, 2 is default. +# This takes precendence over the general LOG_MET_VERBOSITY set in metplus_logging.conf +#LOG_GEN_ENS_PROD_VERBOSITY = 2 + +### +# File I/O +### + +# +# Forecast model input directory for GenEnsProd. +# +GEN_ENS_PROD_INPUT_DIR = {{fcst_input_dir}} +# +# Template for forecast input to GenEnsProd relative to +# GEN_ENS_PROD_INPUT_DIR. +# +# Note that this can be a comma separated list of ensemble members +# or a single line, - filename wildcard characters may be used, ? or *. +# +GEN_ENS_PROD_INPUT_TEMPLATE = {{fcst_input_fn_template}} + +# GEN_ENS_PROD_CTRL_INPUT_DIR = {INPUT_BASE} +# GEN_ENS_PROD_CTRL_INPUT_TEMPLATE = +# {init?fmt=%Y%m%d%H}/mem1/postprd/{ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 + +# +# Output directory for GenEnsProd. +# +# OUTPUT_BASE apparently has to be set to something; it cannot be left +# to its default value. But it is not explicitly used elsewhere in this +# configuration file. +# +OUTPUT_BASE = {{output_base}} +GEN_ENS_PROD_OUTPUT_DIR = {{output_dir}} +# +# Template for output from GenEnsProd relative to +# GEN_ENS_PROD_OUTPUT_DIR. +# +GEN_ENS_PROD_OUTPUT_TEMPLATE = gen_ens_prod_{MODEL}_{{fieldname_in_met_filedir_names}}_{OBTYPE}_{valid?fmt=%Y%m%d_%H%M%S}V.nc +# +# Directory for staging data. +# +STAGING_DIR = {{staging_dir}} +# +# There are n ensembles but 1 is used as control, so specify n-1 members. +# +GEN_ENS_PROD_N_MEMBERS = {{num_ens_members}} + +### +# Field Info +### + +# Ensemble Variables and levels as specified in the ens field dictionary +# of the MET configuration file. Specify as ENS_VARn_NAME, ENS_VARn_LEVELS, +# (optional) ENS_VARn_OPTION +ENS_VAR1_NAME = {{fieldname_in_fcst_input}} +ENS_VAR1_LEVELS = L0 +ENS_VAR1_THRESH = {{field_thresholds}} +# convert from meters to kilofeet: M_to_KFT(x) = x * 3.28084 * 0.001; +ENS_VAR1_OPTIONS = convert(x) = x * 3.28084 * 0.001; + +### +# GenEnsProd +### + +# GEN_ENS_PROD_REGRID_TO_GRID = NONE +# GEN_ENS_PROD_REGRID_METHOD = NEAREST +# GEN_ENS_PROD_REGRID_WIDTH = 1 +# GEN_ENS_PROD_REGRID_VLD_THRESH = 0.5 +# GEN_ENS_PROD_REGRID_SHAPE = SQUARE + +# GEN_ENS_PROD_CENSOR_THRESH = +# GEN_ENS_PROD_CENSOR_VAL = +# GEN_ENS_PROD_CAT_THRESH = +# GEN_ENS_PROD_NC_VAR_STR = + +# Threshold for ratio of valid files to expected files to allow app to run +GEN_ENS_PROD_ENS_THRESH = 0.05 + +GEN_ENS_PROD_NBRHD_PROB_WIDTH = 27 +GEN_ENS_PROD_NBRHD_PROB_SHAPE = CIRCLE +GEN_ENS_PROD_NBRHD_PROB_VLD_THRESH = 0.0 + +# GEN_ENS_PROD_NMEP_SMOOTH_VLD_THRESH = 0.0 +# GEN_ENS_PROD_NMEP_SMOOTH_SHAPE = CIRCLE +# GEN_ENS_PROD_NMEP_SMOOTH_GAUSSIAN_DX = 81.27 +# GEN_ENS_PROD_NMEP_SMOOTH_GAUSSIAN_RADIUS = 120 +# GEN_ENS_PROD_NMEP_SMOOTH_METHOD = GAUSSIAN +# GEN_ENS_PROD_NMEP_SMOOTH_WIDTH = 1 + +# GEN_ENS_PROD_CLIMO_MEAN_FILE_NAME = +# GEN_ENS_PROD_CLIMO_MEAN_FIELD = +# GEN_ENS_PROD_CLIMO_MEAN_REGRID_METHOD = +# GEN_ENS_PROD_CLIMO_MEAN_REGRID_WIDTH = +# GEN_ENS_PROD_CLIMO_MEAN_REGRID_VLD_THRESH = +# GEN_ENS_PROD_CLIMO_MEAN_REGRID_SHAPE = +# GEN_ENS_PROD_CLIMO_MEAN_TIME_INTERP_METHOD = +# GEN_ENS_PROD_CLIMO_MEAN_MATCH_MONTH = +# GEN_ENS_PROD_CLIMO_MEAN_DAY_INTERVAL = 31 +# GEN_ENS_PROD_CLIMO_MEAN_HOUR_INTERVAL = 6 + +# GEN_ENS_PROD_CLIMO_STDEV_FILE_NAME = +# GEN_ENS_PROD_CLIMO_STDEV_FIELD = +# GEN_ENS_PROD_CLIMO_STDEV_REGRID_METHOD = +# GEN_ENS_PROD_CLIMO_STDEV_REGRID_WIDTH = +# GEN_ENS_PROD_CLIMO_STDEV_REGRID_VLD_THRESH = +# GEN_ENS_PROD_CLIMO_STDEV_REGRID_SHAPE = +# GEN_ENS_PROD_CLIMO_STDEV_TIME_INTERP_METHOD = +# GEN_ENS_PROD_CLIMO_STDEV_MATCH_MONTH = +# GEN_ENS_PROD_CLIMO_STDEV_DAY_INTERVAL = 31 +# GEN_ENS_PROD_CLIMO_STDEV_HOUR_INTERVAL = 6 + +GEN_ENS_PROD_ENSEMBLE_FLAG_LATLON = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_MEAN = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_STDEV = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_MINUS = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_PLUS = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_MIN = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_MAX = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_RANGE = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_VLD_COUNT = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_FREQUENCY = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_NEP = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_NMEP = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_CLIMO = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_CLIMO_CDF = FALSE + +# GEN_ENS_PROD_ENS_MEMBER_IDS = +# GEN_ENS_PROD_CONTROL_ID = diff --git a/parm/metplus/GenEnsProd_SFC.conf b/parm/metplus/GenEnsProd_SFC.conf new file mode 100644 index 0000000000..4acf0d4538 --- /dev/null +++ b/parm/metplus/GenEnsProd_SFC.conf @@ -0,0 +1,217 @@ +# GenEnsProd METplus Configuration + +[config] + +# List of applications (tools) to run. +PROCESS_LIST = GenEnsProd + +# time looping - options are INIT, VALID, RETRO, and REALTIME +# If set to INIT or RETRO: +# INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set +# If set to VALID or REALTIME: +# VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set +LOOP_BY = INIT + +# Format of INIT_BEG and INIT_END using % items +# %Y = 4 digit year, %m = 2 digit month, %d = 2 digit day, etc. +# see www.strftime.org for more information +# %Y%m%d%H expands to YYYYMMDDHH +INIT_TIME_FMT = %Y%m%d%H + +# Start time for METplus run - must match INIT_TIME_FMT +INIT_BEG = {{cdate}} + +# End time for METplus run - must match INIT_TIME_FMT +INIT_END = {{cdate}} + +# Increment between METplus runs (in seconds if no units are specified). +# Must be >= 60 seconds. +INIT_INCREMENT = 3600 + +# List of forecast leads to process for each run time (init or valid) +# In hours if units are not specified +# If unset, defaults to 0 (don't loop through forecast leads) +LEAD_SEQ = {{fhr_list}} +# +# Order of loops to process data - Options are times, processes +# Not relevant if only one item is in the PROCESS_LIST +# times = run all wrappers in the PROCESS_LIST for a single run time, then +# increment the run time and run all wrappers again until all times have +# been evaluated. +# processes = run the first wrapper in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST until all +# wrappers have been run +# +LOOP_ORDER = times +# +# Specify the name of the METplus log file. +# +LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} +# +# Specify the location and name of the final METplus conf file. +# +METPLUS_CONF = {GEN_ENS_PROD_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} +# +# Name to identify model (forecast) data in output. +# +MODEL = {{vx_fcst_model_name}} + +GEN_ENS_PROD_DESC = NA +# +# Name to identify observation data in output. +# +OBTYPE = {{obtype}} + +# The METplus GenEnsProd logging level +# 0 quiet to 5 loud, Verbosity setting for MET gen_ens_prod output, 2 is default. +# This takes precendence over the general LOG_MET_VERBOSITY set in metplus_logging.conf +#LOG_GEN_ENS_PROD_VERBOSITY = 2 + +### +# File I/O +### + +# +# Forecast model input directory for GenEnsProd. +# +GEN_ENS_PROD_INPUT_DIR = {{fcst_input_dir}} +# +# Template for forecast input to GenEnsProd relative to +# GEN_ENS_PROD_INPUT_DIR. +# +# Note that this can be a comma separated list of ensemble members +# or a single line, - filename wildcard characters may be used, ? or *. +# +GEN_ENS_PROD_INPUT_TEMPLATE = {{fcst_input_fn_template}} + +# GEN_ENS_PROD_CTRL_INPUT_DIR = {INPUT_BASE} +# GEN_ENS_PROD_CTRL_INPUT_TEMPLATE = +# {init?fmt=%Y%m%d%H}/mem1/postprd/{ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 + +# +# Output directory for GenEnsProd. +# +# OUTPUT_BASE apparently has to be set to something; it cannot be left +# to its default value. But it is not explicitly used elsewhere in this +# configuration file. +# +OUTPUT_BASE = {{output_base}} +GEN_ENS_PROD_OUTPUT_DIR = {{output_dir}} +# +# Template for output from GenEnsProd relative to +# GEN_ENS_PROD_OUTPUT_DIR. +# +GEN_ENS_PROD_OUTPUT_TEMPLATE = gen_ens_prod_{MODEL}_ADP{{fieldname_in_met_filedir_names}}_{OBTYPE}_{valid?fmt=%Y%m%d_%H%M%S}V.nc +# +# Directory for staging data. +# +STAGING_DIR = {{staging_dir}} +# +# There are n ensembles but 1 is used as control, so specify n-1 members. +# +GEN_ENS_PROD_N_MEMBERS = {{num_ens_members}} + +### +# Field Info +### + +# Ensemble Variables and levels as specified in the ens field dictionary +# of the MET configuration file. Specify as ENS_VARn_NAME, ENS_VARn_LEVELS, +# (optional) ENS_VARn_OPTION +ENS_VAR1_NAME = TMP +ENS_VAR1_LEVELS = Z02 +ENS_VAR1_THRESH = >=268, >=273, >=278, >=293, >=298, >=303 + +ENS_VAR2_NAME = DPT +ENS_VAR2_LEVELS = Z2 +ENS_VAR2_THRESH = >=263, >=268, >=273, >=288, >=293, >=298 + +ENS_VAR3_NAME = WIND +ENS_VAR3_LEVELS = Z10 +ENS_VAR3_THRESH = >=5, >=10, >=15 +ENS_VAR3_OPTIONS = GRIB2_pdt = 0; ;; derive instantaneous 10-m wind from U/V components, overriding max 10-m wind + +ENS_VAR4_NAME = TCDC +ENS_VAR4_LEVELS = L0 +ENS_VAR4_THRESH = <25, >75 +ENS_VAR4_OPTIONS = GRIB_lvl_typ = 200; GRIB2_ipdtmpl_index=[27]; GRIB2_ipdtmpl_val=[255]; interp = { type = [ { method = NEAREST; width = 1; } ]; } + +ENS_VAR5_NAME = VIS +ENS_VAR5_LEVELS = L0 +ENS_VAR5_THRESH = <1609, <8045, >=8045 +ENS_VAR5_OPTIONS = interp = { type = [ { method = NEAREST; width = 1; } ]; } + +ENS_VAR6_NAME = HGT +ENS_VAR6_LEVELS = L0 +ENS_VAR6_THRESH = <152, <1520, >=914 +ENS_VAR6_OPTIONS = GRIB_lvl_typ = 215; desc = "CEILING"; + + +### +# GenEnsProd +### + +# GEN_ENS_PROD_REGRID_TO_GRID = NONE +# GEN_ENS_PROD_REGRID_METHOD = NEAREST +# GEN_ENS_PROD_REGRID_WIDTH = 1 +# GEN_ENS_PROD_REGRID_VLD_THRESH = 0.5 +# GEN_ENS_PROD_REGRID_SHAPE = SQUARE + +# GEN_ENS_PROD_CENSOR_THRESH = +# GEN_ENS_PROD_CENSOR_VAL = +# GEN_ENS_PROD_CAT_THRESH = +# GEN_ENS_PROD_NC_VAR_STR = + +# Threshold for ratio of valid files to expected files to allow app to run +GEN_ENS_PROD_ENS_THRESH = 0.05 + +GEN_ENS_PROD_NBRHD_PROB_WIDTH = 27 +GEN_ENS_PROD_NBRHD_PROB_SHAPE = CIRCLE +GEN_ENS_PROD_NBRHD_PROB_VLD_THRESH = 0.0 + +# GEN_ENS_PROD_NMEP_SMOOTH_VLD_THRESH = 0.0 +# GEN_ENS_PROD_NMEP_SMOOTH_SHAPE = CIRCLE +# GEN_ENS_PROD_NMEP_SMOOTH_GAUSSIAN_DX = 81.27 +# GEN_ENS_PROD_NMEP_SMOOTH_GAUSSIAN_RADIUS = 120 +# GEN_ENS_PROD_NMEP_SMOOTH_METHOD = GAUSSIAN +# GEN_ENS_PROD_NMEP_SMOOTH_WIDTH = 1 + +# GEN_ENS_PROD_CLIMO_MEAN_FILE_NAME = +# GEN_ENS_PROD_CLIMO_MEAN_FIELD = +# GEN_ENS_PROD_CLIMO_MEAN_REGRID_METHOD = +# GEN_ENS_PROD_CLIMO_MEAN_REGRID_WIDTH = +# GEN_ENS_PROD_CLIMO_MEAN_REGRID_VLD_THRESH = +# GEN_ENS_PROD_CLIMO_MEAN_REGRID_SHAPE = +# GEN_ENS_PROD_CLIMO_MEAN_TIME_INTERP_METHOD = +# GEN_ENS_PROD_CLIMO_MEAN_MATCH_MONTH = +# GEN_ENS_PROD_CLIMO_MEAN_DAY_INTERVAL = 31 +# GEN_ENS_PROD_CLIMO_MEAN_HOUR_INTERVAL = 6 + +# GEN_ENS_PROD_CLIMO_STDEV_FILE_NAME = +# GEN_ENS_PROD_CLIMO_STDEV_FIELD = +# GEN_ENS_PROD_CLIMO_STDEV_REGRID_METHOD = +# GEN_ENS_PROD_CLIMO_STDEV_REGRID_WIDTH = +# GEN_ENS_PROD_CLIMO_STDEV_REGRID_VLD_THRESH = +# GEN_ENS_PROD_CLIMO_STDEV_REGRID_SHAPE = +# GEN_ENS_PROD_CLIMO_STDEV_TIME_INTERP_METHOD = +# GEN_ENS_PROD_CLIMO_STDEV_MATCH_MONTH = +# GEN_ENS_PROD_CLIMO_STDEV_DAY_INTERVAL = 31 +# GEN_ENS_PROD_CLIMO_STDEV_HOUR_INTERVAL = 6 + +GEN_ENS_PROD_ENSEMBLE_FLAG_LATLON = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_MEAN = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_STDEV = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_MINUS = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_PLUS = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_MIN = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_MAX = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_RANGE = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_VLD_COUNT = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_FREQUENCY = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_NEP = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_NMEP = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_CLIMO = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_CLIMO_CDF = FALSE + +# GEN_ENS_PROD_ENS_MEMBER_IDS = +# GEN_ENS_PROD_CONTROL_ID = diff --git a/parm/metplus/GenEnsProd_UPA.conf b/parm/metplus/GenEnsProd_UPA.conf new file mode 100644 index 0000000000..ea3ebaa4d9 --- /dev/null +++ b/parm/metplus/GenEnsProd_UPA.conf @@ -0,0 +1,237 @@ +# GenEnsProd METplus Configuration + +[config] + +# List of applications (tools) to run. +PROCESS_LIST = GenEnsProd + +# time looping - options are INIT, VALID, RETRO, and REALTIME +# If set to INIT or RETRO: +# INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set +# If set to VALID or REALTIME: +# VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set +LOOP_BY = INIT + +# Format of INIT_BEG and INIT_END using % items +# %Y = 4 digit year, %m = 2 digit month, %d = 2 digit day, etc. +# see www.strftime.org for more information +# %Y%m%d%H expands to YYYYMMDDHH +INIT_TIME_FMT = %Y%m%d%H + +# Start time for METplus run - must match INIT_TIME_FMT +INIT_BEG = {{cdate}} + +# End time for METplus run - must match INIT_TIME_FMT +INIT_END = {{cdate}} + +# Increment between METplus runs (in seconds if no units are specified). +# Must be >= 60 seconds. +INIT_INCREMENT = 3600 + +# List of forecast leads to process for each run time (init or valid) +# In hours if units are not specified +# If unset, defaults to 0 (don't loop through forecast leads) +LEAD_SEQ = {{fhr_list}} +# +# Order of loops to process data - Options are times, processes +# Not relevant if only one item is in the PROCESS_LIST +# times = run all wrappers in the PROCESS_LIST for a single run time, then +# increment the run time and run all wrappers again until all times have +# been evaluated. +# processes = run the first wrapper in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST until all +# wrappers have been run +# +LOOP_ORDER = times +# +# Specify the name of the METplus log file. +# +LOG_METPLUS = {LOG_DIR}/{{metplus_log_fn}} +# +# Specify the location and name of the final METplus conf file. +# +METPLUS_CONF = {GEN_ENS_PROD_OUTPUT_DIR}/metplus_final.{{metplus_config_fn}} +# +# Name to identify model (forecast) data in output. +# +MODEL = {{vx_fcst_model_name}} + +GEN_ENS_PROD_DESC = NA +# +# Name to identify observation data in output. +# +OBTYPE = {{obtype}} + +# The METplus GenEnsProd logging level +# 0 quiet to 5 loud, Verbosity setting for MET gen_ens_prod output, 2 is default. +# This takes precendence over the general LOG_MET_VERBOSITY set in metplus_logging.conf +#LOG_GEN_ENS_PROD_VERBOSITY = 2 + +### +# File I/O +### + +# +# Forecast model input directory for GenEnsProd. +# +GEN_ENS_PROD_INPUT_DIR = {{fcst_input_dir}} +# +# Template for forecast input to GenEnsProd relative to +# GEN_ENS_PROD_INPUT_DIR. +# +# Note that this can be a comma separated list of ensemble members +# or a single line, - filename wildcard characters may be used, ? or *. +# +GEN_ENS_PROD_INPUT_TEMPLATE = {{fcst_input_fn_template}} + +# GEN_ENS_PROD_CTRL_INPUT_DIR = {INPUT_BASE} +# GEN_ENS_PROD_CTRL_INPUT_TEMPLATE = +# {init?fmt=%Y%m%d%H}/mem1/postprd/{ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 + +# +# Output directory for GenEnsProd. +# +# OUTPUT_BASE apparently has to be set to something; it cannot be left +# to its default value. But it is not explicitly used elsewhere in this +# configuration file. +# +OUTPUT_BASE = {{output_base}} +GEN_ENS_PROD_OUTPUT_DIR = {{output_dir}} +# +# Template for output from GenEnsProd relative to +# GEN_ENS_PROD_OUTPUT_DIR. +# +GEN_ENS_PROD_OUTPUT_TEMPLATE = gen_ens_prod_{MODEL}_ADP{{fieldname_in_met_filedir_names}}_{OBTYPE}_{valid?fmt=%Y%m%d_%H%M%S}V.nc +# +# Directory for staging data. +# +STAGING_DIR = {{staging_dir}} +# +# There are n ensembles but 1 is used as control, so specify n-1 members. +# +GEN_ENS_PROD_N_MEMBERS = {{num_ens_members}} + +### +# Field Info +### + +# Ensemble Variables and levels as specified in the ens field dictionary +# of the MET configuration file. Specify as ENS_VARn_NAME, ENS_VARn_LEVELS, +# (optional) ENS_VARn_OPTION +ENS_VAR1_NAME = TMP +ENS_VAR1_LEVELS = P850 +ENS_VAR1_THRESH = >=288, >=293, >=298 + +ENS_VAR2_NAME = TMP +ENS_VAR2_LEVELS = P700 +ENS_VAR2_THRESH = >=273, >=278, >=283 + +ENS_VAR3_NAME = TMP +ENS_VAR3_LEVELS = P500 +ENS_VAR3_THRESH = >=258, >=263, >=268 + +ENS_VAR4_NAME = DPT +ENS_VAR4_LEVELS = P850 +ENS_VAR4_THRESH = >=273, >=278, >=283 + +ENS_VAR5_NAME = DPT +ENS_VAR5_LEVELS = P700 +ENS_VAR5_THRESH = >=263, >=268, >=273 + +ENS_VAR6_NAME = WIND +ENS_VAR6_LEVELS = P850 +ENS_VAR6_THRESH = >=5, >=10, >=15 + +ENS_VAR7_NAME = WIND +ENS_VAR7_LEVELS = P700 +ENS_VAR7_THRESH = >=10, >=15, >=20 + +ENS_VAR8_NAME = WIND +ENS_VAR8_LEVELS = P500 +ENS_VAR8_THRESH = >=15, >=21, >=26 + +ENS_VAR9_NAME = WIND +ENS_VAR9_LEVELS = P250 +ENS_VAR9_THRESH = >=26, >=31, >=36, >=46, >=62 + +ENS_VAR10_NAME = HGT +ENS_VAR10_LEVELS = P500 +ENS_VAR10_THRESH = >=5400, >=5600, >=5880 + +ENS_VAR11_NAME = CAPE +ENS_VAR11_LEVELS = L0 +ENS_VAR11_OPTIONS = cnt_thresh = [ >0 ]; +ENS_VAR11_THRESH = <=1000, >1000&&<2500, >2500&&<4000, >2500 + +ENS_VAR12_NAME = HPBL +ENS_VAR12_LEVELS = Z0 +ENS_VAR12_THRESH = <500, <1500, >1500 + +### +# GenEnsProd +### + +# GEN_ENS_PROD_REGRID_TO_GRID = NONE +# GEN_ENS_PROD_REGRID_METHOD = NEAREST +# GEN_ENS_PROD_REGRID_WIDTH = 1 +# GEN_ENS_PROD_REGRID_VLD_THRESH = 0.5 +# GEN_ENS_PROD_REGRID_SHAPE = SQUARE + +# GEN_ENS_PROD_CENSOR_THRESH = +# GEN_ENS_PROD_CENSOR_VAL = +# GEN_ENS_PROD_CAT_THRESH = +# GEN_ENS_PROD_NC_VAR_STR = + +# Threshold for ratio of valid files to expected files to allow app to run +GEN_ENS_PROD_ENS_THRESH = 0.05 + +GEN_ENS_PROD_NBRHD_PROB_WIDTH = 27 +GEN_ENS_PROD_NBRHD_PROB_SHAPE = CIRCLE +GEN_ENS_PROD_NBRHD_PROB_VLD_THRESH = 0.0 + +# GEN_ENS_PROD_NMEP_SMOOTH_VLD_THRESH = 0.0 +# GEN_ENS_PROD_NMEP_SMOOTH_SHAPE = CIRCLE +# GEN_ENS_PROD_NMEP_SMOOTH_GAUSSIAN_DX = 81.27 +# GEN_ENS_PROD_NMEP_SMOOTH_GAUSSIAN_RADIUS = 120 +# GEN_ENS_PROD_NMEP_SMOOTH_METHOD = GAUSSIAN +# GEN_ENS_PROD_NMEP_SMOOTH_WIDTH = 1 + +# GEN_ENS_PROD_CLIMO_MEAN_FILE_NAME = +# GEN_ENS_PROD_CLIMO_MEAN_FIELD = +# GEN_ENS_PROD_CLIMO_MEAN_REGRID_METHOD = +# GEN_ENS_PROD_CLIMO_MEAN_REGRID_WIDTH = +# GEN_ENS_PROD_CLIMO_MEAN_REGRID_VLD_THRESH = +# GEN_ENS_PROD_CLIMO_MEAN_REGRID_SHAPE = +# GEN_ENS_PROD_CLIMO_MEAN_TIME_INTERP_METHOD = +# GEN_ENS_PROD_CLIMO_MEAN_MATCH_MONTH = +# GEN_ENS_PROD_CLIMO_MEAN_DAY_INTERVAL = 31 +# GEN_ENS_PROD_CLIMO_MEAN_HOUR_INTERVAL = 6 + +# GEN_ENS_PROD_CLIMO_STDEV_FILE_NAME = +# GEN_ENS_PROD_CLIMO_STDEV_FIELD = +# GEN_ENS_PROD_CLIMO_STDEV_REGRID_METHOD = +# GEN_ENS_PROD_CLIMO_STDEV_REGRID_WIDTH = +# GEN_ENS_PROD_CLIMO_STDEV_REGRID_VLD_THRESH = +# GEN_ENS_PROD_CLIMO_STDEV_REGRID_SHAPE = +# GEN_ENS_PROD_CLIMO_STDEV_TIME_INTERP_METHOD = +# GEN_ENS_PROD_CLIMO_STDEV_MATCH_MONTH = +# GEN_ENS_PROD_CLIMO_STDEV_DAY_INTERVAL = 31 +# GEN_ENS_PROD_CLIMO_STDEV_HOUR_INTERVAL = 6 + +GEN_ENS_PROD_ENSEMBLE_FLAG_LATLON = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_MEAN = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_STDEV = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_MINUS = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_PLUS = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_MIN = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_MAX = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_RANGE = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_VLD_COUNT = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_FREQUENCY = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_NEP = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_NMEP = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_CLIMO = FALSE +GEN_ENS_PROD_ENSEMBLE_FLAG_CLIMO_CDF = FALSE + +# GEN_ENS_PROD_ENS_MEMBER_IDS = +# GEN_ENS_PROD_CONTROL_ID = diff --git a/ush/get_met_metplus_tool_name.sh b/ush/get_met_metplus_tool_name.sh index 9ecadfbcb8..3802ded893 100644 --- a/ush/get_met_metplus_tool_name.sh +++ b/ush/get_met_metplus_tool_name.sh @@ -65,8 +65,8 @@ function get_met_metplus_tool_name() { # generic_tool_name=${generic_tool_name,,} valid_vals_generic_tool_name=( \ - "PB2NC" "PCPCOMBINE" "GRIDSTAT" "POINTSTAT" "ENSEMBLESTAT" \ - "pb2nc" "pcpcombine" "gridstat" "pointstat" "ensemblestat" \ + "PB2NC" "PCPCOMBINE" "GRIDSTAT" "POINTSTAT" "GENENSPROD" "ENSEMBLESTAT" \ + "pb2nc" "pcpcombine" "gridstat" "pointstat" "genensprod" "ensemblestat" \ ) check_var_valid_value "generic_tool_name" "valid_vals_generic_tool_name" @@ -87,6 +87,10 @@ function get_met_metplus_tool_name() { _met_tool_name_="point_stat" _metplus_tool_name_="PointStat" ;; + "genensprod") + _met_tool_name_="gen_ens_prod" + _metplus_tool_name_="GenEnsProd" + ;; "ensemblestat") _met_tool_name_="ensemble_stat" _metplus_tool_name_="EnsembleStat" From 8926622366387cb88c28c61c40294f76cbf00e57 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Fri, 24 Mar 2023 08:35:08 -0600 Subject: [PATCH 07/43] Remove lines in EnsembleStat conf files that are for actions that are now done in GenEnsProd (to avoid duplication). --- parm/metplus/EnsembleStat_APCP01h.conf | 28 +++------- parm/metplus/EnsembleStat_APCPgt01h.conf | 28 +++------- parm/metplus/EnsembleStat_REFC.conf | 28 +++------- parm/metplus/EnsembleStat_RETOP.conf | 30 +++-------- parm/metplus/EnsembleStat_SFC.conf | 48 +++-------------- parm/metplus/EnsembleStat_UPA.conf | 69 +++--------------------- 6 files changed, 44 insertions(+), 187 deletions(-) diff --git a/parm/metplus/EnsembleStat_APCP01h.conf b/parm/metplus/EnsembleStat_APCP01h.conf index 8d2277fe10..9feb184f20 100644 --- a/parm/metplus/EnsembleStat_APCP01h.conf +++ b/parm/metplus/EnsembleStat_APCP01h.conf @@ -148,7 +148,7 @@ ENSEMBLE_STAT_ENS_PHIST_BIN_SIZE = 0.05 ENSEMBLE_STAT_CLIMO_CDF_BINS = 1 ENSEMBLE_STAT_CLIMO_CDF_CENTER_BINS = False -ENSEMBLE_STAT_CLIMO_CDF_WRITE_BINS = True +ENSEMBLE_STAT_CLIMO_CDF_WRITE_BINS = False ENSEMBLE_STAT_MASK_GRID = @@ -168,35 +168,21 @@ ENSEMBLE_STAT_OUTPUT_FLAG_ORANK = STAT ENSEMBLE_STAT_OUTPUT_FLAG_SSVAR = STAT ENSEMBLE_STAT_OUTPUT_FLAG_RELP = STAT -ENSEMBLE_STAT_ENSEMBLE_FLAG_LATLON = TRUE -ENSEMBLE_STAT_ENSEMBLE_FLAG_MEAN = TRUE -ENSEMBLE_STAT_ENSEMBLE_FLAG_STDEV = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_LATLON = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_MEAN = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_STDEV = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_MINUS = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_PLUS = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_MIN = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_MAX = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_RANGE = FALSE -ENSEMBLE_STAT_ENSEMBLE_FLAG_VLD_COUNT = TRUE -ENSEMBLE_STAT_ENSEMBLE_FLAG_FREQUENCY = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_VLD_COUNT = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_FREQUENCY = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_NEP = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_NMEP = FALSE -ENSEMBLE_STAT_ENSEMBLE_FLAG_RANK = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_RANK = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_WEIGHT = FALSE # -# Ensemble variables and levels as specified in the ens field dictionary -# of the MET configuration file. Specify as ENS_VARn_NAME, ENS_VARn_LEVELS, -# (optional) ENS_VARn_OPTION. -# -# ENS_VARn_THRESH must be included here in order for the netCDF file -# generated by EnsembleStat to include the ensemble probability (aka -# ensembe frequency) fields corresponding to the specified thresholds. -# These must be included because the corresonding GridStat task for -# ensemble probabilities expects to read them in as input. -# -ENS_VAR1_NAME = {{fieldname_in_fcst_input}} -ENS_VAR1_LEVELS = A{ACCUM_HH} -ENS_VAR1_THRESH = {{field_thresholds}} -# # Precipitation accumulation interval in 2-digit hours. # ACCUM_HH = {{accum_hh}} diff --git a/parm/metplus/EnsembleStat_APCPgt01h.conf b/parm/metplus/EnsembleStat_APCPgt01h.conf index fba99e460f..6e1b77a8b1 100644 --- a/parm/metplus/EnsembleStat_APCPgt01h.conf +++ b/parm/metplus/EnsembleStat_APCPgt01h.conf @@ -155,7 +155,7 @@ ENSEMBLE_STAT_ENS_PHIST_BIN_SIZE = 0.05 ENSEMBLE_STAT_CLIMO_CDF_BINS = 1 ENSEMBLE_STAT_CLIMO_CDF_CENTER_BINS = False -ENSEMBLE_STAT_CLIMO_CDF_WRITE_BINS = True +ENSEMBLE_STAT_CLIMO_CDF_WRITE_BINS = False ENSEMBLE_STAT_MASK_GRID = @@ -175,35 +175,21 @@ ENSEMBLE_STAT_OUTPUT_FLAG_ORANK = STAT ENSEMBLE_STAT_OUTPUT_FLAG_SSVAR = STAT ENSEMBLE_STAT_OUTPUT_FLAG_RELP = STAT -ENSEMBLE_STAT_ENSEMBLE_FLAG_LATLON = TRUE -ENSEMBLE_STAT_ENSEMBLE_FLAG_MEAN = TRUE -ENSEMBLE_STAT_ENSEMBLE_FLAG_STDEV = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_LATLON = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_MEAN = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_STDEV = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_MINUS = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_PLUS = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_MIN = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_MAX = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_RANGE = FALSE -ENSEMBLE_STAT_ENSEMBLE_FLAG_VLD_COUNT = TRUE -ENSEMBLE_STAT_ENSEMBLE_FLAG_FREQUENCY = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_VLD_COUNT = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_FREQUENCY = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_NEP = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_NMEP = FALSE -ENSEMBLE_STAT_ENSEMBLE_FLAG_RANK = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_RANK = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_WEIGHT = FALSE # -# Ensemble variables and levels as specified in the ens field dictionary -# of the MET configuration file. Specify as ENS_VARn_NAME, ENS_VARn_LEVELS, -# (optional) ENS_VARn_OPTION. -# -# ENS_VARn_THRESH must be included here in order for the netCDF file -# generated by EnsembleStat to include the ensemble probability (aka -# ensembe frequency) fields corresponding to the specified thresholds. -# These must be included because the corresonding GridStat task for -# ensemble probabilities expects to read them in as input. -# -ENS_VAR1_NAME = {{fieldname_in_met_output}} -ENS_VAR1_LEVELS = A{ACCUM_HH} -ENS_VAR1_THRESH = {{field_thresholds}} -# # Precipitation accumulation interval in 2-digit hours. # ACCUM_HH = {{accum_hh}} diff --git a/parm/metplus/EnsembleStat_REFC.conf b/parm/metplus/EnsembleStat_REFC.conf index 34ed94929e..d569552119 100644 --- a/parm/metplus/EnsembleStat_REFC.conf +++ b/parm/metplus/EnsembleStat_REFC.conf @@ -150,7 +150,7 @@ ENSEMBLE_STAT_ENS_PHIST_BIN_SIZE = 0.05 ENSEMBLE_STAT_CLIMO_CDF_BINS = 1 ENSEMBLE_STAT_CLIMO_CDF_CENTER_BINS = False -ENSEMBLE_STAT_CLIMO_CDF_WRITE_BINS = True +ENSEMBLE_STAT_CLIMO_CDF_WRITE_BINS = False ENSEMBLE_STAT_MASK_GRID = FULL @@ -170,35 +170,21 @@ ENSEMBLE_STAT_OUTPUT_FLAG_ORANK = STAT ENSEMBLE_STAT_OUTPUT_FLAG_SSVAR = STAT ENSEMBLE_STAT_OUTPUT_FLAG_RELP = STAT -ENSEMBLE_STAT_ENSEMBLE_FLAG_LATLON = TRUE -ENSEMBLE_STAT_ENSEMBLE_FLAG_MEAN = TRUE -ENSEMBLE_STAT_ENSEMBLE_FLAG_STDEV = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_LATLON = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_MEAN = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_STDEV = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_MINUS = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_PLUS = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_MIN = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_MAX = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_RANGE = FALSE -ENSEMBLE_STAT_ENSEMBLE_FLAG_VLD_COUNT = TRUE -ENSEMBLE_STAT_ENSEMBLE_FLAG_FREQUENCY = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_VLD_COUNT = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_FREQUENCY = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_NEP = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_NMEP = FALSE -ENSEMBLE_STAT_ENSEMBLE_FLAG_RANK = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_RANK = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_WEIGHT = FALSE # -# Ensemble variables and levels as specified in the ens field dictionary -# of the MET configuration file. Specify as ENS_VARn_NAME, ENS_VARn_LEVELS, -# (optional) ENS_VARn_OPTION. -# -# ENS_VARn_THRESH must be included here in order for the netCDF file -# generated by EnsembleStat to include the ensemble probability (aka -# ensembe frequency) fields corresponding to the specified thresholds. -# These must be included because the corresonding GridStat task for -# ensemble probabilities expects to read them in as input. -# -ENS_VAR1_NAME = {{fieldname_in_fcst_input}} -ENS_VAR1_LEVELS = L0 -ENS_VAR1_THRESH = {{field_thresholds}} -# # Forecast and observation variables and levels as specified in the fcst # field dictionary of the MET configuration file. Specify as FCST_VARn_NAME, # FCST_VARn_LEVELS, (optional) FCST_VARn_OPTION and OBS_VARn_NAME, diff --git a/parm/metplus/EnsembleStat_RETOP.conf b/parm/metplus/EnsembleStat_RETOP.conf index 28094c1cd5..00bf62b0a6 100644 --- a/parm/metplus/EnsembleStat_RETOP.conf +++ b/parm/metplus/EnsembleStat_RETOP.conf @@ -150,7 +150,7 @@ ENSEMBLE_STAT_ENS_PHIST_BIN_SIZE = 0.05 ENSEMBLE_STAT_CLIMO_CDF_BINS = 1 ENSEMBLE_STAT_CLIMO_CDF_CENTER_BINS = False -ENSEMBLE_STAT_CLIMO_CDF_WRITE_BINS = True +ENSEMBLE_STAT_CLIMO_CDF_WRITE_BINS = False ENSEMBLE_STAT_MASK_GRID = FULL @@ -170,37 +170,21 @@ ENSEMBLE_STAT_OUTPUT_FLAG_ORANK = STAT ENSEMBLE_STAT_OUTPUT_FLAG_SSVAR = STAT ENSEMBLE_STAT_OUTPUT_FLAG_RELP = STAT -ENSEMBLE_STAT_ENSEMBLE_FLAG_LATLON = TRUE -ENSEMBLE_STAT_ENSEMBLE_FLAG_MEAN = TRUE -ENSEMBLE_STAT_ENSEMBLE_FLAG_STDEV = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_LATLON = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_MEAN = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_STDEV = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_MINUS = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_PLUS = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_MIN = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_MAX = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_RANGE = FALSE -ENSEMBLE_STAT_ENSEMBLE_FLAG_VLD_COUNT = TRUE -ENSEMBLE_STAT_ENSEMBLE_FLAG_FREQUENCY = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_VLD_COUNT = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_FREQUENCY = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_NEP = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_NMEP = FALSE -ENSEMBLE_STAT_ENSEMBLE_FLAG_RANK = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_RANK = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_WEIGHT = FALSE # -# Ensemble variables and levels as specified in the ens field dictionary -# of the MET configuration file. Specify as ENS_VARn_NAME, ENS_VARn_LEVELS, -# (optional) ENS_VARn_OPTION. -# -# ENS_VARn_THRESH must be included here in order for the netCDF file -# generated by EnsembleStat to include the ensemble probability (aka -# ensembe frequency) fields corresponding to the specified thresholds. -# These must be included because the corresonding GridStat task for -# ensemble probabilities expects to read them in as input. -# -ENS_VAR1_NAME = {{fieldname_in_fcst_input}} -ENS_VAR1_LEVELS = L0 -ENS_VAR1_THRESH = {{field_thresholds}} -# convert from meters to kilofeet: M_to_KFT(x) = x * 3.28084 * 0.001; -ENS_VAR1_OPTIONS = convert(x) = x * 3.28084 * 0.001; -# # Forecast and observation variables and levels as specified in the fcst # field dictionary of the MET configuration file. Specify as FCST_VARn_NAME, # FCST_VARn_LEVELS, (optional) FCST_VARn_OPTION and OBS_VARn_NAME, diff --git a/parm/metplus/EnsembleStat_SFC.conf b/parm/metplus/EnsembleStat_SFC.conf index 1cb6e4f11e..0bfa03ab1d 100644 --- a/parm/metplus/EnsembleStat_SFC.conf +++ b/parm/metplus/EnsembleStat_SFC.conf @@ -151,7 +151,7 @@ ENSEMBLE_STAT_ENS_PHIST_BIN_SIZE = 0.05 #ENSEMBLE_STAT_CLIMO_CDF_BINS = 1 #ENSEMBLE_STAT_CLIMO_CDF_CENTER_BINS = False -#ENSEMBLE_STAT_CLIMO_CDF_WRITE_BINS = True +ENSEMBLE_STAT_CLIMO_CDF_WRITE_BINS = False ENSEMBLE_STAT_MASK_GRID = @@ -171,53 +171,21 @@ ENSEMBLE_STAT_OUTPUT_FLAG_ORANK = STAT ENSEMBLE_STAT_OUTPUT_FLAG_SSVAR = STAT ENSEMBLE_STAT_OUTPUT_FLAG_RELP = STAT -ENSEMBLE_STAT_ENSEMBLE_FLAG_LATLON = TRUE -ENSEMBLE_STAT_ENSEMBLE_FLAG_MEAN = TRUE -ENSEMBLE_STAT_ENSEMBLE_FLAG_STDEV = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_LATLON = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_MEAN = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_STDEV = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_MINUS = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_PLUS = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_MIN = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_MAX = FALSE -ENSEMBLE_STAT_ENSEMBLE_FLAG_RANGE = TRUE -ENSEMBLE_STAT_ENSEMBLE_FLAG_VLD_COUNT = TRUE -ENSEMBLE_STAT_ENSEMBLE_FLAG_FREQUENCY = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_RANGE = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_VLD_COUNT = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_FREQUENCY = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_NEP = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_NMEP = FALSE -ENSEMBLE_STAT_ENSEMBLE_FLAG_RANK = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_RANK = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_WEIGHT = FALSE # -# Ensemble variables and levels as specified in the ens field dictionary -# of the MET configuration file. Specify as ENS_VARn_NAME, ENS_VARn_LEVELS, -# (optional) ENS_VARn_OPTION. -# -ENS_VAR1_NAME = TMP -ENS_VAR1_LEVELS = Z02 -ENS_VAR1_THRESH = >=268, >=273, >=278, >=293, >=298, >=303 - -ENS_VAR2_NAME = DPT -ENS_VAR2_LEVELS = Z2 -ENS_VAR2_THRESH = >=263, >=268, >=273, >=288, >=293, >=298 - -ENS_VAR3_NAME = WIND -ENS_VAR3_LEVELS = Z10 -ENS_VAR3_THRESH = >=5, >=10, >=15 -ENS_VAR3_OPTIONS = GRIB2_pdt = 0; ;; derive instantaneous 10-m wind from U/V components, overriding max 10-m wind - -ENS_VAR4_NAME = TCDC -ENS_VAR4_LEVELS = L0 -ENS_VAR4_THRESH = <25, >75 -ENS_VAR4_OPTIONS = GRIB_lvl_typ = 200; GRIB2_ipdtmpl_index=[27]; GRIB2_ipdtmpl_val=[255]; interp = { type = [ { method = NEAREST; width = 1; } ]; } - -ENS_VAR5_NAME = VIS -ENS_VAR5_LEVELS = L0 -ENS_VAR5_THRESH = <1609, <8045, >=8045 -ENS_VAR5_OPTIONS = interp = { type = [ { method = NEAREST; width = 1; } ]; } - -ENS_VAR6_NAME = HGT -ENS_VAR6_LEVELS = L0 -ENS_VAR6_THRESH = <152, <1520, >=914 -ENS_VAR6_OPTIONS = GRIB_lvl_typ = 215; desc = "CEILING"; -# # Forecast and observation variables and levels as specified in the fcst # field dictionary of the MET configuration file. Specify as FCST_VARn_NAME, # FCST_VARn_LEVELS, (optional) FCST_VARn_OPTION and OBS_VARn_NAME, diff --git a/parm/metplus/EnsembleStat_UPA.conf b/parm/metplus/EnsembleStat_UPA.conf index 40f11f2d59..92e27272ad 100644 --- a/parm/metplus/EnsembleStat_UPA.conf +++ b/parm/metplus/EnsembleStat_UPA.conf @@ -151,7 +151,7 @@ ENSEMBLE_STAT_ENS_PHIST_BIN_SIZE = 0.05 #ENSEMBLE_STAT_CLIMO_CDF_BINS = 1 #ENSEMBLE_STAT_CLIMO_CDF_CENTER_BINS = False -#ENSEMBLE_STAT_CLIMO_CDF_WRITE_BINS = True +ENSEMBLE_STAT_CLIMO_CDF_WRITE_BINS = False ENSEMBLE_STAT_MASK_GRID = @@ -171,74 +171,21 @@ ENSEMBLE_STAT_OUTPUT_FLAG_ORANK = STAT ENSEMBLE_STAT_OUTPUT_FLAG_SSVAR = STAT ENSEMBLE_STAT_OUTPUT_FLAG_RELP = STAT -ENSEMBLE_STAT_ENSEMBLE_FLAG_LATLON = TRUE -ENSEMBLE_STAT_ENSEMBLE_FLAG_MEAN = TRUE -ENSEMBLE_STAT_ENSEMBLE_FLAG_STDEV = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_LATLON = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_MEAN = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_STDEV = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_MINUS = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_PLUS = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_MIN = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_MAX = FALSE -ENSEMBLE_STAT_ENSEMBLE_FLAG_RANGE = TRUE -ENSEMBLE_STAT_ENSEMBLE_FLAG_VLD_COUNT = TRUE -ENSEMBLE_STAT_ENSEMBLE_FLAG_FREQUENCY = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_RANGE = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_VLD_COUNT = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_FREQUENCY = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_NEP = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_NMEP = FALSE -ENSEMBLE_STAT_ENSEMBLE_FLAG_RANK = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_RANK = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_WEIGHT = FALSE # -# Ensemble variables and levels as specified in the ens field dictionary -# of the MET configuration file. Specify as ENS_VARn_NAME, ENS_VARn_LEVELS, -# (optional) ENS_VARn_OPTION. -# -ENS_VAR1_NAME = TMP -ENS_VAR1_LEVELS = P850 -ENS_VAR1_THRESH = >=288, >=293, >=298 - -ENS_VAR2_NAME = TMP -ENS_VAR2_LEVELS = P700 -ENS_VAR2_THRESH = >=273, >=278, >=283 - -ENS_VAR3_NAME = TMP -ENS_VAR3_LEVELS = P500 -ENS_VAR3_THRESH = >=258, >=263, >=268 - -ENS_VAR4_NAME = DPT -ENS_VAR4_LEVELS = P850 -ENS_VAR4_THRESH = >=273, >=278, >=283 - -ENS_VAR5_NAME = DPT -ENS_VAR5_LEVELS = P700 -ENS_VAR5_THRESH = >=263, >=268, >=273 - -ENS_VAR6_NAME = WIND -ENS_VAR6_LEVELS = P850 -ENS_VAR6_THRESH = >=5, >=10, >=15 - -ENS_VAR7_NAME = WIND -ENS_VAR7_LEVELS = P700 -ENS_VAR7_THRESH = >=10, >=15, >=20 - -ENS_VAR8_NAME = WIND -ENS_VAR8_LEVELS = P500 -ENS_VAR8_THRESH = >=15, >=21, >=26 - -ENS_VAR9_NAME = WIND -ENS_VAR9_LEVELS = P250 -ENS_VAR9_THRESH = >=26, >=31, >=36, >=46, >=62 - -ENS_VAR10_NAME = HGT -ENS_VAR10_LEVELS = P500 -ENS_VAR10_THRESH = >=5400, >=5600, >=5880 - -ENS_VAR11_NAME = CAPE -ENS_VAR11_LEVELS = L0 -ENS_VAR11_OPTIONS = cnt_thresh = [ >0 ]; -ENS_VAR11_THRESH = <=1000, >1000&&<2500, >2500&&<4000, >2500 - -ENS_VAR12_NAME = HPBL -ENS_VAR12_LEVELS = Z0 -ENS_VAR12_THRESH = <500, <1500, >1500 -# # Forecast and observation variables and levels as specified in the fcst # field dictionary of the MET configuration file. Specify as FCST_VARn_NAME, # FCST_VARn_LEVELS, (optional) FCST_VARn_OPTION and OBS_VARn_NAME, From 9d9c2c0faad38acc0b966e41800146ff5bdd97ff Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Fri, 24 Mar 2023 08:37:06 -0600 Subject: [PATCH 08/43] Change input to ensemble mean and probabilistic tasks to come from the output of GenEnsProd (instead of EnsembleStat, which no longer generates the necessary outputs). --- ...nal_run_met_gridstat_or_pointstat_vx_ensmean.sh | 14 ++++---------- ...nal_run_met_gridstat_or_pointstat_vx_ensprob.sh | 14 ++++---------- 2 files changed, 8 insertions(+), 20 deletions(-) diff --git a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh index 15c2e34a56..d94480271b 100755 --- a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh +++ b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh @@ -177,21 +177,15 @@ if [ "${grid_or_point}" = "grid" ]; then esac OBS_INPUT_FN_TEMPLATE=$( eval echo ${OBS_INPUT_FN_TEMPLATE} ) fi -# Keep for when splitting GenEnsProd from EnsembleStat. -# FCST_INPUT_DIR="${vx_output_basedir}${slash_cdate_or_null}/metprd/gen_ens_prod" -# FCST_INPUT_FN_TEMPLATE=$( eval echo 'gen_ens_prod_${VX_FCST_MODEL_NAME}_${FIELDNAME_IN_MET_FILEDIR_NAMES}_${OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V.nc' ) - FCST_INPUT_DIR="${vx_output_basedir}${slash_cdate_or_null}/metprd/EnsembleStat" - FCST_INPUT_FN_TEMPLATE=$( eval echo 'ensemble_stat_${VX_FCST_MODEL_NAME}_${FIELDNAME_IN_MET_FILEDIR_NAMES}_${OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc' ) + FCST_INPUT_DIR="${vx_output_basedir}${slash_cdate_or_null}/metprd/GenEnsProd" + FCST_INPUT_FN_TEMPLATE=$( eval echo 'gen_ens_prod_${VX_FCST_MODEL_NAME}_${FIELDNAME_IN_MET_FILEDIR_NAMES}_${OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V.nc' ) elif [ "${grid_or_point}" = "point" ]; then OBS_INPUT_DIR="${vx_output_basedir}/metprd/Pb2nc_obs" OBS_INPUT_FN_TEMPLATE=$( eval echo ${OBS_NDAS_SFCorUPA_FN_METPROC_TEMPLATE} ) -# Keep for when splitting GenEnsProd from EnsembleStat. -# FCST_INPUT_DIR="${vx_output_basedir}${slash_cdate_or_null}/metprd/gen_ens_prod" -# FCST_INPUT_FN_TEMPLATE=$( eval echo 'gen_ens_prod_${VX_FCST_MODEL_NAME}_ADP${FIELDNAME_IN_MET_FILEDIR_NAMES}_${OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V.nc' ) - FCST_INPUT_DIR="${vx_output_basedir}${slash_cdate_or_null}/metprd/EnsembleStat" - FCST_INPUT_FN_TEMPLATE=$( eval echo 'ensemble_stat_${VX_FCST_MODEL_NAME}_ADP${FIELDNAME_IN_MET_FILEDIR_NAMES}_${OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc' ) + FCST_INPUT_DIR="${vx_output_basedir}${slash_cdate_or_null}/metprd/GenEnsProd" + FCST_INPUT_FN_TEMPLATE=$( eval echo 'gen_ens_prod_${VX_FCST_MODEL_NAME}_ADP${FIELDNAME_IN_MET_FILEDIR_NAMES}_${OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V.nc' ) fi diff --git a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh index fb33760892..f17709729a 100755 --- a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh +++ b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh @@ -136,21 +136,15 @@ if [ "${grid_or_point}" = "grid" ]; then esac OBS_INPUT_FN_TEMPLATE=$( eval echo ${OBS_INPUT_FN_TEMPLATE} ) fi -# Keep for when splitting GenEnsProd from EnsembleStat. -# FCST_INPUT_DIR="${vx_output_basedir}${slash_cdate_or_null}/metprd/gen_ens_prod" -# FCST_INPUT_FN_TEMPLATE=$( eval echo 'gen_ens_prod_${VX_FCST_MODEL_NAME}_${FIELDNAME_IN_MET_FILEDIR_NAMES}_${OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V.nc' ) - FCST_INPUT_DIR="${vx_output_basedir}${slash_cdate_or_null}/metprd/EnsembleStat" - FCST_INPUT_FN_TEMPLATE=$( eval echo 'ensemble_stat_${VX_FCST_MODEL_NAME}_${FIELDNAME_IN_MET_FILEDIR_NAMES}_${OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc' ) + FCST_INPUT_DIR="${vx_output_basedir}${slash_cdate_or_null}/metprd/GenEnsProd" + FCST_INPUT_FN_TEMPLATE=$( eval echo 'gen_ens_prod_${VX_FCST_MODEL_NAME}_${FIELDNAME_IN_MET_FILEDIR_NAMES}_${OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V.nc' ) elif [ "${grid_or_point}" = "point" ]; then OBS_INPUT_DIR="${vx_output_basedir}/metprd/Pb2nc_obs" OBS_INPUT_FN_TEMPLATE=$( eval echo ${OBS_NDAS_SFCorUPA_FN_METPROC_TEMPLATE} ) -# Keep for when splitting GenEnsProd from EnsembleStat. -# FCST_INPUT_DIR="${vx_output_basedir}${slash_cdate_or_null}/metprd/gen_ens_prod" -# FCST_INPUT_FN_TEMPLATE=$( eval echo 'gen_ens_prod_${VX_FCST_MODEL_NAME}_ADP${FIELDNAME_IN_MET_FILEDIR_NAMES}_${OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V.nc' ) - FCST_INPUT_DIR="${vx_output_basedir}${slash_cdate_or_null}/metprd/EnsembleStat" - FCST_INPUT_FN_TEMPLATE=$( eval echo 'ensemble_stat_${VX_FCST_MODEL_NAME}_ADP${FIELDNAME_IN_MET_FILEDIR_NAMES}_${OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc' ) + FCST_INPUT_DIR="${vx_output_basedir}${slash_cdate_or_null}/metprd/GenEnsProd" + FCST_INPUT_FN_TEMPLATE=$( eval echo 'gen_ens_prod_${VX_FCST_MODEL_NAME}_ADP${FIELDNAME_IN_MET_FILEDIR_NAMES}_${OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V.nc' ) fi From 822b633df60bd1cafa54d6a7ae44fed25d98bf51 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Wed, 17 May 2023 15:36:00 -0600 Subject: [PATCH 09/43] Rename metatasks/tasks for clarity; create new metatasks/tasks to make desired dependencies possible; remove unneeded variables; change vx WE2E test config files to be compatible with aforementioned changes. --- parm/wflow/verify.yaml | 143 +++++++++--------- parm/wflow/verify_ensgrid.yaml | 142 +++++++++-------- ...fig.MET_ensemble_verification_only_vx.yaml | 32 +++- .../config.MET_verification_only_vx.yaml | 12 +- 4 files changed, 192 insertions(+), 137 deletions(-) diff --git a/parm/wflow/verify.yaml b/parm/wflow/verify.yaml index 1a676d3fba..315c5b90bc 100644 --- a/parm/wflow/verify.yaml +++ b/parm/wflow/verify.yaml @@ -11,7 +11,6 @@ default_task_verify: &default_task subcyc: !cycstr "@M" LOGDIR: !cycstr "&LOGDIR;" nprocs: '{{ parent.nnodes * parent.ppn }}' - FHR: '{% for h in range(0, workflow.FCST_LEN_HRS+1) %}{{ " %02d" % h }}{% endfor %}' join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;' memory: '{% if user.MACHINE not in ["WCOSS2", "NOAACLOUD"] %}{{ "2G" }}{% endif %}' native: '{{ platform.SCHED_NATIVE_CMD }}' @@ -82,6 +81,76 @@ task_run_MET_Pb2nc_obs: attrs: task: get_obs_ndas +metatask_PcpCombine_obs: + var: + ACCUM_HR: '{% for ah in [1, 3, 6, 24] %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' + task_run_MET_PcpCombine_obs_APCP#ACCUM_HR#h: + <<: *default_task + attrs: + cycledefs: forecast + maxtries: '2' + command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_PCPCOMBINE"' + envars: + <<: *default_vars + VAR: APCP + ACCUM_HH: '#ACCUM_HR#' + obs_or_fcst: obs + OBTYPE: CCPA + OBS_DIR: '&CCPA_OBS_DIR;' + MET_TOOL: 'PCPCOMBINE' + dependency: + or: + not: + taskvalid: + attrs: + task: get_obs_ccpa + and: + taskdep: + attrs: + task: get_obs_ccpa + datadep: + text: "&CCPA_OBS_DIR;" + +metatask_PcpCombine_fcst: + var: + ACCUM_HR: '{% for ah in [1, 3, 6, 24] %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' + metatask_PcpCombine_fcst_APCP#ACCUM_HR#h_all_mems: + var: + mem: '{% if global.DO_ENSEMBLE %}{% for m in range(1, global.NUM_ENS_MEMBERS+1) %}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' + task_run_MET_PcpCombine_fcst_APCP#ACCUM_HR#h_mem#mem#: + <<: *default_task + attrs: + cycledefs: forecast + maxtries: '2' + command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_PCPCOMBINE"' + envars: + <<: *default_vars + VAR: APCP + ACCUM_HH: '#ACCUM_HR#' + obs_or_fcst: fcst + OBTYPE: CCPA + OBS_DIR: '&CCPA_OBS_DIR;' + MET_TOOL: 'PCPCOMBINE' + ENSMEM_INDX: "#mem#" + SLASH_ENSMEM_SUBDIR_OR_NULL: '{% if global.DO_ENSEMBLE %}{{ "/mem#mem#" }}{% endif %}' + dependency: + or_do_post: &post_files_exist + and_run_post: # If post was meant to run, wait on the whole post metatask + taskvalid: + attrs: + task: run_post_mem#mem#_f000 + metataskdep: + attrs: + metatask: run_ens_post + and_inline_post: # If inline post ran, wait on the forecast task to complete + not: + taskvalid: + attrs: + task: run_post_mem#mem#_f000 + taskdep: + attrs: + task: run_fcst_mem#mem# + metatask_vx_ens_member: var: mem: '{% if global.DO_ENSEMBLE %}{% for m in range(1, global.NUM_ENS_MEMBERS+1) %}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' @@ -115,24 +184,10 @@ metatask_vx_ens_member: taskdep: attrs: task: get_obs_mrms - or_do_post: &post_files_exist - and_run_post: # If post was meant to run, wait on the whole post metatask - taskvalid: - attrs: - task: run_post_mem#mem#_f000 - metataskdep: - attrs: - metatask: run_ens_post - and_inline_post: # If inline post ran, wait on the forecast task to complete - not: - taskvalid: - attrs: - task: run_post_mem#mem#_f000 - taskdep: - attrs: - task: run_fcst_mem#mem# + or_do_post: + <<: *post_files_exist - metatask_PointStat_mem#mem#: + metatask_PointStat_NDAS_mem#mem#: var: VAR: SFC UPA task_run_MET_PointStat_vx_#VAR#_mem#mem#: @@ -140,7 +195,6 @@ metatask_vx_ens_member: command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_GRIDSTAT_OR_POINTSTAT_VX"' envars: <<: *default_vars - FHR: '#FCST_HR_LIST#' OBS_DIR: '&NDAS_OBS_DIR;' VAR: '#VAR#' MET_TOOL: 'POINTSTAT' @@ -172,60 +226,10 @@ metatask_vx_ens_member: metatask_GridStat_APCP_acc: var: ACCUM_HR: '{% for ah in [1, 3, 6, 24] %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' - FCST_HR_LIST: '{% for ah in [1, 3, 6, 24] %}{% if workflow.FCST_LEN_HRS >= ah %}{% for h in range(ah, workflow.FCST_LEN_HRS+1, ah) %}{{ "%02d," % h }}{% endfor %}{% endif %} {% endfor %}' - - - task_run_MET_PcpCombine_obs_APCP#ACCUM_HR#h: - <<: *default_task - attrs: - cycledefs: forecast - maxtries: '2' - command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_PCPCOMBINE"' - envars: - <<: *default_vars - VAR: APCP - ACCUM_HH: '#ACCUM_HR#' - obs_or_fcst: obs - OBTYPE: CCPA - OBS_DIR: '&CCPA_OBS_DIR;' - MET_TOOL: 'PCPCOMBINE' - dependency: - or: - not: - taskvalid: - attrs: - task: get_obs_ccpa - and: - taskdep: - attrs: - task: get_obs_ccpa - datadep: - text: "&CCPA_OBS_DIR;" - metatask_vx_ens_member_acc: var: mem: '{% if global.DO_ENSEMBLE %}{% for m in range(1, global.NUM_ENS_MEMBERS+1) %}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' - task_run_MET_PcpCombine_fcst_APCP#ACCUM_HR#h_mem#mem#: - <<: *default_task - attrs: - cycledefs: forecast - maxtries: '2' - command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_PCPCOMBINE"' - envars: - <<: *default_vars - VAR: APCP - ACCUM_HH: '#ACCUM_HR#' - obs_or_fcst: fcst - OBTYPE: CCPA - OBS_DIR: '&CCPA_OBS_DIR;' - MET_TOOL: 'PCPCOMBINE' - ENSMEM_INDX: "#mem#" - SLASH_ENSMEM_SUBDIR_OR_NULL: '{% if global.DO_ENSEMBLE %}{{ "/mem#mem#" }}{% endif %}' - dependency: - or_do_post: - <<: *post_files_exist - task_run_MET_GridStat_vx_APCP#ACCUM_HR#h_mem#mem#: <<: *default_task attrs: @@ -233,7 +237,6 @@ metatask_GridStat_APCP_acc: command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_GRIDSTAT_OR_POINTSTAT_VX"' envars: <<: *default_vars - FHR: '#FCST_HR_LIST#' OBS_DIR: '&CCPA_OBS_DIR;' VAR: APCP ACCUM_HH: '#ACCUM_HR#' diff --git a/parm/wflow/verify_ensgrid.yaml b/parm/wflow/verify_ensgrid.yaml index 4f8dd65ddf..5a13fb2ba7 100644 --- a/parm/wflow/verify_ensgrid.yaml +++ b/parm/wflow/verify_ensgrid.yaml @@ -10,7 +10,6 @@ default_task_verify_ens: &default_task_verify_ens cyc: !cycstr "@H" subcyc: !cycstr "@M" LOGDIR: !cycstr "&LOGDIR;" - FHR: '{% for h in range(0, workflow.FCST_LEN_HRS+1) %}{{ " %02d" % h }}{% endfor %}' nprocs: '{{ parent.nnodes * parent.ppn }}' join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;' memory: '{% if user.MACHINE not in ["WCOSS2", "NOAACLOUD"] %}{{ "2G" }}{% endif %}' @@ -23,56 +22,18 @@ default_task_verify_ens: &default_task_verify_ens queue: '&QUEUE_DEFAULT;' walltime: 01:00:00 -metatask_EnsembleStat: - var: - VAR: REFC RETOP - task_run_MET_EnsembleStat_vx_#VAR#: - <<: *default_task_verify_ens - command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_GENENSPROD_OR_ENSEMBLESTAT"' - envars: - <<: *default_vars - ACCUM_HH: '01' - OBS_DIR: '&MRMS_OBS_DIR;' - VAR: '#VAR#' - MET_TOOL: 'ENSEMBLESTAT' - OBTYPE: 'MRMS' - dependency: - metataskdep: - attrs: - metatask: run_ens_post - -metatask_GridStat_ensprob: - var: - VAR: REFC RETOP - task_run_MET_GridStat_vx_ensprob_#VAR#: - <<: *default_task_verify_ens - command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_GRIDSTAT_OR_POINTSTAT_VX_ENSPROB"' - envars: - <<: *default_vars - ACCUM_HH: '01' - OBS_DIR: '&MRMS_OBS_DIR;' - VAR: '#VAR#' - MET_TOOL: 'GRIDSTAT' - OBTYPE: 'MRMS' - dependency: - taskdep: - attrs: - task: run_MET_EnsembleStat_vx_#VAR# - -metatask_EnsembleStat_acc: +metatask_GEPES_CCPA: var: ACCUM_HR: '{% for ah in [1, 3, 6, 24] %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' - FCST_HR_LIST: '{% for ah in [1, 3, 6, 24] %}{% if workflow.FCST_LEN_HRS >= ah %}{% for h in range(ah, workflow.FCST_LEN_HRS+1, ah) %}{{ "%02d," % h }}{% endfor %}{% endif %} {% endfor %}' - task_run_MET_EnsembleStat_vx_APCP#ACCUM_HR#h: + task_run_MET_GenEnsProd_vx_APCP#ACCUM_HR#h: &task_GenEnsProd_CCPA <<: *default_task_verify_ens command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_GENENSPROD_OR_ENSEMBLESTAT"' - envars: + envars: &envars_GenEnsProd_CCPA <<: *default_vars ACCUM_HH: '#ACCUM_HR#' - FHR: '#FCST_HR_LIST#' OBS_DIR: '&CCPA_OBS_DIR;' VAR: APCP - MET_TOOL: 'ENSEMBLESTAT' + MET_TOOL: 'GENENSPROD' OBTYPE: 'CCPA' dependency: and: @@ -94,7 +55,70 @@ metatask_EnsembleStat_acc: metatask: run_ensemble metataskdep: attrs: - metatask: GridStat_APCP_acc + metatask: PcpCombine_fcst + task_run_MET_EnsembleStat_vx_APCP#ACCUM_HR#h: + <<: *task_GenEnsProd_CCPA + envars: + <<: *envars_GenEnsProd_CCPA + MET_TOOL: 'ENSEMBLESTAT' + dependency: + and: + taskdep: + attrs: + task: run_MET_PcpCombine_obs_APCP#ACCUM_HR#h + taskdep: + attrs: + task: run_MET_GenEnsProd_vx_APCP#ACCUM_HR#h + +metatask_GEPES_MRMS: + var: + VAR: REFC RETOP + task_run_MET_GenEnsProd_vx_#VAR#: &task_GenEnsProd_MRMS + <<: *default_task_verify_ens + command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_GENENSPROD_OR_ENSEMBLESTAT"' + envars: &envars_GenEnsProd_MRMS + <<: *default_vars + ACCUM_HH: '01' + OBS_DIR: '&MRMS_OBS_DIR;' + VAR: '#VAR#' + MET_TOOL: 'GENENSPROD' + OBTYPE: 'MRMS' + dependency: + metataskdep: + attrs: + metatask: run_ens_post + task_run_MET_EnsembleStat_vx_#VAR#: + <<: *task_GenEnsProd_MRMS + envars: + <<: *envars_GenEnsProd_MRMS + MET_TOOL: 'ENSEMBLESTAT' + +metatask_GEPES_NDAS: + var: + VAR: SFC UPA + task_run_MET_GenEnsProd_vx_#VAR#: &task_GenEnsProd_NDAS + <<: *default_task_verify_ens + command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_GENENSPROD_OR_ENSEMBLESTAT"' + envars: &envars_GenEnsProd_NDAS + <<: *default_vars + OBS_DIR: '&NDAS_OBS_DIR;' + VAR: '#VAR#' + MET_TOOL: 'GENENSPROD' + OBTYPE: 'NDAS' + ACCUM_HH: '01' + dependency: + and: + metataskdep: + attrs: + metatask: run_ens_post + taskdep: + attrs: + task: run_MET_Pb2nc_obs + task_run_MET_EnsembleStat_vx_#VAR#: + <<: *task_GenEnsProd_NDAS + envars: + <<: *envars_GenEnsProd_NDAS + MET_TOOL: 'ENSEMBLESTAT' metatask_ensemble_vx_stats: var: @@ -103,18 +127,16 @@ metatask_ensemble_vx_stats: metatask_GridStat_ens_acc: var: ACCUM_HR: '{% for ah in [1, 3, 6, 24] %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' - FCST_HR_LIST: '{% for ah in [1, 3, 6, 24] %}{% if workflow.FCST_LEN_HRS >= ah %}{% for h in range(ah, workflow.FCST_LEN_HRS+1, ah) %}{{ "%02d," % h }}{% endfor %}{% endif %} {% endfor %}' task_run_MET_GridStat_vx_ens#statlc#_APCP#ACCUM_HR#h: <<: *default_task_verify_ens command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_GRIDSTAT_OR_POINTSTAT_VX_ENS#stat#"' envars: <<: *default_vars - ACCUM_HH: '#ACCUM_HR#' - FHR: '#FCST_HR_LIST#' OBS_DIR: '&CCPA_OBS_DIR;' VAR: APCP MET_TOOL: 'GRIDSTAT' OBTYPE: 'CCPA' + ACCUM_HH: '#ACCUM_HR#' dependency: taskdep: attrs: @@ -137,24 +159,20 @@ metatask_ensemble_vx_stats: attrs: task: run_MET_EnsembleStat_vx_#VAR# -metatask_EnsembleStat_point: +metatask_GridStat_ensprob: var: - VAR: SFC UPA - task_run_MET_EnsembleStat_vx_#VAR#: + VAR: REFC RETOP + task_run_MET_GridStat_vx_ensprob_#VAR#: <<: *default_task_verify_ens - command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_GENENSPROD_OR_ENSEMBLESTAT"' + command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_GRIDSTAT_OR_POINTSTAT_VX_ENSPROB"' envars: <<: *default_vars - OBS_DIR: '&NDAS_OBS_DIR;' - VAR: '#VAR#' - MET_TOOL: 'ENSEMBLESTAT' - OBTYPE: 'NDAS' ACCUM_HH: '01' + OBS_DIR: '&MRMS_OBS_DIR;' + VAR: '#VAR#' + MET_TOOL: 'GRIDSTAT' + OBTYPE: 'MRMS' dependency: - and: - metataskdep: - attrs: - metatask: run_ens_post - taskdep: - attrs: - task: run_MET_Pb2nc_obs + taskdep: + attrs: + task: run_MET_EnsembleStat_vx_#VAR# diff --git a/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml b/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml index 0c545761ab..dc27ed7529 100644 --- a/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml +++ b/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml @@ -21,34 +21,58 @@ rocoto: task_get_obs_ccpa: task_get_obs_mrms: task_get_obs_ndas: + metatask_vx_ens_member: metatask_GridStat_MRMS_mem#mem#: task_run_MET_GridStat_vx_#VAR#_mem#mem#: dependency: - metatask_PointStat_mem#mem#: + metatask_PointStat_NDAS_mem#mem#: task_run_MET_PointStat_vx_#VAR#_mem#mem#: dependency: and: or_do_post: or_get_obs: + metatask_PcpCombine_fcst: + metatask_PcpCombine_fcst_APCP#ACCUM_HR#h_all_mems: + task_run_MET_PcpCombine_fcst_APCP#ACCUM_HR#h_mem#mem#: + dependency: + metatask_GridStat_APCP_acc: metatask_vx_ens_member_acc: task_run_MET_PcpCombine_fcst_APCP#ACCUM_HR#h_mem#mem#: dependency: - metatask_EnsembleStat_acc: + + metatask_GEPES_CCPA: task_run_MET_GridStat_vx_ensprob_#VAR#: dependency: + task_run_MET_GenEnsProd_vx_APCP#ACCUM_HR#h: + dependency: + and: + metataskdep: + attrs: + metatask: GridStat_APCP_acc task_run_MET_EnsembleStat_vx_APCP#ACCUM_HR#h: dependency: and: metataskdep: attrs: metatask: GridStat_APCP_acc - metatask_EnsembleStat: + + metatask_GEPES_MRMS: + task_run_MET_GenEnsProd_vx_#VAR#: + dependency: task_run_MET_EnsembleStat_vx_#VAR#: dependency: - metatask_EnsembleStat_point: + + metatask_GEPES_NDAS: + task_run_MET_GenEnsProd_vx_#VAR#: + dependency: + and: + metataskdep: + taskdep: + attrs: + task: run_MET_Pb2nc_obs task_run_MET_EnsembleStat_vx_#VAR#: dependency: and: diff --git a/tests/WE2E/test_configs/verification/config.MET_verification_only_vx.yaml b/tests/WE2E/test_configs/verification/config.MET_verification_only_vx.yaml index 57d1ca3d53..d10a099aba 100644 --- a/tests/WE2E/test_configs/verification/config.MET_verification_only_vx.yaml +++ b/tests/WE2E/test_configs/verification/config.MET_verification_only_vx.yaml @@ -25,11 +25,21 @@ rocoto: task_get_obs_ccpa: task_get_obs_mrms: task_get_obs_ndas: + metatask_PcpCombine_obs: + task_run_MET_PcpCombine_obs_APCP#ACCUM_HR#h: + dependency: + or: + datadep: + text: "&CCPA_OBS_DIR;" + metatask_PcpCombine_fcst: + metatask_PcpCombine_fcst_APCP#ACCUM_HR#h_all_mems: + task_run_MET_PcpCombine_fcst_APCP#ACCUM_HR#h_mem#mem#: + dependency: metatask_vx_ens_member: metatask_GridStat_MRMS_mem#mem#: task_run_MET_GridStat_vx_#VAR#_mem#mem#: dependency: - metatask_PointStat_mem#mem#: + metatask_PointStat_NDAS_mem#mem#: task_run_MET_PointStat_vx_#VAR#_mem#mem#: dependency: and: From 8fb28e70aefb269c94a3a3ab273f74b216d3c451 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Wed, 17 May 2023 15:57:08 -0600 Subject: [PATCH 10/43] Add config variable VX_APCP_ACCUMS_HH to make the accumulation hours for which to perform vx user-specifiable. --- parm/wflow/verify.yaml | 24 +++++++++---------- parm/wflow/verify_ensgrid.yaml | 20 ++++++++-------- ...fig.MET_ensemble_verification_only_vx.yaml | 10 ++++---- .../config.MET_verification_only_vx.yaml | 8 +++---- ush/config_defaults.yaml | 7 ++++++ 5 files changed, 38 insertions(+), 31 deletions(-) diff --git a/parm/wflow/verify.yaml b/parm/wflow/verify.yaml index 315c5b90bc..b238a7eb6e 100644 --- a/parm/wflow/verify.yaml +++ b/parm/wflow/verify.yaml @@ -83,8 +83,8 @@ task_run_MET_Pb2nc_obs: metatask_PcpCombine_obs: var: - ACCUM_HR: '{% for ah in [1, 3, 6, 24] %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' - task_run_MET_PcpCombine_obs_APCP#ACCUM_HR#h: + ACCUM_HH: '{% for ah in verification.VX_APCP_ACCUMS_HH %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' + task_run_MET_PcpCombine_obs_APCP#ACCUM_HH#h: <<: *default_task attrs: cycledefs: forecast @@ -93,7 +93,7 @@ metatask_PcpCombine_obs: envars: <<: *default_vars VAR: APCP - ACCUM_HH: '#ACCUM_HR#' + ACCUM_HH: '#ACCUM_HH#' obs_or_fcst: obs OBTYPE: CCPA OBS_DIR: '&CCPA_OBS_DIR;' @@ -113,11 +113,11 @@ metatask_PcpCombine_obs: metatask_PcpCombine_fcst: var: - ACCUM_HR: '{% for ah in [1, 3, 6, 24] %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' - metatask_PcpCombine_fcst_APCP#ACCUM_HR#h_all_mems: + ACCUM_HH: '{% for ah in verification.VX_APCP_ACCUMS_HH %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' + metatask_PcpCombine_fcst_APCP#ACCUM_HH#h_all_mems: var: mem: '{% if global.DO_ENSEMBLE %}{% for m in range(1, global.NUM_ENS_MEMBERS+1) %}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' - task_run_MET_PcpCombine_fcst_APCP#ACCUM_HR#h_mem#mem#: + task_run_MET_PcpCombine_fcst_APCP#ACCUM_HH#h_mem#mem#: <<: *default_task attrs: cycledefs: forecast @@ -126,7 +126,7 @@ metatask_PcpCombine_fcst: envars: <<: *default_vars VAR: APCP - ACCUM_HH: '#ACCUM_HR#' + ACCUM_HH: '#ACCUM_HH#' obs_or_fcst: fcst OBTYPE: CCPA OBS_DIR: '&CCPA_OBS_DIR;' @@ -225,12 +225,12 @@ metatask_vx_ens_member: metatask_GridStat_APCP_acc: var: - ACCUM_HR: '{% for ah in [1, 3, 6, 24] %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' + ACCUM_HH: '{% for ah in verification.VX_APCP_ACCUMS_HH %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' metatask_vx_ens_member_acc: var: mem: '{% if global.DO_ENSEMBLE %}{% for m in range(1, global.NUM_ENS_MEMBERS+1) %}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' - task_run_MET_GridStat_vx_APCP#ACCUM_HR#h_mem#mem#: + task_run_MET_GridStat_vx_APCP#ACCUM_HH#h_mem#mem#: <<: *default_task attrs: maxtries: '2' @@ -239,7 +239,7 @@ metatask_GridStat_APCP_acc: <<: *default_vars OBS_DIR: '&CCPA_OBS_DIR;' VAR: APCP - ACCUM_HH: '#ACCUM_HR#' + ACCUM_HH: '#ACCUM_HH#' MET_TOOL: 'GRIDSTAT' OBTYPE: 'CCPA' ENSMEM_INDX: "#mem#" @@ -249,7 +249,7 @@ metatask_GridStat_APCP_acc: and: taskdep_obs: attrs: - task: run_MET_PcpCombine_obs_APCP#ACCUM_HR#h + task: run_MET_PcpCombine_obs_APCP#ACCUM_HH#h taskdep_fcst: attrs: - task: run_MET_PcpCombine_fcst_APCP#ACCUM_HR#h_mem#mem# + task: run_MET_PcpCombine_fcst_APCP#ACCUM_HH#h_mem#mem# diff --git a/parm/wflow/verify_ensgrid.yaml b/parm/wflow/verify_ensgrid.yaml index 5a13fb2ba7..607dbcfb89 100644 --- a/parm/wflow/verify_ensgrid.yaml +++ b/parm/wflow/verify_ensgrid.yaml @@ -24,13 +24,13 @@ default_task_verify_ens: &default_task_verify_ens metatask_GEPES_CCPA: var: - ACCUM_HR: '{% for ah in [1, 3, 6, 24] %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' - task_run_MET_GenEnsProd_vx_APCP#ACCUM_HR#h: &task_GenEnsProd_CCPA + ACCUM_HH: '{% for ah in verification.VX_APCP_ACCUMS_HH %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' + task_run_MET_GenEnsProd_vx_APCP#ACCUM_HH#h: &task_GenEnsProd_CCPA <<: *default_task_verify_ens command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_GENENSPROD_OR_ENSEMBLESTAT"' envars: &envars_GenEnsProd_CCPA <<: *default_vars - ACCUM_HH: '#ACCUM_HR#' + ACCUM_HH: '#ACCUM_HH#' OBS_DIR: '&CCPA_OBS_DIR;' VAR: APCP MET_TOOL: 'GENENSPROD' @@ -56,7 +56,7 @@ metatask_GEPES_CCPA: metataskdep: attrs: metatask: PcpCombine_fcst - task_run_MET_EnsembleStat_vx_APCP#ACCUM_HR#h: + task_run_MET_EnsembleStat_vx_APCP#ACCUM_HH#h: <<: *task_GenEnsProd_CCPA envars: <<: *envars_GenEnsProd_CCPA @@ -65,10 +65,10 @@ metatask_GEPES_CCPA: and: taskdep: attrs: - task: run_MET_PcpCombine_obs_APCP#ACCUM_HR#h + task: run_MET_PcpCombine_obs_APCP#ACCUM_HH#h taskdep: attrs: - task: run_MET_GenEnsProd_vx_APCP#ACCUM_HR#h + task: run_MET_GenEnsProd_vx_APCP#ACCUM_HH#h metatask_GEPES_MRMS: var: @@ -126,8 +126,8 @@ metatask_ensemble_vx_stats: statlc: mean prob metatask_GridStat_ens_acc: var: - ACCUM_HR: '{% for ah in [1, 3, 6, 24] %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' - task_run_MET_GridStat_vx_ens#statlc#_APCP#ACCUM_HR#h: + ACCUM_HH: '{% for ah in verification.VX_APCP_ACCUMS_HH %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' + task_run_MET_GridStat_vx_ens#statlc#_APCP#ACCUM_HH#h: <<: *default_task_verify_ens command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_GRIDSTAT_OR_POINTSTAT_VX_ENS#stat#"' envars: @@ -136,11 +136,11 @@ metatask_ensemble_vx_stats: VAR: APCP MET_TOOL: 'GRIDSTAT' OBTYPE: 'CCPA' - ACCUM_HH: '#ACCUM_HR#' + ACCUM_HH: '#ACCUM_HH#' dependency: taskdep: attrs: - task: run_MET_EnsembleStat_vx_APCP#ACCUM_HR#h + task: run_MET_EnsembleStat_vx_APCP#ACCUM_HH#h metatask_PointStat_ens: var: VAR: SFC UPA diff --git a/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml b/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml index dc27ed7529..89b4182723 100644 --- a/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml +++ b/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml @@ -34,25 +34,25 @@ rocoto: or_get_obs: metatask_PcpCombine_fcst: - metatask_PcpCombine_fcst_APCP#ACCUM_HR#h_all_mems: - task_run_MET_PcpCombine_fcst_APCP#ACCUM_HR#h_mem#mem#: + metatask_PcpCombine_fcst_APCP#ACCUM_HH#h_all_mems: + task_run_MET_PcpCombine_fcst_APCP#ACCUM_HH#h_mem#mem#: dependency: metatask_GridStat_APCP_acc: metatask_vx_ens_member_acc: - task_run_MET_PcpCombine_fcst_APCP#ACCUM_HR#h_mem#mem#: + task_run_MET_PcpCombine_fcst_APCP#ACCUM_HH#h_mem#mem#: dependency: metatask_GEPES_CCPA: task_run_MET_GridStat_vx_ensprob_#VAR#: dependency: - task_run_MET_GenEnsProd_vx_APCP#ACCUM_HR#h: + task_run_MET_GenEnsProd_vx_APCP#ACCUM_HH#h: dependency: and: metataskdep: attrs: metatask: GridStat_APCP_acc - task_run_MET_EnsembleStat_vx_APCP#ACCUM_HR#h: + task_run_MET_EnsembleStat_vx_APCP#ACCUM_HH#h: dependency: and: metataskdep: diff --git a/tests/WE2E/test_configs/verification/config.MET_verification_only_vx.yaml b/tests/WE2E/test_configs/verification/config.MET_verification_only_vx.yaml index d10a099aba..48e6166832 100644 --- a/tests/WE2E/test_configs/verification/config.MET_verification_only_vx.yaml +++ b/tests/WE2E/test_configs/verification/config.MET_verification_only_vx.yaml @@ -26,14 +26,14 @@ rocoto: task_get_obs_mrms: task_get_obs_ndas: metatask_PcpCombine_obs: - task_run_MET_PcpCombine_obs_APCP#ACCUM_HR#h: + task_run_MET_PcpCombine_obs_APCP#ACCUM_HH#h: dependency: or: datadep: text: "&CCPA_OBS_DIR;" metatask_PcpCombine_fcst: - metatask_PcpCombine_fcst_APCP#ACCUM_HR#h_all_mems: - task_run_MET_PcpCombine_fcst_APCP#ACCUM_HR#h_mem#mem#: + metatask_PcpCombine_fcst_APCP#ACCUM_HH#h_all_mems: + task_run_MET_PcpCombine_fcst_APCP#ACCUM_HH#h_mem#mem#: dependency: metatask_vx_ens_member: metatask_GridStat_MRMS_mem#mem#: @@ -47,7 +47,7 @@ rocoto: or_get_obs: metatask_GridStat_APCP_acc: metatask_vx_ens_member_acc: - task_run_MET_PcpCombine_fcst_APCP#ACCUM_HR#h_mem#mem#: + task_run_MET_PcpCombine_fcst_APCP#ACCUM_HH#h_mem#mem#: dependency: verification: diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index e8ad654430..31b41adcf3 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -2331,7 +2331,14 @@ verification: # This is used in forming the names of the verification output files as # well as in the contents of those files. # + # VX_APCP_ACCUMS_HH: + # The 2-digit accumulation periods (in units of hours) to consider for + # APCP (accumulated precipitation). If VX_FIELDS contains "APCP", then + # VX_APCP_ACCUMS_HH must contain at least one element. If not, + # VX_APCP_ACCUMS_HH will be ignored. + # VX_FCST_MODEL_NAME: '{{ nco.NET }}.{{ task_run_post.POST_OUTPUT_DOMAIN_NAME }}' + VX_APCP_ACCUMS_HH: [ 1, 3, 6, 24 ] # # VX_FCST_INPUT_BASEDIR: # Template for top-level directory containing forecast (but not obs) From 54db2502d2a1c63e98936495bb343dc2b55a69f7 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Thu, 18 May 2023 08:24:09 -0600 Subject: [PATCH 11/43] Add the experiment configuration parameter VX_FIELDS that specifies which fields or group of fields on which to perform verification. --- parm/wflow/verify.yaml | 6 ++++-- parm/wflow/verify_ensgrid.yaml | 8 ++++---- ush/config_defaults.yaml | 4 ++++ 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/parm/wflow/verify.yaml b/parm/wflow/verify.yaml index b238a7eb6e..4a1efeeb9d 100644 --- a/parm/wflow/verify.yaml +++ b/parm/wflow/verify.yaml @@ -156,7 +156,9 @@ metatask_vx_ens_member: mem: '{% if global.DO_ENSEMBLE %}{% for m in range(1, global.NUM_ENS_MEMBERS+1) %}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' metatask_GridStat_MRMS_mem#mem#: var: - VAR: REFC RETOP + #VAR: REFC RETOP + #VAR: '{% for var in verification.VX_FIELDS %}{% if var in ["REFC", "RETOP"] %}{{ "%s " % var }}{% endif %}{% endfor %}' + VAR: '{% for var in verification.VX_FIELDS %}{% if var in ["REFC", "RETOP"] %}{{ "%s " % var }}{% endif %}{% endfor %}' task_run_MET_GridStat_vx_#VAR#_mem#mem#: <<: *default_task command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_GRIDSTAT_OR_POINTSTAT_VX"' @@ -189,7 +191,7 @@ metatask_vx_ens_member: metatask_PointStat_NDAS_mem#mem#: var: - VAR: SFC UPA + VAR: '{% for var in verification.VX_FIELDS %}{% if var in ["SFC", "UPA"] %}{{ "%s " % var }}{% endif %}{% endfor %}' task_run_MET_PointStat_vx_#VAR#_mem#mem#: <<: *default_task command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_GRIDSTAT_OR_POINTSTAT_VX"' diff --git a/parm/wflow/verify_ensgrid.yaml b/parm/wflow/verify_ensgrid.yaml index 607dbcfb89..ccd813f23c 100644 --- a/parm/wflow/verify_ensgrid.yaml +++ b/parm/wflow/verify_ensgrid.yaml @@ -72,7 +72,7 @@ metatask_GEPES_CCPA: metatask_GEPES_MRMS: var: - VAR: REFC RETOP + VAR: '{% for var in verification.VX_FIELDS %}{% if var in ["REFC", "RETOP"] %}{{ "%s " % var }}{% endif %}{% endfor %}' task_run_MET_GenEnsProd_vx_#VAR#: &task_GenEnsProd_MRMS <<: *default_task_verify_ens command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_GENENSPROD_OR_ENSEMBLESTAT"' @@ -95,7 +95,7 @@ metatask_GEPES_MRMS: metatask_GEPES_NDAS: var: - VAR: SFC UPA + VAR: '{% for var in verification.VX_FIELDS %}{% if var in ["SFC", "UPA"] %}{{ "%s " % var }}{% endif %}{% endfor %}' task_run_MET_GenEnsProd_vx_#VAR#: &task_GenEnsProd_NDAS <<: *default_task_verify_ens command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_GENENSPROD_OR_ENSEMBLESTAT"' @@ -143,7 +143,7 @@ metatask_ensemble_vx_stats: task: run_MET_EnsembleStat_vx_APCP#ACCUM_HH#h metatask_PointStat_ens: var: - VAR: SFC UPA + VAR: '{% for var in verification.VX_FIELDS %}{% if var in ["SFC", "UPA"] %}{{ "%s " % var }}{% endif %}{% endfor %}' task_run_MET_PointStat_vx_ens#statlc#_#VAR#: <<: *default_task_verify_ens command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_GRIDSTAT_OR_POINTSTAT_VX_ENS#stat#"' @@ -161,7 +161,7 @@ metatask_ensemble_vx_stats: metatask_GridStat_ensprob: var: - VAR: REFC RETOP + VAR: '{% for var in verification.VX_FIELDS %}{% if var in ["REFC", "RETOP"] %}{{ "%s " % var }}{% endif %}{% endfor %}' task_run_MET_GridStat_vx_ensprob_#VAR#: <<: *default_task_verify_ens command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_GRIDSTAT_OR_POINTSTAT_VX_ENSPROB"' diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 31b41adcf3..84ea0226ae 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -2331,6 +2331,9 @@ verification: # This is used in forming the names of the verification output files as # well as in the contents of those files. # + # VX_FIELDS: + # The fields or groups of fields on which to run verification. + # # VX_APCP_ACCUMS_HH: # The 2-digit accumulation periods (in units of hours) to consider for # APCP (accumulated precipitation). If VX_FIELDS contains "APCP", then @@ -2338,6 +2341,7 @@ verification: # VX_APCP_ACCUMS_HH will be ignored. # VX_FCST_MODEL_NAME: '{{ nco.NET }}.{{ task_run_post.POST_OUTPUT_DOMAIN_NAME }}' + VX_FIELDS: [ "APCP", "REFC", "RETOP", "SFC", "UPA" ] VX_APCP_ACCUMS_HH: [ 1, 3, 6, 24 ] # # VX_FCST_INPUT_BASEDIR: From d31dac6bc9671e3136338d72615975c4193ac195 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Thu, 18 May 2023 10:28:57 -0600 Subject: [PATCH 12/43] Rename and add vx metatasks for clarity. --- parm/wflow/verify.yaml | 14 +++++++------- parm/wflow/verify_ensgrid.yaml | 13 +++++++++---- .../config.MET_ensemble_verification_only_vx.yaml | 14 ++++++++------ .../config.MET_verification_only_vx.yaml | 15 ++++++++++----- 4 files changed, 34 insertions(+), 22 deletions(-) diff --git a/parm/wflow/verify.yaml b/parm/wflow/verify.yaml index 4a1efeeb9d..a70f01a8a0 100644 --- a/parm/wflow/verify.yaml +++ b/parm/wflow/verify.yaml @@ -111,7 +111,7 @@ metatask_PcpCombine_obs: datadep: text: "&CCPA_OBS_DIR;" -metatask_PcpCombine_fcst: +metatask_PcpCombine_fcst_all_accums_all_mems: var: ACCUM_HH: '{% for ah in verification.VX_APCP_ACCUMS_HH %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' metatask_PcpCombine_fcst_APCP#ACCUM_HH#h_all_mems: @@ -151,13 +151,11 @@ metatask_PcpCombine_fcst: attrs: task: run_fcst_mem#mem# -metatask_vx_ens_member: +metatask_GridStat_MRMS_all_mems: var: mem: '{% if global.DO_ENSEMBLE %}{% for m in range(1, global.NUM_ENS_MEMBERS+1) %}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' metatask_GridStat_MRMS_mem#mem#: var: - #VAR: REFC RETOP - #VAR: '{% for var in verification.VX_FIELDS %}{% if var in ["REFC", "RETOP"] %}{{ "%s " % var }}{% endif %}{% endfor %}' VAR: '{% for var in verification.VX_FIELDS %}{% if var in ["REFC", "RETOP"] %}{{ "%s " % var }}{% endif %}{% endfor %}' task_run_MET_GridStat_vx_#VAR#_mem#mem#: <<: *default_task @@ -189,6 +187,9 @@ metatask_vx_ens_member: or_do_post: <<: *post_files_exist +metatask_PointStat_NDAS_all_mems: + var: + mem: '{% if global.DO_ENSEMBLE %}{% for m in range(1, global.NUM_ENS_MEMBERS+1) %}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' metatask_PointStat_NDAS_mem#mem#: var: VAR: '{% for var in verification.VX_FIELDS %}{% if var in ["SFC", "UPA"] %}{{ "%s " % var }}{% endif %}{% endfor %}' @@ -225,13 +226,12 @@ metatask_vx_ens_member: attrs: task: get_obs_ndas -metatask_GridStat_APCP_acc: +metatask_GridStat_CCPA_all_accums_all_mems: var: ACCUM_HH: '{% for ah in verification.VX_APCP_ACCUMS_HH %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' - metatask_vx_ens_member_acc: + metatask_GridStat_CCPA_APCP#ACCUM_HH#h_all_mems: var: mem: '{% if global.DO_ENSEMBLE %}{% for m in range(1, global.NUM_ENS_MEMBERS+1) %}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' - task_run_MET_GridStat_vx_APCP#ACCUM_HH#h_mem#mem#: <<: *default_task attrs: diff --git a/parm/wflow/verify_ensgrid.yaml b/parm/wflow/verify_ensgrid.yaml index ccd813f23c..c8540b5b7d 100644 --- a/parm/wflow/verify_ensgrid.yaml +++ b/parm/wflow/verify_ensgrid.yaml @@ -120,11 +120,11 @@ metatask_GEPES_NDAS: <<: *envars_GenEnsProd_NDAS MET_TOOL: 'ENSEMBLESTAT' -metatask_ensemble_vx_stats: +metatask_GridStat_CCPA_ensmeanprob_all_accums: var: stat: MEAN PROB statlc: mean prob - metatask_GridStat_ens_acc: + metatask_GridStat_CCPA_ens#statlc#_all_accums: var: ACCUM_HH: '{% for ah in verification.VX_APCP_ACCUMS_HH %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' task_run_MET_GridStat_vx_ens#statlc#_APCP#ACCUM_HH#h: @@ -141,7 +141,12 @@ metatask_ensemble_vx_stats: taskdep: attrs: task: run_MET_EnsembleStat_vx_APCP#ACCUM_HH#h - metatask_PointStat_ens: + +metatask_PointStat_NDAS_ensmeanprob: + var: + stat: MEAN PROB + statlc: mean prob + metatask_PointStat_NDAS_ens#statlc#: var: VAR: '{% for var in verification.VX_FIELDS %}{% if var in ["SFC", "UPA"] %}{{ "%s " % var }}{% endif %}{% endfor %}' task_run_MET_PointStat_vx_ens#statlc#_#VAR#: @@ -159,7 +164,7 @@ metatask_ensemble_vx_stats: attrs: task: run_MET_EnsembleStat_vx_#VAR# -metatask_GridStat_ensprob: +metatask_GridStat_CCPA_ensprob: var: VAR: '{% for var in verification.VX_FIELDS %}{% if var in ["REFC", "RETOP"] %}{{ "%s " % var }}{% endif %}{% endfor %}' task_run_MET_GridStat_vx_ensprob_#VAR#: diff --git a/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml b/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml index 89b4182723..594d8fda5f 100644 --- a/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml +++ b/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml @@ -22,10 +22,12 @@ rocoto: task_get_obs_mrms: task_get_obs_ndas: - metatask_vx_ens_member: + metatask_GridStat_MRMS_all_mems: metatask_GridStat_MRMS_mem#mem#: task_run_MET_GridStat_vx_#VAR#_mem#mem#: dependency: + + metatask_PointStat_NDAS_all_mems: metatask_PointStat_NDAS_mem#mem#: task_run_MET_PointStat_vx_#VAR#_mem#mem#: dependency: @@ -33,13 +35,13 @@ rocoto: or_do_post: or_get_obs: - metatask_PcpCombine_fcst: + metatask_PcpCombine_fcst_all_accums_all_mems: metatask_PcpCombine_fcst_APCP#ACCUM_HH#h_all_mems: task_run_MET_PcpCombine_fcst_APCP#ACCUM_HH#h_mem#mem#: dependency: - metatask_GridStat_APCP_acc: - metatask_vx_ens_member_acc: + metatask_GridStat_CCPA_all_accums_all_mems: + metatask_GridStat_CCPA_APCP#ACCUM_HH#h_all_mems: task_run_MET_PcpCombine_fcst_APCP#ACCUM_HH#h_mem#mem#: dependency: @@ -51,13 +53,13 @@ rocoto: and: metataskdep: attrs: - metatask: GridStat_APCP_acc + metatask: GridStat_CCPA_all_accums_all_mems task_run_MET_EnsembleStat_vx_APCP#ACCUM_HH#h: dependency: and: metataskdep: attrs: - metatask: GridStat_APCP_acc + metatask: GridStat_CCPA_all_accums_all_mems metatask_GEPES_MRMS: task_run_MET_GenEnsProd_vx_#VAR#: diff --git a/tests/WE2E/test_configs/verification/config.MET_verification_only_vx.yaml b/tests/WE2E/test_configs/verification/config.MET_verification_only_vx.yaml index 48e6166832..b22fea69b1 100644 --- a/tests/WE2E/test_configs/verification/config.MET_verification_only_vx.yaml +++ b/tests/WE2E/test_configs/verification/config.MET_verification_only_vx.yaml @@ -25,32 +25,37 @@ rocoto: task_get_obs_ccpa: task_get_obs_mrms: task_get_obs_ndas: + metatask_PcpCombine_obs: task_run_MET_PcpCombine_obs_APCP#ACCUM_HH#h: dependency: or: datadep: text: "&CCPA_OBS_DIR;" - metatask_PcpCombine_fcst: + + metatask_PcpCombine_fcst_all_accums_all_mems: metatask_PcpCombine_fcst_APCP#ACCUM_HH#h_all_mems: task_run_MET_PcpCombine_fcst_APCP#ACCUM_HH#h_mem#mem#: dependency: - metatask_vx_ens_member: + + metatask_GridStat_MRMS_all_mems: metatask_GridStat_MRMS_mem#mem#: task_run_MET_GridStat_vx_#VAR#_mem#mem#: dependency: + + metatask_PointStat_NDAS_all_mems: metatask_PointStat_NDAS_mem#mem#: task_run_MET_PointStat_vx_#VAR#_mem#mem#: dependency: and: or_do_post: or_get_obs: - metatask_GridStat_APCP_acc: - metatask_vx_ens_member_acc: + + metatask_GridStat_CCPA_all_accums_all_mems: + metatask_GridStat_CCPA_APCP#ACCUM_HH#h_all_mems: task_run_MET_PcpCombine_fcst_APCP#ACCUM_HH#h_mem#mem#: dependency: verification: VX_FCST_MODEL_NAME: FV3_GFS_v15p2_CONUS_25km VX_FCST_INPUT_BASEDIR: '{{ platform.get("TEST_VX_FCST_INPUT_BASEDIR") }}' - From d6dfb447d5284fbd183f2eb3dbcd2271f55d13b0 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Thu, 18 May 2023 11:36:53 -0600 Subject: [PATCH 13/43] Bug fix in metatask name. --- parm/wflow/verify_ensgrid.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/wflow/verify_ensgrid.yaml b/parm/wflow/verify_ensgrid.yaml index c8540b5b7d..7f93306080 100644 --- a/parm/wflow/verify_ensgrid.yaml +++ b/parm/wflow/verify_ensgrid.yaml @@ -164,7 +164,7 @@ metatask_PointStat_NDAS_ensmeanprob: attrs: task: run_MET_EnsembleStat_vx_#VAR# -metatask_GridStat_CCPA_ensprob: +metatask_GridStat_MRMS_ensprob: var: VAR: '{% for var in verification.VX_FIELDS %}{% if var in ["REFC", "RETOP"] %}{{ "%s " % var }}{% endif %}{% endfor %}' task_run_MET_GridStat_vx_ensprob_#VAR#: From 255a2fc99074df09af5a99efd242ccf5ce5b711d Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Thu, 18 May 2023 12:50:59 -0600 Subject: [PATCH 14/43] Move CCPA/APCP metatask to before MRMS and NDAS once since the former field(s) are usually come first in vx. --- parm/wflow/verify.yaml | 60 +++++++++++++++++++++--------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/parm/wflow/verify.yaml b/parm/wflow/verify.yaml index a70f01a8a0..6f6571b8ac 100644 --- a/parm/wflow/verify.yaml +++ b/parm/wflow/verify.yaml @@ -151,6 +151,36 @@ metatask_PcpCombine_fcst_all_accums_all_mems: attrs: task: run_fcst_mem#mem# +metatask_GridStat_CCPA_all_accums_all_mems: + var: + ACCUM_HH: '{% for ah in verification.VX_APCP_ACCUMS_HH %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' + metatask_GridStat_CCPA_APCP#ACCUM_HH#h_all_mems: + var: + mem: '{% if global.DO_ENSEMBLE %}{% for m in range(1, global.NUM_ENS_MEMBERS+1) %}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' + task_run_MET_GridStat_vx_APCP#ACCUM_HH#h_mem#mem#: + <<: *default_task + attrs: + maxtries: '2' + command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_GRIDSTAT_OR_POINTSTAT_VX"' + envars: + <<: *default_vars + OBS_DIR: '&CCPA_OBS_DIR;' + VAR: APCP + ACCUM_HH: '#ACCUM_HH#' + MET_TOOL: 'GRIDSTAT' + OBTYPE: 'CCPA' + ENSMEM_INDX: "#mem#" + SLASH_ENSMEM_SUBDIR_OR_NULL: '{% if global.DO_ENSEMBLE %}{{ "/mem#mem#" }}{% endif %}' + walltime: 02:00:00 + dependency: + and: + taskdep_obs: + attrs: + task: run_MET_PcpCombine_obs_APCP#ACCUM_HH#h + taskdep_fcst: + attrs: + task: run_MET_PcpCombine_fcst_APCP#ACCUM_HH#h_mem#mem# + metatask_GridStat_MRMS_all_mems: var: mem: '{% if global.DO_ENSEMBLE %}{% for m in range(1, global.NUM_ENS_MEMBERS+1) %}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' @@ -225,33 +255,3 @@ metatask_PointStat_NDAS_all_mems: taskdep: attrs: task: get_obs_ndas - -metatask_GridStat_CCPA_all_accums_all_mems: - var: - ACCUM_HH: '{% for ah in verification.VX_APCP_ACCUMS_HH %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' - metatask_GridStat_CCPA_APCP#ACCUM_HH#h_all_mems: - var: - mem: '{% if global.DO_ENSEMBLE %}{% for m in range(1, global.NUM_ENS_MEMBERS+1) %}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' - task_run_MET_GridStat_vx_APCP#ACCUM_HH#h_mem#mem#: - <<: *default_task - attrs: - maxtries: '2' - command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_GRIDSTAT_OR_POINTSTAT_VX"' - envars: - <<: *default_vars - OBS_DIR: '&CCPA_OBS_DIR;' - VAR: APCP - ACCUM_HH: '#ACCUM_HH#' - MET_TOOL: 'GRIDSTAT' - OBTYPE: 'CCPA' - ENSMEM_INDX: "#mem#" - SLASH_ENSMEM_SUBDIR_OR_NULL: '{% if global.DO_ENSEMBLE %}{{ "/mem#mem#" }}{% endif %}' - walltime: 02:00:00 - dependency: - and: - taskdep_obs: - attrs: - task: run_MET_PcpCombine_obs_APCP#ACCUM_HH#h - taskdep_fcst: - attrs: - task: run_MET_PcpCombine_fcst_APCP#ACCUM_HH#h_mem#mem# From b17871b2f8b8cd1e8c227a3cba6a46e848d8d905 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Thu, 18 May 2023 13:40:59 -0600 Subject: [PATCH 15/43] Remove rocoto verification metatasks from workflow when there are no fields for those metatasks. --- ush/setup.py | 42 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/ush/setup.py b/ush/setup.py index 002ccbf2df..2948dd5074 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -540,7 +540,47 @@ def remove_tag(tasks, tag): post_meta = rocoto_tasks.get("metatask_run_ens_post", {}) post_meta.pop("metatask_run_sub_hourly_post", None) post_meta.pop("metatask_sub_hourly_last_hour_post", None) - + # + # ----------------------------------------------------------------------- + # + # Remove all verification metatasks for which no fields are specified. + # + # ----------------------------------------------------------------------- + # + vx_fields_all = {} + vx_metatasks_all = {} + + vx_fields_all["CCPA"] = ["APCP"] + vx_metatasks_all["CCPA"] = ["metatask_PcpCombine_obs", + "metatask_PcpCombine_fcst_all_accums_all_mems", + "metatask_GridStat_CCPA_all_accums_all_mems", + "metatask_GEPES_CCPA", + "metatask_GridStat_CCPA_ensmeanprob_all_accums"] + + vx_fields_all["MRMS"] = ["REFC", "RETOP"] + vx_metatasks_all["MRMS"] = ["metatask_GridStat_MRMS_all_mems", + "metatask_GEPES_MRMS", + "metatask_GridStat_MRMS_ensprob"] + + vx_fields_all["NDAS"] = ["SFC", "UPA"] + vx_metatasks_all["NDAS"] = ["task_run_MET_Pb2nc_obs", + "metatask_PointStat_NDAS_all_mems", + "metatask_GEPES_NDAS", + "metatask_PointStat_NDAS_ensmeanprob"] + + vx_fields_config = expt_config["verification"]["VX_FIELDS"] + for obstype in vx_fields_all: + vx_fields_obstype = [field for field in vx_fields_config if field in vx_fields_all[obstype]] + if not vx_fields_obstype: + for metatask in vx_metatasks_all[obstype]: + logging.info(dedent( + f""" + Removing verification metatask + {metatask} + since no {obstype} fields are specified for verification. + """ + )) + rocoto_config['tasks'].pop(metatask, None) # # ----------------------------------------------------------------------- # From a4db400450b951301953581bded2d0ad396d2eb2 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Thu, 18 May 2023 17:26:07 -0600 Subject: [PATCH 16/43] Clean up vx [meta]task dependencies. --- parm/wflow/verify.yaml | 4 +- parm/wflow/verify_ensgrid.yaml | 24 ++++++---- ...fig.MET_ensemble_verification_only_vx.yaml | 48 +++++-------------- .../config.MET_verification_only_vx.yaml | 5 -- 4 files changed, 29 insertions(+), 52 deletions(-) diff --git a/parm/wflow/verify.yaml b/parm/wflow/verify.yaml index 6f6571b8ac..ab36d990aa 100644 --- a/parm/wflow/verify.yaml +++ b/parm/wflow/verify.yaml @@ -174,10 +174,10 @@ metatask_GridStat_CCPA_all_accums_all_mems: walltime: 02:00:00 dependency: and: - taskdep_obs: + taskdep_pcpcombine_obs: attrs: task: run_MET_PcpCombine_obs_APCP#ACCUM_HH#h - taskdep_fcst: + taskdep_pcpcombine_fcst: attrs: task: run_MET_PcpCombine_fcst_APCP#ACCUM_HH#h_mem#mem# diff --git a/parm/wflow/verify_ensgrid.yaml b/parm/wflow/verify_ensgrid.yaml index 7f93306080..783adc91f2 100644 --- a/parm/wflow/verify_ensgrid.yaml +++ b/parm/wflow/verify_ensgrid.yaml @@ -55,7 +55,7 @@ metatask_GEPES_CCPA: metatask: run_ensemble metataskdep: attrs: - metatask: PcpCombine_fcst + metatask: PcpCombine_fcst_APCP#ACCUM_HH#h_all_mems task_run_MET_EnsembleStat_vx_APCP#ACCUM_HH#h: <<: *task_GenEnsProd_CCPA envars: @@ -63,10 +63,10 @@ metatask_GEPES_CCPA: MET_TOOL: 'ENSEMBLESTAT' dependency: and: - taskdep: + taskdep_pcpcombine_obs: attrs: task: run_MET_PcpCombine_obs_APCP#ACCUM_HH#h - taskdep: + taskdep_genensprod: attrs: task: run_MET_GenEnsProd_vx_APCP#ACCUM_HH#h @@ -107,18 +107,22 @@ metatask_GEPES_NDAS: OBTYPE: 'NDAS' ACCUM_HH: '01' dependency: - and: - metataskdep: - attrs: - metatask: run_ens_post - taskdep: - attrs: - task: run_MET_Pb2nc_obs + metataskdep: + attrs: + metatask: run_ens_post task_run_MET_EnsembleStat_vx_#VAR#: <<: *task_GenEnsProd_NDAS envars: <<: *envars_GenEnsProd_NDAS MET_TOOL: 'ENSEMBLESTAT' + dependency: + and: + taskdep_pb2nc: + attrs: + task: run_MET_Pb2nc_obs + taskdep_genensprod: + attrs: + task: run_MET_GenEnsProd_vx_#VAR# metatask_GridStat_CCPA_ensmeanprob_all_accums: var: diff --git a/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml b/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml index 594d8fda5f..7b2f5e7cfa 100644 --- a/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml +++ b/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml @@ -22,6 +22,18 @@ rocoto: task_get_obs_mrms: task_get_obs_ndas: + metatask_PcpCombine_obs: + task_run_MET_PcpCombine_obs_APCP#ACCUM_HH#h: + dependency: + or: + datadep: + text: "&CCPA_OBS_DIR;" + + metatask_PcpCombine_fcst_all_accums_all_mems: + metatask_PcpCombine_fcst_APCP#ACCUM_HH#h_all_mems: + task_run_MET_PcpCombine_fcst_APCP#ACCUM_HH#h_mem#mem#: + dependency: + metatask_GridStat_MRMS_all_mems: metatask_GridStat_MRMS_mem#mem#: task_run_MET_GridStat_vx_#VAR#_mem#mem#: @@ -35,53 +47,19 @@ rocoto: or_do_post: or_get_obs: - metatask_PcpCombine_fcst_all_accums_all_mems: - metatask_PcpCombine_fcst_APCP#ACCUM_HH#h_all_mems: - task_run_MET_PcpCombine_fcst_APCP#ACCUM_HH#h_mem#mem#: - dependency: - - metatask_GridStat_CCPA_all_accums_all_mems: - metatask_GridStat_CCPA_APCP#ACCUM_HH#h_all_mems: - task_run_MET_PcpCombine_fcst_APCP#ACCUM_HH#h_mem#mem#: - dependency: - metatask_GEPES_CCPA: - task_run_MET_GridStat_vx_ensprob_#VAR#: - dependency: task_run_MET_GenEnsProd_vx_APCP#ACCUM_HH#h: dependency: and: - metataskdep: - attrs: - metatask: GridStat_CCPA_all_accums_all_mems - task_run_MET_EnsembleStat_vx_APCP#ACCUM_HH#h: - dependency: - and: - metataskdep: - attrs: - metatask: GridStat_CCPA_all_accums_all_mems + or_do_post: metatask_GEPES_MRMS: task_run_MET_GenEnsProd_vx_#VAR#: dependency: - task_run_MET_EnsembleStat_vx_#VAR#: - dependency: metatask_GEPES_NDAS: task_run_MET_GenEnsProd_vx_#VAR#: dependency: - and: - metataskdep: - taskdep: - attrs: - task: run_MET_Pb2nc_obs - task_run_MET_EnsembleStat_vx_#VAR#: - dependency: - and: - metataskdep: - taskdep: - attrs: - task: run_MET_Pb2nc_obs global: DO_ENSEMBLE: true diff --git a/tests/WE2E/test_configs/verification/config.MET_verification_only_vx.yaml b/tests/WE2E/test_configs/verification/config.MET_verification_only_vx.yaml index b22fea69b1..c04a517f36 100644 --- a/tests/WE2E/test_configs/verification/config.MET_verification_only_vx.yaml +++ b/tests/WE2E/test_configs/verification/config.MET_verification_only_vx.yaml @@ -51,11 +51,6 @@ rocoto: or_do_post: or_get_obs: - metatask_GridStat_CCPA_all_accums_all_mems: - metatask_GridStat_CCPA_APCP#ACCUM_HH#h_all_mems: - task_run_MET_PcpCombine_fcst_APCP#ACCUM_HH#h_mem#mem#: - dependency: - verification: VX_FCST_MODEL_NAME: FV3_GFS_v15p2_CONUS_25km VX_FCST_INPUT_BASEDIR: '{{ platform.get("TEST_VX_FCST_INPUT_BASEDIR") }}' From a095d4279f63ffe51a19ccf630a40bfcb1db00fb Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Thu, 18 May 2023 17:28:09 -0600 Subject: [PATCH 17/43] Add new experiment flags RUN_VX_DET and RUN_VX_ENS that specify whether or not to run deterministic and ensemble verification, respectively. --- ...fig.MET_ensemble_verification_only_vx.yaml | 2 + .../config.MET_verification_only_vx.yaml | 1 + ush/config_defaults.yaml | 17 +++++ ush/setup.py | 67 ++++++++++++++++--- 4 files changed, 78 insertions(+), 9 deletions(-) diff --git a/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml b/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml index 7b2f5e7cfa..e5e79d7694 100644 --- a/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml +++ b/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml @@ -68,3 +68,5 @@ global: verification: VX_FCST_MODEL_NAME: FV3_GFS_v15p2_CONUS_25km VX_FCST_INPUT_BASEDIR: '{{ platform.get("TEST_VX_FCST_INPUT_BASEDIR") }}' + RUN_VX_DET: True + RUN_VX_ENS: True diff --git a/tests/WE2E/test_configs/verification/config.MET_verification_only_vx.yaml b/tests/WE2E/test_configs/verification/config.MET_verification_only_vx.yaml index c04a517f36..d57d6b3b10 100644 --- a/tests/WE2E/test_configs/verification/config.MET_verification_only_vx.yaml +++ b/tests/WE2E/test_configs/verification/config.MET_verification_only_vx.yaml @@ -54,3 +54,4 @@ rocoto: verification: VX_FCST_MODEL_NAME: FV3_GFS_v15p2_CONUS_25km VX_FCST_INPUT_BASEDIR: '{{ platform.get("TEST_VX_FCST_INPUT_BASEDIR") }}' + RUN_VX_DET: True diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 84ea0226ae..0e7b4b86ee 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -2326,6 +2326,23 @@ verification: OBS_NDAS_SFCorUPA_FN_TEMPLATE: 'prepbufr.ndas.{valid?fmt=%Y%m%d%H}' OBS_NDAS_SFCorUPA_FN_METPROC_TEMPLATE: '${OBS_NDAS_SFCorUPA_FN_TEMPLATE}.nc' # + # RUN_VX_DET: + # Flag that specifies whether to run deterministic verification. If set + # to True, this will run deterministic vx on the post-processed forecast + # output. This post-processed output may consist of a single forecast + # or an ensemble of foreasts, and it may be staged from previous runs of + # the SRW App or may be generated by running the TN_RUN_FCST task as part + # of the current SRW-App-generated experiment. + # + # RUN_VX_ENS: + # Flag that specifies whether to run ensemble verification. The ensemble + # forecast output on which vx will be run may be staged or generated by + # running an ensemble of forecasts with the weather model as part of the + # current SRW-App-generated experiment. + # + RUN_VX_DET: False + RUN_VX_ENS: False + # # VX_FCST_MODEL_NAME: # String that specifies a descriptive name for the model being verified. # This is used in forming the names of the verification output files as diff --git a/ush/setup.py b/ush/setup.py index 2948dd5074..bda5c86b15 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -543,7 +543,7 @@ def remove_tag(tasks, tag): # # ----------------------------------------------------------------------- # - # Remove all verification metatasks for which no fields are specified. + # Remove all verification [meta]tasks for which no fields are specified. # # ----------------------------------------------------------------------- # @@ -573,14 +573,63 @@ def remove_tag(tasks, tag): vx_fields_obstype = [field for field in vx_fields_config if field in vx_fields_all[obstype]] if not vx_fields_obstype: for metatask in vx_metatasks_all[obstype]: - logging.info(dedent( - f""" - Removing verification metatask - {metatask} - since no {obstype} fields are specified for verification. - """ - )) - rocoto_config['tasks'].pop(metatask, None) + if metatask in rocoto_config['tasks']: + logging.info(dedent( + f""" + Removing verification [meta]task + "{metatask}" + from workflow since no fields belonging to observation type "{obstype}" + are specified for verification.""" + )) + rocoto_config['tasks'].pop(metatask, None) + # + # ----------------------------------------------------------------------- + # + # Remove deterministic and/or ensemble verification [meta]tasks depending + # on the values of the experiment configuration flags RUN_VX_DET and + # RUN_VX_ENS. + # + # ----------------------------------------------------------------------- + # + run_vx_det = expt_config["verification"]["RUN_VX_DET"] + run_vx_ens = expt_config["verification"]["RUN_VX_ENS"] + + vx_flags = {} + vx_metatasks = {} + + vx_flags["det_or_ens"] = run_vx_det or run_vx_ens + vx_metatasks["det_or_ens"] = ["metatask_PcpCombine_obs", + "metatask_PcpCombine_fcst_all_accums_all_mems", + "task_run_MET_Pb2nc_obs"] + + vx_flags["det"] = run_vx_det + vx_metatasks["det"] = ["metatask_GridStat_CCPA_all_accums_all_mems", + "metatask_GridStat_MRMS_all_mems", + "metatask_PointStat_NDAS_all_mems"] + + vx_flags["ens"] = run_vx_ens + vx_metatasks["ens"] = ["metatask_GEPES_CCPA", + "metatask_GEPES_MRMS", + "metatask_GEPES_NDAS", + "metatask_GridStat_CCPA_ensmeanprob_all_accums", + "metatask_GridStat_MRMS_ensprob", + "metatask_PointStat_NDAS_ensmeanprob"] + + run_vx_det = expt_config["verification"]["RUN_VX_DET"] + run_vx_ens = expt_config["verification"]["RUN_VX_ENS"] + + for key,flag in vx_flags.items(): + if not flag: + for metatask in vx_metatasks[key]: + if metatask in rocoto_config['tasks']: + logging.info(dedent( + f""" + Removing verification [meta]task + "{metatask}" + from workflow since vx_flags["{key}"] is set to {flag}.""" + )) + rocoto_config['tasks'].pop(metatask, None) + # # ----------------------------------------------------------------------- # From d4941be04015de4d66946ce63bd983807319aab4 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Thu, 18 May 2023 17:56:00 -0600 Subject: [PATCH 18/43] Bug fix for GEPES_MRMS metatask. --- parm/wflow/verify_ensgrid.yaml | 8 ++++++++ .../config.MET_ensemble_verification_only_vx.yaml | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/parm/wflow/verify_ensgrid.yaml b/parm/wflow/verify_ensgrid.yaml index 783adc91f2..578f9d8188 100644 --- a/parm/wflow/verify_ensgrid.yaml +++ b/parm/wflow/verify_ensgrid.yaml @@ -92,6 +92,14 @@ metatask_GEPES_MRMS: envars: <<: *envars_GenEnsProd_MRMS MET_TOOL: 'ENSEMBLESTAT' + dependency: + and: + taskdep_getobsmrms: + attrs: + task: get_obs_mrms + taskdep_genensprod: + attrs: + task: run_MET_GenEnsProd_vx_#VAR# metatask_GEPES_NDAS: var: diff --git a/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml b/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml index e5e79d7694..560b3b840d 100644 --- a/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml +++ b/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml @@ -56,6 +56,10 @@ rocoto: metatask_GEPES_MRMS: task_run_MET_GenEnsProd_vx_#VAR#: dependency: + task_run_MET_EnsembleStat_vx_#VAR#: + dependency: + and: + taskdep_getobsmrms: metatask_GEPES_NDAS: task_run_MET_GenEnsProd_vx_#VAR#: From 27f1fa28b200e6ba099cc929225a3f4ceed19815 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Fri, 19 May 2023 11:50:26 -0600 Subject: [PATCH 19/43] Bug fixes to ensemble verification task dependencies. --- parm/wflow/verify_ensgrid.yaml | 41 +++++++++++-------- ...fig.MET_ensemble_verification_only_vx.yaml | 2 + 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/parm/wflow/verify_ensgrid.yaml b/parm/wflow/verify_ensgrid.yaml index 578f9d8188..9008a18d2f 100644 --- a/parm/wflow/verify_ensgrid.yaml +++ b/parm/wflow/verify_ensgrid.yaml @@ -53,7 +53,13 @@ metatask_GEPES_CCPA: metataskdep: attrs: metatask: run_ensemble - metataskdep: + # The PcpCombine task for obs must be complete because this GenEnsProd + # task checks to see the forecast hours for which obs are available before + # processing the forecast for those hours. + taskdep_pcpcombine_obs: + attrs: + task: run_MET_PcpCombine_obs_APCP#ACCUM_HH#h + metataskdep_pcpcombine_fcst: attrs: metatask: PcpCombine_fcst_APCP#ACCUM_HH#h_all_mems task_run_MET_EnsembleStat_vx_APCP#ACCUM_HH#h: @@ -62,13 +68,9 @@ metatask_GEPES_CCPA: <<: *envars_GenEnsProd_CCPA MET_TOOL: 'ENSEMBLESTAT' dependency: - and: - taskdep_pcpcombine_obs: - attrs: - task: run_MET_PcpCombine_obs_APCP#ACCUM_HH#h - taskdep_genensprod: - attrs: - task: run_MET_GenEnsProd_vx_APCP#ACCUM_HH#h + taskdep_genensprod: + attrs: + task: run_MET_GenEnsProd_vx_APCP#ACCUM_HH#h metatask_GEPES_MRMS: var: @@ -115,22 +117,25 @@ metatask_GEPES_NDAS: OBTYPE: 'NDAS' ACCUM_HH: '01' dependency: - metataskdep: - attrs: - metatask: run_ens_post + and: + # The Pb2nc task (which is run only for obs) must be complete because + # this GenEnsProd task checks to see the forecast hours for which obs + # are available before processing the forecast for those hours. + taskdep_pb2nc: + attrs: + task: run_MET_Pb2nc_obs + metataskdep: + attrs: + metatask: run_ens_post task_run_MET_EnsembleStat_vx_#VAR#: <<: *task_GenEnsProd_NDAS envars: <<: *envars_GenEnsProd_NDAS MET_TOOL: 'ENSEMBLESTAT' dependency: - and: - taskdep_pb2nc: - attrs: - task: run_MET_Pb2nc_obs - taskdep_genensprod: - attrs: - task: run_MET_GenEnsProd_vx_#VAR# + taskdep_genensprod: + attrs: + task: run_MET_GenEnsProd_vx_#VAR# metatask_GridStat_CCPA_ensmeanprob_all_accums: var: diff --git a/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml b/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml index 560b3b840d..29da2c50cc 100644 --- a/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml +++ b/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml @@ -64,6 +64,8 @@ rocoto: metatask_GEPES_NDAS: task_run_MET_GenEnsProd_vx_#VAR#: dependency: + and: + metataskdep: global: DO_ENSEMBLE: true From f31436604ddd8bd742225e751d2a7a3b42c180c5 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Fri, 19 May 2023 16:04:27 -0600 Subject: [PATCH 20/43] Bug fixes to ensemble task dependencies. --- parm/wflow/verify_ensgrid.yaml | 20 +++++++++---------- ...fig.MET_ensemble_verification_only_vx.yaml | 4 ---- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/parm/wflow/verify_ensgrid.yaml b/parm/wflow/verify_ensgrid.yaml index 9008a18d2f..47f08725a8 100644 --- a/parm/wflow/verify_ensgrid.yaml +++ b/parm/wflow/verify_ensgrid.yaml @@ -86,22 +86,22 @@ metatask_GEPES_MRMS: MET_TOOL: 'GENENSPROD' OBTYPE: 'MRMS' dependency: - metataskdep: - attrs: - metatask: run_ens_post + and: + taskdep: + attrs: + task: get_obs_mrms + metataskdep: + attrs: + metatask: run_ens_post task_run_MET_EnsembleStat_vx_#VAR#: <<: *task_GenEnsProd_MRMS envars: <<: *envars_GenEnsProd_MRMS MET_TOOL: 'ENSEMBLESTAT' dependency: - and: - taskdep_getobsmrms: - attrs: - task: get_obs_mrms - taskdep_genensprod: - attrs: - task: run_MET_GenEnsProd_vx_#VAR# + taskdep: + attrs: + task: run_MET_GenEnsProd_vx_#VAR# metatask_GEPES_NDAS: var: diff --git a/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml b/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml index 29da2c50cc..eab198042b 100644 --- a/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml +++ b/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml @@ -56,10 +56,6 @@ rocoto: metatask_GEPES_MRMS: task_run_MET_GenEnsProd_vx_#VAR#: dependency: - task_run_MET_EnsembleStat_vx_#VAR#: - dependency: - and: - taskdep_getobsmrms: metatask_GEPES_NDAS: task_run_MET_GenEnsProd_vx_#VAR#: From 7da364da48682b3826434f12cc16b2ff8d627838 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Fri, 19 May 2023 16:05:28 -0600 Subject: [PATCH 21/43] Modifications to vx WE2E tests that don't use staged forecasts to get them to work properly with latest changes. --- ...RFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.yaml | 1 + ...RFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GFS_v16.yaml | 11 ++++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.yaml b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.yaml index 2d2ee706a1..aabaea1c7b 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.yaml +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.yaml @@ -35,3 +35,4 @@ task_get_extrn_lbcs: USE_USER_STAGED_EXTRN_FILES: true verification: VX_FCST_MODEL_NAME: FV3_GFS_v15p2_CONUS_25km + RUN_VX_DET: True diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GFS_v16.yaml b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GFS_v16.yaml index 9ec3925c59..7fc98f79e7 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GFS_v16.yaml +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GFS_v16.yaml @@ -17,9 +17,17 @@ workflow: rocoto: tasks: taskgroups: '{{ ["parm/wflow/prep.yaml", "parm/wflow/coldstart.yaml", "parm/wflow/post.yaml", "parm/wflow/verify.yaml", "parm/wflow/verify_ensgrid.yaml"]|include }}' + task_get_obs_ccpa: + task_get_obs_mrms: + task_get_obs_ndas: metatask_run_ensemble: task_run_fcst_mem#mem#: walltime: 01:00:00 + metatask_GEPES_MRMS: + task_run_MET_GenEnsProd_vx_#VAR#: + dependency: + and: + taskdep: task_get_extrn_ics: EXTRN_MDL_NAME_ICS: NAM USE_USER_STAGED_EXTRN_FILES: true @@ -32,4 +40,5 @@ global: NUM_ENS_MEMBERS: 2 verification: VX_FCST_MODEL_NAME: FV3_GFS_v15p2_CONUS_25km - + RUN_VX_DET: True + RUN_VX_ENS: True From e99d3bad213f2d3a1eac0e079a81e622509e71f5 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Fri, 19 May 2023 16:29:00 -0600 Subject: [PATCH 22/43] Rename vx workflow yaml configuration file names for clarity and accuracy. --- parm/wflow/{verify.yaml => verify_det.yaml} | 0 parm/wflow/{verify_ensgrid.yaml => verify_ens.yaml} | 0 ...id_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.yaml | 2 +- ...id_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GFS_v16.yaml | 8 ++++---- .../config.MET_ensemble_verification_only_vx.yaml | 2 +- .../verification/config.MET_verification_only_vx.yaml | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) rename parm/wflow/{verify.yaml => verify_det.yaml} (100%) rename parm/wflow/{verify_ensgrid.yaml => verify_ens.yaml} (100%) diff --git a/parm/wflow/verify.yaml b/parm/wflow/verify_det.yaml similarity index 100% rename from parm/wflow/verify.yaml rename to parm/wflow/verify_det.yaml diff --git a/parm/wflow/verify_ensgrid.yaml b/parm/wflow/verify_ens.yaml similarity index 100% rename from parm/wflow/verify_ensgrid.yaml rename to parm/wflow/verify_ens.yaml diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.yaml b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.yaml index aabaea1c7b..f323164536 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.yaml +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.yaml @@ -17,7 +17,7 @@ workflow: PREEXISTING_DIR_METHOD: rename rocoto: tasks: - taskgroups: '{{ ["parm/wflow/prep.yaml", "parm/wflow/coldstart.yaml", "parm/wflow/post.yaml", "parm/wflow/verify.yaml"]|include }}' + taskgroups: '{{ ["parm/wflow/prep.yaml", "parm/wflow/coldstart.yaml", "parm/wflow/post.yaml", "parm/wflow/verify_det.yaml"]|include }}' metatask_run_ensemble: task_run_fcst_mem#mem#: walltime: 01:00:00 diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GFS_v16.yaml b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GFS_v16.yaml index 7fc98f79e7..351cad85b8 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GFS_v16.yaml +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GFS_v16.yaml @@ -16,13 +16,13 @@ workflow: PREEXISTING_DIR_METHOD: rename rocoto: tasks: - taskgroups: '{{ ["parm/wflow/prep.yaml", "parm/wflow/coldstart.yaml", "parm/wflow/post.yaml", "parm/wflow/verify.yaml", "parm/wflow/verify_ensgrid.yaml"]|include }}' - task_get_obs_ccpa: - task_get_obs_mrms: - task_get_obs_ndas: + taskgroups: '{{ ["parm/wflow/prep.yaml", "parm/wflow/coldstart.yaml", "parm/wflow/post.yaml", "parm/wflow/verify_det.yaml", "parm/wflow/verify_ens.yaml"]|include }}' metatask_run_ensemble: task_run_fcst_mem#mem#: walltime: 01:00:00 + task_get_obs_ccpa: + task_get_obs_mrms: + task_get_obs_ndas: metatask_GEPES_MRMS: task_run_MET_GenEnsProd_vx_#VAR#: dependency: diff --git a/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml b/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml index eab198042b..5f76123bee 100644 --- a/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml +++ b/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml @@ -17,7 +17,7 @@ workflow: rocoto: tasks: - taskgroups: '{{ ["parm/wflow/verify.yaml", "parm/wflow/verify_ensgrid.yaml"]|include }}' + taskgroups: '{{ ["parm/wflow/verify_det.yaml", "parm/wflow/verify_ens.yaml"]|include }}' task_get_obs_ccpa: task_get_obs_mrms: task_get_obs_ndas: diff --git a/tests/WE2E/test_configs/verification/config.MET_verification_only_vx.yaml b/tests/WE2E/test_configs/verification/config.MET_verification_only_vx.yaml index d57d6b3b10..d2e64d8ba0 100644 --- a/tests/WE2E/test_configs/verification/config.MET_verification_only_vx.yaml +++ b/tests/WE2E/test_configs/verification/config.MET_verification_only_vx.yaml @@ -21,7 +21,7 @@ workflow: # It also assumes staged observations, so turn off those tasks. rocoto: tasks: - taskgroups: '{{ ["parm/wflow/verify.yaml"]|include }}' + taskgroups: '{{ ["parm/wflow/verify_det.yaml"]|include }}' task_get_obs_ccpa: task_get_obs_mrms: task_get_obs_ndas: From a920213e90a963fd38b6c79e1cd8a98c9f817f74 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Fri, 19 May 2023 17:00:27 -0600 Subject: [PATCH 23/43] For consistency with the way other variables are passed into the METplus config files (and in order for the final config files to have the correct values), change the way the variable ENSMEM_INDX is passed into METplus config files from an environment variable to a jinja variable. --- parm/metplus/GridStat_APCP01h.conf | 2 +- parm/metplus/GridStat_APCPgt01h.conf | 2 +- parm/metplus/GridStat_REFC.conf | 2 +- parm/metplus/GridStat_RETOP.conf | 2 +- parm/metplus/PointStat_SFC.conf | 2 +- parm/metplus/PointStat_UPA.conf | 2 +- scripts/exregional_run_met_genensprod_or_ensemblestat.sh | 1 + scripts/exregional_run_met_gridstat_or_pointstat_vx.sh | 1 + scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh | 1 + scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh | 1 + scripts/exregional_run_met_pb2nc_obs.sh | 1 + scripts/exregional_run_met_pcpcombine.sh | 1 + 12 files changed, 12 insertions(+), 6 deletions(-) diff --git a/parm/metplus/GridStat_APCP01h.conf b/parm/metplus/GridStat_APCP01h.conf index 9f23de0ea2..e563753d42 100644 --- a/parm/metplus/GridStat_APCP01h.conf +++ b/parm/metplus/GridStat_APCP01h.conf @@ -88,7 +88,7 @@ GRID_STAT_REGRID_SHAPE = SQUARE # suffix to MODEL that identifies the forecast ensemble member. This # makes it easier to identify each curve. # -MODEL = {{vx_fcst_model_name}}_mem{ENV[ENSMEM_INDX]} +MODEL = {{vx_fcst_model_name}}_mem{{ensmem_indx}} FCST_NATIVE_DATA_TYPE = GRIB # diff --git a/parm/metplus/GridStat_APCPgt01h.conf b/parm/metplus/GridStat_APCPgt01h.conf index 464374f965..9544f9a8cc 100644 --- a/parm/metplus/GridStat_APCPgt01h.conf +++ b/parm/metplus/GridStat_APCPgt01h.conf @@ -88,7 +88,7 @@ GRID_STAT_REGRID_SHAPE = SQUARE # suffix to MODEL that identifies the forecast ensemble member. This # makes it easier to identify each curve. # -MODEL = {{vx_fcst_model_name}}_mem{ENV[ENSMEM_INDX]} +MODEL = {{vx_fcst_model_name}}_mem{{ensmem_indx}} FCST_NATIVE_DATA_TYPE = GRIB # diff --git a/parm/metplus/GridStat_REFC.conf b/parm/metplus/GridStat_REFC.conf index 0bb4a35ecd..0c6102b113 100644 --- a/parm/metplus/GridStat_REFC.conf +++ b/parm/metplus/GridStat_REFC.conf @@ -87,7 +87,7 @@ GRID_STAT_GRID_WEIGHT_FLAG = NONE # suffix to MODEL that identifies the forecast ensemble member. This # makes it easier to identify each curve. # -MODEL = {{vx_fcst_model_name}}_mem{ENV[ENSMEM_INDX]} +MODEL = {{vx_fcst_model_name}}_mem{{ensmem_indx}} FCST_NATIVE_DATA_TYPE = GRIB # diff --git a/parm/metplus/GridStat_RETOP.conf b/parm/metplus/GridStat_RETOP.conf index 134d10bee9..d38288a74e 100644 --- a/parm/metplus/GridStat_RETOP.conf +++ b/parm/metplus/GridStat_RETOP.conf @@ -87,7 +87,7 @@ GRID_STAT_GRID_WEIGHT_FLAG = NONE # suffix to MODEL that identifies the forecast ensemble member. This # makes it easier to identify each curve. # -MODEL = {{vx_fcst_model_name}}_mem{ENV[ENSMEM_INDX]} +MODEL = {{vx_fcst_model_name}}_mem{{ensmem_indx}} FCST_NATIVE_DATA_TYPE = GRIB # diff --git a/parm/metplus/PointStat_SFC.conf b/parm/metplus/PointStat_SFC.conf index 58c7d9c228..0c1ae647bc 100644 --- a/parm/metplus/PointStat_SFC.conf +++ b/parm/metplus/PointStat_SFC.conf @@ -123,7 +123,7 @@ POINT_STAT_OFFSETS = 0 # suffix to MODEL that identifies the forecast ensemble member. This # makes it easier to identify each curve. # -MODEL = {{vx_fcst_model_name}}_mem{ENV[ENSMEM_INDX]} +MODEL = {{vx_fcst_model_name}}_mem{{ensmem_indx}} # # Name to identify observation data in output. # diff --git a/parm/metplus/PointStat_UPA.conf b/parm/metplus/PointStat_UPA.conf index 06bc948c6d..367979f81a 100644 --- a/parm/metplus/PointStat_UPA.conf +++ b/parm/metplus/PointStat_UPA.conf @@ -123,7 +123,7 @@ POINT_STAT_OFFSETS = 0 # suffix to MODEL that identifies the forecast ensemble member. This # makes it easier to identify each curve. # -MODEL = {{vx_fcst_model_name}}_mem{ENV[ENSMEM_INDX]} +MODEL = {{vx_fcst_model_name}}_mem{{ensmem_indx}} # # Name to identify observation data in output. # diff --git a/scripts/exregional_run_met_genensprod_or_ensemblestat.sh b/scripts/exregional_run_met_genensprod_or_ensemblestat.sh index 9f5ddef0b7..4b6129e7f6 100755 --- a/scripts/exregional_run_met_genensprod_or_ensemblestat.sh +++ b/scripts/exregional_run_met_genensprod_or_ensemblestat.sh @@ -368,6 +368,7 @@ settings="\ # Ensemble and member-specific information. # 'num_ens_members': '${NUM_ENS_MEMBERS}' + 'ensmem_indx': '${ENSMEM_INDX:-}' 'time_lag': '${time_lag:-}' # # Field information. diff --git a/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh b/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh index 62e5961f4e..6be745086e 100755 --- a/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh +++ b/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh @@ -348,6 +348,7 @@ settings="\ # Ensemble and member-specific information. # 'num_ens_members': '${NUM_ENS_MEMBERS}' + 'ensmem_indx': '${ENSMEM_INDX:-}' 'time_lag': '${time_lag:-}' # # Field information. diff --git a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh index d94480271b..6ceaeb67d4 100755 --- a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh +++ b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh @@ -328,6 +328,7 @@ settings="\ # Ensemble and member-specific information. # 'num_ens_members': '${NUM_ENS_MEMBERS}' + 'ensmem_indx': '${ENSMEM_INDX:-}' 'time_lag': '${time_lag:-}' # # Field information. diff --git a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh index f17709729a..c761f15135 100755 --- a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh +++ b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh @@ -290,6 +290,7 @@ settings="\ # Ensemble and member-specific information. # 'num_ens_members': '${NUM_ENS_MEMBERS}' + 'ensmem_indx': '${ENSMEM_INDX:-}' 'time_lag': '${time_lag:-}' # # Field information. diff --git a/scripts/exregional_run_met_pb2nc_obs.sh b/scripts/exregional_run_met_pb2nc_obs.sh index 528e7743dc..3ab2003717 100755 --- a/scripts/exregional_run_met_pb2nc_obs.sh +++ b/scripts/exregional_run_met_pb2nc_obs.sh @@ -265,6 +265,7 @@ settings="\ # Ensemble and member-specific information. # 'num_ens_members': '${NUM_ENS_MEMBERS}' + 'ensmem_indx': '${ENSMEM_INDX:-}' 'time_lag': '${time_lag:-}' # # Field information. diff --git a/scripts/exregional_run_met_pcpcombine.sh b/scripts/exregional_run_met_pcpcombine.sh index 0ff30d270a..d40180dc13 100755 --- a/scripts/exregional_run_met_pcpcombine.sh +++ b/scripts/exregional_run_met_pcpcombine.sh @@ -318,6 +318,7 @@ settings="\ # Ensemble and member-specific information. # 'num_ens_members': '${NUM_ENS_MEMBERS}' + 'ensmem_indx': '${ENSMEM_INDX:-}' 'time_lag': '${time_lag:-}' # # Field information. From 3c2af8da80ede038144488baf0614ad98f7d4b46 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Tue, 23 May 2023 00:38:14 -0600 Subject: [PATCH 24/43] Changes to make vx tests work in NCO mode: 1) Replace "mem${ensmem_indx}" with new variable "${ensmem_name}". 2) Remove DOT_ENSMEM_OR_NULL since ensmem_name can now be used instead. 3) Replace SLASH_ENSMEM_SUBDIR_OR_NULL with slash_ensmem_subdir_or_null. 4) Remove unneeded code from exregional_run_met_pb2nc_obs.sh. --- parm/metplus/GridStat_APCP01h.conf | 2 +- parm/metplus/GridStat_APCPgt01h.conf | 2 +- parm/metplus/GridStat_REFC.conf | 2 +- parm/metplus/GridStat_RETOP.conf | 2 +- parm/metplus/PointStat_SFC.conf | 2 +- parm/metplus/PointStat_UPA.conf | 2 +- parm/wflow/verify_det.yaml | 1 + ...onal_run_met_genensprod_or_ensemblestat.sh | 5 +-- ...gional_run_met_gridstat_or_pointstat_vx.sh | 34 ++++++++++++----- ...un_met_gridstat_or_pointstat_vx_ensmean.sh | 2 +- ...un_met_gridstat_or_pointstat_vx_ensprob.sh | 2 +- scripts/exregional_run_met_pb2nc_obs.sh | 12 +----- scripts/exregional_run_met_pcpcombine.sh | 37 ++++++++++++------- ush/config_defaults.yaml | 6 +-- 14 files changed, 62 insertions(+), 49 deletions(-) diff --git a/parm/metplus/GridStat_APCP01h.conf b/parm/metplus/GridStat_APCP01h.conf index e563753d42..0ba9399203 100644 --- a/parm/metplus/GridStat_APCP01h.conf +++ b/parm/metplus/GridStat_APCP01h.conf @@ -88,7 +88,7 @@ GRID_STAT_REGRID_SHAPE = SQUARE # suffix to MODEL that identifies the forecast ensemble member. This # makes it easier to identify each curve. # -MODEL = {{vx_fcst_model_name}}_mem{{ensmem_indx}} +MODEL = {{vx_fcst_model_name}}_{{ensmem_name}} FCST_NATIVE_DATA_TYPE = GRIB # diff --git a/parm/metplus/GridStat_APCPgt01h.conf b/parm/metplus/GridStat_APCPgt01h.conf index 9544f9a8cc..d56bd21e23 100644 --- a/parm/metplus/GridStat_APCPgt01h.conf +++ b/parm/metplus/GridStat_APCPgt01h.conf @@ -88,7 +88,7 @@ GRID_STAT_REGRID_SHAPE = SQUARE # suffix to MODEL that identifies the forecast ensemble member. This # makes it easier to identify each curve. # -MODEL = {{vx_fcst_model_name}}_mem{{ensmem_indx}} +MODEL = {{vx_fcst_model_name}}_{{ensmem_name}} FCST_NATIVE_DATA_TYPE = GRIB # diff --git a/parm/metplus/GridStat_REFC.conf b/parm/metplus/GridStat_REFC.conf index 0c6102b113..ce77614033 100644 --- a/parm/metplus/GridStat_REFC.conf +++ b/parm/metplus/GridStat_REFC.conf @@ -87,7 +87,7 @@ GRID_STAT_GRID_WEIGHT_FLAG = NONE # suffix to MODEL that identifies the forecast ensemble member. This # makes it easier to identify each curve. # -MODEL = {{vx_fcst_model_name}}_mem{{ensmem_indx}} +MODEL = {{vx_fcst_model_name}}_{{ensmem_name}} FCST_NATIVE_DATA_TYPE = GRIB # diff --git a/parm/metplus/GridStat_RETOP.conf b/parm/metplus/GridStat_RETOP.conf index d38288a74e..67ed1e8b2b 100644 --- a/parm/metplus/GridStat_RETOP.conf +++ b/parm/metplus/GridStat_RETOP.conf @@ -87,7 +87,7 @@ GRID_STAT_GRID_WEIGHT_FLAG = NONE # suffix to MODEL that identifies the forecast ensemble member. This # makes it easier to identify each curve. # -MODEL = {{vx_fcst_model_name}}_mem{{ensmem_indx}} +MODEL = {{vx_fcst_model_name}}_{{ensmem_name}} FCST_NATIVE_DATA_TYPE = GRIB # diff --git a/parm/metplus/PointStat_SFC.conf b/parm/metplus/PointStat_SFC.conf index 0c1ae647bc..62f59328fe 100644 --- a/parm/metplus/PointStat_SFC.conf +++ b/parm/metplus/PointStat_SFC.conf @@ -123,7 +123,7 @@ POINT_STAT_OFFSETS = 0 # suffix to MODEL that identifies the forecast ensemble member. This # makes it easier to identify each curve. # -MODEL = {{vx_fcst_model_name}}_mem{{ensmem_indx}} +MODEL = {{vx_fcst_model_name}}_{{ensmem_name}} # # Name to identify observation data in output. # diff --git a/parm/metplus/PointStat_UPA.conf b/parm/metplus/PointStat_UPA.conf index 367979f81a..385b5387a2 100644 --- a/parm/metplus/PointStat_UPA.conf +++ b/parm/metplus/PointStat_UPA.conf @@ -123,7 +123,7 @@ POINT_STAT_OFFSETS = 0 # suffix to MODEL that identifies the forecast ensemble member. This # makes it easier to identify each curve. # -MODEL = {{vx_fcst_model_name}}_mem{{ensmem_indx}} +MODEL = {{vx_fcst_model_name}}_{{ensmem_name}} # # Name to identify observation data in output. # diff --git a/parm/wflow/verify_det.yaml b/parm/wflow/verify_det.yaml index e9b5bd6b3d..445b0d5329 100644 --- a/parm/wflow/verify_det.yaml +++ b/parm/wflow/verify_det.yaml @@ -98,6 +98,7 @@ metatask_PcpCombine_obs: OBTYPE: CCPA OBS_DIR: '&CCPA_OBS_DIR;' MET_TOOL: 'PCPCOMBINE' + ENSMEM_INDX: '' dependency: or: not: diff --git a/scripts/exregional_run_met_genensprod_or_ensemblestat.sh b/scripts/exregional_run_met_genensprod_or_ensemblestat.sh index 4b6129e7f6..0bff6499c1 100755 --- a/scripts/exregional_run_met_genensprod_or_ensemblestat.sh +++ b/scripts/exregional_run_met_genensprod_or_ensemblestat.sh @@ -211,14 +211,13 @@ for (( i=0; i<${NUM_ENS_MEMBERS}; i++ )); do mem_indx=$(($i+1)) mem_indx_fmt=$(printf "%0${NDIGITS_ENSMEM_NAMES}d" "${mem_indx}") + ensmem_name="mem${mem_indx_fmt}" time_lag=$( bc -l <<< "${ENS_TIME_LAG_HRS[$i]}*${SECS_PER_HOUR}" ) if [ "${RUN_ENVIR}" = "nco" ]; then cdate_ensmem_subdir_or_null="" - DOT_ENSMEM_OR_NULL=".mem${mem_indx_fmt}" else cdate_ensmem_subdir_or_null="${CDATE}/mem${mem_indx_fmt}" - DOT_ENSMEM_OR_NULL="" fi if [ "${field_is_APCPgt01h}" = "TRUE" ]; then @@ -227,7 +226,7 @@ for (( i=0; i<${NUM_ENS_MEMBERS}; i++ )); do template="${FCST_SUBDIR_TEMPLATE}/${FCST_FN_TEMPLATE}" fi - SLASH_ENSMEM_SUBDIR_OR_NULL="/mem${mem_indx_fmt}" + slash_ensmem_subdir_or_null="/${ensmem_name}" if [ -z "${FCST_INPUT_FN_TEMPLATE}" ]; then FCST_INPUT_FN_TEMPLATE=" $(eval echo ${template})" else diff --git a/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh b/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh index 6be745086e..f27090952b 100755 --- a/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh +++ b/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh @@ -172,15 +172,29 @@ fi # vx_fcst_input_basedir=$( eval echo "${VX_FCST_INPUT_BASEDIR}" ) vx_output_basedir=$( eval echo "${VX_OUTPUT_BASEDIR}" ) +ensmem_name="mem${ENSMEM_INDX}" if [ "${RUN_ENVIR}" = "nco" ]; then - if [[ ${DO_ENSEMBLE} == "TRUE" ]]; then - ENSMEM=$( echo ${SLASH_ENSMEM_SUBDIR_OR_NULL} | cut -d"/" -f2 ) - DOT_ENSMEM_OR_NULL=".$ENSMEM" + slash_cdate_or_null="" + slash_ensmem_subdir_or_null="" +else + slash_cdate_or_null="/${CDATE}" +# +# Since other aspects of a deterministic run use the "mem000" string (e.g. +# in rocoto workflow task names, in log file names), it seems reasonable +# that a deterministic run create a "mem000" subdirectory under the $CDATE +# directory. But since that is currently not the case in in the run_fcst +# task, we need the following if-statement. If and when such a modification +# is made for the run_fcst task, we would remove this if-statement and +# simply set +# slash_ensmem_subdir_or_null="/${ensmem_name}" +# or, better, just remove this variale and code "/${ensmem_name}" where +# slash_ensmem_subdir_or_null currently appears below. +# + if [ "${DO_ENSEMBLE}" = "TRUE" ]; then + slash_ensmem_subdir_or_null="/${ensmem_name}" else - DOT_ENSMEM_OR_NULL="" + slash_ensmem_subdir_or_null="" fi -else - DOT_ENSMEM_OR_NULL="" fi if [ "${grid_or_point}" = "grid" ]; then @@ -189,7 +203,7 @@ if [ "${grid_or_point}" = "grid" ]; then if [ "${field_is_APCPgt01h}" = "TRUE" ]; then OBS_INPUT_DIR="${vx_output_basedir}/metprd/PcpCombine_obs" OBS_INPUT_FN_TEMPLATE=$( eval echo ${OBS_CCPA_APCPgt01h_FN_TEMPLATE} ) - FCST_INPUT_DIR="${vx_output_basedir}/${CDATE}/mem${ENSMEM_INDX}/metprd/PcpCombine_fcst" + FCST_INPUT_DIR="${vx_output_basedir}${slash_cdate_or_null}/${slash_ensmem_subdir_or_null}/metprd/PcpCombine_fcst" FCST_INPUT_FN_TEMPLATE=$( eval echo ${FCST_FN_METPROC_TEMPLATE} ) else OBS_INPUT_DIR="${OBS_DIR}" @@ -218,7 +232,7 @@ elif [ "${grid_or_point}" = "point" ]; then fi -OUTPUT_BASE="${vx_output_basedir}/${CDATE}/mem${ENSMEM_INDX}" +OUTPUT_BASE="${vx_output_basedir}${slash_cdate_or_null}/${slash_ensmem_subdir_or_null}" OUTPUT_DIR="${OUTPUT_BASE}/metprd/${metplus_tool_name}" STAGING_DIR="${OUTPUT_BASE}/stage/${FIELDNAME_IN_MET_FILEDIR_NAMES}" # @@ -300,7 +314,7 @@ else metplus_config_tmpl_fn="${FIELDNAME_IN_MET_FILEDIR_NAMES}" fi metplus_config_tmpl_fn="${metplus_tool_name}_${metplus_config_tmpl_fn}" -metplus_config_fn="${metplus_tool_name}_${FIELDNAME_IN_MET_FILEDIR_NAMES}_mem${ENSMEM_INDX}" +metplus_config_fn="${metplus_tool_name}_${FIELDNAME_IN_MET_FILEDIR_NAMES}_${ensmem_name}" metplus_log_fn="${metplus_config_fn}" # # Add prefixes and suffixes (extensions) to the base file names. @@ -348,7 +362,7 @@ settings="\ # Ensemble and member-specific information. # 'num_ens_members': '${NUM_ENS_MEMBERS}' - 'ensmem_indx': '${ENSMEM_INDX:-}' + 'ensmem_name': '${ensmem_name:-}' 'time_lag': '${time_lag:-}' # # Field information. diff --git a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh index 6ceaeb67d4..1cf3138c75 100755 --- a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh +++ b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh @@ -328,7 +328,7 @@ settings="\ # Ensemble and member-specific information. # 'num_ens_members': '${NUM_ENS_MEMBERS}' - 'ensmem_indx': '${ENSMEM_INDX:-}' + 'ensmem_name': '${ensmem_name:-}' 'time_lag': '${time_lag:-}' # # Field information. diff --git a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh index c761f15135..3a31968693 100755 --- a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh +++ b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh @@ -290,7 +290,7 @@ settings="\ # Ensemble and member-specific information. # 'num_ens_members': '${NUM_ENS_MEMBERS}' - 'ensmem_indx': '${ENSMEM_INDX:-}' + 'ensmem_name': '${ensmem_name:-}' 'time_lag': '${time_lag:-}' # # Field information. diff --git a/scripts/exregional_run_met_pb2nc_obs.sh b/scripts/exregional_run_met_pb2nc_obs.sh index 3ab2003717..78a7bea4ed 100755 --- a/scripts/exregional_run_met_pb2nc_obs.sh +++ b/scripts/exregional_run_met_pb2nc_obs.sh @@ -108,16 +108,6 @@ set_vx_params \ #----------------------------------------------------------------------- # vx_output_basedir=$( eval echo "${VX_OUTPUT_BASEDIR}" ) -if [ "${RUN_ENVIR}" = "nco" ]; then - if [[ ${DO_ENSEMBLE} == "TRUE" ]]; then - ENSMEM=$( echo ${SLASH_ENSMEM_SUBDIR_OR_NULL} | cut -d"/" -f2 ) - DOT_ENSMEM_OR_NULL=".$ENSMEM" - else - DOT_ENSMEM_OR_NULL="" - fi -else - DOT_ENSMEM_OR_NULL="" -fi OBS_INPUT_DIR="${OBS_DIR}" OBS_INPUT_FN_TEMPLATE=$( eval echo ${OBS_NDAS_SFCorUPA_FN_TEMPLATE} ) @@ -265,7 +255,7 @@ settings="\ # Ensemble and member-specific information. # 'num_ens_members': '${NUM_ENS_MEMBERS}' - 'ensmem_indx': '${ENSMEM_INDX:-}' + 'ensmem_name': '${ensmem_name:-}' 'time_lag': '${time_lag:-}' # # Field information. diff --git a/scripts/exregional_run_met_pcpcombine.sh b/scripts/exregional_run_met_pcpcombine.sh index d40180dc13..e1789fbba6 100755 --- a/scripts/exregional_run_met_pcpcombine.sh +++ b/scripts/exregional_run_met_pcpcombine.sh @@ -136,15 +136,29 @@ fi # vx_fcst_input_basedir=$( eval echo "${VX_FCST_INPUT_BASEDIR}" ) vx_output_basedir=$( eval echo "${VX_OUTPUT_BASEDIR}" ) +ensmem_name="mem${ENSMEM_INDX}" if [ "${RUN_ENVIR}" = "nco" ]; then - if [[ ${DO_ENSEMBLE} == "TRUE" ]]; then - ENSMEM=$( echo ${SLASH_ENSMEM_SUBDIR_OR_NULL} | cut -d"/" -f2 ) - DOT_ENSMEM_OR_NULL=".$ENSMEM" + slash_cdate_or_null="" + slash_ensmem_subdir_or_null="" +else + slash_cdate_or_null="/${CDATE}" +# +# Since other aspects of a deterministic run use the "mem000" string (e.g. +# in rocoto workflow task names, in log file names), it seems reasonable +# that a deterministic run create a "mem000" subdirectory under the $CDATE +# directory. But since that is currently not the case in in the run_fcst +# task, we need the following if-statement. If and when such a modification +# is made for the run_fcst task, we would remove this if-statement and +# simply set +# slash_ensmem_subdir_or_null="/${ensmem_name}" +# or, better, just remove this variale and code "/${ensmem_name}" where +# slash_ensmem_subdir_or_null currently appears below. +# + if [ "${DO_ENSEMBLE}" = "TRUE" ]; then + slash_ensmem_subdir_or_null="/${ensmem_name}" else - DOT_ENSMEM_OR_NULL="" + slash_ensmem_subdir_or_null="" fi -else - DOT_ENSMEM_OR_NULL="" fi OBS_INPUT_DIR="" @@ -167,7 +181,7 @@ elif [ "${obs_or_fcst}" = "fcst" ]; then FCST_INPUT_DIR="${vx_fcst_input_basedir}" FCST_INPUT_FN_TEMPLATE=$( eval echo ${FCST_SUBDIR_TEMPLATE:+${FCST_SUBDIR_TEMPLATE}/}${FCST_FN_TEMPLATE} ) - OUTPUT_BASE="${vx_output_basedir}/${CDATE}/mem${ENSMEM_INDX}" + OUTPUT_BASE="${vx_output_basedir}${slash_cdate_or_null}/${slash_ensmem_subdir_or_null}" OUTPUT_DIR="${OUTPUT_BASE}/metprd/${metplus_tool_name}_fcst" OUTPUT_FN_TEMPLATE=$( eval echo ${FCST_FN_METPROC_TEMPLATE} ) STAGING_DIR="${OUTPUT_BASE}/stage/${FIELDNAME_IN_MET_FILEDIR_NAMES}" @@ -252,15 +266,10 @@ fi # #----------------------------------------------------------------------- # -uscore_ensmem_name_or_null="" -if [ "${obs_or_fcst}" = "fcst" ]; then - uscore_ensmem_name_or_null="_mem${ENSMEM_INDX}" -fi -# # First, set the base file names. # metplus_config_tmpl_fn="${metplus_tool_name}_${obs_or_fcst}" -metplus_config_fn="${metplus_config_tmpl_fn}_${FIELDNAME_IN_MET_FILEDIR_NAMES}${uscore_ensmem_name_or_null}" +metplus_config_fn="${metplus_config_tmpl_fn}_${FIELDNAME_IN_MET_FILEDIR_NAMES}${ENSMEM_INDX:+_${ensmem_name}}" metplus_log_fn="${metplus_config_fn}_$CDATE" # # If operating on observation files, append the cycle date to the name @@ -318,7 +327,7 @@ settings="\ # Ensemble and member-specific information. # 'num_ens_members': '${NUM_ENS_MEMBERS}' - 'ensmem_indx': '${ENSMEM_INDX:-}' + 'ensmem_name': '${ensmem_name:-}' 'time_lag': '${time_lag:-}' # # Field information. diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 3df8bd402e..09d774b068 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -2475,9 +2475,9 @@ verification: # # File name and path templates used in the verification tasks. # - FCST_SUBDIR_TEMPLATE: '{% if user.RUN_ENVIR == "nco" %}${NET}.{init?fmt=%Y%m%d?shift=-${time_lag}}/{init?fmt=%H?shift=-${time_lag}}{% else %}{init?fmt=%Y%m%d%H?shift=-${time_lag}}${SLASH_ENSMEM_SUBDIR_OR_NULL}/postprd{% endif %}' - FCST_FN_TEMPLATE: '${NET}.t{init?fmt=%H?shift=-${time_lag}}z{% if user.RUN_ENVIR == "nco" %}${DOT_ENSMEM_OR_NULL}{% endif %}.prslev.f{lead?fmt=%HHH?shift=${time_lag}}.${POST_OUTPUT_DOMAIN_NAME}.grib2' - FCST_FN_METPROC_TEMPLATE: '${NET}.t{init?fmt=%H}z{% if user.RUN_ENVIR == "nco" %}${DOT_ENSMEM_OR_NULL}{% endif %}.prslev.f{lead?fmt=%HHH}.${POST_OUTPUT_DOMAIN_NAME}_a${ACCUM_HH}h.nc' + FCST_SUBDIR_TEMPLATE: '{% if user.RUN_ENVIR == "nco" %}${NET}.{init?fmt=%Y%m%d?shift=-${time_lag}}/{init?fmt=%H?shift=-${time_lag}}{% else %}{init?fmt=%Y%m%d%H?shift=-${time_lag}}{% if global.DO_ENSEMBLE %}/${ensmem_name}{% endif %}/postprd{% endif %}' + FCST_FN_TEMPLATE: '${NET}.t{init?fmt=%H?shift=-${time_lag}}z{% if user.RUN_ENVIR == "nco" and global.DO_ENSEMBLE %}.${ensmem_name}{% endif %}.prslev.f{lead?fmt=%HHH?shift=${time_lag}}.${POST_OUTPUT_DOMAIN_NAME}.grib2' + FCST_FN_METPROC_TEMPLATE: '${NET}.t{init?fmt=%H}z{% if user.RUN_ENVIR == "nco" and global.DO_ENSEMBLE %}.${ensmem_name}{% endif %}.prslev.f{lead?fmt=%HHH}.${POST_OUTPUT_DOMAIN_NAME}_a${ACCUM_HH}h.nc' # # For verification tasks that need observational data, this specifies # the maximum number of observation files that may be missing. If more From b534d7368efc7e703cdbbb80a49a5120e4383a89 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Tue, 23 May 2023 02:05:19 -0600 Subject: [PATCH 25/43] Change variable name for clarity. --- parm/wflow/verify_det.yaml | 6 +++--- parm/wflow/verify_ens.yaml | 4 ++-- ush/config_defaults.yaml | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/parm/wflow/verify_det.yaml b/parm/wflow/verify_det.yaml index 445b0d5329..9ea734ffb1 100644 --- a/parm/wflow/verify_det.yaml +++ b/parm/wflow/verify_det.yaml @@ -83,7 +83,7 @@ task_run_MET_Pb2nc_obs: metatask_PcpCombine_obs: var: - ACCUM_HH: '{% for ah in verification.VX_APCP_ACCUMS_HH %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' + ACCUM_HH: '{% for ah in verification.VX_APCP_ACCUMS_HRS %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' task_run_MET_PcpCombine_obs_APCP#ACCUM_HH#h: <<: *default_task attrs: @@ -114,7 +114,7 @@ metatask_PcpCombine_obs: metatask_PcpCombine_fcst_all_accums_all_mems: var: - ACCUM_HH: '{% for ah in verification.VX_APCP_ACCUMS_HH %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' + ACCUM_HH: '{% for ah in verification.VX_APCP_ACCUMS_HRS %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' metatask_PcpCombine_fcst_APCP#ACCUM_HH#h_all_mems: var: mem: '{% if global.DO_ENSEMBLE %}{% for m in range(1, global.NUM_ENS_MEMBERS+1) %}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' @@ -154,7 +154,7 @@ metatask_PcpCombine_fcst_all_accums_all_mems: metatask_GridStat_CCPA_all_accums_all_mems: var: - ACCUM_HH: '{% for ah in verification.VX_APCP_ACCUMS_HH %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' + ACCUM_HH: '{% for ah in verification.VX_APCP_ACCUMS_HRS %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' metatask_GridStat_CCPA_APCP#ACCUM_HH#h_all_mems: var: mem: '{% if global.DO_ENSEMBLE %}{% for m in range(1, global.NUM_ENS_MEMBERS+1) %}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' diff --git a/parm/wflow/verify_ens.yaml b/parm/wflow/verify_ens.yaml index 47f08725a8..10eab47d90 100644 --- a/parm/wflow/verify_ens.yaml +++ b/parm/wflow/verify_ens.yaml @@ -24,7 +24,7 @@ default_task_verify_ens: &default_task_verify_ens metatask_GEPES_CCPA: var: - ACCUM_HH: '{% for ah in verification.VX_APCP_ACCUMS_HH %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' + ACCUM_HH: '{% for ah in verification.VX_APCP_ACCUMS_HRS %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' task_run_MET_GenEnsProd_vx_APCP#ACCUM_HH#h: &task_GenEnsProd_CCPA <<: *default_task_verify_ens command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_GENENSPROD_OR_ENSEMBLESTAT"' @@ -143,7 +143,7 @@ metatask_GridStat_CCPA_ensmeanprob_all_accums: statlc: mean prob metatask_GridStat_CCPA_ens#statlc#_all_accums: var: - ACCUM_HH: '{% for ah in verification.VX_APCP_ACCUMS_HH %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' + ACCUM_HH: '{% for ah in verification.VX_APCP_ACCUMS_HRS %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' task_run_MET_GridStat_vx_ens#statlc#_APCP#ACCUM_HH#h: <<: *default_task_verify_ens command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_GRIDSTAT_OR_POINTSTAT_VX_ENS#stat#"' diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 09d774b068..d77bf6d849 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -2452,15 +2452,15 @@ verification: # VX_FIELDS: # The fields or groups of fields on which to run verification. # - # VX_APCP_ACCUMS_HH: + # VX_APCP_ACCUMS_HRS: # The 2-digit accumulation periods (in units of hours) to consider for # APCP (accumulated precipitation). If VX_FIELDS contains "APCP", then - # VX_APCP_ACCUMS_HH must contain at least one element. If not, - # VX_APCP_ACCUMS_HH will be ignored. + # VX_APCP_ACCUMS_HRS must contain at least one element. If not, + # VX_APCP_ACCUMS_HRS will be ignored. # VX_FCST_MODEL_NAME: '{{ nco.NET }}.{{ task_run_post.POST_OUTPUT_DOMAIN_NAME }}' VX_FIELDS: [ "APCP", "REFC", "RETOP", "SFC", "UPA" ] - VX_APCP_ACCUMS_HH: [ 1, 3, 6, 24 ] + VX_APCP_ACCUMS_HRS: [ 1, 3, 6, 24 ] # # VX_FCST_INPUT_BASEDIR: # Template for top-level directory containing forecast (but not obs) From e6bbad3060f824d35bbe4022af10dd6f87aefaf1 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Tue, 23 May 2023 03:01:25 -0600 Subject: [PATCH 26/43] Add code to allow each element of a list variable to be checked for valid values; specify valid sets of values for VX_FIELDS and VX_APCP_ACCUMS_HRS. --- ush/setup.py | 26 +++++++++++++++++++------- ush/valid_param_vals.yaml | 2 ++ 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/ush/setup.py b/ush/setup.py index bda5c86b15..b650d3bbdd 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -1619,13 +1619,25 @@ def dict_find(user_dict, substring): if v is None or v == "": continue vkey = "valid_vals_" + k - if (vkey in cfg_v) and not (v in cfg_v[vkey]): - raise Exception( - f""" - The variable {k}={v} in the user's configuration - does not have a valid value. Possible values are: - {k} = {cfg_v[vkey]}""" - ) + if (vkey in cfg_v): + if (type(v) == list): + if not(all(ele in cfg_v[vkey] for ele in v)): + raise Exception( + dedent(f""" + The variable + {k} = {v} + in the user's configuration has at least one invalid value. Possible values are: + {k} = {cfg_v[vkey]}""" + )) + else: + if not (v in cfg_v[vkey]): + raise Exception( + dedent(f""" + The variable + {k} = {v} + in the user's configuration does not have a valid value. Possible values are: + {k} = {cfg_v[vkey]}""" + )) return expt_config diff --git a/ush/valid_param_vals.yaml b/ush/valid_param_vals.yaml index 98fbfd09d9..f839dcb858 100644 --- a/ush/valid_param_vals.yaml +++ b/ush/valid_param_vals.yaml @@ -86,3 +86,5 @@ valid_vals_DO_AQM_GEFS_LBCS: [True, False] valid_vals_DO_AQM_SAVE_AIRNOW_HIST: [True, False] valid_vals_COLDSTART: [True, False] valid_vals_DO_NLDN_LGHT: [True, False] +valid_vals_VX_FIELDS: [ "APCP", "REFC", "RETOP", "SFC", "UPA" ] +valid_vals_VX_APCP_ACCUMS_HRS: [ 1, 3, 6, 24 ] From 017bc5c4c40a8d1d2f251577f8c23efd9fb64c92 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Wed, 24 May 2023 20:32:40 -0600 Subject: [PATCH 27/43] Change argument list to function set_vx_fhr_list() for clarity and to allow separate max number of missing files for obs vs. forecast. --- ...ional_run_met_genensprod_or_ensemblestat.sh | 3 ++- ...egional_run_met_gridstat_or_pointstat_vx.sh | 3 ++- ...run_met_gridstat_or_pointstat_vx_ensmean.sh | 3 ++- ...run_met_gridstat_or_pointstat_vx_ensprob.sh | 3 ++- scripts/exregional_run_met_pb2nc_obs.sh | 3 ++- scripts/exregional_run_met_pcpcombine.sh | 5 ++++- ush/config_defaults.yaml | 7 +++++++ ush/set_vx_fhr_list.sh | 18 +++++++++++------- 8 files changed, 32 insertions(+), 13 deletions(-) diff --git a/scripts/exregional_run_met_genensprod_or_ensemblestat.sh b/scripts/exregional_run_met_genensprod_or_ensemblestat.sh index 0bff6499c1..8464b5ad7b 100755 --- a/scripts/exregional_run_met_genensprod_or_ensemblestat.sh +++ b/scripts/exregional_run_met_genensprod_or_ensemblestat.sh @@ -254,7 +254,8 @@ set_vx_fhr_list \ accum_hh="${ACCUM_HH}" \ base_dir="${OBS_INPUT_DIR}" \ fn_template="${OBS_INPUT_FN_TEMPLATE}" \ - check_hourly_files="FALSE" \ + check_accum_contrib_files="FALSE" \ + num_missing_files_max="${NUM_MISSING_OBS_FILES_MAX}" \ outvarname_fhr_list="FHR_LIST" # #----------------------------------------------------------------------- diff --git a/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh b/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh index f27090952b..c6395db119 100755 --- a/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh +++ b/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh @@ -249,7 +249,8 @@ set_vx_fhr_list \ accum_hh="${ACCUM_HH}" \ base_dir="${OBS_INPUT_DIR}" \ fn_template="${OBS_INPUT_FN_TEMPLATE}" \ - check_hourly_files="FALSE" \ + check_accum_contrib_files="FALSE" \ + num_missing_files_max="${NUM_MISSING_OBS_FILES_MAX}" \ outvarname_fhr_list="FHR_LIST" # #----------------------------------------------------------------------- diff --git a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh index 1cf3138c75..3ca0fcdb8e 100755 --- a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh +++ b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh @@ -206,7 +206,8 @@ set_vx_fhr_list \ accum_hh="${ACCUM_HH}" \ base_dir="${OBS_INPUT_DIR}" \ fn_template="${OBS_INPUT_FN_TEMPLATE}" \ - check_hourly_files="FALSE" \ + check_accum_contrib_files="FALSE" \ + num_missing_files_max="${NUM_MISSING_OBS_FILES_MAX}" \ outvarname_fhr_list="FHR_LIST" # #----------------------------------------------------------------------- diff --git a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh index 3a31968693..7efa2c1fe4 100755 --- a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh +++ b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh @@ -165,7 +165,8 @@ set_vx_fhr_list \ accum_hh="${ACCUM_HH}" \ base_dir="${OBS_INPUT_DIR}" \ fn_template="${OBS_INPUT_FN_TEMPLATE}" \ - check_hourly_files="FALSE" \ + check_accum_contrib_files="FALSE" \ + num_missing_files_max="${NUM_MISSING_OBS_FILES_MAX}" \ outvarname_fhr_list="FHR_LIST" # #----------------------------------------------------------------------- diff --git a/scripts/exregional_run_met_pb2nc_obs.sh b/scripts/exregional_run_met_pb2nc_obs.sh index 78a7bea4ed..95feb701f4 100755 --- a/scripts/exregional_run_met_pb2nc_obs.sh +++ b/scripts/exregional_run_met_pb2nc_obs.sh @@ -130,7 +130,8 @@ set_vx_fhr_list \ accum_hh="${ACCUM_HH}" \ base_dir="${OBS_INPUT_DIR}" \ fn_template="${OBS_INPUT_FN_TEMPLATE}" \ - check_hourly_files="FALSE" \ + check_accum_contrib_files="FALSE" \ + num_missing_files_max="${NUM_MISSING_OBS_FILES_MAX}" \ outvarname_fhr_list="FHR_LIST" # #----------------------------------------------------------------------- diff --git a/scripts/exregional_run_met_pcpcombine.sh b/scripts/exregional_run_met_pcpcombine.sh index e1789fbba6..3b19335fc8 100755 --- a/scripts/exregional_run_met_pcpcombine.sh +++ b/scripts/exregional_run_met_pcpcombine.sh @@ -197,9 +197,11 @@ fi if [ "${obs_or_fcst}" = "obs" ]; then base_dir="${OBS_INPUT_DIR}" fn_template="${OBS_INPUT_FN_TEMPLATE}" + num_missing_files_max="${NUM_MISSING_OBS_FILES_MAX}" elif [ "${obs_or_fcst}" = "fcst" ]; then base_dir="${FCST_INPUT_DIR}" fn_template="${FCST_INPUT_FN_TEMPLATE}" + num_missing_files_max="${NUM_MISSING_FCST_FILES_MAX}" fi set_vx_fhr_list \ @@ -209,7 +211,8 @@ set_vx_fhr_list \ accum_hh="${ACCUM_HH}" \ base_dir="${base_dir}" \ fn_template="${fn_template}" \ - check_hourly_files="TRUE" \ + check_accum_contrib_files="TRUE" \ + num_missing_files_max="${num_missing_files_max}" \ outvarname_fhr_list="FHR_LIST" # #----------------------------------------------------------------------- diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index d77bf6d849..609782b90d 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -2491,6 +2491,13 @@ verification: # number missing divided by the number that are expected to exist). # NUM_MISSING_OBS_FILES_MAX: 2 + # + # For verification tasks that need forecast data, this specifies the + # maximum number of post-processed forecast files that may be missing. + # If more than this number are missing, the verification task will not + # be run. + # + NUM_MISSING_FCST_FILES_MAX: 0 #---------------------------- # CPL_AQM config parameters diff --git a/ush/set_vx_fhr_list.sh b/ush/set_vx_fhr_list.sh index d3403d09f2..2e79da7e12 100644 --- a/ush/set_vx_fhr_list.sh +++ b/ush/set_vx_fhr_list.sh @@ -54,7 +54,8 @@ function set_vx_fhr_list() { "accum_hh" \ "base_dir" \ "fn_template" \ - "check_hourly_files" \ + "check_accum_contrib_files" \ + "num_missing_files_max" \ "outvarname_fhr_list" \ ) process_args valid_args "$@" @@ -153,7 +154,7 @@ is: fhr_orig="${fhr_array[$i]}" - if [ "${check_hourly_files}" = "TRUE" ]; then + if [ "${check_accum_contrib_files}" = "TRUE" ]; then fhr=$(( ${fhr_orig} - ${accum_hh} + 1 )) num_back_hrs=${accum_hh} else @@ -165,6 +166,9 @@ is: for (( j=0; j<${num_back_hrs}; j++ )); do # # Use the provided template to set the name of/relative path to the file +# Note that the while-loop below is over all METplus time string templates +# of the form {...} in the template fn_template; it continues until all +# such templates have been evaluated to actual time strings. # fn="${fn_template}" regex_search_tmpl="(.*)(\{.*\})(.*)" @@ -247,18 +251,18 @@ Final (i.e. after filtering for missing files) set of foreast hours is # #----------------------------------------------------------------------- # -# If the number of missing files is greater than the user-specified -# variable NUM_MISSING_OBS_FILES_MAX, print out an error message and +# If the number of missing files is greater than the maximum allowed +# (specified by num_missing_files_max), print out an error message and # exit. # #----------------------------------------------------------------------- # - if [ "${num_missing_files}" -gt "${NUM_MISSING_OBS_FILES_MAX}" ]; then + if [ "${num_missing_files}" -gt "${num_missing_files_max}" ]; then print_err_msg_exit "\ The number of missing files (num_missing_files) is greater than the -maximum allowed number (NUM_MISSING_OBS_FILES_MAX): +maximum allowed number (num_missing_files_max): num_missing_files = ${num_missing_files} - NUM_MISSING_OBS_FILES_MAX = ${NUM_MISSING_OBS_FILES_MAX}" + num_missing_files_max = ${num_missing_files_max}" fi # #----------------------------------------------------------------------- From 1e908c78c1b3a0cbf0195fea987b183a4dc76daa Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Wed, 24 May 2023 20:38:48 -0600 Subject: [PATCH 28/43] Minor spacing edits. --- parm/wflow/coldstart.yaml | 2 +- parm/wflow/post.yaml | 2 +- parm/wflow/verify_det.yaml | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/parm/wflow/coldstart.yaml b/parm/wflow/coldstart.yaml index 98aa2e98fd..c64b62bbe3 100644 --- a/parm/wflow/coldstart.yaml +++ b/parm/wflow/coldstart.yaml @@ -75,7 +75,7 @@ task_get_extrn_lbcs: metatask_run_ensemble: var: - mem: '{% if global.DO_ENSEMBLE %}{%- for m in range(1, global.NUM_ENS_MEMBERS+1) -%}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' + mem: '{% if global.DO_ENSEMBLE %}{%- for m in range(1, global.NUM_ENS_MEMBERS+1) -%}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' task_make_ics_mem#mem#: <<: *default_task command: '&LOAD_MODULES_RUN_TASK_FP; "make_ics" "&JOBSdir;/JREGIONAL_MAKE_ICS"' diff --git a/parm/wflow/post.yaml b/parm/wflow/post.yaml index 782b21b020..17f957d5d0 100644 --- a/parm/wflow/post.yaml +++ b/parm/wflow/post.yaml @@ -25,7 +25,7 @@ default_task_post: &default_task metatask_run_ens_post: var: - mem: '{% if global.DO_ENSEMBLE %}{%- for m in range(1, global.NUM_ENS_MEMBERS+1) -%}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' + mem: '{% if global.DO_ENSEMBLE %}{%- for m in range(1, global.NUM_ENS_MEMBERS+1) -%}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' metatask_run_post: var: fhr: '{% for h in range(0, workflow.LONG_FCST_LEN+1) %}{{ " %03d" % h }}{% endfor %}' diff --git a/parm/wflow/verify_det.yaml b/parm/wflow/verify_det.yaml index 9ea734ffb1..6284324958 100644 --- a/parm/wflow/verify_det.yaml +++ b/parm/wflow/verify_det.yaml @@ -117,7 +117,7 @@ metatask_PcpCombine_fcst_all_accums_all_mems: ACCUM_HH: '{% for ah in verification.VX_APCP_ACCUMS_HRS %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' metatask_PcpCombine_fcst_APCP#ACCUM_HH#h_all_mems: var: - mem: '{% if global.DO_ENSEMBLE %}{% for m in range(1, global.NUM_ENS_MEMBERS+1) %}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' + mem: '{% if global.DO_ENSEMBLE %}{% for m in range(1, global.NUM_ENS_MEMBERS+1) %}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' task_run_MET_PcpCombine_fcst_APCP#ACCUM_HH#h_mem#mem#: <<: *default_task attrs: @@ -157,7 +157,7 @@ metatask_GridStat_CCPA_all_accums_all_mems: ACCUM_HH: '{% for ah in verification.VX_APCP_ACCUMS_HRS %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' metatask_GridStat_CCPA_APCP#ACCUM_HH#h_all_mems: var: - mem: '{% if global.DO_ENSEMBLE %}{% for m in range(1, global.NUM_ENS_MEMBERS+1) %}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' + mem: '{% if global.DO_ENSEMBLE %}{% for m in range(1, global.NUM_ENS_MEMBERS+1) %}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' task_run_MET_GridStat_vx_APCP#ACCUM_HH#h_mem#mem#: <<: *default_task attrs: @@ -184,7 +184,7 @@ metatask_GridStat_CCPA_all_accums_all_mems: metatask_GridStat_MRMS_all_mems: var: - mem: '{% if global.DO_ENSEMBLE %}{% for m in range(1, global.NUM_ENS_MEMBERS+1) %}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' + mem: '{% if global.DO_ENSEMBLE %}{% for m in range(1, global.NUM_ENS_MEMBERS+1) %}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' metatask_GridStat_MRMS_mem#mem#: var: VAR: '{% for var in verification.VX_FIELDS %}{% if var in ["REFC", "RETOP"] %}{{ "%s " % var }}{% endif %}{% endfor %}' @@ -220,7 +220,7 @@ metatask_GridStat_MRMS_all_mems: metatask_PointStat_NDAS_all_mems: var: - mem: '{% if global.DO_ENSEMBLE %}{% for m in range(1, global.NUM_ENS_MEMBERS+1) %}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' + mem: '{% if global.DO_ENSEMBLE %}{% for m in range(1, global.NUM_ENS_MEMBERS+1) %}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' metatask_PointStat_NDAS_mem#mem#: var: VAR: '{% for var in verification.VX_FIELDS %}{% if var in ["SFC", "UPA"] %}{{ "%s " % var }}{% endif %}{% endfor %}' From cf7bff0aa4bca8d0fb729447e47ae39ad49aff3b Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Thu, 25 May 2023 03:54:42 -0600 Subject: [PATCH 29/43] Split out vx preprocessing tasks that are needed by both deterministic and ensemble vx tasks into a new workflow task group file (verify_pre.yaml); add new vx preprocessing metatask (check_post_output) that checks that all post-processed forecast output files exist; fix up vx task dependencies. --- parm/wflow/post.yaml | 2 +- parm/wflow/verify_det.yaml | 172 ++------------ parm/wflow/verify_ens.yaml | 43 ++-- parm/wflow/verify_pre.yaml | 219 ++++++++++++++++++ ...S_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.yaml | 2 +- ...S_25km_ics_NAM_lbcs_NAM_suite_GFS_v16.yaml | 7 +- ...fig.MET_ensemble_verification_only_vx.yaml | 43 +--- .../config.MET_verification_only_vx.yaml | 27 +-- ush/setup.py | 12 +- 9 files changed, 261 insertions(+), 266 deletions(-) create mode 100644 parm/wflow/verify_pre.yaml diff --git a/parm/wflow/post.yaml b/parm/wflow/post.yaml index 17f957d5d0..ae02b4025b 100644 --- a/parm/wflow/post.yaml +++ b/parm/wflow/post.yaml @@ -26,7 +26,7 @@ default_task_post: &default_task metatask_run_ens_post: var: mem: '{% if global.DO_ENSEMBLE %}{%- for m in range(1, global.NUM_ENS_MEMBERS+1) -%}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' - metatask_run_post: + metatask_run_post_mem#mem#_all_fhrs: var: fhr: '{% for h in range(0, workflow.LONG_FCST_LEN+1) %}{{ " %03d" % h }}{% endfor %}' cycledef: '{% for h in range(0, workflow.LONG_FCST_LEN+1) %}{% if h <= workflow.FCST_LEN_CYCL|min %}forecast {% else %}long_forecast {% endif %}{% endfor %}' diff --git a/parm/wflow/verify_det.yaml b/parm/wflow/verify_det.yaml index 6284324958..804cc06107 100644 --- a/parm/wflow/verify_det.yaml +++ b/parm/wflow/verify_det.yaml @@ -1,4 +1,4 @@ -default_task_verify: &default_task +default_task_verify_det: &default_task_verify_det account: '&ACCOUNT;' attrs: cycledefs: forecast @@ -22,136 +22,6 @@ default_task_verify: &default_task queue: '&QUEUE_DEFAULT;' walltime: 00:30:00 -task_get_obs_ccpa: - <<: *default_task - command: '&LOAD_MODULES_RUN_TASK_FP; "get_obs" "&JOBSdir;/JREGIONAL_GET_OBS_CCPA"' - envars: - <<: *default_vars - ACCUM_HH: '01' - OBS_DIR: '&CCPA_OBS_DIR;' - partition: '{% if platform.get("PARTITION_HPSS") %}&PARTITION_HPSS;{% else %}None{% endif %}' - queue: "&QUEUE_HPSS;" - walltime: 00:45:00 - -task_get_obs_mrms: - <<: *default_task - command: '&LOAD_MODULES_RUN_TASK_FP; "get_obs" "&JOBSdir;/JREGIONAL_GET_OBS_MRMS"' - envars: - <<: *default_vars - OBS_DIR: '&MRMS_OBS_DIR;' - VAR: 'REFC RETOP' - partition: '{% if platform.get("PARTITION_HPSS") %}&PARTITION_HPSS;{% else %}None{% endif %}' - queue: "&QUEUE_HPSS;" - walltime: 00:45:00 - -task_get_obs_ndas: - <<: *default_task - envars: - <<: *default_vars - OBS_DIR: '&NDAS_OBS_DIR;' - command: '&LOAD_MODULES_RUN_TASK_FP; "get_obs" "&JOBSdir;/JREGIONAL_GET_OBS_NDAS"' - queue: "&QUEUE_HPSS;" - partition: '{% if platform.get("PARTITION_HPSS") %}&PARTITION_HPSS;{% else %}None{% endif %}' - walltime: 02:00:00 - -task_run_MET_Pb2nc_obs: - <<: *default_task - attrs: - cycledefs: forecast - maxtries: '2' - command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_PB2NC_OBS"' - envars: - <<: *default_vars - VAR: SFC - ACCUM_HH: '01' - obs_or_fcst: obs - OBTYPE: NDAS - OBS_DIR: '&NDAS_OBS_DIR;' - MET_TOOL: 'PB2NC' - dependency: - and: - datadep: - text: "&NDAS_OBS_DIR;" - or: - not: - taskvalid: - attrs: - task: get_obs_ndas - taskdep: - attrs: - task: get_obs_ndas - -metatask_PcpCombine_obs: - var: - ACCUM_HH: '{% for ah in verification.VX_APCP_ACCUMS_HRS %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' - task_run_MET_PcpCombine_obs_APCP#ACCUM_HH#h: - <<: *default_task - attrs: - cycledefs: forecast - maxtries: '2' - command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_PCPCOMBINE"' - envars: - <<: *default_vars - VAR: APCP - ACCUM_HH: '#ACCUM_HH#' - obs_or_fcst: obs - OBTYPE: CCPA - OBS_DIR: '&CCPA_OBS_DIR;' - MET_TOOL: 'PCPCOMBINE' - ENSMEM_INDX: '' - dependency: - or: - not: - taskvalid: - attrs: - task: get_obs_ccpa - and: - taskdep: - attrs: - task: get_obs_ccpa - datadep: - text: "&CCPA_OBS_DIR;" - -metatask_PcpCombine_fcst_all_accums_all_mems: - var: - ACCUM_HH: '{% for ah in verification.VX_APCP_ACCUMS_HRS %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' - metatask_PcpCombine_fcst_APCP#ACCUM_HH#h_all_mems: - var: - mem: '{% if global.DO_ENSEMBLE %}{% for m in range(1, global.NUM_ENS_MEMBERS+1) %}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' - task_run_MET_PcpCombine_fcst_APCP#ACCUM_HH#h_mem#mem#: - <<: *default_task - attrs: - cycledefs: forecast - maxtries: '2' - command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_PCPCOMBINE"' - envars: - <<: *default_vars - VAR: APCP - ACCUM_HH: '#ACCUM_HH#' - obs_or_fcst: fcst - OBTYPE: CCPA - OBS_DIR: '&CCPA_OBS_DIR;' - MET_TOOL: 'PCPCOMBINE' - ENSMEM_INDX: "#mem#" - SLASH_ENSMEM_SUBDIR_OR_NULL: '{% if global.DO_ENSEMBLE %}{{ "/mem#mem#" }}{% endif %}' - dependency: - or_do_post: &post_files_exist - and_run_post: # If post was meant to run, wait on the whole post metatask - taskvalid: - attrs: - task: run_post_mem#mem#_f000 - metataskdep: - attrs: - metatask: run_ens_post - and_inline_post: # If inline post ran, wait on the forecast task to complete - not: - taskvalid: - attrs: - task: run_post_mem#mem#_f000 - taskdep: - attrs: - task: run_fcst_mem#mem# - metatask_GridStat_CCPA_all_accums_all_mems: var: ACCUM_HH: '{% for ah in verification.VX_APCP_ACCUMS_HRS %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' @@ -159,7 +29,7 @@ metatask_GridStat_CCPA_all_accums_all_mems: var: mem: '{% if global.DO_ENSEMBLE %}{% for m in range(1, global.NUM_ENS_MEMBERS+1) %}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' task_run_MET_GridStat_vx_APCP#ACCUM_HH#h_mem#mem#: - <<: *default_task + <<: *default_task_verify_det attrs: maxtries: '2' command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_GRIDSTAT_OR_POINTSTAT_VX"' @@ -189,7 +59,7 @@ metatask_GridStat_MRMS_all_mems: var: VAR: '{% for var in verification.VX_FIELDS %}{% if var in ["REFC", "RETOP"] %}{{ "%s " % var }}{% endif %}{% endfor %}' task_run_MET_GridStat_vx_#VAR#_mem#mem#: - <<: *default_task + <<: *default_task_verify_det command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_GRIDSTAT_OR_POINTSTAT_VX"' envars: <<: *default_vars @@ -208,15 +78,13 @@ metatask_GridStat_MRMS_all_mems: taskvalid: attrs: task: get_obs_mrms - and: - taskvalid: - attrs: - task: get_obs_mrms - taskdep: - attrs: - task: get_obs_mrms - or_do_post: - <<: *post_files_exist + taskdep: + attrs: + task: get_obs_mrms + datadep_post_files_exist: + attrs: + age: 00:00:00:30 + text: !cycstr '{{ workflow.EXPTDIR }}/@Y@m@d@H/post_files_exist_mem#mem#.txt' metatask_PointStat_NDAS_all_mems: var: @@ -225,7 +93,7 @@ metatask_PointStat_NDAS_all_mems: var: VAR: '{% for var in verification.VX_FIELDS %}{% if var in ["SFC", "UPA"] %}{{ "%s " % var }}{% endif %}{% endfor %}' task_run_MET_PointStat_vx_#VAR#_mem#mem#: - <<: *default_task + <<: *default_task_verify_det command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_GRIDSTAT_OR_POINTSTAT_VX"' envars: <<: *default_vars @@ -242,17 +110,7 @@ metatask_PointStat_NDAS_all_mems: taskdep_pb2nc: attrs: task: run_MET_Pb2nc_obs - or_do_post: - <<: *post_files_exist - or_get_obs: # Get obs complete if turned on - not: - taskvalid: - attrs: - task: get_obs_ndas - and: - taskvalid: - attrs: - task: get_obs_ndas - taskdep: - attrs: - task: get_obs_ndas + datadep_post_files_exist: + attrs: + age: 00:00:00:30 + text: !cycstr '{{ workflow.EXPTDIR }}/@Y@m@d@H/post_files_exist_mem#mem#.txt' diff --git a/parm/wflow/verify_ens.yaml b/parm/wflow/verify_ens.yaml index 10eab47d90..4fdf8e6964 100644 --- a/parm/wflow/verify_ens.yaml +++ b/parm/wflow/verify_ens.yaml @@ -22,7 +22,7 @@ default_task_verify_ens: &default_task_verify_ens queue: '&QUEUE_DEFAULT;' walltime: 01:00:00 -metatask_GEPES_CCPA: +metatask_GenEnsProd_EnsembleStat_CCPA: var: ACCUM_HH: '{% for ah in verification.VX_APCP_ACCUMS_HRS %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' task_run_MET_GenEnsProd_vx_APCP#ACCUM_HH#h: &task_GenEnsProd_CCPA @@ -37,22 +37,6 @@ metatask_GEPES_CCPA: OBTYPE: 'CCPA' dependency: and: - or_do_post: &post_files_exist - and_run_post: # If post was meant to run, wait on the whole post metatask - taskvalid: - attrs: - task: run_post_mem001_f000 - metataskdep: - attrs: - metatask: run_ens_post - and_inline_post: # If inline post ran, wait on the forecast ensemble to complete - not: - taskvalid: - attrs: - task: run_post_mem001_f000 - metataskdep: - attrs: - metatask: run_ensemble # The PcpCombine task for obs must be complete because this GenEnsProd # task checks to see the forecast hours for which obs are available before # processing the forecast for those hours. @@ -72,7 +56,7 @@ metatask_GEPES_CCPA: attrs: task: run_MET_GenEnsProd_vx_APCP#ACCUM_HH#h -metatask_GEPES_MRMS: +metatask_GenEnsProd_EnsembleStat_MRMS: var: VAR: '{% for var in verification.VX_FIELDS %}{% if var in ["REFC", "RETOP"] %}{{ "%s " % var }}{% endif %}{% endfor %}' task_run_MET_GenEnsProd_vx_#VAR#: &task_GenEnsProd_MRMS @@ -87,12 +71,18 @@ metatask_GEPES_MRMS: OBTYPE: 'MRMS' dependency: and: - taskdep: - attrs: - task: get_obs_mrms - metataskdep: + or_get_obs: # Ensure get_obs task is complete if it's turned on + not: + taskvalid: + attrs: + task: get_obs_mrms + taskdep: + attrs: + task: get_obs_mrms + metataskdep_post_files_exist: &post_files_exist attrs: - metatask: run_ens_post + metatask: check_post_output_all_mems + task_run_MET_EnsembleStat_vx_#VAR#: <<: *task_GenEnsProd_MRMS envars: @@ -103,7 +93,7 @@ metatask_GEPES_MRMS: attrs: task: run_MET_GenEnsProd_vx_#VAR# -metatask_GEPES_NDAS: +metatask_GenEnsProd_EnsembleStat_NDAS: var: VAR: '{% for var in verification.VX_FIELDS %}{% if var in ["SFC", "UPA"] %}{{ "%s " % var }}{% endif %}{% endfor %}' task_run_MET_GenEnsProd_vx_#VAR#: &task_GenEnsProd_NDAS @@ -124,9 +114,8 @@ metatask_GEPES_NDAS: taskdep_pb2nc: attrs: task: run_MET_Pb2nc_obs - metataskdep: - attrs: - metatask: run_ens_post + metataskdep_post_files_exist: + <<: *post_files_exist task_run_MET_EnsembleStat_vx_#VAR#: <<: *task_GenEnsProd_NDAS envars: diff --git a/parm/wflow/verify_pre.yaml b/parm/wflow/verify_pre.yaml new file mode 100644 index 0000000000..088a0e3345 --- /dev/null +++ b/parm/wflow/verify_pre.yaml @@ -0,0 +1,219 @@ +default_task_verify_pre: &default_task_verify_pre + account: '&ACCOUNT;' + attrs: + cycledefs: forecast + maxtries: '1' + envars: &default_vars + GLOBAL_VAR_DEFNS_FP: '&GLOBAL_VAR_DEFNS_FP;' + USHdir: '&USHdir;' + PDY: !cycstr '@Y@m@d' + cyc: !cycstr "@H" + subcyc: !cycstr "@M" + LOGDIR: !cycstr "&LOGDIR;" + nprocs: '{{ parent.nnodes * parent.ppn }}' + join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;' + memory: '{% if user.MACHINE not in ["NOAACLOUD"] %}{{ "2G" }}{% endif %}' + native: '{{ platform.SCHED_NATIVE_CMD }}' + nnodes: 1 + nodes: '{{ nnodes }}:ppn={{ ppn }}' + nodesize: '&NCORES_PER_NODE;' + partition: '{% if platform.get("PARTITION_DEFAULT") %}&PARTITION_DEFAULT;{% else %}None{% endif %}' + ppn: 1 + queue: '&QUEUE_DEFAULT;' + walltime: 00:30:00 + +task_get_obs_ccpa: + <<: *default_task_verify_pre + command: '&LOAD_MODULES_RUN_TASK_FP; "get_obs" "&JOBSdir;/JREGIONAL_GET_OBS_CCPA"' + envars: + <<: *default_vars + ACCUM_HH: '01' + OBS_DIR: '&CCPA_OBS_DIR;' + partition: '{% if platform.get("PARTITION_HPSS") %}&PARTITION_HPSS;{% else %}None{% endif %}' + queue: "&QUEUE_HPSS;" + walltime: 00:45:00 + +task_get_obs_mrms: + <<: *default_task_verify_pre + command: '&LOAD_MODULES_RUN_TASK_FP; "get_obs" "&JOBSdir;/JREGIONAL_GET_OBS_MRMS"' + envars: + <<: *default_vars + OBS_DIR: '&MRMS_OBS_DIR;' + VAR: 'REFC RETOP' + partition: '{% if platform.get("PARTITION_HPSS") %}&PARTITION_HPSS;{% else %}None{% endif %}' + queue: "&QUEUE_HPSS;" + walltime: 00:45:00 + +task_get_obs_ndas: + <<: *default_task_verify_pre + envars: + <<: *default_vars + OBS_DIR: '&NDAS_OBS_DIR;' + command: '&LOAD_MODULES_RUN_TASK_FP; "get_obs" "&JOBSdir;/JREGIONAL_GET_OBS_NDAS"' + queue: "&QUEUE_HPSS;" + partition: '{% if platform.get("PARTITION_HPSS") %}&PARTITION_HPSS;{% else %}None{% endif %}' + walltime: 02:00:00 + +task_run_MET_Pb2nc_obs: + <<: *default_task_verify_pre + attrs: + cycledefs: forecast + maxtries: '2' + command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_PB2NC_OBS"' + envars: + <<: *default_vars + VAR: SFC + ACCUM_HH: '01' + obs_or_fcst: obs + OBTYPE: NDAS + OBS_DIR: '&NDAS_OBS_DIR;' + MET_TOOL: 'PB2NC' + dependency: + and: + datadep: + text: "&NDAS_OBS_DIR;" + or: + not: + taskvalid: + attrs: + task: get_obs_ndas + taskdep: + attrs: + task: get_obs_ndas + +metatask_PcpCombine_obs: + var: + ACCUM_HH: '{% for ah in verification.VX_APCP_ACCUMS_HRS %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' + task_run_MET_PcpCombine_obs_APCP#ACCUM_HH#h: + <<: *default_task_verify_pre + attrs: + cycledefs: forecast + maxtries: '2' + command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_PCPCOMBINE"' + envars: + <<: *default_vars + VAR: APCP + ACCUM_HH: '#ACCUM_HH#' + obs_or_fcst: obs + OBTYPE: CCPA + OBS_DIR: '&CCPA_OBS_DIR;' + MET_TOOL: 'PCPCOMBINE' + ENSMEM_INDX: '' + dependency: + and: + datadep: + text: "&CCPA_OBS_DIR;" + or: + not: + taskvalid: + attrs: + task: get_obs_ccpa + taskdep: + attrs: + task: get_obs_ccpa + +metatask_check_post_output_all_mems: + var: + mem: '{% if global.DO_ENSEMBLE %}{% for m in range(1, global.NUM_ENS_MEMBERS+1) %}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' + task_check_post_output_mem#mem#: + <<: *default_task_verify_pre + attrs: + cycledefs: forecast + maxtries: '1' + command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_CHECK_POST_OUTPUT"' + envars: + <<: *default_vars + VAR: APCP + ACCUM_HH: '01' + ENSMEM_INDX: '#mem#' + dependency: + # This "or" checks that the necessary stand-alone post tasks or forecast + # tasks with inline post have run to generate the post files that are + # needed as forecast input to various verification tasks. + or_post_files_exist: &post_files_exist + # This "and" is to check whether post is being run stand-alone (i.e. outside + # the weather model in tasks separate from the forecast task(s)), and if + # so, to ensure that the stand-alone post metatask for the current member + # has completed for all forecast hours. + and_standalone_post: + # This "or" is to check whether a stand-alane post task exists [either for + # mem000 (if running a deterministic forecast) or for mem001 (if running + # an ensemble forecast)]. Checking for forecast hour 000 is sufficient. + # If so, it implies post is being run stand-alone. + or_post_task_exists: &post_task_exists + taskvalid_mem000: + attrs: + task: run_post_mem000_f000 + taskvalid_mem001: + attrs: + task: run_post_mem001_f000 + # We would like a dependency here on the run_post_#mem#_all_fhrs metatask, + # but that is apparently not possible in rocoto with a metataskvalid/ + # metataskdep combination (currently, metataskvalid is not a defined + # dependency type in ROCOTO); that approach only works for tasks. Thus, + # here we use jinja directives to explicitly list all the tasks in the + # run_post_#mem#_all_fhrs metatask. + #or_post_metatask: + # not: + # metataskvalid: + # attrs: + # metatask: run_post_mem#mem#_all_fhrs + # metataskdep: + # attrs: + # metatask: run_post_mem#mem#_all_fhrs + taskdep: + attrs: + task: '{% for h in range(0, workflow.LONG_FCST_LEN+1) %}{% if h > 0 %}{{" \n"}}{% endif %}{%- endfor -%}' + # This "and" is to check whether post is being run inline (i.e. as part of + # the weather model), and if so, to ensure that the forecast task for the + # current member has completed. + and_inline_post: + # This "not" is to check whether a stand-alone post task DOESN'T exist + # [either for mem000 (if running a deterministic forecast) or for mem001 + # (if running an ensemble forecast)]. If so, it implies that either + # post is being run inline (as part of the weather model) or is not being + # run at all. + not: + or_post_task_exists: + <<: *post_task_exists + # This "or" is to verify that either the forecast task for the current + # member exists and has completed, or that it doesn't exist. + or_run_fcst: + and: + taskvalid: &fcst_task_exists + attrs: + task: run_fcst_mem#mem# + taskdep: + attrs: + task: run_fcst_mem#mem# + not: + taskvalid: + <<: *fcst_task_exists + +metatask_PcpCombine_fcst_all_accums_all_mems: + var: + ACCUM_HH: '{% for ah in verification.VX_APCP_ACCUMS_HRS %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' + metatask_PcpCombine_fcst_APCP#ACCUM_HH#h_all_mems: + var: + mem: '{% if global.DO_ENSEMBLE %}{% for m in range(1, global.NUM_ENS_MEMBERS+1) %}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' + task_run_MET_PcpCombine_fcst_APCP#ACCUM_HH#h_mem#mem#: + <<: *default_task_verify_pre + attrs: + cycledefs: forecast + maxtries: '2' + command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_PCPCOMBINE"' + envars: + <<: *default_vars + VAR: APCP + ACCUM_HH: '#ACCUM_HH#' + obs_or_fcst: fcst + OBTYPE: CCPA + OBS_DIR: '&CCPA_OBS_DIR;' + MET_TOOL: 'PCPCOMBINE' + ENSMEM_INDX: "#mem#" + dependency: + datadep: + attrs: + age: 00:00:00:30 + text: !cycstr '{{ workflow.EXPTDIR }}/@Y@m@d@H/post_files_exist_mem#mem#.txt' + walltime: 00:05:00 diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.yaml b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.yaml index f323164536..5b5785c3ae 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.yaml +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.yaml @@ -17,7 +17,7 @@ workflow: PREEXISTING_DIR_METHOD: rename rocoto: tasks: - taskgroups: '{{ ["parm/wflow/prep.yaml", "parm/wflow/coldstart.yaml", "parm/wflow/post.yaml", "parm/wflow/verify_det.yaml"]|include }}' + taskgroups: '{{ ["parm/wflow/prep.yaml", "parm/wflow/coldstart.yaml", "parm/wflow/post.yaml", "parm/wflow/verify_pre.yaml", "parm/wflow/verify_det.yaml"]|include }}' metatask_run_ensemble: task_run_fcst_mem#mem#: walltime: 01:00:00 diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GFS_v16.yaml b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GFS_v16.yaml index 351cad85b8..53fc7bfc29 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GFS_v16.yaml +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GFS_v16.yaml @@ -16,18 +16,13 @@ workflow: PREEXISTING_DIR_METHOD: rename rocoto: tasks: - taskgroups: '{{ ["parm/wflow/prep.yaml", "parm/wflow/coldstart.yaml", "parm/wflow/post.yaml", "parm/wflow/verify_det.yaml", "parm/wflow/verify_ens.yaml"]|include }}' + taskgroups: '{{ ["parm/wflow/prep.yaml", "parm/wflow/coldstart.yaml", "parm/wflow/post.yaml", "parm/wflow/verify_pre.yaml", "parm/wflow/verify_det.yaml", "parm/wflow/verify_ens.yaml"]|include }}' metatask_run_ensemble: task_run_fcst_mem#mem#: walltime: 01:00:00 task_get_obs_ccpa: task_get_obs_mrms: task_get_obs_ndas: - metatask_GEPES_MRMS: - task_run_MET_GenEnsProd_vx_#VAR#: - dependency: - and: - taskdep: task_get_extrn_ics: EXTRN_MDL_NAME_ICS: NAM USE_USER_STAGED_EXTRN_FILES: true diff --git a/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml b/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml index 5f76123bee..f1e1ff5b91 100644 --- a/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml +++ b/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml @@ -17,52 +17,11 @@ workflow: rocoto: tasks: - taskgroups: '{{ ["parm/wflow/verify_det.yaml", "parm/wflow/verify_ens.yaml"]|include }}' + taskgroups: '{{ ["parm/wflow/verify_pre.yaml", "parm/wflow/verify_det.yaml", "parm/wflow/verify_ens.yaml"]|include }}' task_get_obs_ccpa: task_get_obs_mrms: task_get_obs_ndas: - metatask_PcpCombine_obs: - task_run_MET_PcpCombine_obs_APCP#ACCUM_HH#h: - dependency: - or: - datadep: - text: "&CCPA_OBS_DIR;" - - metatask_PcpCombine_fcst_all_accums_all_mems: - metatask_PcpCombine_fcst_APCP#ACCUM_HH#h_all_mems: - task_run_MET_PcpCombine_fcst_APCP#ACCUM_HH#h_mem#mem#: - dependency: - - metatask_GridStat_MRMS_all_mems: - metatask_GridStat_MRMS_mem#mem#: - task_run_MET_GridStat_vx_#VAR#_mem#mem#: - dependency: - - metatask_PointStat_NDAS_all_mems: - metatask_PointStat_NDAS_mem#mem#: - task_run_MET_PointStat_vx_#VAR#_mem#mem#: - dependency: - and: - or_do_post: - or_get_obs: - - metatask_GEPES_CCPA: - task_run_MET_GenEnsProd_vx_APCP#ACCUM_HH#h: - dependency: - and: - or_do_post: - - metatask_GEPES_MRMS: - task_run_MET_GenEnsProd_vx_#VAR#: - dependency: - - metatask_GEPES_NDAS: - task_run_MET_GenEnsProd_vx_#VAR#: - dependency: - and: - metataskdep: - global: DO_ENSEMBLE: true NUM_ENS_MEMBERS: 2 diff --git a/tests/WE2E/test_configs/verification/config.MET_verification_only_vx.yaml b/tests/WE2E/test_configs/verification/config.MET_verification_only_vx.yaml index d2e64d8ba0..7ed3426b4b 100644 --- a/tests/WE2E/test_configs/verification/config.MET_verification_only_vx.yaml +++ b/tests/WE2E/test_configs/verification/config.MET_verification_only_vx.yaml @@ -21,36 +21,11 @@ workflow: # It also assumes staged observations, so turn off those tasks. rocoto: tasks: - taskgroups: '{{ ["parm/wflow/verify_det.yaml"]|include }}' + taskgroups: '{{ ["parm/wflow/verify_pre.yaml", "parm/wflow/verify_det.yaml"]|include }}' task_get_obs_ccpa: task_get_obs_mrms: task_get_obs_ndas: - metatask_PcpCombine_obs: - task_run_MET_PcpCombine_obs_APCP#ACCUM_HH#h: - dependency: - or: - datadep: - text: "&CCPA_OBS_DIR;" - - metatask_PcpCombine_fcst_all_accums_all_mems: - metatask_PcpCombine_fcst_APCP#ACCUM_HH#h_all_mems: - task_run_MET_PcpCombine_fcst_APCP#ACCUM_HH#h_mem#mem#: - dependency: - - metatask_GridStat_MRMS_all_mems: - metatask_GridStat_MRMS_mem#mem#: - task_run_MET_GridStat_vx_#VAR#_mem#mem#: - dependency: - - metatask_PointStat_NDAS_all_mems: - metatask_PointStat_NDAS_mem#mem#: - task_run_MET_PointStat_vx_#VAR#_mem#mem#: - dependency: - and: - or_do_post: - or_get_obs: - verification: VX_FCST_MODEL_NAME: FV3_GFS_v15p2_CONUS_25km VX_FCST_INPUT_BASEDIR: '{{ platform.get("TEST_VX_FCST_INPUT_BASEDIR") }}' diff --git a/ush/setup.py b/ush/setup.py index b650d3bbdd..0a7e872c6c 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -554,18 +554,18 @@ def remove_tag(tasks, tag): vx_metatasks_all["CCPA"] = ["metatask_PcpCombine_obs", "metatask_PcpCombine_fcst_all_accums_all_mems", "metatask_GridStat_CCPA_all_accums_all_mems", - "metatask_GEPES_CCPA", + "metatask_GenEnsProd_EnsembleStat_CCPA", "metatask_GridStat_CCPA_ensmeanprob_all_accums"] vx_fields_all["MRMS"] = ["REFC", "RETOP"] vx_metatasks_all["MRMS"] = ["metatask_GridStat_MRMS_all_mems", - "metatask_GEPES_MRMS", + "metatask_GenEnsProd_EnsembleStat_MRMS", "metatask_GridStat_MRMS_ensprob"] vx_fields_all["NDAS"] = ["SFC", "UPA"] vx_metatasks_all["NDAS"] = ["task_run_MET_Pb2nc_obs", "metatask_PointStat_NDAS_all_mems", - "metatask_GEPES_NDAS", + "metatask_GenEnsProd_EnsembleStat_NDAS", "metatask_PointStat_NDAS_ensmeanprob"] vx_fields_config = expt_config["verification"]["VX_FIELDS"] @@ -608,9 +608,9 @@ def remove_tag(tasks, tag): "metatask_PointStat_NDAS_all_mems"] vx_flags["ens"] = run_vx_ens - vx_metatasks["ens"] = ["metatask_GEPES_CCPA", - "metatask_GEPES_MRMS", - "metatask_GEPES_NDAS", + vx_metatasks["ens"] = ["metatask_GenEnsProd_EnsembleStat_CCPA", + "metatask_GenEnsProd_EnsembleStat_MRMS", + "metatask_GenEnsProd_EnsembleStat_NDAS", "metatask_GridStat_CCPA_ensmeanprob_all_accums", "metatask_GridStat_MRMS_ensprob", "metatask_PointStat_NDAS_ensmeanprob"] From 72440f53448fe2df6b9e29ca91db0b0781b79501 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Thu, 25 May 2023 04:08:50 -0600 Subject: [PATCH 30/43] Remove experiment variables RUN_VX_DET and RUN_VX_ENS since their functionality is replaced by inclusion/exclusion of workflow taskgroup yaml configuration files. --- ...S_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.yaml | 1 - ...S_25km_ics_NAM_lbcs_NAM_suite_GFS_v16.yaml | 2 - ...fig.MET_ensemble_verification_only_vx.yaml | 2 - .../config.MET_verification_only_vx.yaml | 1 - ush/config_defaults.yaml | 17 ------- ush/setup.py | 47 ------------------- 6 files changed, 70 deletions(-) diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.yaml b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.yaml index 5b5785c3ae..41da098f00 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.yaml +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.yaml @@ -35,4 +35,3 @@ task_get_extrn_lbcs: USE_USER_STAGED_EXTRN_FILES: true verification: VX_FCST_MODEL_NAME: FV3_GFS_v15p2_CONUS_25km - RUN_VX_DET: True diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GFS_v16.yaml b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GFS_v16.yaml index 53fc7bfc29..25099ce9b8 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GFS_v16.yaml +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GFS_v16.yaml @@ -35,5 +35,3 @@ global: NUM_ENS_MEMBERS: 2 verification: VX_FCST_MODEL_NAME: FV3_GFS_v15p2_CONUS_25km - RUN_VX_DET: True - RUN_VX_ENS: True diff --git a/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml b/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml index f1e1ff5b91..ff7b05a906 100644 --- a/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml +++ b/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml @@ -29,5 +29,3 @@ global: verification: VX_FCST_MODEL_NAME: FV3_GFS_v15p2_CONUS_25km VX_FCST_INPUT_BASEDIR: '{{ platform.get("TEST_VX_FCST_INPUT_BASEDIR") }}' - RUN_VX_DET: True - RUN_VX_ENS: True diff --git a/tests/WE2E/test_configs/verification/config.MET_verification_only_vx.yaml b/tests/WE2E/test_configs/verification/config.MET_verification_only_vx.yaml index 7ed3426b4b..1fc04a622a 100644 --- a/tests/WE2E/test_configs/verification/config.MET_verification_only_vx.yaml +++ b/tests/WE2E/test_configs/verification/config.MET_verification_only_vx.yaml @@ -29,4 +29,3 @@ rocoto: verification: VX_FCST_MODEL_NAME: FV3_GFS_v15p2_CONUS_25km VX_FCST_INPUT_BASEDIR: '{{ platform.get("TEST_VX_FCST_INPUT_BASEDIR") }}' - RUN_VX_DET: True diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 609782b90d..5a8cd63cad 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -2427,23 +2427,6 @@ verification: OBS_NDAS_SFCorUPA_FN_TEMPLATE: 'prepbufr.ndas.{valid?fmt=%Y%m%d%H}' OBS_NDAS_SFCorUPA_FN_METPROC_TEMPLATE: '${OBS_NDAS_SFCorUPA_FN_TEMPLATE}.nc' # - # RUN_VX_DET: - # Flag that specifies whether to run deterministic verification. If set - # to True, this will run deterministic vx on the post-processed forecast - # output. This post-processed output may consist of a single forecast - # or an ensemble of foreasts, and it may be staged from previous runs of - # the SRW App or may be generated by running the TN_RUN_FCST task as part - # of the current SRW-App-generated experiment. - # - # RUN_VX_ENS: - # Flag that specifies whether to run ensemble verification. The ensemble - # forecast output on which vx will be run may be staged or generated by - # running an ensemble of forecasts with the weather model as part of the - # current SRW-App-generated experiment. - # - RUN_VX_DET: False - RUN_VX_ENS: False - # # VX_FCST_MODEL_NAME: # String that specifies a descriptive name for the model being verified. # This is used in forming the names of the verification output files as diff --git a/ush/setup.py b/ush/setup.py index 0a7e872c6c..f483c62cc9 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -582,53 +582,6 @@ def remove_tag(tasks, tag): are specified for verification.""" )) rocoto_config['tasks'].pop(metatask, None) - # - # ----------------------------------------------------------------------- - # - # Remove deterministic and/or ensemble verification [meta]tasks depending - # on the values of the experiment configuration flags RUN_VX_DET and - # RUN_VX_ENS. - # - # ----------------------------------------------------------------------- - # - run_vx_det = expt_config["verification"]["RUN_VX_DET"] - run_vx_ens = expt_config["verification"]["RUN_VX_ENS"] - - vx_flags = {} - vx_metatasks = {} - - vx_flags["det_or_ens"] = run_vx_det or run_vx_ens - vx_metatasks["det_or_ens"] = ["metatask_PcpCombine_obs", - "metatask_PcpCombine_fcst_all_accums_all_mems", - "task_run_MET_Pb2nc_obs"] - - vx_flags["det"] = run_vx_det - vx_metatasks["det"] = ["metatask_GridStat_CCPA_all_accums_all_mems", - "metatask_GridStat_MRMS_all_mems", - "metatask_PointStat_NDAS_all_mems"] - - vx_flags["ens"] = run_vx_ens - vx_metatasks["ens"] = ["metatask_GenEnsProd_EnsembleStat_CCPA", - "metatask_GenEnsProd_EnsembleStat_MRMS", - "metatask_GenEnsProd_EnsembleStat_NDAS", - "metatask_GridStat_CCPA_ensmeanprob_all_accums", - "metatask_GridStat_MRMS_ensprob", - "metatask_PointStat_NDAS_ensmeanprob"] - - run_vx_det = expt_config["verification"]["RUN_VX_DET"] - run_vx_ens = expt_config["verification"]["RUN_VX_ENS"] - - for key,flag in vx_flags.items(): - if not flag: - for metatask in vx_metatasks[key]: - if metatask in rocoto_config['tasks']: - logging.info(dedent( - f""" - Removing verification [meta]task - "{metatask}" - from workflow since vx_flags["{key}"] is set to {flag}.""" - )) - rocoto_config['tasks'].pop(metatask, None) # # ----------------------------------------------------------------------- From 74975f0d3aeef39c0f33b90db066066db278429c Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Thu, 25 May 2023 04:37:43 -0600 Subject: [PATCH 31/43] If no vx fields are specified, remove the metatask "check_post_output_all_mems" from the workflow. --- ush/setup.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/ush/setup.py b/ush/setup.py index f483c62cc9..775d1db20e 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -568,7 +568,17 @@ def remove_tag(tasks, tag): "metatask_GenEnsProd_EnsembleStat_NDAS", "metatask_PointStat_NDAS_ensmeanprob"] + # Get the vx fields specified in the experiment configuration. vx_fields_config = expt_config["verification"]["VX_FIELDS"] + + # If there are no vx fields specified, remove those tasks that are necessary + # for all observation types. + if not vx_fields_config: + metatask = "metatask_check_post_output_all_mems" + rocoto_config['tasks'].pop(metatask) + + # If for a given obstype no fields are specified, remove all vx metatasks + # for that obstype. for obstype in vx_fields_all: vx_fields_obstype = [field for field in vx_fields_config if field in vx_fields_all[obstype]] if not vx_fields_obstype: @@ -581,7 +591,7 @@ def remove_tag(tasks, tag): from workflow since no fields belonging to observation type "{obstype}" are specified for verification.""" )) - rocoto_config['tasks'].pop(metatask, None) + rocoto_config['tasks'].pop(metatask) # # ----------------------------------------------------------------------- @@ -1292,8 +1302,8 @@ def get_location(xcs, fmt, expt_cfg): raise Exception( f''' Ensemble verification can not be run unless running in ensemble mode: - DO_ENSEMBLE = \"{do_ensemble}\" - Ensemble verification tasks: {task_str} + DO_ENSEMBLE = \"{do_ensemble}\" + Ensemble verification tasks: {task_str} ''' ) From 71db8465843f9c687b26e3826996409aac47b23c Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Thu, 25 May 2023 12:50:54 -0600 Subject: [PATCH 32/43] Clean up check of consistency between DO_ENSEMBLE flag and inclusion of ensemble vx tasks in the workflow (by making sure that the names of the ensemble vx tasks are read in from the appropriate yaml file instead of being hard-coded). --- ush/setup.py | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/ush/setup.py b/ush/setup.py index 775d1db20e..6d387c0374 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -1287,25 +1287,35 @@ def get_location(xcs, fmt, expt_cfg): # # ----------------------------------------------------------------------- # - + # Get list of all top-level tasks and metatasks in the workflow. task_defs = rocoto_config.get('tasks') + all_tasks = [task for task in task_defs] - # Ensemble verification can only be run in ensemble mode - do_ensemble = global_sect["DO_ENSEMBLE"] + # Get list of all valid top-level tasks and metatasks pertaining to ensemble + # verification. + ens_vx_task_defns = load_config_file( + os.path.join(USHdir, os.pardir, "parm", "wflow", "verify_ens.yaml")) + ens_vx_valid_tasks = [task for task in ens_vx_task_defns] + # Get list of all valid top-level tasks and metatasks in the workflow that + # pertain to ensemble verification. + ens_vx_tasks = [task for task in ens_vx_valid_tasks if task in all_tasks] - # Gather all the tasks/metatasks that are defined for verifying - # ensembles - ens_vx_tasks = [task for task in task_defs if "MET_GridStat_vx_ens" in task] + # Get the value of the configuration flag for ensemble mode (DO_ENSEMBLE) + # and ensure that it is set to True if ensemble vx tasks are included in + # the workflow (or vice-versa). + do_ensemble = global_sect["DO_ENSEMBLE"] if (not do_ensemble) and ens_vx_tasks: - task_str = "\n".join(ens_vx_tasks) - raise Exception( - f''' - Ensemble verification can not be run unless running in ensemble mode: - DO_ENSEMBLE = \"{do_ensemble}\" - Ensemble verification tasks: {task_str} - ''' - ) + task_str = " " + "\n ".join(ens_vx_tasks) + msg = dedent(f""" + Ensemble verification can not be run unless running in ensemble mode: + DO_ENSEMBLE = \"{do_ensemble}\" + Ensemble verification tasks: + """) + msg = "".join([msg, task_str, dedent(f""" + Please set DO_ENSEMBLE to True or remove ensemble vx tasks from the + workflow.""")]) + raise Exception(msg) # # ----------------------------------------------------------------------- From e0b46a82a78672244de6861fa65f7c4b2a583d01 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Thu, 25 May 2023 16:31:15 -0600 Subject: [PATCH 33/43] Fix the way the time-lag array ENS_TIME_LAG_HRS is set (for both ensemble and deterministic forecasts) and the way the time lag is calculated for a given ensemble member (or the deterministic forecast). --- .../exregional_run_met_genensprod_or_ensemblestat.sh | 10 +++++----- scripts/exregional_run_met_gridstat_or_pointstat_vx.sh | 9 +++++---- scripts/exregional_run_met_pcpcombine.sh | 9 +++++---- ush/config_defaults.yaml | 6 ++++-- 4 files changed, 19 insertions(+), 15 deletions(-) diff --git a/scripts/exregional_run_met_genensprod_or_ensemblestat.sh b/scripts/exregional_run_met_genensprod_or_ensemblestat.sh index 8464b5ad7b..132471968c 100755 --- a/scripts/exregional_run_met_genensprod_or_ensemblestat.sh +++ b/scripts/exregional_run_met_genensprod_or_ensemblestat.sh @@ -209,17 +209,17 @@ NDIGITS_ENSMEM_NAMES=3 FCST_INPUT_FN_TEMPLATE="" for (( i=0; i<${NUM_ENS_MEMBERS}; i++ )); do - mem_indx=$(($i+1)) - mem_indx_fmt=$(printf "%0${NDIGITS_ENSMEM_NAMES}d" "${mem_indx}") - ensmem_name="mem${mem_indx_fmt}" - time_lag=$( bc -l <<< "${ENS_TIME_LAG_HRS[$i]}*${SECS_PER_HOUR}" ) + ensmem_indx=$(printf "%0${NDIGITS_ENSMEM_NAMES}d" "$((i+1))") + ensmem_name="mem${ensmem_indx}" if [ "${RUN_ENVIR}" = "nco" ]; then cdate_ensmem_subdir_or_null="" else - cdate_ensmem_subdir_or_null="${CDATE}/mem${mem_indx_fmt}" + cdate_ensmem_subdir_or_null="${CDATE}/${ensmem_name}" fi + time_lag=$( bc -l <<< "${ENS_TIME_LAG_HRS[$i]}*${SECS_PER_HOUR}" ) + if [ "${field_is_APCPgt01h}" = "TRUE" ]; then template="${cdate_ensmem_subdir_or_null:+${cdate_ensmem_subdir_or_null}/}metprd/PcpCombine_fcst/${FCST_FN_METPROC_TEMPLATE}" else diff --git a/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh b/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh index c6395db119..2a62b9d1d3 100755 --- a/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh +++ b/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh @@ -117,10 +117,11 @@ set_vx_params \ # #----------------------------------------------------------------------- # -time_lag=$(( (${MEM_INDX_OR_NULL:+${ENS_TIME_LAG_HRS[${MEM_INDX_OR_NULL}-1]}}+0) )) -# Convert to seconds. We do this as a separate step using bc because -# bash's $((...)) arithmetic operator can't handle floats well. -time_lag=$( bc -l <<< "${time_lag}*${SECS_PER_HOUR}" ) +i="0" +if [ "${DO_ENSEMBLE}" = "TRUE" ]; then + i=$( bc -l <<< "${ENSMEM_INDX}-1" ) +fi +time_lag=$( bc -l <<< "${ENS_TIME_LAG_HRS[$i]}*${SECS_PER_HOUR}" ) # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_run_met_pcpcombine.sh b/scripts/exregional_run_met_pcpcombine.sh index 3b19335fc8..021f8b15da 100755 --- a/scripts/exregional_run_met_pcpcombine.sh +++ b/scripts/exregional_run_met_pcpcombine.sh @@ -121,10 +121,11 @@ set_vx_params \ # time_lag="0" if [ "${obs_or_fcst}" = "fcst" ]; then - time_lag=$(( (${MEM_INDX_OR_NULL:+${ENS_TIME_LAG_HRS[${MEM_INDX_OR_NULL}-1]}}+0) )) -# Convert to seconds. We do this as a separate step using bc because -# bash's $((...)) arithmetic operator can't handle floats well. - time_lag=$( bc -l <<< "${time_lag}*${SECS_PER_HOUR}" ) + i="0" + if [ "${DO_ENSEMBLE}" = "TRUE" ]; then + i=$( bc -l <<< "${ENSMEM_INDX}-1" ) + fi + time_lag=$( bc -l <<< "${ENS_TIME_LAG_HRS[$i]}*${SECS_PER_HOUR}" ) fi # #----------------------------------------------------------------------- diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 5a8cd63cad..ec7468bdc3 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -2297,7 +2297,9 @@ global: # experiment directory # # ENS_TIME_LAG_HRS: - # Time lag (in hours) to use for each ensemble member. + # Time lag (in hours) to use for each ensemble member. For a deterministic + # forecast, this is a one-element array. Default values of array elements + # are zero. # #----------------------------------------------------------------------- # @@ -2305,7 +2307,7 @@ global: NUM_ENS_MEMBERS: 0 ENSMEM_NAMES: '{% for m in range(NUM_ENS_MEMBERS) %}{{ "mem%03d, " % m }}{% endfor %}' FV3_NML_ENSMEM_FPS: '{% for mem in ENSMEM_NAMES %}{{ [EXPTDIR, "%s_%s" % FV3_NML_FN, mem]|path_join }}{% endfor %}' - ENS_TIME_LAG_HRS: '[ {% for m in range(NUM_ENS_MEMBERS) %} 0, {% endfor %} ]' + ENS_TIME_LAG_HRS: '[ {% for m in range([1,NUM_ENS_MEMBERS]|max) %} 0, {% endfor %} ]' # #----------------------------------------------------------------------- # From c02dfe47ff73b9d2584d2a9215314cf2c8c788ea Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Fri, 26 May 2023 10:20:06 -0600 Subject: [PATCH 34/43] Add missing files. --- jobs/JREGIONAL_CHECK_POST_OUTPUT | 99 ++++++++++++++++++++ scripts/exregional_check_post_output.sh | 119 ++++++++++++++++++++++++ 2 files changed, 218 insertions(+) create mode 100755 jobs/JREGIONAL_CHECK_POST_OUTPUT create mode 100755 scripts/exregional_check_post_output.sh diff --git a/jobs/JREGIONAL_CHECK_POST_OUTPUT b/jobs/JREGIONAL_CHECK_POST_OUTPUT new file mode 100755 index 0000000000..b9c1868282 --- /dev/null +++ b/jobs/JREGIONAL_CHECK_POST_OUTPUT @@ -0,0 +1,99 @@ +#!/bin/bash + +# +#----------------------------------------------------------------------- +# +# +# +#----------------------------------------------------------------------- +# + +# +#----------------------------------------------------------------------- +# +# Source the variable definitions file and the bash utility functions. +# +#----------------------------------------------------------------------- +# +. $USHdir/source_util_funcs.sh +source_config_for_task "" ${GLOBAL_VAR_DEFNS_FP} +. $USHdir/job_preamble.sh +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# +{ save_shell_opts; . $USHdir/preamble.sh; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Get the full path to the file in which this script/function is located +# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in +# which the file is located (scrfunc_dir). +# +#----------------------------------------------------------------------- +# +scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) +scrfunc_fn=$( basename "${scrfunc_fp}" ) +scrfunc_dir=$( dirname "${scrfunc_fp}" ) +# +#----------------------------------------------------------------------- +# +# Print message indicating entry into script. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +Entering script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" + +This is the J-job script for the task that checks that all the post- +processed output files in fact exist and are at least a certain age. +These files may have been generated by UPP as part of the current SRW +App workflow, or they may be user-staged. +========================================================================" +# +#----------------------------------------------------------------------- +# +# Call the ex-script for this J-job and pass to it the necessary varia- +# bles. +# +#----------------------------------------------------------------------- +# +$SCRIPTSdir/exregional_check_post_output.sh || \ +print_err_msg_exit "\ +Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." +# +#----------------------------------------------------------------------- +# +# Create a flag file to make rocoto aware that the make_grid task has +# successfully completed (so that other tasks that depend on it can be +# launched). +# +#----------------------------------------------------------------------- +# +ensmem_name="mem${ENSMEM_INDX}" +cycle_dir="$EXPTDIR/$CDATE" +mkdir_vrfy -p "${cycle_dir}" +touch "${cycle_dir}/post_files_exist_${ensmem_name}.txt" +# +#----------------------------------------------------------------------- +# +# Run job postamble. +# +#----------------------------------------------------------------------- +# +job_postamble +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/func- +# tion. +# +#----------------------------------------------------------------------- +# +{ restore_shell_opts; } > /dev/null 2>&1 diff --git a/scripts/exregional_check_post_output.sh b/scripts/exregional_check_post_output.sh new file mode 100755 index 0000000000..ee01bc5404 --- /dev/null +++ b/scripts/exregional_check_post_output.sh @@ -0,0 +1,119 @@ +#!/bin/bash + +# +#----------------------------------------------------------------------- +# +# Source the variable definitions file and the bash utility functions. +# +#----------------------------------------------------------------------- +# +. $USHdir/source_util_funcs.sh +source_config_for_task "task_run_met_pcpcombine|task_run_post" ${GLOBAL_VAR_DEFNS_FP} +# +#----------------------------------------------------------------------- +# +# Source files defining auxiliary functions for verification. +# +#----------------------------------------------------------------------- +# +. $USHdir/set_vx_fhr_list.sh +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# +{ save_shell_opts; . $USHdir/preamble.sh; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Get the full path to the file in which this script/function is located +# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in +# which the file is located (scrfunc_dir). +# +#----------------------------------------------------------------------- +# +scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) +scrfunc_fn=$( basename "${scrfunc_fp}" ) +scrfunc_dir=$( dirname "${scrfunc_fp}" ) +# +#----------------------------------------------------------------------- +# +# Print message indicating entry into script. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +Entering script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" + +This is the ex-script for the task that checks that all the post-processed +output files in fact exist and are at least a certain age. These files +may have been generated by UPP as part of the current SRW App workflow, +or they may be user-staged. +========================================================================" +# +#----------------------------------------------------------------------- +# +# Get the time lag for the current ensemble member. +# +#----------------------------------------------------------------------- +# +i="0" +if [ "${DO_ENSEMBLE}" = "TRUE" ]; then + i=$( bc -l <<< "${ENSMEM_INDX}-1" ) +fi +time_lag=$( bc -l <<< "${ENS_TIME_LAG_HRS[$i]}*${SECS_PER_HOUR}" ) +# +#----------------------------------------------------------------------- +# +# Get the list of forecast hours for which there is a post-processed +# output file. Note that: +# +# 1) CDATE (in YYYYMMDDHH format) is already available via the call to +# the job_preamble.sh script in the j-job of this ex-script. +# 2) VAR is set to "APCP" and ACCUM_HH is set to "01" because we assume +# the output files are hourly, so these settings will result in the +# function set_vx_fhr_list checking for existence of hourly post output +# files. +# +#----------------------------------------------------------------------- +# +ensmem_name="mem${ENSMEM_INDX}" +FCST_INPUT_FN_TEMPLATE=$( eval echo ${FCST_SUBDIR_TEMPLATE:+${FCST_SUBDIR_TEMPLATE}/}${FCST_FN_TEMPLATE} ) +set_vx_fhr_list \ + cdate="${CDATE}" \ + fcst_len_hrs="${FCST_LEN_HRS}" \ + field="$VAR" \ + accum_hh="${ACCUM_HH}" \ + base_dir="${VX_FCST_INPUT_BASEDIR}" \ + fn_template="${FCST_INPUT_FN_TEMPLATE}" \ + check_accum_contrib_files="FALSE" \ + num_missing_files_max="${NUM_MISSING_FCST_FILES_MAX}" \ + outvarname_fhr_list="FHR_LIST" +# +#----------------------------------------------------------------------- +# +# Print message indicating successful completion of script. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +Done checking for existence of post-processed files for ensemble member ${ENSMEM_INDX}. + +Exiting script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" +========================================================================" +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/func- +# tion. +# +#----------------------------------------------------------------------- +# +{ restore_shell_opts; } > /dev/null 2>&1 From a304f501bc5f15af2e09b0ee8b950394ee45d8e5 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Fri, 26 May 2023 13:28:18 -0600 Subject: [PATCH 35/43] Modifications to get verification to work on time-lagged staged forecast files. --- scripts/exregional_check_post_output.sh | 3 ++- scripts/exregional_run_met_genensprod_or_ensemblestat.sh | 3 +-- scripts/exregional_run_met_gridstat_or_pointstat_vx.sh | 3 ++- scripts/exregional_run_met_pcpcombine.sh | 3 ++- ush/bash_utils/eval_METplus_timestr_tmpl.sh | 6 ++++-- ush/config_defaults.yaml | 7 +++++++ 6 files changed, 18 insertions(+), 7 deletions(-) diff --git a/scripts/exregional_check_post_output.sh b/scripts/exregional_check_post_output.sh index ee01bc5404..d088e218cd 100755 --- a/scripts/exregional_check_post_output.sh +++ b/scripts/exregional_check_post_output.sh @@ -82,7 +82,8 @@ time_lag=$( bc -l <<< "${ENS_TIME_LAG_HRS[$i]}*${SECS_PER_HOUR}" ) # #----------------------------------------------------------------------- # -ensmem_name="mem${ENSMEM_INDX}" +ensmem_indx=$(printf "%0${VX_NDIGITS_ENSMEM_NAMES}d" "${ENSMEM_INDX}") +ensmem_name="mem${ensmem_indx}" FCST_INPUT_FN_TEMPLATE=$( eval echo ${FCST_SUBDIR_TEMPLATE:+${FCST_SUBDIR_TEMPLATE}/}${FCST_FN_TEMPLATE} ) set_vx_fhr_list \ cdate="${CDATE}" \ diff --git a/scripts/exregional_run_met_genensprod_or_ensemblestat.sh b/scripts/exregional_run_met_genensprod_or_ensemblestat.sh index 132471968c..49e2dfdd91 100755 --- a/scripts/exregional_run_met_genensprod_or_ensemblestat.sh +++ b/scripts/exregional_run_met_genensprod_or_ensemblestat.sh @@ -205,11 +205,10 @@ fi # tasks). This will be exported to the environment and read by the # METplus configuration files. # -NDIGITS_ENSMEM_NAMES=3 FCST_INPUT_FN_TEMPLATE="" for (( i=0; i<${NUM_ENS_MEMBERS}; i++ )); do - ensmem_indx=$(printf "%0${NDIGITS_ENSMEM_NAMES}d" "$((i+1))") + ensmem_indx=$(printf "%0${VX_NDIGITS_ENSMEM_NAMES}d" "$((i+1))") ensmem_name="mem${ensmem_indx}" if [ "${RUN_ENVIR}" = "nco" ]; then diff --git a/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh b/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh index 2a62b9d1d3..a6a30671b6 100755 --- a/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh +++ b/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh @@ -173,7 +173,8 @@ fi # vx_fcst_input_basedir=$( eval echo "${VX_FCST_INPUT_BASEDIR}" ) vx_output_basedir=$( eval echo "${VX_OUTPUT_BASEDIR}" ) -ensmem_name="mem${ENSMEM_INDX}" +ensmem_indx=$(printf "%0${VX_NDIGITS_ENSMEM_NAMES}d" "${ENSMEM_INDX}") +ensmem_name="mem${ensmem_indx}" if [ "${RUN_ENVIR}" = "nco" ]; then slash_cdate_or_null="" slash_ensmem_subdir_or_null="" diff --git a/scripts/exregional_run_met_pcpcombine.sh b/scripts/exregional_run_met_pcpcombine.sh index 021f8b15da..a40bece103 100755 --- a/scripts/exregional_run_met_pcpcombine.sh +++ b/scripts/exregional_run_met_pcpcombine.sh @@ -137,7 +137,8 @@ fi # vx_fcst_input_basedir=$( eval echo "${VX_FCST_INPUT_BASEDIR}" ) vx_output_basedir=$( eval echo "${VX_OUTPUT_BASEDIR}" ) -ensmem_name="mem${ENSMEM_INDX}" +ensmem_indx=$(printf "%0${VX_NDIGITS_ENSMEM_NAMES}d" "${ENSMEM_INDX}") +ensmem_name="mem${ensmem_indx}" if [ "${RUN_ENVIR}" = "nco" ]; then slash_cdate_or_null="" slash_ensmem_subdir_or_null="" diff --git a/ush/bash_utils/eval_METplus_timestr_tmpl.sh b/ush/bash_utils/eval_METplus_timestr_tmpl.sh index 0f5db76153..245369509b 100644 --- a/ush/bash_utils/eval_METplus_timestr_tmpl.sh +++ b/ush/bash_utils/eval_METplus_timestr_tmpl.sh @@ -185,8 +185,10 @@ METplus time string template passed to this function is: METplus_timestr_tmpl = \"${METplus_timestr_tmpl}\"" ;; esac - - time_shift_str=$(( ${METplus_time_shift} + 0 ))" seconds" +# +# Calculate the time shift as an integer in units of seconds. +# + time_shift_str=$(( $(printf "%.0f" "${METplus_time_shift}") + 0 ))" seconds" # #----------------------------------------------------------------------- # diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index ec7468bdc3..5d458b05d1 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -2458,6 +2458,13 @@ verification: VX_FCST_INPUT_BASEDIR: '{% if user.RUN_ENVIR == "nco" %}$COMOUT/../..{% else %}{{ workflow.EXPTDIR }}{% endif %}' VX_OUTPUT_BASEDIR: '{% if user.RUN_ENVIR == "nco" %}$COMOUT/metout{% else %}{{ workflow.EXPTDIR }}{% endif %}' # + # Number of digits in the ensember member names. This is a configurable + # variable to allow users to change its value (e.g. to go from "mem004" + # to "mem02") when using staged forecast files that do not use the same + # number of digits as the SRW App. + # + VX_NDIGITS_ENSMEM_NAMES: 3 + # # File name and path templates used in the verification tasks. # FCST_SUBDIR_TEMPLATE: '{% if user.RUN_ENVIR == "nco" %}${NET}.{init?fmt=%Y%m%d?shift=-${time_lag}}/{init?fmt=%H?shift=-${time_lag}}{% else %}{init?fmt=%Y%m%d%H?shift=-${time_lag}}{% if global.DO_ENSEMBLE %}/${ensmem_name}{% endif %}/postprd{% endif %}' From b8dd48104a8955dc5f0bbdcc45a1f87516248da3 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Fri, 26 May 2023 15:16:14 -0600 Subject: [PATCH 36/43] Add WE2E test for time-lagging. --- ...nsemble_verification_only_vx_time_lag.yaml | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx_time_lag.yaml diff --git a/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx_time_lag.yaml b/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx_time_lag.yaml new file mode 100644 index 0000000000..caa24d2eee --- /dev/null +++ b/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx_time_lag.yaml @@ -0,0 +1,43 @@ +metadata: + description: |- + This test is to ensure that the workflow running in community mode can + successfully run both the MET deterministic and ensemble verification + tasks using staged observation and forecast files (i.e. with other tasks + such as pre-processing, forecast, and post-processing deactivated) AND + assuming one of the ensemble members is time-lagged. + +user: + RUN_ENVIR: community + +workflow: + PREDEF_GRID_NAME: RRFS_CONUS_25km + DATE_FIRST_CYCL: '2021050500' + DATE_LAST_CYCL: '2021050500' + FCST_LEN_HRS: 6 + PREEXISTING_DIR_METHOD: rename + +rocoto: + tasks: + taskgroups: '{{ ["parm/wflow/verify_pre.yaml", "parm/wflow/verify_det.yaml", "parm/wflow/verify_ens.yaml"]|include }}' + task_get_obs_ccpa: + task_get_obs_mrms: + task_get_obs_ndas: + +nco: + NET: 'RRFSE_CONUS' + +global: + DO_ENSEMBLE: true + NUM_ENS_MEMBERS: 2 + ENS_TIME_LAG_HRS: '[ 0, 12 ]' + +platform: + CCPA_OBS_DIR: '/scratch2/BMC/fv3lam/ens_design_RRFS/obs_data/ccpa/proc' + MRMS_OBS_DIR: '/scratch2/BMC/fv3lam/ens_design_RRFS/obs_data/mrms/proc' + NDAS_OBS_DIR: '/scratch2/BMC/fv3lam/ens_design_RRFS/obs_data/ndas/proc' + +verification: + VX_FCST_MODEL_NAME: FV3_GFS_v15p2_CONUS_25km + VX_FCST_INPUT_BASEDIR: '/scratch1/BMC/hmtb/beck/ens_design_RRFS/time_lagging_data' + VX_NDIGITS_ENSMEM_NAMES: 1 + FCST_FN_TEMPLATE: '${NET}.t{init?fmt=%H?shift=-${time_lag}}z{% if user.RUN_ENVIR == "nco" and global.DO_ENSEMBLE %}.${ensmem_name}{% endif %}.bgdawpf{lead?fmt=%HHH?shift=${time_lag}}.tm00.grib2' From 77056d2084bdc6c85c0d0c87ad69e90893b51629 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Tue, 30 May 2023 18:39:15 -0600 Subject: [PATCH 37/43] Change the test configuration for the WE2E test "MET_ensemble_verification_only_vx_time_lag" to look for the staged forecast data from the default location for the machine; also, change the predefined grid name to the one on which this staged data is provided (this is necessary to form the correct path to the data). --- .../config.MET_ensemble_verification_only_vx_time_lag.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx_time_lag.yaml b/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx_time_lag.yaml index caa24d2eee..d4a95812bb 100644 --- a/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx_time_lag.yaml +++ b/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx_time_lag.yaml @@ -10,7 +10,7 @@ user: RUN_ENVIR: community workflow: - PREDEF_GRID_NAME: RRFS_CONUS_25km + PREDEF_GRID_NAME: RRFS_CONUS_3km DATE_FIRST_CYCL: '2021050500' DATE_LAST_CYCL: '2021050500' FCST_LEN_HRS: 6 @@ -38,6 +38,6 @@ platform: verification: VX_FCST_MODEL_NAME: FV3_GFS_v15p2_CONUS_25km - VX_FCST_INPUT_BASEDIR: '/scratch1/BMC/hmtb/beck/ens_design_RRFS/time_lagging_data' + VX_FCST_INPUT_BASEDIR: '{{ platform.get("TEST_VX_FCST_INPUT_BASEDIR") }}' VX_NDIGITS_ENSMEM_NAMES: 1 FCST_FN_TEMPLATE: '${NET}.t{init?fmt=%H?shift=-${time_lag}}z{% if user.RUN_ENVIR == "nco" and global.DO_ENSEMBLE %}.${ensmem_name}{% endif %}.bgdawpf{lead?fmt=%HHH?shift=${time_lag}}.tm00.grib2' From 76420d6c13f56481d81daa1cb37f907a71f4236b Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Tue, 30 May 2023 18:40:53 -0600 Subject: [PATCH 38/43] Add missing vx WE2E tests to the list of comprehensive tests. --- tests/WE2E/machine_suites/comprehensive | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/WE2E/machine_suites/comprehensive b/tests/WE2E/machine_suites/comprehensive index 07ea344271..ce5baaff91 100644 --- a/tests/WE2E/machine_suites/comprehensive +++ b/tests/WE2E/machine_suites/comprehensive @@ -55,6 +55,8 @@ grid_SUBCONUS_Ind_3km_ics_NAM_lbcs_NAM_suite_GFS_v16 grid_SUBCONUS_Ind_3km_ics_RAP_lbcs_RAP_suite_RRFS_v1beta_plot GST_release_public_v1 MET_verification_only_vx +MET_ensemble_verification_only_vx +MET_ensemble_verification_only_vx_time_lag nco nco_ensemble nco_grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 From 1a6bb6d72eb234f38d5fa1df29e490c114438d42 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Wed, 31 May 2023 17:47:20 -0600 Subject: [PATCH 39/43] Add forecast lead time (based on nominal cycle start time disregarding any time-lagging) to names of GenEnsProd and EnsembleStat output files. --- parm/metplus/EnsembleStat_APCP01h.conf | 6 ++---- parm/metplus/EnsembleStat_APCPgt01h.conf | 7 +++---- parm/metplus/EnsembleStat_REFC.conf | 6 ++---- parm/metplus/EnsembleStat_RETOP.conf | 6 ++---- parm/metplus/EnsembleStat_SFC.conf | 6 ++---- parm/metplus/EnsembleStat_UPA.conf | 6 ++---- parm/metplus/GenEnsProd_APCP01h.conf | 5 ++--- parm/metplus/GenEnsProd_APCPgt01h.conf | 5 ++--- parm/metplus/GenEnsProd_REFC.conf | 5 ++--- parm/metplus/GenEnsProd_RETOP.conf | 5 ++--- parm/metplus/GenEnsProd_SFC.conf | 5 ++--- parm/metplus/GenEnsProd_UPA.conf | 5 ++--- .../exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh | 4 ++-- .../exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh | 4 ++-- 14 files changed, 29 insertions(+), 46 deletions(-) diff --git a/parm/metplus/EnsembleStat_APCP01h.conf b/parm/metplus/EnsembleStat_APCP01h.conf index 9feb184f20..aa85f5dc51 100644 --- a/parm/metplus/EnsembleStat_APCP01h.conf +++ b/parm/metplus/EnsembleStat_APCP01h.conf @@ -93,8 +93,6 @@ ENSEMBLE_STAT_ENS_THRESH = 0.05 # ens.vld_thresh value in the MET config file ENSEMBLE_STAT_ENS_VLD_THRESH = 1.0 -ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_{FIELDNAME_IN_MET_FILEDIR_NAMES}_{OBTYPE} - FIELDNAME_IN_MET_FILEDIR_NAMES = {{fieldname_in_met_filedir_names}} # ENSEMBLE_STAT_MET_OBS_ERR_TABLE is not required. @@ -258,10 +256,10 @@ OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = {{obs_input_fn_template}} # FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}} # -# Template for output from EnsembleStat relative to -# ENSEMBLE_STAT_OUTPUT_DIR. +# Template for output from EnsembleStat relative to ENSEMBLE_STAT_OUTPUT_DIR. # ENSEMBLE_STAT_OUTPUT_TEMPLATE = +ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_{FIELDNAME_IN_MET_FILEDIR_NAMES}_{OBTYPE}_{lead?fmt=%H%M%S}L # # Template for climatology input to EnsembleStat relative to # ENSEMBLE_STAT_CLIMO_MEAN_INPUT_DIR. Not used in this example. diff --git a/parm/metplus/EnsembleStat_APCPgt01h.conf b/parm/metplus/EnsembleStat_APCPgt01h.conf index 6e1b77a8b1..e748642b9b 100644 --- a/parm/metplus/EnsembleStat_APCPgt01h.conf +++ b/parm/metplus/EnsembleStat_APCPgt01h.conf @@ -100,8 +100,6 @@ ENSEMBLE_STAT_ENS_THRESH = 0.05 # ens.vld_thresh value in the MET config file ENSEMBLE_STAT_ENS_VLD_THRESH = 1.0 -ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_{FIELDNAME_IN_MET_FILEDIR_NAMES}_{OBTYPE} - FIELDNAME_IN_MET_FILEDIR_NAMES = {{fieldname_in_met_filedir_names}} # ENSEMBLE_STAT_MET_OBS_ERR_TABLE is not required. @@ -265,10 +263,11 @@ OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = {{obs_input_fn_template}} # FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}} # -# Template for output from EnsembleStat relative to -# ENSEMBLE_STAT_OUTPUT_DIR. +# Template for output from EnsembleStat relative to ENSEMBLE_STAT_OUTPUT_DIR. # ENSEMBLE_STAT_OUTPUT_TEMPLATE = +ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_{FIELDNAME_IN_MET_FILEDIR_NAMES}_{OBTYPE}_{lead?fmt=%H%M%S}L + # # Template for climatology input to EnsembleStat relative to # ENSEMBLE_STAT_CLIMO_MEAN_INPUT_DIR. Not used in this example. diff --git a/parm/metplus/EnsembleStat_REFC.conf b/parm/metplus/EnsembleStat_REFC.conf index d569552119..d6fe4874c8 100644 --- a/parm/metplus/EnsembleStat_REFC.conf +++ b/parm/metplus/EnsembleStat_REFC.conf @@ -93,8 +93,6 @@ ENSEMBLE_STAT_ENS_THRESH = 0.05 # ens.vld_thresh value in the MET config file ENSEMBLE_STAT_ENS_VLD_THRESH = 1.0 -ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_{FIELDNAME_IN_MET_FILEDIR_NAMES}_{OBTYPE} - FIELDNAME_IN_MET_FILEDIR_NAMES = {{fieldname_in_met_filedir_names}} # ENSEMBLE_STAT_MET_OBS_ERR_TABLE is not required. @@ -256,10 +254,10 @@ OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = {{obs_input_fn_template}} # FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}} # -# Template for output from EnsembleStat relative to -# ENSEMBLE_STAT_OUTPUT_DIR. +# Template for output from EnsembleStat relative to ENSEMBLE_STAT_OUTPUT_DIR. # ENSEMBLE_STAT_OUTPUT_TEMPLATE = +ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_{FIELDNAME_IN_MET_FILEDIR_NAMES}_{OBTYPE}_{lead?fmt=%H%M%S}L # # Template for climatology input to EnsembleStat relative to # ENSEMBLE_STAT_CLIMO_MEAN_INPUT_DIR. Not used in this example. diff --git a/parm/metplus/EnsembleStat_RETOP.conf b/parm/metplus/EnsembleStat_RETOP.conf index 00bf62b0a6..55ac5fc8b1 100644 --- a/parm/metplus/EnsembleStat_RETOP.conf +++ b/parm/metplus/EnsembleStat_RETOP.conf @@ -93,8 +93,6 @@ ENSEMBLE_STAT_ENS_THRESH = 0.05 # ens.vld_thresh value in the MET config file ENSEMBLE_STAT_ENS_VLD_THRESH = 1.0 -ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_{FIELDNAME_IN_MET_FILEDIR_NAMES}_{OBTYPE} - FIELDNAME_IN_MET_FILEDIR_NAMES = {{fieldname_in_met_filedir_names}} # ENSEMBLE_STAT_MET_OBS_ERR_TABLE is not required. @@ -258,10 +256,10 @@ OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = {{obs_input_fn_template}} # FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}} # -# Template for output from EnsembleStat relative to -# ENSEMBLE_STAT_OUTPUT_DIR. +# Template for output from EnsembleStat relative to ENSEMBLE_STAT_OUTPUT_DIR. # ENSEMBLE_STAT_OUTPUT_TEMPLATE = +ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_{FIELDNAME_IN_MET_FILEDIR_NAMES}_{OBTYPE}_{lead?fmt=%H%M%S}L # # Template for climatology input to EnsembleStat relative to # ENSEMBLE_STAT_CLIMO_MEAN_INPUT_DIR. Not used in this example. diff --git a/parm/metplus/EnsembleStat_SFC.conf b/parm/metplus/EnsembleStat_SFC.conf index 0bfa03ab1d..14c5c7b85a 100644 --- a/parm/metplus/EnsembleStat_SFC.conf +++ b/parm/metplus/EnsembleStat_SFC.conf @@ -93,8 +93,6 @@ ENSEMBLE_STAT_ENS_THRESH = 0.05 # ens.vld_thresh value in the MET config file ENSEMBLE_STAT_ENS_VLD_THRESH = 1.0 -ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_ADP{FIELDNAME_IN_MET_FILEDIR_NAMES}_{OBTYPE} - FIELDNAME_IN_MET_FILEDIR_NAMES = {{fieldname_in_met_filedir_names}} ENSEMBLE_STAT_OBS_QUALITY_INC = 0, 1, 2, 3, 9, NA @@ -305,10 +303,10 @@ OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = # FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}} # -# Template for output from EnsembleStat relative to -# ENSEMBLE_STAT_OUTPUT_DIR. +# Template for output from EnsembleStat relative to ENSEMBLE_STAT_OUTPUT_DIR. # ENSEMBLE_STAT_OUTPUT_TEMPLATE = +ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_ADP{FIELDNAME_IN_MET_FILEDIR_NAMES}_{OBTYPE}_{lead?fmt=%H%M%S}L # # Template for climatology input to EnsembleStat relative to # ENSEMBLE_STAT_CLIMO_MEAN_INPUT_DIR. Not used in this example. diff --git a/parm/metplus/EnsembleStat_UPA.conf b/parm/metplus/EnsembleStat_UPA.conf index 92e27272ad..4bc2bad3d0 100644 --- a/parm/metplus/EnsembleStat_UPA.conf +++ b/parm/metplus/EnsembleStat_UPA.conf @@ -93,8 +93,6 @@ ENSEMBLE_STAT_ENS_THRESH = 0.05 # ens.vld_thresh value in the MET config file ENSEMBLE_STAT_ENS_VLD_THRESH = 1.0 -ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_ADP{FIELDNAME_IN_MET_FILEDIR_NAMES}_{OBTYPE} - FIELDNAME_IN_MET_FILEDIR_NAMES = {{fieldname_in_met_filedir_names}} ENSEMBLE_STAT_OBS_QUALITY_INC = 0, 1, 2, 3, 9, NA @@ -347,10 +345,10 @@ OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = # FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = {{fcst_input_fn_template}} # -# Template for output from EnsembleStat relative to -# ENSEMBLE_STAT_OUTPUT_DIR. +# Template for output from EnsembleStat relative to ENSEMBLE_STAT_OUTPUT_DIR. # ENSEMBLE_STAT_OUTPUT_TEMPLATE = +ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_ADP{FIELDNAME_IN_MET_FILEDIR_NAMES}_{OBTYPE}_{lead?fmt=%H%M%S}L # # Template for climatology input to EnsembleStat relative to # ENSEMBLE_STAT_CLIMO_MEAN_INPUT_DIR. Not used in this example. diff --git a/parm/metplus/GenEnsProd_APCP01h.conf b/parm/metplus/GenEnsProd_APCP01h.conf index 1731306975..d60c8d3ce0 100644 --- a/parm/metplus/GenEnsProd_APCP01h.conf +++ b/parm/metplus/GenEnsProd_APCP01h.conf @@ -98,10 +98,9 @@ GEN_ENS_PROD_INPUT_TEMPLATE = {{fcst_input_fn_template}} OUTPUT_BASE = {{output_base}} GEN_ENS_PROD_OUTPUT_DIR = {{output_dir}} # -# Template for output from GenEnsProd relative to -# GEN_ENS_PROD_OUTPUT_DIR. +# Template for output from GenEnsProd relative to GEN_ENS_PROD_OUTPUT_DIR. # -GEN_ENS_PROD_OUTPUT_TEMPLATE = gen_ens_prod_{MODEL}_{{fieldname_in_met_filedir_names}}_{OBTYPE}_{valid?fmt=%Y%m%d_%H%M%S}V.nc +GEN_ENS_PROD_OUTPUT_TEMPLATE = gen_ens_prod_{MODEL}_{{fieldname_in_met_filedir_names}}_{OBTYPE}_{lead?fmt=%H%M%S}L_{valid?fmt=%Y%m%d_%H%M%S}V.nc # # Directory for staging data. # diff --git a/parm/metplus/GenEnsProd_APCPgt01h.conf b/parm/metplus/GenEnsProd_APCPgt01h.conf index e0df15ed68..08886087d9 100644 --- a/parm/metplus/GenEnsProd_APCPgt01h.conf +++ b/parm/metplus/GenEnsProd_APCPgt01h.conf @@ -101,10 +101,9 @@ FCST_IS_PROB = false OUTPUT_BASE = {{output_base}} GEN_ENS_PROD_OUTPUT_DIR = {{output_dir}} # -# Template for output from GenEnsProd relative to -# GEN_ENS_PROD_OUTPUT_DIR. +# Template for output from GenEnsProd relative to GEN_ENS_PROD_OUTPUT_DIR. # -GEN_ENS_PROD_OUTPUT_TEMPLATE = gen_ens_prod_{MODEL}_{{fieldname_in_met_filedir_names}}_{OBTYPE}_{valid?fmt=%Y%m%d_%H%M%S}V.nc +GEN_ENS_PROD_OUTPUT_TEMPLATE = gen_ens_prod_{MODEL}_{{fieldname_in_met_filedir_names}}_{OBTYPE}_{lead?fmt=%H%M%S}L_{valid?fmt=%Y%m%d_%H%M%S}V.nc # # Directory for staging data. # diff --git a/parm/metplus/GenEnsProd_REFC.conf b/parm/metplus/GenEnsProd_REFC.conf index 08dea7b571..8c02f71d64 100644 --- a/parm/metplus/GenEnsProd_REFC.conf +++ b/parm/metplus/GenEnsProd_REFC.conf @@ -98,10 +98,9 @@ GEN_ENS_PROD_INPUT_TEMPLATE = {{fcst_input_fn_template}} OUTPUT_BASE = {{output_base}} GEN_ENS_PROD_OUTPUT_DIR = {{output_dir}} # -# Template for output from GenEnsProd relative to -# GEN_ENS_PROD_OUTPUT_DIR. +# Template for output from GenEnsProd relative to GEN_ENS_PROD_OUTPUT_DIR. # -GEN_ENS_PROD_OUTPUT_TEMPLATE = gen_ens_prod_{MODEL}_{{fieldname_in_met_filedir_names}}_{OBTYPE}_{valid?fmt=%Y%m%d_%H%M%S}V.nc +GEN_ENS_PROD_OUTPUT_TEMPLATE = gen_ens_prod_{MODEL}_{{fieldname_in_met_filedir_names}}_{OBTYPE}_{lead?fmt=%H%M%S}L_{valid?fmt=%Y%m%d_%H%M%S}V.nc # # Directory for staging data. # diff --git a/parm/metplus/GenEnsProd_RETOP.conf b/parm/metplus/GenEnsProd_RETOP.conf index 205fc3eb75..111b3bfccc 100644 --- a/parm/metplus/GenEnsProd_RETOP.conf +++ b/parm/metplus/GenEnsProd_RETOP.conf @@ -98,10 +98,9 @@ GEN_ENS_PROD_INPUT_TEMPLATE = {{fcst_input_fn_template}} OUTPUT_BASE = {{output_base}} GEN_ENS_PROD_OUTPUT_DIR = {{output_dir}} # -# Template for output from GenEnsProd relative to -# GEN_ENS_PROD_OUTPUT_DIR. +# Template for output from GenEnsProd relative to GEN_ENS_PROD_OUTPUT_DIR. # -GEN_ENS_PROD_OUTPUT_TEMPLATE = gen_ens_prod_{MODEL}_{{fieldname_in_met_filedir_names}}_{OBTYPE}_{valid?fmt=%Y%m%d_%H%M%S}V.nc +GEN_ENS_PROD_OUTPUT_TEMPLATE = gen_ens_prod_{MODEL}_{{fieldname_in_met_filedir_names}}_{OBTYPE}_{lead?fmt=%H%M%S}L_{valid?fmt=%Y%m%d_%H%M%S}V.nc # # Directory for staging data. # diff --git a/parm/metplus/GenEnsProd_SFC.conf b/parm/metplus/GenEnsProd_SFC.conf index 4acf0d4538..47b49b9adb 100644 --- a/parm/metplus/GenEnsProd_SFC.conf +++ b/parm/metplus/GenEnsProd_SFC.conf @@ -98,10 +98,9 @@ GEN_ENS_PROD_INPUT_TEMPLATE = {{fcst_input_fn_template}} OUTPUT_BASE = {{output_base}} GEN_ENS_PROD_OUTPUT_DIR = {{output_dir}} # -# Template for output from GenEnsProd relative to -# GEN_ENS_PROD_OUTPUT_DIR. +# Template for output from GenEnsProd relative to GEN_ENS_PROD_OUTPUT_DIR. # -GEN_ENS_PROD_OUTPUT_TEMPLATE = gen_ens_prod_{MODEL}_ADP{{fieldname_in_met_filedir_names}}_{OBTYPE}_{valid?fmt=%Y%m%d_%H%M%S}V.nc +GEN_ENS_PROD_OUTPUT_TEMPLATE = gen_ens_prod_{MODEL}_ADP{{fieldname_in_met_filedir_names}}_{OBTYPE}_{lead?fmt=%H%M%S}L_{valid?fmt=%Y%m%d_%H%M%S}V.nc # # Directory for staging data. # diff --git a/parm/metplus/GenEnsProd_UPA.conf b/parm/metplus/GenEnsProd_UPA.conf index ea3ebaa4d9..ee1ceb42ca 100644 --- a/parm/metplus/GenEnsProd_UPA.conf +++ b/parm/metplus/GenEnsProd_UPA.conf @@ -98,10 +98,9 @@ GEN_ENS_PROD_INPUT_TEMPLATE = {{fcst_input_fn_template}} OUTPUT_BASE = {{output_base}} GEN_ENS_PROD_OUTPUT_DIR = {{output_dir}} # -# Template for output from GenEnsProd relative to -# GEN_ENS_PROD_OUTPUT_DIR. +# Template for output from GenEnsProd relative to GEN_ENS_PROD_OUTPUT_DIR. # -GEN_ENS_PROD_OUTPUT_TEMPLATE = gen_ens_prod_{MODEL}_ADP{{fieldname_in_met_filedir_names}}_{OBTYPE}_{valid?fmt=%Y%m%d_%H%M%S}V.nc +GEN_ENS_PROD_OUTPUT_TEMPLATE = gen_ens_prod_{MODEL}_ADP{{fieldname_in_met_filedir_names}}_{OBTYPE}_{lead?fmt=%H%M%S}L_{valid?fmt=%Y%m%d_%H%M%S}V.nc # # Directory for staging data. # diff --git a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh index 3ca0fcdb8e..c8cb4d9cac 100755 --- a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh +++ b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh @@ -178,14 +178,14 @@ if [ "${grid_or_point}" = "grid" ]; then OBS_INPUT_FN_TEMPLATE=$( eval echo ${OBS_INPUT_FN_TEMPLATE} ) fi FCST_INPUT_DIR="${vx_output_basedir}${slash_cdate_or_null}/metprd/GenEnsProd" - FCST_INPUT_FN_TEMPLATE=$( eval echo 'gen_ens_prod_${VX_FCST_MODEL_NAME}_${FIELDNAME_IN_MET_FILEDIR_NAMES}_${OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V.nc' ) + FCST_INPUT_FN_TEMPLATE=$( eval echo 'gen_ens_prod_${VX_FCST_MODEL_NAME}_${FIELDNAME_IN_MET_FILEDIR_NAMES}_${OBTYPE}_{lead?fmt=%H%M%S}L_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V.nc' ) elif [ "${grid_or_point}" = "point" ]; then OBS_INPUT_DIR="${vx_output_basedir}/metprd/Pb2nc_obs" OBS_INPUT_FN_TEMPLATE=$( eval echo ${OBS_NDAS_SFCorUPA_FN_METPROC_TEMPLATE} ) FCST_INPUT_DIR="${vx_output_basedir}${slash_cdate_or_null}/metprd/GenEnsProd" - FCST_INPUT_FN_TEMPLATE=$( eval echo 'gen_ens_prod_${VX_FCST_MODEL_NAME}_ADP${FIELDNAME_IN_MET_FILEDIR_NAMES}_${OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V.nc' ) + FCST_INPUT_FN_TEMPLATE=$( eval echo 'gen_ens_prod_${VX_FCST_MODEL_NAME}_ADP${FIELDNAME_IN_MET_FILEDIR_NAMES}_${OBTYPE}_{lead?fmt=%H%M%S}L_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V.nc' ) fi diff --git a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh index 7efa2c1fe4..473a17dcf5 100755 --- a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh +++ b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh @@ -137,14 +137,14 @@ if [ "${grid_or_point}" = "grid" ]; then OBS_INPUT_FN_TEMPLATE=$( eval echo ${OBS_INPUT_FN_TEMPLATE} ) fi FCST_INPUT_DIR="${vx_output_basedir}${slash_cdate_or_null}/metprd/GenEnsProd" - FCST_INPUT_FN_TEMPLATE=$( eval echo 'gen_ens_prod_${VX_FCST_MODEL_NAME}_${FIELDNAME_IN_MET_FILEDIR_NAMES}_${OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V.nc' ) + FCST_INPUT_FN_TEMPLATE=$( eval echo 'gen_ens_prod_${VX_FCST_MODEL_NAME}_${FIELDNAME_IN_MET_FILEDIR_NAMES}_${OBTYPE}_{lead?fmt=%H%M%S}L_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V.nc' ) elif [ "${grid_or_point}" = "point" ]; then OBS_INPUT_DIR="${vx_output_basedir}/metprd/Pb2nc_obs" OBS_INPUT_FN_TEMPLATE=$( eval echo ${OBS_NDAS_SFCorUPA_FN_METPROC_TEMPLATE} ) FCST_INPUT_DIR="${vx_output_basedir}${slash_cdate_or_null}/metprd/GenEnsProd" - FCST_INPUT_FN_TEMPLATE=$( eval echo 'gen_ens_prod_${VX_FCST_MODEL_NAME}_ADP${FIELDNAME_IN_MET_FILEDIR_NAMES}_${OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V.nc' ) + FCST_INPUT_FN_TEMPLATE=$( eval echo 'gen_ens_prod_${VX_FCST_MODEL_NAME}_ADP${FIELDNAME_IN_MET_FILEDIR_NAMES}_${OBTYPE}_{lead?fmt=%H%M%S}L_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V.nc' ) fi From 38244374360d93dd913ad46f7f9fe0ae48150c99 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Wed, 31 May 2023 17:49:22 -0600 Subject: [PATCH 40/43] Bug fix: Change dependence of ensmean and ensprob tasks to be on the GenEnsProd task, not the EnsembleStat task. Also, change order of metatasks so that the one for MRMS comes before the one for NDAS. --- parm/wflow/verify_ens.yaml | 40 +++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/parm/wflow/verify_ens.yaml b/parm/wflow/verify_ens.yaml index 4fdf8e6964..3dbf924ce8 100644 --- a/parm/wflow/verify_ens.yaml +++ b/parm/wflow/verify_ens.yaml @@ -146,7 +146,25 @@ metatask_GridStat_CCPA_ensmeanprob_all_accums: dependency: taskdep: attrs: - task: run_MET_EnsembleStat_vx_APCP#ACCUM_HH#h + task: run_MET_GenEnsProd_vx_APCP#ACCUM_HH#h + +metatask_GridStat_MRMS_ensprob: + var: + VAR: '{% for var in verification.VX_FIELDS %}{% if var in ["REFC", "RETOP"] %}{{ "%s " % var }}{% endif %}{% endfor %}' + task_run_MET_GridStat_vx_ensprob_#VAR#: + <<: *default_task_verify_ens + command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_GRIDSTAT_OR_POINTSTAT_VX_ENSPROB"' + envars: + <<: *default_vars + ACCUM_HH: '01' + OBS_DIR: '&MRMS_OBS_DIR;' + VAR: '#VAR#' + MET_TOOL: 'GRIDSTAT' + OBTYPE: 'MRMS' + dependency: + taskdep: + attrs: + task: run_MET_GenEnsProd_vx_#VAR# metatask_PointStat_NDAS_ensmeanprob: var: @@ -168,22 +186,4 @@ metatask_PointStat_NDAS_ensmeanprob: dependency: taskdep: attrs: - task: run_MET_EnsembleStat_vx_#VAR# - -metatask_GridStat_MRMS_ensprob: - var: - VAR: '{% for var in verification.VX_FIELDS %}{% if var in ["REFC", "RETOP"] %}{{ "%s " % var }}{% endif %}{% endfor %}' - task_run_MET_GridStat_vx_ensprob_#VAR#: - <<: *default_task_verify_ens - command: '&LOAD_MODULES_RUN_TASK_FP; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_GRIDSTAT_OR_POINTSTAT_VX_ENSPROB"' - envars: - <<: *default_vars - ACCUM_HH: '01' - OBS_DIR: '&MRMS_OBS_DIR;' - VAR: '#VAR#' - MET_TOOL: 'GRIDSTAT' - OBTYPE: 'MRMS' - dependency: - taskdep: - attrs: - task: run_MET_EnsembleStat_vx_#VAR# + task: run_MET_GenEnsProd_vx_#VAR# From 7d86881b4287715f37781a4359b048859ae0955c Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Wed, 31 May 2023 19:39:38 -0600 Subject: [PATCH 41/43] For MRMS and NDAS fields, include hour 0 in the verification. --- ush/set_vx_fhr_list.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ush/set_vx_fhr_list.sh b/ush/set_vx_fhr_list.sh index 2e79da7e12..490c7f5374 100644 --- a/ush/set_vx_fhr_list.sh +++ b/ush/set_vx_fhr_list.sh @@ -106,19 +106,19 @@ function set_vx_fhr_list() { fhr_int="${accum_hh}" ;; "REFC") - fhr_min="01" + fhr_min="00" fhr_int="01" ;; "RETOP") - fhr_min="01" + fhr_min="00" fhr_int="01" ;; "SFC") - fhr_min="01" + fhr_min="00" fhr_int="01" ;; "UPA") - fhr_min="06" + fhr_min="00" fhr_int="06" ;; *) From 141f9e75777082b942315232714238c386c023df Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Thu, 1 Jun 2023 04:28:30 -0600 Subject: [PATCH 42/43] Bug fix requested by Michelle in get_obs_... ex-scripts. --- scripts/exregional_get_obs_ccpa.sh | 20 ++++++++++---------- scripts/exregional_get_obs_mrms.sh | 4 ++-- scripts/exregional_get_obs_ndas.sh | 16 ++++++++-------- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/scripts/exregional_get_obs_ccpa.sh b/scripts/exregional_get_obs_ccpa.sh index d3f53c8d5a..9b54e49d56 100755 --- a/scripts/exregional_get_obs_ccpa.sh +++ b/scripts/exregional_get_obs_ccpa.sh @@ -162,15 +162,15 @@ while [[ ${current_fcst} -le ${fcst_length} ]]; do if [[ ${vhh_noZero} -ge 19 && ${vhh_noZero} -le 23 ]]; then cd_vrfy $ccpa_raw/${vyyyymmdd_p1} # Pull CCPA data from HPSS - TarCommand="htar -xvf ${TarFile_p1} \`htar -tf ${TarFile_p1} | egrep \"ccpa.t${vhh}z.${accum}h.hrap.conus.gb2\" | awk '{print $7}'\`" + TarCommand="htar -xvf ${TarFile_p1} \`htar -tf ${TarFile_p1} | egrep \"ccpa.t${vhh}z.${accum}h.hrap.conus.gb2\" | awk '{print \$7}'\`" echo "CALLING: ${TarCommand}" - htar -xvf ${TarFile_p1} `htar -tf ${TarFile_p1} | egrep "ccpa.t${vhh}z.${accum}h.hrap.conus.gb2" | awk '{print $7}'` + htar -xvf ${TarFile_p1} `htar -tf ${TarFile_p1} | egrep "ccpa.t${vhh}z.${accum}h.hrap.conus.gb2" | awk '{print \$7}'` else cd_vrfy $ccpa_raw/${vyyyymmdd} # Pull CCPA data from HPSS - TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"ccpa.t${vhh}z.${accum}h.hrap.conus.gb2\" | awk '{print $7}'\`" + TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"ccpa.t${vhh}z.${accum}h.hrap.conus.gb2\" | awk '{print \$7}'\`" echo "CALLING: ${TarCommand}" - htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "ccpa.t${vhh}z.${accum}h.hrap.conus.gb2" | awk '{print $7}'` + htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "ccpa.t${vhh}z.${accum}h.hrap.conus.gb2" | awk '{print \$7}'` fi # One hour CCPA files have incorrect metadeta in the files under the "00" directory from 20180718 to 20210504. @@ -200,15 +200,15 @@ while [[ ${current_fcst} -le ${fcst_length} ]]; do if [[ ${vhh_noZero} -ne 21 ]]; then cd_vrfy $ccpa_raw/${vyyyymmdd} # Pull CCPA data from HPSS - TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"ccpa.t${vhh}z.${accum}h.hrap.conus.gb2\" | awk '{print $7}'\`" + TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"ccpa.t${vhh}z.${accum}h.hrap.conus.gb2\" | awk '{print \$7}'\`" echo "CALLING: ${TarCommand}" - htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "ccpa.t${vhh}z.${accum}h.hrap.conus.gb2" | awk '{print $7}'` + htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "ccpa.t${vhh}z.${accum}h.hrap.conus.gb2" | awk '{print \$7}'` elif [[ ${vhh_noZero} -eq 21 ]]; then cd_vrfy $ccpa_raw/${vyyyymmdd_p1} # Pull CCPA data from HPSS - TarCommand="htar -xvf ${TarFile_p1} \`htar -tf ${TarFile_p1} | egrep \"ccpa.t${vhh}z.${accum}h.hrap.conus.gb2\" | awk '{print $7}'\`" + TarCommand="htar -xvf ${TarFile_p1} \`htar -tf ${TarFile_p1} | egrep \"ccpa.t${vhh}z.${accum}h.hrap.conus.gb2\" | awk '{print \$7}'\`" echo "CALLING: ${TarCommand}" - htar -xvf ${TarFile_p1} `htar -tf ${TarFile_p1} | egrep "ccpa.t${vhh}z.${accum}h.hrap.conus.gb2" | awk '{print $7}'` + htar -xvf ${TarFile_p1} `htar -tf ${TarFile_p1} | egrep "ccpa.t${vhh}z.${accum}h.hrap.conus.gb2" | awk '{print \$7}'` fi if [[ ${vhh_noZero} -eq 0 ]]; then @@ -226,9 +226,9 @@ while [[ ${current_fcst} -le ${fcst_length} ]]; do elif [[ ${accum} == "06" ]]; then cd_vrfy $ccpa_raw/${vyyyymmdd} # Pull CCPA data from HPSS - TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"ccpa.t${vhh}z.${accum}h.hrap.conus.gb2\" | awk '{print $7}'\`" + TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"ccpa.t${vhh}z.${accum}h.hrap.conus.gb2\" | awk '{print \$7}'\`" echo "CALLING: ${TarCommand}" - htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "ccpa.t${vhh}z.${accum}h.hrap.conus.gb2" | awk '{print $7}'` + htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "ccpa.t${vhh}z.${accum}h.hrap.conus.gb2" | awk '{print \$7}'` if [[ ${vhh_noZero} -eq 0 ]]; then cp_vrfy $ccpa_raw/${vyyyymmdd}/00/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} diff --git a/scripts/exregional_get_obs_mrms.sh b/scripts/exregional_get_obs_mrms.sh index 2f2d7afb4f..2d4ecac82f 100755 --- a/scripts/exregional_get_obs_mrms.sh +++ b/scripts/exregional_get_obs_mrms.sh @@ -128,8 +128,8 @@ while [[ ${cur_ut} -le ${end_valid_ut} ]]; do echo "TAR FILE:${TarFile}" - TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"${field_base_name}${level}${vyyyy}${vmm}${vdd}-[0-9][0-9][0-9][0-9][0-9][0-9].grib2.gz\" | awk '{print $7}'\`" - htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "${field_base_name}${level}${vyyyy}${vmm}${vdd}-[0-9][0-9][0-9][0-9][0-9][0-9].grib2.gz" | awk '{print $7}'` + TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"${field_base_name}${level}${vyyyy}${vmm}${vdd}-[0-9][0-9][0-9][0-9][0-9][0-9].grib2.gz\" | awk '{print \$7}'\`" + htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "${field_base_name}${level}${vyyyy}${vmm}${vdd}-[0-9][0-9][0-9][0-9][0-9][0-9].grib2.gz" | awk '{print \$7}'` Status=$? if [[ ${Status} != 0 ]]; then diff --git a/scripts/exregional_get_obs_ndas.sh b/scripts/exregional_get_obs_ndas.sh index 401013a9ce..4249faafca 100755 --- a/scripts/exregional_get_obs_ndas.sh +++ b/scripts/exregional_get_obs_ndas.sh @@ -91,24 +91,24 @@ echo "vhh_noZero=$vhh_noZero" # Name of NDAS tar file on HPSS is dependent on date. Logic accounts for files from 2019 until July 2020. if [[ ${vyyyymmdd} -ge 20190101 && ${vyyyymmdd} -le 20190820 ]]; then TarFile="/NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/com2_nam_prod_nam.${vyyyy}${vmm}${vdd}${vhh}.bufr.tar" - TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"prepbufr.tm[0-9][0-9].nr\" | awk '{print $7}'\`" + TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"prepbufr.tm[0-9][0-9].nr\" | awk '{print \$7}'\`" echo "CALLING: ${TarCommand}" - htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "prepbufr.tm[0-9][0-9].nr" | awk '{print $7}'` + htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "prepbufr.tm[0-9][0-9].nr" | awk '{print \$7}'` elif [[ ${vyyyymmdd} -ge 20190821 && ${vyyyymmdd} -le 20200226 ]]; then TarFile="/NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/gpfs_dell1_nco_ops_com_nam_prod_nam.${vyyyy}${vmm}${vdd}${vhh}.bufr.tar" - TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"prepbufr.tm[0-9][0-9].nr\" | awk '{print $7}'\`" + TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"prepbufr.tm[0-9][0-9].nr\" | awk '{print \$7}'\`" echo "CALLING: ${TarCommand}" - htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "prepbufr.tm[0-9][0-9].nr" | awk '{print $7}'` + htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "prepbufr.tm[0-9][0-9].nr" | awk '{print \$7}'` elif [[ ${vyyyymmdd} -gt 20200226 && ${vyyyymmdd} -le 20220627 ]]; then TarFile="/NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/com_nam_prod_nam.${vyyyy}${vmm}${vdd}${vhh}.bufr.tar" - TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"prepbufr.tm[0-9][0-9].nr\" | awk '{print $7}'\`" + TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"prepbufr.tm[0-9][0-9].nr\" | awk '{print \$7}'\`" echo "CALLING: ${TarCommand}" - htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "prepbufr.tm[0-9][0-9].nr" | awk '{print $7}'` + htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "prepbufr.tm[0-9][0-9].nr" | awk '{print \$7}'` else TarFile="/NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/com_obsproc_v1.0_nam.${vyyyy}${vmm}${vdd}${vhh}.bufr.tar" - TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"prepbufr.tm[0-9][0-9].nr\" | awk '{print $7}'\`" + TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"prepbufr.tm[0-9][0-9].nr\" | awk '{print \$7}'\`" echo "CALLING: ${TarCommand}" - htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "prepbufr.tm[0-9][0-9].nr" | awk '{print $7}'` + htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "prepbufr.tm[0-9][0-9].nr" | awk '{print \$7}'` fi if [[ ! -d "$ndas_proc" ]]; then From 6b163fccaba9f05f194593d33c97d74c14dc6509 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Thu, 1 Jun 2023 04:29:43 -0600 Subject: [PATCH 43/43] Second bug fix requested by Michelle: add FHR back into get_obs_... tasks. --- parm/wflow/verify_pre.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/parm/wflow/verify_pre.yaml b/parm/wflow/verify_pre.yaml index 088a0e3345..2eccbf8027 100644 --- a/parm/wflow/verify_pre.yaml +++ b/parm/wflow/verify_pre.yaml @@ -29,6 +29,7 @@ task_get_obs_ccpa: <<: *default_vars ACCUM_HH: '01' OBS_DIR: '&CCPA_OBS_DIR;' + FHR: '{% for h in range(0, workflow.FCST_LEN_HRS+1) %}{{ " %02d" % h }}{% endfor %}' partition: '{% if platform.get("PARTITION_HPSS") %}&PARTITION_HPSS;{% else %}None{% endif %}' queue: "&QUEUE_HPSS;" walltime: 00:45:00 @@ -40,6 +41,7 @@ task_get_obs_mrms: <<: *default_vars OBS_DIR: '&MRMS_OBS_DIR;' VAR: 'REFC RETOP' + FHR: '{% for h in range(0, workflow.FCST_LEN_HRS+1) %}{{ " %02d" % h }}{% endfor %}' partition: '{% if platform.get("PARTITION_HPSS") %}&PARTITION_HPSS;{% else %}None{% endif %}' queue: "&QUEUE_HPSS;" walltime: 00:45:00 @@ -49,6 +51,7 @@ task_get_obs_ndas: envars: <<: *default_vars OBS_DIR: '&NDAS_OBS_DIR;' + FHR: '{% for h in range(0, workflow.FCST_LEN_HRS+1) %}{{ " %02d" % h }}{% endfor %}' command: '&LOAD_MODULES_RUN_TASK_FP; "get_obs" "&JOBSdir;/JREGIONAL_GET_OBS_NDAS"' queue: "&QUEUE_HPSS;" partition: '{% if platform.get("PARTITION_HPSS") %}&PARTITION_HPSS;{% else %}None{% endif %}'