diff --git a/.github/parm/use_case_groups.json b/.github/parm/use_case_groups.json index 3bb687f2c8..0e9a05056c 100644 --- a/.github/parm/use_case_groups.json +++ b/.github/parm/use_case_groups.json @@ -117,7 +117,7 @@ { "category": "precipitation", "index_list": "3-7", - "run": true + "run": false }, { "category": "precipitation", diff --git a/docs/Contributors_Guide/create_wrapper.rst b/docs/Contributors_Guide/create_wrapper.rst index 9ce44eceee..d9849dcc20 100644 --- a/docs/Contributors_Guide/create_wrapper.rst +++ b/docs/Contributors_Guide/create_wrapper.rst @@ -22,7 +22,7 @@ For example, the new_tool wrapper would be named **NewToolWrapper**. Add Entry to LOWER_TO_WRAPPER_NAME Dictionary --------------------------------------------- -In *metplus/util/doc_util.py*, add entries to the LOWER_TO_WRAPPER_NAME +In *metplus/util/constants.py*, add entries to the LOWER_TO_WRAPPER_NAME dictionary so that the wrapper can be found in the PROCESS_LIST even if it is formatted differently. The key should be the wrapper name in all lower-case letters without any underscores. The value should be the class name diff --git a/docs/Users_Guide/glossary.rst b/docs/Users_Guide/glossary.rst index dbe419ef6e..9c7ad67f9b 100644 --- a/docs/Users_Guide/glossary.rst +++ b/docs/Users_Guide/glossary.rst @@ -1620,12 +1620,12 @@ METplus Configuration Glossary .. warning:: **DEPRECATED:** Please use :term:`FCST_THRESH_LIST` instead. FCST_THRESH_LIST - Specify the values of the FCST_THRESH column in the MET .stat file to use. This is optional in the METplus configuration file for running with :term:`LOOP_ORDER` = times. + Specify the values of the FCST_THRESH column in the MET .stat file to use. | *Used by:* StatAnalysis OBS_THRESH_LIST - Specify the values of the OBS_THRESH column in the MET .stat file to use. This is optional in the METplus configuration file for running with :term:`LOOP_ORDER` = times. + Specify the values of the OBS_THRESH column in the MET .stat file to use. | *Used by:* StatAnalysis @@ -1647,7 +1647,7 @@ METplus Configuration Glossary .. warning:: **DEPRECATED:** Please use :term:`FCST_LEVEL_LIST` instead. FCST_LEVEL_LIST - Specify the values of the FCST_LEV column in the MET .stat file to use. This is optional in the METplus configuration file for running with :term:`LOOP_ORDER` = times. + Specify the values of the FCST_LEV column in the MET .stat file to use. | *Used by:* StatAnalysis @@ -1655,12 +1655,12 @@ METplus Configuration Glossary .. warning:: **DEPRECATED:** Please use :term:`FCST_VAR_LIST` instead. FCST_VAR_LIST - Specify the values of the FCST_VAR column in the MET .stat file to use. This is optional in the METplus configuration file for running with :term:`LOOP_ORDER` = times. + Specify the values of the FCST_VAR column in the MET .stat file to use. | *Used by:* StatAnalysis FCST_UNITS_LIST - Specify the values of the FCST_UNITS column in the MET .stat file to use. This is optional in the METplus configuration file for running with :term:`LOOP_ORDER` = times. + Specify the values of the FCST_UNITS column in the MET .stat file to use. | *Used by:* StatAnalysis @@ -2319,7 +2319,7 @@ METplus Configuration Glossary LINE_TYPE_LIST - Specify the MET STAT line types to be considered. For TCMPRPlotter, this is optional in the METplus configuration file for running with :term:`LOOP_ORDER` = times. + Specify the MET STAT line types to be considered. | *Used by:* MakePlots, StatAnalysis, TCMPRPlotter @@ -2395,7 +2395,7 @@ METplus Configuration Glossary .. warning:: **DEPRECATED:** Please use :term:`LOOP_BY` instead. LOOP_ORDER - Control the looping order for METplus. Valid options are "times" or "processes". "times" runs all items in the :term:`PROCESS_LIST` for a single run time, then repeat until all times have been evaluated. "processes" runs each item in the :term:`PROCESS_LIST` for all times specified, then repeat for the next item in the :term:`PROCESS_LIST`. + .. warning:: **DEPRECATED:** This previously controlled the looping order for METplus. This was removed in v5.0.0. The wrappers will always execute the logic that was previously run when LOOP_ORDER = processes, which runs each item in the :term:`PROCESS_LIST` for all times specified, then repeat for the next item in the :term:`PROCESS_LIST`. | *Used by:* All @@ -3207,7 +3207,7 @@ METplus Configuration Glossary .. warning:: **DEPRECATED:** Please use :term:`OBS_LEVEL_LIST` instead. OBS_LEVEL_LIST - Specify the values of the OBS_LEV column in the MET .stat file to use. This is optional in the METplus configuration file for running with :term:`LOOP_ORDER` = times. + Specify the values of the OBS_LEV column in the MET .stat file to use. | *Used by:* StatAnalysis @@ -3215,12 +3215,12 @@ METplus Configuration Glossary .. warning:: **DEPRECATED:** Please use :term:`OBS_VAR_LIST` instead. OBS_VAR_LIST - Specify the values of the OBS_VAR column in the MET .stat file to use. This is optional in the METplus configuration file for running with :term:`LOOP_ORDER` = times. + Specify the values of the OBS_VAR column in the MET .stat file to use. | *Used by:* StatAnalysis OBS_UNITS_LIST - Specify the values of the OBS_UNITS column in the MET .stat file to use. This is optional in the METplus configuration file for running with :term:`LOOP_ORDER` = times. + Specify the values of the OBS_UNITS column in the MET .stat file to use. | *Used by:* StatAnalysis @@ -3837,7 +3837,7 @@ METplus Configuration Glossary .. warning:: **DEPRECATED:** Please use :term:`MODEL_STAT_ANALYSIS_DUMP_ROW_TEMPLATE` instead. MODEL_STAT_ANALYSIS_DUMP_ROW_TEMPLATE - Specify the template to use for the stat_analysis dump_row file. A user customized template to use for the dump_row file. If left blank and a dump_row file is requested, a default version will be used. This is optional in the METplus configuration file for running with :term:`LOOP_ORDER` = times. + Specify the template to use for the stat_analysis dump_row file. A user customized template to use for the dump_row file. If left blank and a dump_row file is requested, a default version will be used. | *Used by:* StatAnalysis @@ -3853,7 +3853,7 @@ METplus Configuration Glossary .. warning:: **DEPRECATED:** Please use :term:`MODEL_STAT_ANALYSIS_OUT_STAT_TEMPLATE` instead. MODEL_STAT_ANALYSIS_OUT_STAT_TEMPLATE - Specify the template to use for the stat_analysis out_stat file. A user customized template to use for the out_stat file. If left blank and a out_stat file is requested, a default version will be used. This is optional in the METplus configuration file for running with :term:`LOOP_ORDER` = times. + Specify the template to use for the stat_analysis out_stat file. A user customized template to use for the out_stat file. If left blank and a out_stat file is requested, a default version will be used. | *Used by:* StatAnalysis @@ -7636,12 +7636,11 @@ METplus Configuration Glossary | *Used by:* UserScript TC_PAIRS_RUN_ONCE - If True and LOOP_ORDER = processes, TCPairs will be run once using the + If True, TCPairs will be run once using the INIT_BEG or VALID_BEG value (depending on the value of LOOP_BY). This is the default setting and preserves the original logic of the wrapper. If this variable is set to False, then TCPairs will run once - for each run time iteration. If LOOP_ORDER = times, then TCPairs will - still run for each run time. The preferred configuration settings to + for each run time iteration. The preferred configuration settings to run TCPairs once for a range of init or valid times is to set INIT_BEG to INIT_END (if LOOP_BY = INIT) and define the range of init times to filter the data inside TCPairs with TC_PAIRS_INIT_BEG and diff --git a/docs/Users_Guide/systemconfiguration.rst b/docs/Users_Guide/systemconfiguration.rst index e9e3868f2d..c0ff3ef4dd 100644 --- a/docs/Users_Guide/systemconfiguration.rst +++ b/docs/Users_Guide/systemconfiguration.rst @@ -1078,38 +1078,14 @@ the output directory. Loop Order ---------- -The METplus wrappers can be configured to loop first by times then -processes or vice-versa. Looping by times first will run each process in -the process list for a given run time, increment to the next run time, run -each process in the process list, and so on. Looping by processes first -will run all times for the first process, then run all times for the -second process, and so on. +The METplus wrappers will run all times for the first process defined in the +:term:`PROCESS_LIST`, then run all times for the second process, and so on. +The :term:`LOOP_ORDER` variable has been deprecated in v5.0.0. +This is the behavior that was previously executed when LOOP_ORDER = processes. -**Example 1 Configuration**:: - - [config] - LOOP_ORDER = times - - PROCESS_LIST = PCPCombine, GridStat - - VALID_BEG = 20190201 - VALID_END = 20190203 - VALID_INCREMENT = 1d - -will run in the following order:: - - * PCPCombine at 2019-02-01 - * GridStat at 2019-02-01 - * PCPCombine at 2019-02-02 - * GridStat at 2019-02-02 - * PCPCombine at 2019-02-03 - * GridStat at 2019-02-03 - - -**Example 2 Configuration**:: +**Example Configuration**:: [config] - LOOP_ORDER = processes PROCESS_LIST = PCPCombine, GridStat @@ -1126,12 +1102,7 @@ will run in the following order:: * GridStat at 2019-02-02 * GridStat at 2019-02-03 -.. note:: - If running a MET tool that processes data over a time range, such as - SeriesAnalysis or StatAnalysis, the tool must be run with - LOOP_ORDER = processes. - .. _Custom_Looping: Custom Looping @@ -1782,15 +1753,13 @@ The possible values for the \*_RUNTIME_FREQ variables are: (init or valid and forecast lead combination). All filename templates are substituted with values. -Note that :term:`LOOP_ORDER` must be set to processes to run these wrappers. -Also note that the following example may not contain all of the configuration +Note that the following example may not contain all of the configuration variables that are required for a successful run. The are intended to show how these variables affect how the data is processed. **SeriesAnalysis Examples**:: [config] - LOOP_ORDER = processes LOOP_BY = INIT INIT_TIME_FMT = %Y%m%d%H diff --git a/docs/Users_Guide/wrappers.rst b/docs/Users_Guide/wrappers.rst index 0956b88bef..8850e47abe 100644 --- a/docs/Users_Guide/wrappers.rst +++ b/docs/Users_Guide/wrappers.rst @@ -3693,9 +3693,9 @@ Description ----------- The MakePlots wrapper creates various statistical plots using python -scripts for the various METplus Wrappers use cases. This can only be run -following StatAnalysis wrapper when LOOP_ORDER = processes. To run -MakePlots wrapper, include MakePlots in PROCESS_LIST. +scripts for the various METplus Wrappers use cases. +This can only be run following StatAnalysis wrapper. +To run MakePlots wrapper, include MakePlots in PROCESS_LIST. METplus Configuration --------------------- @@ -6829,74 +6829,18 @@ Description The StatAnalysis wrapper encapsulates the behavior of the MET stat_analysis tool. It provides the infrastructure to summarize and -filter the MET .stat files. StatAnalysis wrapper can be run in two -different methods. First is to look at the STAT lines for a single date, -to use this method set LOOP_ORDER = times. Second is to look at the STAT -lines over a span of dates, to use this method set LOOP_ORDER = -processes. To run StatAnalysis wrapper, include StatAnalysis in -PROCESS_LIST. +filter the MET .stat files. METplus Configuration --------------------- -The following values must be defined in the METplus Wrappers -configuration file for running with LOOP_ORDER = times: +The following values **must** be defined in the METplus configuration file: | :term:`STAT_ANALYSIS_OUTPUT_DIR` -| :term:`MODEL_STAT_ANALYSIS_DUMP_ROW_TEMPLATE` -| :term:`MODEL_STAT_ANALYSIS_OUT_STAT_TEMPLATE` | :term:`LOG_STAT_ANALYSIS_VERBOSITY` | :term:`MODEL\` | :term:`MODEL_OBTYPE` | :term:`MODEL_STAT_ANALYSIS_LOOKIN_DIR` -| :term:`MODEL_LIST` -| :term:`GROUP_LIST_ITEMS` -| :term:`LOOP_LIST_ITEMS` -| :term:`STAT_ANALYSIS_CONFIG_FILE` -| :term:`STAT_ANALYSIS_JOB_NAME` -| :term:`STAT_ANALYSIS_JOB_ARGS` -| :term:`STAT_ANALYSIS_MET_CONFIG_OVERRIDES` -| - -The following values are **optional** in the METplus Wrappers -configuration file for running with LOOP_ORDER = times: - -| :term:`DESC_LIST` -| :term:`FCST_VALID_HOUR_LIST` -| :term:`OBS_VALID_HOUR_LIST` -| :term:`FCST_INIT_HOUR_LIST` -| :term:`OBS_INIT_HOUR_LIST` -| :term:`FCST_VAR_LIST` -| :term:`OBS_VAR_LIST` -| :term:`FCST_LEVEL_LIST` -| :term:`OBS_LEVEL_LIST` -| :term:`FCST_UNITS_LIST` -| :term:`OBS_UNITS_LIST` -| :term:`FCST_THRESH_LIST` -| :term:`OBS_THRESH_LIST` -| :term:`FCST_LEAD_LIST` -| :term:`OBS_LEAD_LIST` -| :term:`VX_MASK_LIST` -| :term:`INTERP_MTHD_LIST` -| :term:`INTERP_PNTS_LIST` -| :term:`ALPHA_LIST` -| :term:`COV_THRESH_LIST` -| :term:`LINE_TYPE_LIST` -| :term:`STAT_ANALYSIS_SKIP_IF_OUTPUT_EXISTS` -| :term:`STAT_ANALYSIS_HSS_EC_VALUE` -| :term:`STAT_ANALYSIS_OUTPUT_TEMPLATE` -| - -The following values **must** be defined in the METplus Wrappers -configuration file for running with LOOP_ORDER = processes: - -| :term:`STAT_ANALYSIS_OUTPUT_DIR` -| :term:`LOG_STAT_ANALYSIS_VERBOSITY` -| :term:`DATE_TYPE` -| :term:`STAT_ANALYSIS_CONFIG_FILE` -| :term:`MODEL\` -| :term:`MODEL_OBTYPE` -| :term:`MODEL_STAT_ANALYSIS_LOOKIN_DIR` | :term:`MODEL_REFERENCE_NAME` | :term:`GROUP_LIST_ITEMS` | :term:`LOOP_LIST_ITEMS` @@ -6904,11 +6848,14 @@ configuration file for running with LOOP_ORDER = processes: | :term:`VX_MASK_LIST` | :term:`FCST_LEAD_LIST` | :term:`LINE_TYPE_LIST` +| :term:`STAT_ANALYSIS_JOB_NAME` +| :term:`STAT_ANALYSIS_JOB_ARGS` +| :term:`STAT_ANALYSIS_MET_CONFIG_OVERRIDES` | -The following values are optional in the METplus Wrappers configuration -file for running with LOOP_ORDER = processes: +The following values are optional in the METplus configuration file: +| :term:`STAT_ANALYSIS_CONFIG_FILE` | :term:`VAR_FOURIER_DECOMP` | :term:`VAR_WAVE_NUM_LIST` | :term:`FCST_VALID_HOUR_LIST` @@ -6923,6 +6870,8 @@ file for running with LOOP_ORDER = processes: | :term:`ALPHA_LIST` | :term:`STAT_ANALYSIS_HSS_EC_VALUE` | :term:`STAT_ANALYSIS_OUTPUT_TEMPLATE` +| :term:`MODEL_STAT_ANALYSIS_DUMP_ROW_TEMPLATE` +| :term:`MODEL_STAT_ANALYSIS_OUT_STAT_TEMPLATE` | .. warning:: **DEPRECATED:** diff --git a/internal/tests/pytests/plotting/make_plots/test_make_plots_wrapper.py b/internal/tests/pytests/plotting/make_plots/test_make_plots_wrapper.py index 5ee62c781a..5c6b08c910 100644 --- a/internal/tests/pytests/plotting/make_plots/test_make_plots_wrapper.py +++ b/internal/tests/pytests/plotting/make_plots/test_make_plots_wrapper.py @@ -47,7 +47,6 @@ def test_create_c_dict(metplus_config): mp = make_plots_wrapper(metplus_config) # Test 1 c_dict = mp.create_c_dict() - assert(c_dict['LOOP_ORDER'] == 'processes') # NOTE: MakePlots relies on output from StatAnalysis # so its input resides in the output of StatAnalysis assert(c_dict['INPUT_BASE_DIR'] == mp.config.getdir('OUTPUT_BASE') diff --git a/internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py b/internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py index 5259e03ef1..d588d2425f 100644 --- a/internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py +++ b/internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py @@ -313,9 +313,13 @@ def test_series_analysis_single_field(metplus_config, config_overrides, config_file = wrapper.c_dict.get('CONFIG_FILE') out_dir = wrapper.c_dict.get('OUTPUT_DIR') + prefix = 'series_analysis_files_' + suffix = '_init_20050807000000_valid_ALL_lead_ALL.txt' + fcst_file = f'{prefix}fcst{suffix}' + obs_file = f'{prefix}obs{suffix}' expected_cmds = [(f"{app_path} " - f"-fcst {out_dir}/FCST_FILES " - f"-obs {out_dir}/OBS_FILES " + f"-fcst {out_dir}/{fcst_file} " + f"-obs {out_dir}/{obs_file} " f"-out {out_dir}/2005080700 " f"-config {config_file} {verbosity}"), ] @@ -356,8 +360,6 @@ def test_get_fcst_file_info(metplus_config): expected_beg = '000' expected_end = '048' - time_info = {'storm_id': storm_id, 'lead': 0, 'valid': '', 'init': ''} - wrapper = series_analysis_wrapper(metplus_config) wrapper.c_dict['FCST_INPUT_DIR'] = '/fake/path/of/file' wrapper.c_dict['FCST_INPUT_TEMPLATE'] = ( @@ -404,49 +406,26 @@ def test_get_storms_list(metplus_config): assert storm_list == expected_storm_list -# added list of all files for reference for creating subsets -all_fake_fcst = ['fcst/20141214_00/ML1201072014/FCST_TILE_F000_gfs_4_20141214_0000_000.nc', - 'fcst/20141214_00/ML1221072014/FCST_TILE_F000_gfs_4_20141214_0000_000.nc', - 'fcst/20141214_00/ML1201072014/FCST_TILE_F006_gfs_4_20141214_0000_006.nc', - 'fcst/20141214_00/ML1221072014/FCST_TILE_F006_gfs_4_20141214_0000_006.nc', - 'fcst/20141214_00/ML1201072014/FCST_TILE_F012_gfs_4_20141214_0000_012.nc', - 'fcst/20141214_00/ML1221072014/FCST_TILE_F012_gfs_4_20141214_0000_012.nc', - 'fcst/20141215_00/ML1291072014/FCST_TILE_F000_gfs_4_20141215_0000_000.nc', - 'fcst/20141215_00/ML1291072014/FCST_TILE_F006_gfs_4_20141215_0000_006.nc', - 'fcst/20141215_00/ML1291072014/FCST_TILE_F012_gfs_4_20141215_0000_012.nc', - ] -all_fake_obs = ['obs/20141214_00/ML1201072014/OBS_TILE_F000_gfs_4_20141214_0000_000.nc', - 'obs/20141214_00/ML1221072014/OBS_TILE_F000_gfs_4_20141214_0000_000.nc', - 'obs/20141214_00/ML1201072014/OBS_TILE_F006_gfs_4_20141214_0000_006.nc', - 'obs/20141214_00/ML1221072014/OBS_TILE_F006_gfs_4_20141214_0000_006.nc', - 'obs/20141214_00/ML1201072014/OBS_TILE_F012_gfs_4_20141214_0000_012.nc', - 'obs/20141214_00/ML1221072014/OBS_TILE_F012_gfs_4_20141214_0000_012.nc', - 'obs/20141215_00/ML1291072014/OBS_TILE_F000_gfs_4_20141215_0000_000.nc', - 'obs/20141215_00/ML1291072014/OBS_TILE_F006_gfs_4_20141215_0000_006.nc', - 'obs/20141215_00/ML1291072014/OBS_TILE_F012_gfs_4_20141215_0000_012.nc', - ] - - @pytest.mark.parametrize( 'time_info, expect_fcst_subset, expect_obs_subset', [ - # filter by init all storms + # 0: filter by init all storms ({'init': datetime(2014, 12, 14, 0, 0), 'valid': '*', 'lead': '*', 'storm_id': '*'}, ['fcst/20141214_00/ML1201072014/FCST_TILE_F000_gfs_4_20141214_0000_000.nc', - 'fcst/20141214_00/ML1221072014/FCST_TILE_F000_gfs_4_20141214_0000_000.nc', 'fcst/20141214_00/ML1201072014/FCST_TILE_F006_gfs_4_20141214_0000_006.nc', - 'fcst/20141214_00/ML1221072014/FCST_TILE_F006_gfs_4_20141214_0000_006.nc', 'fcst/20141214_00/ML1201072014/FCST_TILE_F012_gfs_4_20141214_0000_012.nc', + 'fcst/20141214_00/ML1221072014/FCST_TILE_F000_gfs_4_20141214_0000_000.nc', + 'fcst/20141214_00/ML1221072014/FCST_TILE_F006_gfs_4_20141214_0000_006.nc', 'fcst/20141214_00/ML1221072014/FCST_TILE_F012_gfs_4_20141214_0000_012.nc',], ['obs/20141214_00/ML1201072014/OBS_TILE_F000_gfs_4_20141214_0000_000.nc', - 'obs/20141214_00/ML1221072014/OBS_TILE_F000_gfs_4_20141214_0000_000.nc', 'obs/20141214_00/ML1201072014/OBS_TILE_F006_gfs_4_20141214_0000_006.nc', - 'obs/20141214_00/ML1221072014/OBS_TILE_F006_gfs_4_20141214_0000_006.nc', 'obs/20141214_00/ML1201072014/OBS_TILE_F012_gfs_4_20141214_0000_012.nc', + 'obs/20141214_00/ML1221072014/OBS_TILE_F000_gfs_4_20141214_0000_000.nc', + 'obs/20141214_00/ML1221072014/OBS_TILE_F006_gfs_4_20141214_0000_006.nc', 'obs/20141214_00/ML1221072014/OBS_TILE_F012_gfs_4_20141214_0000_012.nc',]), - # filter by init single storm + # 1: filter by init single storm ({'init': datetime(2014, 12, 14, 0, 0), 'valid': '*', 'lead': '*', @@ -461,7 +440,7 @@ def test_get_storms_list(metplus_config): 'obs/20141214_00/ML1201072014/OBS_TILE_F006_gfs_4_20141214_0000_006.nc', 'obs/20141214_00/ML1201072014/OBS_TILE_F012_gfs_4_20141214_0000_012.nc', ]), - # filter by init another single storm + # 2: filter by init another single storm ({'init': datetime(2014, 12, 14, 0, 0), 'valid': '*', 'lead': '*', @@ -476,7 +455,7 @@ def test_get_storms_list(metplus_config): 'obs/20141214_00/ML1221072014/OBS_TILE_F006_gfs_4_20141214_0000_006.nc', 'obs/20141214_00/ML1221072014/OBS_TILE_F012_gfs_4_20141214_0000_012.nc', ]), - # filter by lead all storms + # 3: filter by lead all storms ({'init': '*', 'valid': '*', 'lead': 21600, @@ -510,7 +489,6 @@ def test_get_all_files_and_subset(metplus_config, time_info, expect_fcst_subset, stat_input_dir, tile_input_dir = get_input_dirs(wrapper.config) stat_input_template = 'another_fake_filter_{init?fmt=%Y%m%d_%H}.tcst' - wrapper.c_dict['RUN_ONCE_PER_STORM_ID'] = True wrapper.c_dict['TC_STAT_INPUT_DIR'] = stat_input_dir wrapper.c_dict['TC_STAT_INPUT_TEMPLATE'] = stat_input_template @@ -522,16 +500,24 @@ def test_get_all_files_and_subset(metplus_config, time_info, expect_fcst_subset, wrapper.c_dict['FCST_INPUT_DIR'] = fcst_input_dir wrapper.c_dict['OBS_INPUT_DIR'] = obs_input_dir - assert wrapper.get_all_files() + if time_info['storm_id'] == '*': + wrapper.c_dict['RUN_ONCE_PER_STORM_ID'] = False + else: + wrapper.c_dict['RUN_ONCE_PER_STORM_ID'] = True + assert wrapper.get_all_files() + print(f"ALL FILES: {wrapper.c_dict['ALL_FILES']}") expected_fcst = [ 'fcst/20141214_00/ML1201072014/FCST_TILE_F000_gfs_4_20141214_0000_000.nc', - 'fcst/20141214_00/ML1221072014/FCST_TILE_F000_gfs_4_20141214_0000_000.nc', 'fcst/20141214_00/ML1201072014/FCST_TILE_F006_gfs_4_20141214_0000_006.nc', - 'fcst/20141214_00/ML1221072014/FCST_TILE_F006_gfs_4_20141214_0000_006.nc', 'fcst/20141214_00/ML1201072014/FCST_TILE_F012_gfs_4_20141214_0000_012.nc', + 'fcst/20141214_00/ML1221072014/FCST_TILE_F000_gfs_4_20141214_0000_000.nc', + 'fcst/20141214_00/ML1221072014/FCST_TILE_F006_gfs_4_20141214_0000_006.nc', 'fcst/20141214_00/ML1221072014/FCST_TILE_F012_gfs_4_20141214_0000_012.nc', ] + if time_info['storm_id'] != '*': + expected_fcst = [item for item in expected_fcst + if time_info['storm_id'] in item] expected_fcst_files = [] for expected in expected_fcst: expected_fcst_files.append(os.path.join(tile_input_dir, expected)) @@ -539,26 +525,44 @@ def test_get_all_files_and_subset(metplus_config, time_info, expect_fcst_subset, expected_obs = [ 'obs/20141214_00/ML1201072014/OBS_TILE_F000_gfs_4_20141214_0000_000.nc', - 'obs/20141214_00/ML1221072014/OBS_TILE_F000_gfs_4_20141214_0000_000.nc', 'obs/20141214_00/ML1201072014/OBS_TILE_F006_gfs_4_20141214_0000_006.nc', - 'obs/20141214_00/ML1221072014/OBS_TILE_F006_gfs_4_20141214_0000_006.nc', 'obs/20141214_00/ML1201072014/OBS_TILE_F012_gfs_4_20141214_0000_012.nc', + 'obs/20141214_00/ML1221072014/OBS_TILE_F000_gfs_4_20141214_0000_000.nc', + 'obs/20141214_00/ML1221072014/OBS_TILE_F006_gfs_4_20141214_0000_006.nc', 'obs/20141214_00/ML1221072014/OBS_TILE_F012_gfs_4_20141214_0000_012.nc', ] + if time_info['storm_id'] != '*': + expected_obs = [item for item in expected_obs + if time_info['storm_id'] in item] expected_obs_files = [] for expected in expected_obs: expected_obs_files.append(os.path.join(tile_input_dir, expected)) + fcst_key, obs_key = wrapper._get_fcst_obs_keys(time_info['storm_id']) + fcst_files = [item[fcst_key] for item in wrapper.c_dict['ALL_FILES'] + if fcst_key in item] + obs_files = [item[obs_key] for item in wrapper.c_dict['ALL_FILES'] + if obs_key in item] # convert list of lists into a single list to compare to expected results - fcst_files = [item['fcst'] for item in wrapper.c_dict['ALL_FILES']] fcst_files = [item for sub in fcst_files for item in sub] - obs_files = [item['obs'] for item in wrapper.c_dict['ALL_FILES']] obs_files = [item for sub in obs_files for item in sub] - + fcst_files.sort() + obs_files.sort() assert fcst_files == expected_fcst_files assert obs_files == expected_obs_files - fcst_files_sub, obs_files_sub = wrapper.subset_input_files(time_info) + list_file_dict = wrapper.subset_input_files(time_info) + fcst_files_sub = [] + obs_files_sub = [] + for key, value in list_file_dict.items(): + if key.startswith('fcst'): + with open(value, 'r') as file_handle: + fcst_files_sub.extend(file_handle.read().splitlines()[1:]) + if key.startswith('obs'): + with open(value, 'r') as file_handle: + obs_files_sub.extend(file_handle.read().splitlines()[1:]) + fcst_files_sub.sort() + obs_files_sub.sort() assert fcst_files_sub and obs_files_sub assert len(fcst_files_sub) == len(obs_files_sub) @@ -571,7 +575,7 @@ def test_get_all_files_and_subset(metplus_config, time_info, expect_fcst_subset, @pytest.mark.parametrize( 'config_overrides, time_info, storm_id, lead_group, expect_fcst_subset, expect_obs_subset', [ - # filter by init all storms + # 0: filter by init all storms ({'LEAD_SEQ': '0H, 6H, 12H', 'SERIES_ANALYSIS_OUTPUT_TEMPLATE': "{init?fmt=%Y%m%d_%H}/{storm_id}/series_{fcst_name}_{fcst_level}.nc", 'TEST_OUTPUT_DIRNAME': 'byinitallstorms'}, @@ -593,7 +597,7 @@ def test_get_all_files_and_subset(metplus_config, time_info, expect_fcst_subset, 'obs/20141214_00/ML1221072014/OBS_TILE_F006_gfs_4_20141214_0000_006.nc', 'obs/20141214_00/ML1201072014/OBS_TILE_F012_gfs_4_20141214_0000_012.nc', 'obs/20141214_00/ML1221072014/OBS_TILE_F012_gfs_4_20141214_0000_012.nc',]), - # filter by init single storm + # 1: filter by init single storm ({'LEAD_SEQ': '0H, 6H, 12H', 'SERIES_ANALYSIS_OUTPUT_TEMPLATE': "{init?fmt=%Y%m%d_%H}/{storm_id}/series_{fcst_name}_{fcst_level}.nc", 'TEST_OUTPUT_DIRNAME': 'byinitstormA'}, @@ -611,7 +615,7 @@ def test_get_all_files_and_subset(metplus_config, time_info, expect_fcst_subset, 'obs/20141214_00/ML1201072014/OBS_TILE_F006_gfs_4_20141214_0000_006.nc', 'obs/20141214_00/ML1201072014/OBS_TILE_F012_gfs_4_20141214_0000_012.nc', ]), - # filter by init another single storm + # 2: filter by init another single storm ({'LEAD_SEQ': '0H, 6H, 12H', 'SERIES_ANALYSIS_OUTPUT_TEMPLATE': "{init?fmt=%Y%m%d_%H}/{storm_id}/series_{fcst_name}_{fcst_level}.nc", 'TEST_OUTPUT_DIRNAME': 'byinitstormB'}, @@ -629,7 +633,7 @@ def test_get_all_files_and_subset(metplus_config, time_info, expect_fcst_subset, 'obs/20141214_00/ML1221072014/OBS_TILE_F006_gfs_4_20141214_0000_006.nc', 'obs/20141214_00/ML1221072014/OBS_TILE_F012_gfs_4_20141214_0000_012.nc', ]), - # filter by lead all storms + # 3: filter by lead all storms ({'LEAD_SEQ': '0H, 6H, 12H', 'SERIES_ANALYSIS_OUTPUT_TEMPLATE': "series_{fcst_name}_{fcst_level}.nc", 'TEST_OUTPUT_DIRNAME': 'byleadallstorms'}, @@ -645,7 +649,7 @@ def test_get_all_files_and_subset(metplus_config, time_info, expect_fcst_subset, ['obs/20141214_00/ML1201072014/OBS_TILE_F006_gfs_4_20141214_0000_006.nc', 'obs/20141214_00/ML1221072014/OBS_TILE_F006_gfs_4_20141214_0000_006.nc', ]), - # filter by lead 1 storm + # 4: filter by lead 1 storm ({'LEAD_SEQ': '0H, 6H, 12H', 'SERIES_ANALYSIS_OUTPUT_TEMPLATE': "series_{fcst_name}_{fcst_level}.nc", 'TEST_OUTPUT_DIRNAME': 'byleadstormA'}, @@ -659,7 +663,7 @@ def test_get_all_files_and_subset(metplus_config, time_info, expect_fcst_subset, ], ['obs/20141214_00/ML1201072014/OBS_TILE_F006_gfs_4_20141214_0000_006.nc', ]), - # filter by lead another storm + # 5: filter by lead another storm ({'LEAD_SEQ': '0H, 6H, 12H', 'SERIES_ANALYSIS_OUTPUT_TEMPLATE': "series_{fcst_name}_{fcst_level}.nc", 'TEST_OUTPUT_DIRNAME': 'byleadstormB'}, @@ -673,7 +677,7 @@ def test_get_all_files_and_subset(metplus_config, time_info, expect_fcst_subset, ], ['obs/20141214_00/ML1221072014/OBS_TILE_F006_gfs_4_20141214_0000_006.nc', ]), - # filter by lead groups A all storms + # 6: filter by lead groups A all storms ({'LEAD_SEQ_1': '0H, 6H', 'LEAD_SEQ_1_LABEL': 'Group1', 'LEAD_SEQ_2': '12H', @@ -695,7 +699,7 @@ def test_get_all_files_and_subset(metplus_config, time_info, expect_fcst_subset, 'obs/20141214_00/ML1201072014/OBS_TILE_F006_gfs_4_20141214_0000_006.nc', 'obs/20141214_00/ML1221072014/OBS_TILE_F006_gfs_4_20141214_0000_006.nc', ]), - # filter by lead groups B all storms + # 7: filter by lead groups B all storms ({'LEAD_SEQ_1': '0H, 6H', 'LEAD_SEQ_1_LABEL': 'Group1', 'LEAD_SEQ_2': '12H', @@ -730,32 +734,31 @@ def test_get_fcst_and_obs_path(metplus_config, config_overrides, all_config_overrides.update(config_overrides) wrapper = series_analysis_wrapper(metplus_config, all_config_overrides) stat_input_dir, tile_input_dir = get_input_dirs(wrapper.config) - fcst_input_dir = os.path.join(tile_input_dir, - 'fcst') - obs_input_dir = os.path.join(tile_input_dir, - 'obs') + fcst_input_dir = os.path.join(tile_input_dir, 'fcst') + obs_input_dir = os.path.join(tile_input_dir, 'obs') stat_input_template = 'another_fake_filter_{init?fmt=%Y%m%d_%H}.tcst' wrapper.c_dict['TC_STAT_INPUT_DIR'] = stat_input_dir wrapper.c_dict['TC_STAT_INPUT_TEMPLATE'] = stat_input_template - wrapper.c_dict['RUN_ONCE_PER_STORM_ID'] = True wrapper.c_dict['FCST_INPUT_DIR'] = fcst_input_dir wrapper.c_dict['OBS_INPUT_DIR'] = obs_input_dir test_out_dirname = wrapper.config.getstr('config', 'TEST_OUTPUT_DIRNAME') output_dir = os.path.join(wrapper.config.getdir('OUTPUT_BASE'), - 'series_by', - 'output', - test_out_dirname) + 'series_by', 'output', test_out_dirname) wrapper.c_dict['OUTPUT_DIR'] = output_dir - assert wrapper.get_all_files() + fcst_id, obs_id = wrapper._get_fcst_obs_keys(storm_id) # read output files and compare to expected list if storm_id == '*': storm_dir = 'all_storms' + wrapper.c_dict['RUN_ONCE_PER_STORM_ID'] = False else: storm_dir = storm_id + wrapper.c_dict['RUN_ONCE_PER_STORM_ID'] = True + + assert wrapper.get_all_files() templates = config_overrides['SERIES_ANALYSIS_OUTPUT_TEMPLATE'].split('/') if len(templates) == 1: @@ -763,21 +766,13 @@ def test_get_fcst_and_obs_path(metplus_config, config_overrides, else: output_prefix = os.path.join('20141214_00', storm_dir) - if lead_group: - leads = lead_group[1] - else: - leads = None - fcst_list_file = wrapper._get_ascii_filename('FCST', storm_id, leads) - fcst_file_path = os.path.join(output_dir, - output_prefix, - fcst_list_file) + fcst_list_file = wrapper.get_list_file_name(time_info, fcst_id) + fcst_file_path = os.path.join(output_dir, output_prefix, fcst_list_file) if os.path.exists(fcst_file_path): os.remove(fcst_file_path) - obs_list_file = wrapper._get_ascii_filename('OBS', storm_id, leads) - obs_file_path = os.path.join(output_dir, - output_prefix, - obs_list_file) + obs_list_file = wrapper.get_list_file_name(time_info, obs_id) + obs_file_path = os.path.join(output_dir, output_prefix, obs_list_file) if os.path.exists(obs_file_path): os.remove(obs_file_path) @@ -790,6 +785,7 @@ def test_get_fcst_and_obs_path(metplus_config, config_overrides, actual_fcsts = file_handle.readlines() actual_fcsts = [item.strip() for item in actual_fcsts[1:]] + assert len(actual_fcsts) == len(expect_fcst_subset) for actual_file, expected_file in zip(actual_fcsts, expect_fcst_subset): actual_file = actual_file.replace(tile_input_dir, '').lstrip('/') assert actual_file == expected_file @@ -798,61 +794,12 @@ def test_get_fcst_and_obs_path(metplus_config, config_overrides, actual_obs_files = file_handle.readlines() actual_obs_files = [item.strip() for item in actual_obs_files[1:]] + assert len(actual_obs_files) == len(expect_obs_subset) for actual_file, expected_file in zip(actual_obs_files, expect_obs_subset): actual_file = actual_file.replace(tile_input_dir, '').lstrip('/') assert actual_file == expected_file -@pytest.mark.parametrize( - 'storm_id, leads, expected_result', [ - # storm ID, no leads - ('ML1221072014', None, '_FILES_ML1221072014'), - # no storm ID no leads - ('*', None, '_FILES'), - # storm ID, 1 lead - ('ML1221072014', [relativedelta(hours=12)], '_FILES_ML1221072014_F012'), - # no storm ID, 1 lead - ('*', [relativedelta(hours=12)], '_FILES_F012'), - # storm ID, 2 leads - ('ML1221072014', [relativedelta(hours=18), - relativedelta(hours=12)], - '_FILES_ML1221072014_F012_to_F018'), - # no storm ID, 2 leads - ('*', [relativedelta(hours=18), - relativedelta(hours=12)], - '_FILES_F012_to_F018'), - # storm ID, 3 leads - ('ML1221072014', [relativedelta(hours=15), - relativedelta(hours=18), - relativedelta(hours=12)], - '_FILES_ML1221072014_F012_to_F018'), - # no storm ID, 3 leads - ('*', [relativedelta(hours=15), - relativedelta(hours=18), - relativedelta(hours=12)], - '_FILES_F012_to_F018'), - ] -) -@pytest.mark.wrapper_a -def test_get_ascii_filename(metplus_config, storm_id, leads, - expected_result): - wrapper = series_analysis_wrapper(metplus_config) - for data_type in ['FCST', 'OBS']: - actual_result = wrapper._get_ascii_filename(data_type, - storm_id, - leads) - assert actual_result == f"{data_type}{expected_result}" - - if leads is None: - return - - lead_seconds = [ti_get_seconds_from_lead(item) for item in leads] - actual_result = wrapper._get_ascii_filename(data_type, - storm_id, - lead_seconds) - assert actual_result == f"{data_type}{expected_result}" - - @pytest.mark.parametrize( # no storm ID, label 'template, storm_id, label, expected_result', [ diff --git a/internal/tests/pytests/wrappers/stat_analysis/test_stat_analysis.py b/internal/tests/pytests/wrappers/stat_analysis/test_stat_analysis.py index 054e167bc5..8a5755d263 100644 --- a/internal/tests/pytests/wrappers/stat_analysis/test_stat_analysis.py +++ b/internal/tests/pytests/wrappers/stat_analysis/test_stat_analysis.py @@ -54,7 +54,6 @@ def test_create_c_dict(metplus_config): st = stat_analysis_wrapper(metplus_config) # Test 1 c_dict = st.create_c_dict() - assert c_dict['LOOP_ORDER'] == 'times' assert(os.path.realpath(c_dict['CONFIG_FILE']) == (METPLUS_BASE+'/internal/tests/' +'config/STATAnalysisConfig')) assert(c_dict['OUTPUT_DIR'] == (st.config.getdir('OUTPUT_BASE') diff --git a/metplus/util/config_metplus.py b/metplus/util/config_metplus.py index feea916bc2..654063212a 100644 --- a/metplus/util/config_metplus.py +++ b/metplus/util/config_metplus.py @@ -1000,7 +1000,6 @@ def check_for_deprecated_config(config): # modify the code to handle both variables accordingly deprecated_dict = { 'LOOP_BY_INIT' : {'sec' : 'config', 'alt' : 'LOOP_BY', 'copy': False}, - 'LOOP_METHOD' : {'sec' : 'config', 'alt' : 'LOOP_ORDER'}, 'PREPBUFR_DIR_REGEX' : {'sec' : 'regex_pattern', 'alt' : None}, 'PREPBUFR_FILE_REGEX' : {'sec' : 'regex_pattern', 'alt' : None}, 'OBS_INPUT_DIR_REGEX' : {'sec' : 'regex_pattern', 'alt' : 'OBS_POINT_STAT_INPUT_DIR', 'copy': False}, diff --git a/metplus/util/constants.py b/metplus/util/constants.py index 7e7d9cd9f0..d2a5380b18 100644 --- a/metplus/util/constants.py +++ b/metplus/util/constants.py @@ -1,3 +1,45 @@ +# Constant variables used throughout the METplus wrappers source code + +# dictionary used by get_wrapper_name function to easily convert wrapper +# name in many formats to the correct name of the wrapper class +LOWER_TO_WRAPPER_NAME = { + 'ascii2nc': 'ASCII2NC', + 'cycloneplotter': 'CyclonePlotter', + 'ensemblestat': 'EnsembleStat', + 'example': 'Example', + 'extracttiles': 'ExtractTiles', + 'gempaktocf': 'GempakToCF', + 'genvxmask': 'GenVxMask', + 'genensprod': 'GenEnsProd', + 'gfdltracker': 'GFDLTracker', + 'griddiag': 'GridDiag', + 'gridstat': 'GridStat', + 'ioda2nc': 'IODA2NC', + 'makeplots': 'MakePlots', + 'metdbload': 'METDbLoad', + 'mode': 'MODE', + 'mtd': 'MTD', + 'modetimedomain': 'MTD', + 'pb2nc': 'PB2NC', + 'pcpcombine': 'PCPCombine', + 'plotdataplane': 'PlotDataPlane', + 'plotpointobs': 'PlotPointObs', + 'point2grid': 'Point2Grid', + 'pointtogrid': 'Point2Grid', + 'pointstat': 'PointStat', + 'pyembedingest': 'PyEmbedIngest', + 'regriddataplane': 'RegridDataPlane', + 'seriesanalysis': 'SeriesAnalysis', + 'statanalysis': 'StatAnalysis', + 'tcgen': 'TCGen', + 'tcpairs': 'TCPairs', + 'tcrmw': 'TCRMW', + 'tcstat': 'TCStat', + 'tcmprplotter': 'TCMPRPlotter', + 'usage': 'Usage', + 'userscript': 'UserScript', +} + # supported file extensions that will automatically be uncompressed COMPRESSION_EXTENSIONS = [ '.gz', diff --git a/metplus/util/doc_util.py b/metplus/util/doc_util.py index 44ee50ad8a..d32e42f778 100755 --- a/metplus/util/doc_util.py +++ b/metplus/util/doc_util.py @@ -3,45 +3,8 @@ import sys import os -# dictionary used by get_wrapper_name function to easily convert wrapper -# name in many formats to the correct name of the wrapper class -LOWER_TO_WRAPPER_NAME = { - 'ascii2nc': 'ASCII2NC', - 'cycloneplotter': 'CyclonePlotter', - 'ensemblestat': 'EnsembleStat', - 'example': 'Example', - 'extracttiles': 'ExtractTiles', - 'gempaktocf': 'GempakToCF', - 'genvxmask': 'GenVxMask', - 'genensprod': 'GenEnsProd', - 'gfdltracker': 'GFDLTracker', - 'griddiag': 'GridDiag', - 'gridstat': 'GridStat', - 'ioda2nc': 'IODA2NC', - 'makeplots': 'MakePlots', - 'metdbload': 'METDbLoad', - 'mode': 'MODE', - 'mtd': 'MTD', - 'modetimedomain': 'MTD', - 'pb2nc': 'PB2NC', - 'pcpcombine': 'PCPCombine', - 'plotdataplane': 'PlotDataPlane', - 'plotpointobs': 'PlotPointObs', - 'point2grid': 'Point2Grid', - 'pointtogrid': 'Point2Grid', - 'pointstat': 'PointStat', - 'pyembedingest': 'PyEmbedIngest', - 'regriddataplane': 'RegridDataPlane', - 'seriesanalysis': 'SeriesAnalysis', - 'statanalysis': 'StatAnalysis', - 'tcgen': 'TCGen', - 'tcpairs': 'TCPairs', - 'tcrmw': 'TCRMW', - 'tcstat': 'TCStat', - 'tcmprplotter': 'TCMPRPlotter', - 'usage': 'Usage', - 'userscript': 'UserScript', -} +from . import LOWER_TO_WRAPPER_NAME + def get_wrapper_name(process_name): """! Determine name of wrapper from string that may not contain the correct diff --git a/metplus/util/met_util.py b/metplus/util/met_util.py index a8c337b8f4..32eecab662 100644 --- a/metplus/util/met_util.py +++ b/metplus/util/met_util.py @@ -136,21 +136,11 @@ def run_metplus(config, process_list): logger.info("Refer to ERROR messages above to resolve issues.") return 1 - loop_order = config.getstr('config', 'LOOP_ORDER', '').lower() - - if loop_order == "processes": - all_commands = [] - for process in processes: - new_commands = process.run_all_times() - if new_commands: - all_commands.extend(new_commands) - - elif loop_order == "times": - all_commands = loop_over_times_and_call(config, processes) - else: - logger.error("Invalid LOOP_ORDER defined. " - "Options are processes, times") - return 1 + all_commands = [] + for process in processes: + new_commands = process.run_all_times() + if new_commands: + all_commands.extend(new_commands) # if process list contains any wrapper that should run commands if any([item[0] not in NO_COMMAND_WRAPPERS for item in process_list]): @@ -396,12 +386,6 @@ def write_final_conf(config): @param config METplusConfig object to write to file """ - # write out os environment to file for debugging - env_file = os.path.join(config.getdir('LOG_DIR'), '.metplus_user_env') - with open(env_file, 'w') as env_file: - for key, value in os.environ.items(): - env_file.write('{}={}\n'.format(key, value)) - final_conf = config.getstr('config', 'METPLUS_CONF') # remove variables that start with CURRENT diff --git a/metplus/wrappers/command_builder.py b/metplus/wrappers/command_builder.py index f16dc01eff..31cf9da942 100755 --- a/metplus/wrappers/command_builder.py +++ b/metplus/wrappers/command_builder.py @@ -30,6 +30,7 @@ from ..util import get_wrapped_met_config_file, add_met_config_item, format_met_config from ..util import remove_quotes from ..util import get_field_info, format_field_info +from ..util import get_wrapper_name from ..util.met_config import add_met_config_dict, handle_climo_dict # pylint:disable=pointless-string-statement @@ -88,10 +89,7 @@ def __init__(self, config, instance=None): ) self.instance = instance - - self.env = os.environ.copy() - if hasattr(config, 'env'): - self.env = config.env + self.env = config.env if hasattr(config, 'env') else os.environ.copy() # populate c_dict dictionary self.c_dict = self.create_c_dict() @@ -1280,16 +1278,6 @@ def run_command(self, cmd, cmd_name=None): return True - # argument needed to match call - # pylint:disable=unused-argument - def run_at_time(self, input_dict): - """! Used to output error and exit if wrapper is attempted to be run - with LOOP_ORDER = times and the run_at_time method is not implemented - """ - self.log_error(f'run_at_time not implemented for {self.log_name} ' - 'wrapper. Cannot run with LOOP_ORDER = times') - return None - def run_all_times(self, custom=None): """! Loop over time range specified in conf file and call METplus wrapper for each time @@ -1627,3 +1615,9 @@ def handle_climo_cdf_dict(self, write_bins=True): items['direct_prob'] = 'bool' self.add_met_config_dict('climo_cdf', items) + + def get_wrapper_instance_name(self): + wrapper_name = get_wrapper_name(self.app_name) + if not self.instance: + return wrapper_name + return f'{wrapper_name}({self.instance})' diff --git a/metplus/wrappers/grid_diag_wrapper.py b/metplus/wrappers/grid_diag_wrapper.py index afe63f4480..d44026af25 100755 --- a/metplus/wrappers/grid_diag_wrapper.py +++ b/metplus/wrappers/grid_diag_wrapper.py @@ -152,26 +152,6 @@ def get_command(self): return cmd def run_at_time_once(self, time_info): - """! Process runtime and try to build command to run ascii2nc - Args: - @param time_info dictionary containing timing information - """ - - # if custom is already set in time info, run for only that item - # if not, loop over the CUSTOM_LOOP_LIST and process once for each - if 'custom' in time_info: - custom_loop_list = [time_info['custom']] - else: - custom_loop_list = self.c_dict['CUSTOM_LOOP_LIST'] - - for custom_string in custom_loop_list: - if custom_string: - self.logger.info(f"Processing custom string: {custom_string}") - - time_info['custom'] = custom_string - self.run_at_time_custom(time_info) - - def run_at_time_custom(self, time_info): self.clear() # subset input files as appropriate diff --git a/metplus/wrappers/make_plots_wrapper.py b/metplus/wrappers/make_plots_wrapper.py index 29bbcc8aac..88ba564fb7 100755 --- a/metplus/wrappers/make_plots_wrapper.py +++ b/metplus/wrappers/make_plots_wrapper.py @@ -105,7 +105,6 @@ def create_c_dict(self): self.config.getstr('config', 'LOG_MAKE_PLOTS_VERBOSITY', c_dict['VERBOSITY']) ) - c_dict['LOOP_ORDER'] = self.config.getstr('config', 'LOOP_ORDER') c_dict['INPUT_BASE_DIR'] = self.config.getdir('MAKE_PLOTS_INPUT_DIR') c_dict['OUTPUT_BASE_DIR'] = self.config.getdir('MAKE_PLOTS_OUTPUT_DIR') c_dict['SCRIPTS_BASE_DIR'] = self.config.getdir('MAKE_PLOTS_SCRIPTS_DIR') diff --git a/metplus/wrappers/met_db_load_wrapper.py b/metplus/wrappers/met_db_load_wrapper.py index e5a8783935..1ed678cb9f 100755 --- a/metplus/wrappers/met_db_load_wrapper.py +++ b/metplus/wrappers/met_db_load_wrapper.py @@ -94,6 +94,9 @@ def create_c_dict(self): self.log_error(f"Must set MET_DB_LOAD_MV_{name}") c_dict[f'MV_{name}'] = value + # set variable to skip finding input files + c_dict['FIND_FILES'] = False + return c_dict def get_command(self): @@ -111,49 +114,30 @@ def run_at_time_once(self, time_info): """ success = True - # if custom is already set in time info, run for only that item - # if not, loop over the CUSTOM_LOOP_LIST and process once for each - if 'custom' in time_info: - custom_loop_list = [time_info['custom']] - else: - custom_loop_list = self.c_dict['CUSTOM_LOOP_LIST'] - - for custom_string in custom_loop_list: - if custom_string: - self.logger.info(f"Processing custom string: {custom_string}") + # if lead and either init or valid are set, compute other string sub + if time_info.get('lead') != '*': + if (time_info.get('init') != '*' + or time_info.get('valid') != '*'): + time_info = time_util.ti_calculate(time_info) - time_info['custom'] = custom_string - # if lead and either init or valid are set, compute other string sub - if time_info.get('lead') != '*': - if (time_info.get('init') != '*' - or time_info.get('valid') != '*'): - time_info = time_util.ti_calculate(time_info) + self.set_environment_variables(time_info) - self.set_environment_variables(time_info) + if not self.replace_values_in_xml(time_info): + return - if not self.replace_values_in_xml(time_info): - return + # run command + if not self.build(): + success = False - # run command - if not self.build(): - success = False - - # remove tmp file - if self.c_dict.get('REMOVE_TMP_XML', True): - xml_file = self.c_dict.get('XML_TMP_FILE') - if xml_file and os.path.exists(xml_file): - self.logger.debug(f"Removing tmp file: {xml_file}") - os.remove(xml_file) + # remove tmp file + if self.c_dict.get('REMOVE_TMP_XML', True): + xml_file = self.c_dict.get('XML_TMP_FILE') + if xml_file and os.path.exists(xml_file): + self.logger.debug(f"Removing tmp file: {xml_file}") + os.remove(xml_file) return success - def get_all_files(self, custom=None): - """! Don't get list of all files for METdbLoad wrapper - - @returns True to report that no failures occurred - """ - return True - def get_stat_directories(self, input_paths): """! Traverse through files under input path and find all directories that contain .stat, .tcst, mode*.txt, and mtd*.txt files. diff --git a/metplus/wrappers/plot_point_obs_wrapper.py b/metplus/wrappers/plot_point_obs_wrapper.py index 3648003c87..2a0ec4373b 100755 --- a/metplus/wrappers/plot_point_obs_wrapper.py +++ b/metplus/wrappers/plot_point_obs_wrapper.py @@ -14,10 +14,10 @@ from ..util import do_string_sub, ti_calculate, get_lead_sequence from ..util import skip_time -from . import RuntimeFreqWrapper +from . import LoopTimesWrapper -class PlotPointObsWrapper(RuntimeFreqWrapper): +class PlotPointObsWrapper(LoopTimesWrapper): """! Wrapper used to build commands to call plot_point_obs """ WRAPPER_ENV_VAR_KEYS = [ @@ -56,10 +56,6 @@ def create_c_dict(self): c_dict = super().create_c_dict() app = self.app_name.upper() - # set default runtime frequency if unset explicitly - if not c_dict['RUNTIME_FREQ']: - c_dict['RUNTIME_FREQ'] = 'RUN_ONCE_FOR_EACH' - c_dict['VERBOSITY'] = self.config.getstr('config', f'LOG_{app}_VERBOSITY', c_dict['VERBOSITY']) @@ -70,8 +66,8 @@ def create_c_dict(self): c_dict['INPUT_DIR'] = self.config.getdir(f'{app}_INPUT_DIR', '') if not c_dict['INPUT_TEMPLATE']: - self.logger.warning(f'{app}_INPUT_TEMPLATE is required ' - 'to run PlotPointObs wrapper.') + self.log_error(f'{app}_INPUT_TEMPLATE is required ' + 'to run PlotPointObs wrapper.') # get optional grid input files c_dict['GRID_INPUT_TEMPLATE'] = self.config.getraw( diff --git a/metplus/wrappers/runtime_freq_wrapper.py b/metplus/wrappers/runtime_freq_wrapper.py index 8488f2072a..2812e81e15 100755 --- a/metplus/wrappers/runtime_freq_wrapper.py +++ b/metplus/wrappers/runtime_freq_wrapper.py @@ -28,11 +28,12 @@ class RuntimeFreqWrapper(CommandBuilder): # valid options for run frequency - FREQ_OPTIONS = ['RUN_ONCE', - 'RUN_ONCE_PER_INIT_OR_VALID', - 'RUN_ONCE_PER_LEAD', - 'RUN_ONCE_FOR_EACH' - ] + FREQ_OPTIONS = [ + 'RUN_ONCE', + 'RUN_ONCE_PER_INIT_OR_VALID', + 'RUN_ONCE_PER_LEAD', + 'RUN_ONCE_FOR_EACH' + ] def __init__(self, config, instance=None): super().__init__(config, instance=instance) @@ -100,14 +101,8 @@ def run_all_times(self): f" {', '.join(self.FREQ_OPTIONS)}") return None - # if not running once for each runtime and loop order is not set to - # 'processes' report an error - if self.c_dict['RUNTIME_FREQ'] != 'RUN_ONCE_FOR_EACH': - loop_order = self.config.getstr('config', 'LOOP_ORDER', '').lower() - if loop_order != 'processes': - self.log_error(f"Cannot run using {self.c_dict['RUNTIME_FREQ']} " - "mode unless LOOP_ORDER = processes") - return None + wrapper_instance_name = self.get_wrapper_instance_name() + self.logger.info(f'Running wrapper: {wrapper_instance_name}') # loop over all custom strings for custom_string in self.c_dict['CUSTOM_LOOP_LIST']: @@ -125,11 +120,6 @@ def run_all_times_custom(self, custom): @returns True on success, False on failure """ - # get a list of all input files that are available - if not self.get_all_files(custom): - self.log_error("A problem occurred trying to obtain input files") - return None - runtime_freq = self.c_dict['RUNTIME_FREQ'] if runtime_freq == 'RUN_ONCE': self.run_once(custom) @@ -138,7 +128,7 @@ def run_all_times_custom(self, custom): elif runtime_freq == 'RUN_ONCE_PER_LEAD': self.run_once_per_lead(custom) elif runtime_freq == 'RUN_ONCE_FOR_EACH': - self.all_commands = super().run_all_times(custom) + self.run_once_for_each(custom) def run_once(self, custom): self.logger.debug("Running once for all files") @@ -155,6 +145,10 @@ def run_once(self, custom): time_input['valid'] = '*' time_input['lead'] = '*' + if not self.get_all_files(custom): + self.log_error("A problem occurred trying to obtain input files") + return None + return self.run_at_time_once(time_input) def run_once_per_init_or_valid(self, custom): @@ -178,6 +172,8 @@ def run_once_per_init_or_valid(self, custom): time_input['lead'] = '*' + self.c_dict['ALL_FILES'] = self.get_all_files_from_leads(time_input) + self.clear() if not self.run_at_time_once(time_input): success = False @@ -205,19 +201,36 @@ def run_once_per_lead(self, custom): time_input['init'] = '*' time_input['valid'] = '*' + self.c_dict['ALL_FILES'] = self.get_all_files_for_lead(time_input) + self.clear() if not self.run_at_time_once(time_input): success = False return success - def run_at_time(self, input_dict): - """! Runs the command for a given run time. This function loops - over the list of forecast leads and list of custom loops - and runs once for each combination + def run_once_for_each(self, custom): + self.logger.debug(f"Running once for each init/valid and lead time") - @param input_dict dictionary containing time information - """ + success = True + for time_input in time_generator(self.config): + if time_input is None: + success = False + continue + + log_runtime_banner(self.config, time_input, self) + add_to_time_input(time_input, + instance=self.instance, + custom=custom) + + # loop of forecast leads and process each + if not self.run_at_time(time_input): + success = False + + return success + + def run_at_time(self, input_dict): + success = True # loop of forecast leads and process each lead_seq = get_lead_sequence(self.config, input_dict) for lead in lead_seq: @@ -248,7 +261,10 @@ def run_at_time(self, input_dict): # Run for given init/valid time and forecast lead combination self.clear() - self.run_at_time_once(time_info) + if not self.run_at_time_once(time_info): + success = False + + return success def get_all_files(self, custom=None): """! Get all files that can be processed with the app. @@ -256,6 +272,9 @@ def get_all_files(self, custom=None): i.e. fcst or obs, and the value is a list of files that fit in that category """ + if not self.c_dict.get('FIND_FILES', True): + return True + self.logger.debug("Finding all input files") all_files = [] @@ -268,27 +287,8 @@ def get_all_files(self, custom=None): instance=self.instance, custom=custom) - # loop over all forecast leads - wildcard_if_empty = self.c_dict.get('WILDCARD_LEAD_IF_EMPTY', - False) - lead_seq = get_lead_sequence(self.config, - time_input, - wildcard_if_empty=wildcard_if_empty) - for lead in lead_seq: - time_input['lead'] = lead - - # set current lead time config and environment variables - time_info = time_util.ti_calculate(time_input) - - if skip_time(time_info, self.c_dict.get('SKIP_TIMES')): - continue - - file_dict = self.get_files_from_time(time_info) - if file_dict: - if isinstance(file_dict, list): - all_files.extend(file_dict) - else: - all_files.append(file_dict) + lead_files = self.get_all_files_from_leads(time_input) + all_files.extend(lead_files) if not all_files: return False @@ -296,7 +296,66 @@ def get_all_files(self, custom=None): self.c_dict['ALL_FILES'] = all_files return True - def get_files_from_time(self, time_info): + def get_all_files_from_leads(self, time_input): + if not self.c_dict.get('FIND_FILES', True): + return True + + lead_files = [] + # loop over all forecast leads + wildcard_if_empty = self.c_dict.get('WILDCARD_LEAD_IF_EMPTY', + False) + lead_seq = get_lead_sequence(self.config, + time_input, + wildcard_if_empty=wildcard_if_empty) + for lead in lead_seq: + current_time_input = time_input.copy() + current_time_input['lead'] = lead + + # set current lead time config and environment variables + time_info = time_util.ti_calculate(current_time_input) + + if skip_time(time_info, self.c_dict.get('SKIP_TIMES')): + continue + + file_dict = self.get_files_from_time(time_info) + if file_dict: + if isinstance(file_dict, list): + lead_files.extend(file_dict) + else: + lead_files.append(file_dict) + + return lead_files + + def get_all_files_for_lead(self, time_input): + if not self.c_dict.get('FIND_FILES', True): + return True + + new_files = [] + for run_time in time_generator(self.config): + if run_time is None: + continue + + current_time_input = time_input.copy() + if 'valid' in run_time: + current_time_input['valid'] = run_time['valid'] + del current_time_input['init'] + elif 'init' in run_time: + current_time_input['init'] = run_time['init'] + del current_time_input['valid'] + time_info = time_util.ti_calculate(current_time_input) + if skip_time(time_info, self.c_dict.get('SKIP_TIMES')): + continue + file_dict = self.get_files_from_time(time_info) + if file_dict: + if isinstance(file_dict, list): + new_files.extend(file_dict) + else: + new_files.append(file_dict) + + return new_files + + @staticmethod + def get_files_from_time(time_info): """! Create dictionary containing time information (key time_info) and any relevant files for that runtime. @param time_info dictionary containing time information @@ -307,7 +366,8 @@ def get_files_from_time(self, time_info): file_dict['time_info'] = time_info.copy() return file_dict - def compare_time_info(self, runtime, filetime): + @staticmethod + def compare_time_info(runtime, filetime): """! Compare current runtime dictionary to current file time dictionary If runtime value for init, valid, or lead is not a wildcard and it doesn't match the file's time value, return False. Otherwise @@ -318,12 +378,13 @@ def compare_time_info(self, runtime, filetime): @returns True if file's info matches the requirements for current runtime or False if not. """ + # False if init/valid is not wildcard and the file time doesn't match for time_val in ['init', 'valid']: if (runtime[time_val] != '*' and filetime[time_val] != runtime[time_val]): return False - if runtime['lead'] == '*': + if runtime.get('lead', '*') == '*': return True # convert each value to seconds to compare @@ -373,7 +434,7 @@ def find_input_files(self, time_info, fill_missing=False): return all_input_files - def subset_input_files(self, time_info): + def subset_input_files(self, time_info, output_dir=None, leads=None): """! Obtain a subset of input files from the c_dict ALL_FILES based on the time information for the current run. @@ -386,21 +447,34 @@ def subset_input_files(self, time_info): if not self.c_dict.get('ALL_FILES'): return all_input_files + if leads is None: + lead_loop = [None] + else: + lead_loop = leads + for file_dict in self.c_dict['ALL_FILES']: - # compare time information for each input file - # add file to list of files to use if it matches - if not self.compare_time_info(time_info, file_dict['time_info']): - continue + for lead in lead_loop: + if lead is not None: + current_time_info = time_info.copy() + current_time_info['lead'] = lead + else: + current_time_info = time_info - for input_key in file_dict: - # skip time info key - if input_key == 'time_info': + # compare time information for each input file + # add file to list of files to use if it matches + if not self.compare_time_info(current_time_info, + file_dict['time_info']): continue - if input_key not in all_input_files: - all_input_files[input_key] = [] + for input_key in file_dict: + # skip time info key + if input_key == 'time_info': + continue + + if input_key not in all_input_files: + all_input_files[input_key] = [] - all_input_files[input_key].extend(file_dict[input_key]) + all_input_files[input_key].extend(file_dict[input_key]) # return None if no matching input files were found if not all_input_files: @@ -410,7 +484,9 @@ def subset_input_files(self, time_info): list_file_dict = {} for identifier, input_files in all_input_files.items(): list_file_name = self.get_list_file_name(time_info, identifier) - list_file_path = self.write_list_file(list_file_name, input_files) + list_file_path = self.write_list_file(list_file_name, + input_files, + output_dir=output_dir) list_file_dict[identifier] = list_file_path return list_file_dict @@ -435,11 +511,16 @@ def get_list_file_name(self, time_info, identifier): else: valid = time_info['valid'].strftime('%Y%m%d%H%M%S') - if time_info['lead'] == '*': + if time_info.get('lead', '*') == '*': lead = 'ALL' else: lead = time_util.ti_get_seconds_from_lead(time_info['lead'], time_info['valid']) + # use lead with letter if seconds cannot be computed e.g. 3m + if lead is None: + lead = time_util.ti_get_lead_string(time_info['lead'], + plural=False, + letter_only=True) return (f"{self.app_name}_files_{identifier}_" f"init_{init}_valid_{valid}_lead_{lead}.txt") diff --git a/metplus/wrappers/series_analysis_wrapper.py b/metplus/wrappers/series_analysis_wrapper.py index cc45658217..8d24ea0c30 100755 --- a/metplus/wrappers/series_analysis_wrapper.py +++ b/metplus/wrappers/series_analysis_wrapper.py @@ -26,7 +26,8 @@ from ..util import do_string_sub, parse_template, get_tags from ..util import get_lead_sequence, get_lead_sequence_groups from ..util import ti_get_hours_from_lead, ti_get_seconds_from_lead -from ..util import ti_get_lead_string +from ..util import ti_get_lead_string, ti_calculate +from ..util import ti_get_seconds_from_relativedelta from ..util import parse_var_list from ..util import add_to_time_input from ..util import field_read_prob_info @@ -465,11 +466,29 @@ def run_once_per_lead(self, custom): self.logger.debug(f"Processing {lead_group[0]} - forecast leads: " f"{', '.join(lead_hours)}") + + self.c_dict['ALL_FILES'] = ( + self.get_all_files_for_leads(input_dict, lead_group[1]) + ) + + # if only 1 forecast lead is being processed, set it in time dict + if len(lead_group[1]) == 1: + input_dict['lead'] = lead_group[1][0] + if not self.run_at_time_once(input_dict, lead_group): success = False return success + def get_all_files_for_leads(self, input_dict, leads): + all_files = [] + current_input_dict = input_dict.copy() + for lead in leads: + current_input_dict['lead'] = lead + new_files = self.get_all_files_for_lead(current_input_dict) + all_files.extend(new_files) + return all_files + def run_at_time_once(self, time_info, lead_group=None): """! Attempt to build series_analysis command for run time @@ -577,12 +596,23 @@ def get_files_from_time(self, time_info): if fcst_files is None or obs_files is None: return None - file_dict['fcst'] = fcst_files - file_dict['obs'] = obs_files + fcst_key, obs_key = self._get_fcst_obs_keys(storm_id) + + file_dict[fcst_key] = fcst_files + file_dict[obs_key] = obs_files file_dict_list.append(file_dict) return file_dict_list + @staticmethod + def _get_fcst_obs_keys(storm_id): + fcst_key = 'fcst' + obs_key = 'obs' + if storm_id != '*': + fcst_key = f'{fcst_key}_{storm_id}' + obs_key = f'{obs_key}_{storm_id}' + return fcst_key, obs_key + def find_input_files(self, time_info, data_type): """! Loop over list of input templates and find files for each @@ -596,27 +626,6 @@ def find_input_files(self, time_info, data_type): mandatory=False) return input_files - def subset_input_files(self, time_info): - """! Obtain a subset of input files from the c_dict ALL_FILES based on - the time information for the current run. - - @param time_info dictionary containing time information - @returns the path to a ascii file containing the list of files - or None if could not find any files - """ - fcst_files = [] - obs_files = [] - for file_dict in self.c_dict['ALL_FILES']: - # compare time information for each input file - # add file to list of files to use if it matches - if not self.compare_time_info(time_info, file_dict['time_info']): - continue - - fcst_files.extend(file_dict['fcst']) - obs_files.extend(file_dict['obs']) - - return fcst_files, obs_files - def compare_time_info(self, runtime, filetime): """! Call parents implementation then if the current run time and file time may potentially still not match, use storm_id to check @@ -696,48 +705,21 @@ def _get_fcst_and_obs_path(self, time_info, storm_id, lead_group): return fcst_path, obs_path - all_fcst_files = [] - all_obs_files = [] - lead_loop = leads if leads else [None] - for lead in lead_loop: - if lead is not None: - time_info['lead'] = lead - - fcst_files, obs_files = self.subset_input_files(time_info) - if fcst_files and obs_files: - all_fcst_files.extend(fcst_files) - all_obs_files.extend(obs_files) - - # skip if no files were found - if not all_fcst_files or not all_obs_files: - return None, None - output_dir = self.get_output_dir(time_info, storm_id, label) - # create forecast (or both) file list - if self.c_dict['USING_BOTH']: - data_type = 'BOTH' - else: - data_type = 'FCST' - - fcst_ascii_filename = self._get_ascii_filename(data_type, - storm_id, - leads) - fcst_path = self.write_list_file(fcst_ascii_filename, - all_fcst_files, - output_dir=output_dir) + list_file_dict = self.subset_input_files(time_info, + output_dir=output_dir, + leads=leads) + if not list_file_dict: + return None, None + # add storm_id and label to time_info for output filename + self._add_storm_id_and_label(time_info, storm_id, label) + fcst_key, obs_key = self._get_fcst_obs_keys(storm_id) + fcst_path = list_file_dict[fcst_key] if self.c_dict['USING_BOTH']: return fcst_path, fcst_path - - # create analysis file list - obs_ascii_filename = self._get_ascii_filename('OBS', - storm_id, - leads) - obs_path = self.write_list_file(obs_ascii_filename, - all_obs_files, - output_dir=output_dir) - + obs_path = list_file_dict[obs_key] return fcst_path, obs_path def _check_python_embedding(self): @@ -758,50 +740,6 @@ def _check_python_embedding(self): return True - @staticmethod - def _get_ascii_filename(data_type, storm_id, leads=None): - """! Build filename for ASCII file list file - - @param data_type FCST, OBS, or BOTH - @param storm_id current storm ID or wildcard character - @param leads list of forecast leads to use add the forecast hour - string to the filename or the minimum and maximum forecast hour - strings if there are more than one lead - @returns string containing filename to use - """ - prefix = f"{data_type}_FILES" - - # of storm ID is set (not wildcard), then add it to filename - if storm_id == '*': - filename = '' - else: - filename = f"_{storm_id}" - - # add forecast leads if specified - if leads is not None: - lead_hours_list = [] - for lead in leads: - lead_hours = ti_get_hours_from_lead(lead) - if lead_hours is None: - lead_hours = ti_get_lead_string(lead, - letter_only=True) - lead_hours_list.append(lead_hours) - - # get first forecast lead, convert to hours, and add to filename - lead_hours = min(lead_hours_list) - - lead_str = str(lead_hours).zfill(3) - filename += f"_F{lead_str}" - - # if list of forecast leads, get min and max and add them to name - if len(lead_hours_list) > 1: - max_lead_hours = max(lead_hours_list) - max_lead_str = str(max_lead_hours).zfill(3) - filename += f"_to_F{max_lead_str}" - - ascii_filename = f"{prefix}{filename}" - return ascii_filename - def get_output_dir(self, time_info, storm_id, label): """! Determine directory that will contain output data from the OUTPUT_DIR and OUTPUT_TEMPLATE. This will include any @@ -818,17 +756,23 @@ def get_output_dir(self, time_info, storm_id, label): output_dir_template = os.path.join(self.c_dict['OUTPUT_DIR'], self.c_dict['OUTPUT_TEMPLATE']) output_dir_template = os.path.dirname(output_dir_template) + + # get output directory including storm ID and label + current_time_info = time_info.copy() + self._add_storm_id_and_label(current_time_info, storm_id, label) + output_dir = do_string_sub(output_dir_template, + **current_time_info) + return output_dir + + @staticmethod + def _add_storm_id_and_label(time_info, storm_id, label): if storm_id == '*': storm_id_out = 'all_storms' else: storm_id_out = storm_id - # get output directory including storm ID and label time_info['storm_id'] = storm_id_out time_info['label'] = label - output_dir = do_string_sub(output_dir_template, - **time_info) - return output_dir def build_and_run_series_request(self, time_info, fcst_path, obs_path): """! Build up the -obs, -fcst, -out necessary for running the diff --git a/metplus/wrappers/stat_analysis_wrapper.py b/metplus/wrappers/stat_analysis_wrapper.py index de313014ca..ffde04030a 100755 --- a/metplus/wrappers/stat_analysis_wrapper.py +++ b/metplus/wrappers/stat_analysis_wrapper.py @@ -158,7 +158,6 @@ def create_c_dict(self): self.config.getstr('config', 'LOG_STAT_ANALYSIS_VERBOSITY', c_dict['VERBOSITY']) ) - c_dict['LOOP_ORDER'] = self.config.getstr('config', 'LOOP_ORDER') # STATAnalysis config file is optional, so # don't provide wrapped config file name as default value @@ -436,7 +435,7 @@ def set_lists_loop_or_group(self, c_dict): for missing_config in missing_config_list: # if running MakePlots - if (c_dict['LOOP_ORDER'] == 'processes' and self.runMakePlots): + if self.runMakePlots: # if LINE_TYPE_LIST is missing, add it to group list if missing_config == 'LINE_TYPE_LIST': diff --git a/metplus/wrappers/tc_pairs_wrapper.py b/metplus/wrappers/tc_pairs_wrapper.py index e4870b7e06..c86cc01ab8 100755 --- a/metplus/wrappers/tc_pairs_wrapper.py +++ b/metplus/wrappers/tc_pairs_wrapper.py @@ -285,7 +285,7 @@ def create_c_dict(self): False) ) - # if LOOP_ORDER = processes, only run once if True + # only run once if True c_dict['RUN_ONCE'] = self.config.getbool('config', 'TC_PAIRS_RUN_ONCE', True) diff --git a/metplus/wrappers/user_script_wrapper.py b/metplus/wrappers/user_script_wrapper.py index a64cfbdefa..50384c0190 100755 --- a/metplus/wrappers/user_script_wrapper.py +++ b/metplus/wrappers/user_script_wrapper.py @@ -63,43 +63,25 @@ def run_at_time_once(self, time_info): @param time_info dictionary containing time information @returns True if command was run successfully, False otherwise """ - success = True - - # if custom is already set in time info, run for only that item - # if not, loop over the CUSTOM_LOOP_LIST and process once for each - if 'custom' in time_info: - custom_loop_list = [time_info['custom']] - else: - custom_loop_list = self.c_dict['CUSTOM_LOOP_LIST'] - - for custom_string in custom_loop_list: - if custom_string: - self.logger.info(f"Processing custom string: {custom_string}") - - time_info['custom'] = custom_string - # if lead and either init or valid are set, compute other string sub - if time_info.get('lead') != '*': - if (time_info.get('init') != '*' - or time_info.get('valid') != '*'): - time_info = time_util.ti_calculate(time_info) - - # create file list text files for the current run time criteria - # set c_dict to the input file dict to set as environment vars - self.c_dict['INPUT_LIST_DICT'] = self.subset_input_files(time_info) - - self.set_environment_variables(time_info) - - # substitute values from dictionary into command - self.c_dict['COMMAND'] = ( - do_string_sub(self.c_dict['COMMAND_TEMPLATE'], - **time_info) - ) - - # run command - if not self.build(): - success = False - - return success + # if lead and either init or valid are set, compute other string sub + if time_info.get('lead') != '*': + if (time_info.get('init') != '*' + or time_info.get('valid') != '*'): + time_info = time_util.ti_calculate(time_info) + + # create file list text files for the current run time criteria + # set c_dict to the input file dict to set as environment vars + self.c_dict['INPUT_LIST_DICT'] = self.subset_input_files(time_info) + + self.set_environment_variables(time_info) + + # substitute values from dictionary into command + self.c_dict['COMMAND'] = ( + do_string_sub(self.c_dict['COMMAND_TEMPLATE'], + **time_info) + ) + + return self.build() def get_files_from_time(self, time_info): """! Create dictionary containing time information (key time_info) and