diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index f5ac63a66d..bd1feac438 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -238,7 +238,6 @@ jobs: needs: use_case_tests if: ${{ always() && needs.use_case_tests.result == 'failure' }} steps: - - uses: actions/checkout@v4 - name: Check for error logs id: check-for-error-logs run: | diff --git a/docs/Users_Guide/release-notes.rst b/docs/Users_Guide/release-notes.rst index 6042b24fb1..0bd253b36d 100644 --- a/docs/Users_Guide/release-notes.rst +++ b/docs/Users_Guide/release-notes.rst @@ -30,6 +30,48 @@ When applicable, release notes are followed by the `GitHub issue `__ number which describes the bugfix, enhancement, or new feature. +METplus Version 6.0.0 Beta 3 Release Notes (2024-02-08) +------------------------------------------------------- + + .. dropdown:: Enhancements + + * Add support for MET land-mask settings in Point-Stat + (`#2334 `_) + * Enhance the TC-Pairs wrapper to support the new diag_required and diag_min_req configuration options + (`#2430 `_) + * Enhance the TC-Diag wrapper to support new configuration options added in MET-12.0.0-beta2 + (`#2432 `_) + * Prevent error if some input files are missing + (`#2460 `_) + + .. dropdown:: Bugfix + + NONE + + .. dropdown:: New Wrappers + + NONE + + .. dropdown:: New Use Cases + + * Verify Total Column Ozone against NASA's OMI dataset + (`#1989 `_) + * RRFS reformatting, aggregating, and plotting use case + (`#2406 `_) + * Satellite Altimetry data + (`#2383 `_) + + .. dropdown:: Documentation + + * Create video to demonstrate how to update use cases that use deprecated environment variables + (`#2371 `_) + + .. dropdown:: Internal + + * Update Documentation Overview and Conventions + (`#2454 `_) + + METplus Version 6.0.0 Beta 2 Release Notes (2023-11-14) ------------------------------------------------------- diff --git a/internal/tests/pytests/wrappers/pb2nc/__init__.py b/internal/tests/data/ascii/precip24_2010010112.ascii similarity index 100% rename from internal/tests/pytests/wrappers/pb2nc/__init__.py rename to internal/tests/data/ascii/precip24_2010010112.ascii diff --git a/internal/tests/data/ascii/precip24_2010010118.ascii b/internal/tests/data/ascii/precip24_2010010118.ascii new file mode 100644 index 0000000000..e69de29bb2 diff --git a/internal/tests/data/ens/2009123106/arw-fer-gep1/d01_2009123106_02400.grib b/internal/tests/data/ens/2009123106/arw-fer-gep1/d01_2009123106_02400.grib new file mode 100644 index 0000000000..e69de29bb2 diff --git a/internal/tests/data/ens/2009123106/arw-fer-gep5/d01_2009123106_02400.grib b/internal/tests/data/ens/2009123106/arw-fer-gep5/d01_2009123106_02400.grib new file mode 100644 index 0000000000..e69de29bb2 diff --git a/internal/tests/data/ens/2009123106/arw-sch-gep6/d01_2009123106_02400.grib b/internal/tests/data/ens/2009123106/arw-sch-gep6/d01_2009123106_02400.grib new file mode 100644 index 0000000000..e69de29bb2 diff --git a/internal/tests/data/ens/2009123106/arw-tom-gep3/d01_2009123106_02400.grib b/internal/tests/data/ens/2009123106/arw-tom-gep3/d01_2009123106_02400.grib new file mode 100644 index 0000000000..e69de29bb2 diff --git a/internal/tests/data/ens/2009123106/arw-tom-gep7/d01_2009123106_02400.grib b/internal/tests/data/ens/2009123106/arw-tom-gep7/d01_2009123106_02400.grib new file mode 100644 index 0000000000..e69de29bb2 diff --git a/internal/tests/data/obs/2010010106_obs_file b/internal/tests/data/obs/2010010106_obs_file new file mode 100644 index 0000000000..e69de29bb2 diff --git a/internal/tests/data/obs/2010010112_obs_file b/internal/tests/data/obs/2010010112_obs_file new file mode 100644 index 0000000000..e69de29bb2 diff --git a/internal/tests/data/obs/2010010118_obs_file b/internal/tests/data/obs/2010010118_obs_file new file mode 100644 index 0000000000..e69de29bb2 diff --git a/internal/tests/data/obs/2010010218_obs_file b/internal/tests/data/obs/2010010218_obs_file new file mode 100644 index 0000000000..e69de29bb2 diff --git a/internal/tests/data/tc_gen/track/track_fake_2018020100 b/internal/tests/data/tc_gen/track/track_fake_2018020100 new file mode 100644 index 0000000000..e69de29bb2 diff --git a/internal/tests/data/tc_gen/track/track_fake_2018020112 b/internal/tests/data/tc_gen/track/track_fake_2018020112 new file mode 100644 index 0000000000..e69de29bb2 diff --git a/internal/tests/data/tc_gen/track/track_fake_2018020200 b/internal/tests/data/tc_gen/track/track_fake_2018020200 new file mode 100644 index 0000000000..e69de29bb2 diff --git a/internal/tests/data/tc_gen/track/track_fake_2018020212 b/internal/tests/data/tc_gen/track/track_fake_2018020212 new file mode 100644 index 0000000000..e69de29bb2 diff --git a/internal/tests/data/tc_gen/track/track_fake_2018103100 b/internal/tests/data/tc_gen/track/track_fake_2018103100 new file mode 100644 index 0000000000..e69de29bb2 diff --git a/internal/tests/data/tc_gen/track/track_fake_2018103112 b/internal/tests/data/tc_gen/track/track_fake_2018103112 new file mode 100644 index 0000000000..e69de29bb2 diff --git a/internal/tests/pytests/conftest.py b/internal/tests/pytests/conftest.py index 2b7e7dfbab..9853f78fb4 100644 --- a/internal/tests/pytests/conftest.py +++ b/internal/tests/pytests/conftest.py @@ -113,6 +113,14 @@ def test_example(metplus_config): if len(msg.args) != 0] print("Tests raised the following errors:") print("\n".join(err_msgs)) + if config.logger.warning.call_args_list: + warn_msgs = [ + str(msg.args[0]) + for msg + in config.logger.warning.call_args_list + if len(msg.args) != 0] + print("\nTests raised the following warnings:") + print("\n".join(warn_msgs)) config.logger = old_logger # don't remove output base if test fails if request.node.rep_call.failed: @@ -185,3 +193,16 @@ def make_nc(tmp_path, lon, lat, z, data, variable='Temp', file_name='fake.nc'): temp[0, :, :, :] = data return file_name + + +@pytest.fixture(scope="function") +def get_test_data_dir(): + """!Get path to directory containing test data. + """ + def get_test_data_path(subdir): + internal_tests_dir = os.path.abspath( + os.path.join(os.path.dirname(__file__), os.pardir) + ) + return os.path.join(internal_tests_dir, 'data', subdir) + + return get_test_data_path diff --git a/internal/tests/pytests/util/run_util/test_run_util.py b/internal/tests/pytests/util/run_util/test_run_util.py index 661d106c56..6815e0b9d2 100644 --- a/internal/tests/pytests/util/run_util/test_run_util.py +++ b/internal/tests/pytests/util/run_util/test_run_util.py @@ -48,6 +48,8 @@ 'METPLUS_BASE', 'PARM_BASE', 'METPLUS_VERSION', + 'ALLOW_MISSING_INPUTS', + 'INPUT_THRESH', ] diff --git a/internal/tests/pytests/wrappers/ascii2nc/test_ascii2nc_wrapper.py b/internal/tests/pytests/wrappers/ascii2nc/test_ascii2nc_wrapper.py index 0202f14b8d..341f93112c 100644 --- a/internal/tests/pytests/wrappers/ascii2nc/test_ascii2nc_wrapper.py +++ b/internal/tests/pytests/wrappers/ascii2nc/test_ascii2nc_wrapper.py @@ -17,8 +17,8 @@ def ascii2nc_wrapper(metplus_config, config_overrides=None): 'LOOP_BY': 'VALID', 'VALID_TIME_FMT': '%Y%m%d%H', 'VALID_BEG': '2010010112', - 'VALID_END': '2010010112', - 'VALID_INCREMENT': '1M', + 'VALID_END': '2010010118', + 'VALID_INCREMENT': '6H', 'ASCII2NC_INPUT_TEMPLATE': '{INPUT_BASE}/met_test/data/sample_obs/ascii/precip24_{valid?fmt=%Y%m%d%H}.ascii', 'ASCII2NC_OUTPUT_TEMPLATE': '{OUTPUT_BASE}/ascii2nc/precip24_{valid?fmt=%Y%m%d%H}.nc', 'ASCII2NC_CONFIG_FILE': '{PARM_BASE}/met_config/Ascii2NcConfig_wrapped', @@ -47,6 +47,36 @@ def ascii2nc_wrapper(metplus_config, config_overrides=None): return ASCII2NCWrapper(config, instance=instance) +@pytest.mark.parametrize( + 'missing, run, thresh, errors, allow_missing', [ + (1, 3, 0.5, 0, True), + (1, 3, 0.8, 1, True), + (1, 3, 0.5, 1, False), + ] +) +@pytest.mark.wrapper +def test_ascii2nc_missing_inputs(metplus_config, get_test_data_dir, + missing, run, thresh, errors, allow_missing): + config_overrides = { + 'INPUT_MUST_EXIST': True, + 'ASCII2NC_ALLOW_MISSING_INPUTS': allow_missing, + 'ASCII2NC_INPUT_THRESH': thresh, + 'ASCII2NC_INPUT_TEMPLATE': os.path.join(get_test_data_dir('ascii'), 'precip24_{valid?fmt=%Y%m%d%H}.ascii'), + 'VALID_END': '2010010200', + } + wrapper = ascii2nc_wrapper(metplus_config, config_overrides) + assert wrapper.isOK + + all_cmds = wrapper.run_all_times() + for cmd, _ in all_cmds: + print(cmd) + + print(f'missing: {wrapper.missing_input_count} / {wrapper.run_count}, errors: {wrapper.errors}') + assert wrapper.missing_input_count == missing + assert wrapper.run_count == run + assert wrapper.errors == errors + + @pytest.mark.parametrize( 'config_overrides, env_var_values', [ ({}, @@ -163,11 +193,13 @@ def test_ascii2nc_wrapper(metplus_config, config_overrides, input_path = wrapper.config.getraw('config', 'ASCII2NC_INPUT_TEMPLATE') input_dir = os.path.dirname(input_path) - input_file = 'precip24_2010010112.ascii' + input_file1 = 'precip24_2010010112.ascii' + input_file2 = 'precip24_2010010118.ascii' output_path = wrapper.config.getraw('config', 'ASCII2NC_OUTPUT_TEMPLATE') output_dir = os.path.dirname(output_path) - output_file = 'precip24_2010010112.nc' + output_file1 = 'precip24_2010010112.nc' + output_file2 = 'precip24_2010010118.nc' all_commands = wrapper.run_all_times() print(f"ALL COMMANDS: {all_commands}") @@ -177,13 +209,17 @@ def test_ascii2nc_wrapper(metplus_config, config_overrides, verbosity = f"-v {wrapper.c_dict['VERBOSITY']}" config_file = wrapper.c_dict.get('CONFIG_FILE') - expected_cmd = (f"{app_path} " - f"{input_dir}/{input_file} " - f"{output_dir}/{output_file} " - f"-config {config_file} " - f"{verbosity}") + expected_cmds = [ + (f"{app_path} {input_dir}/{input_file1} {output_dir}/{output_file1} " + f"-config {config_file} {verbosity}"), + (f"{app_path} {input_dir}/{input_file2} {output_dir}/{output_file2} " + f"-config {config_file} {verbosity}"), + ] - assert all_commands[0][0] == expected_cmd + assert len(all_commands) == len(expected_cmds) + for (cmd, _), expected_cmd in zip(all_commands, expected_cmds): + # ensure commands are generated as expected + assert cmd == expected_cmd env_vars = all_commands[0][1] diff --git a/internal/tests/pytests/wrappers/command_builder/test_command_builder.py b/internal/tests/pytests/wrappers/command_builder/test_command_builder.py index 89fd7fef36..f18cb0c5a3 100644 --- a/internal/tests/pytests/wrappers/command_builder/test_command_builder.py +++ b/internal/tests/pytests/wrappers/command_builder/test_command_builder.py @@ -1199,17 +1199,20 @@ def test_errors_and_defaults(metplus_config): assert actual == False assert _in_last_err('Could not generate command', cb.logger) - # test python embedding error + # test python embedding check with mock.patch.object(cb_wrapper, 'is_python_script', return_value=True): actual = cb.check_for_python_embedding('FCST',{'fcst_name':'pyEmbed'}) - assert actual == None - assert _in_last_err('must be set to a valid Python Embedding type', cb.logger) + assert actual == 'python_embedding' - cb.c_dict['FCST_INPUT_DATATYPE'] = 'PYTHON_XARRAY' + cb.env_var_dict['METPLUS_FCST_FILE_TYPE'] = "PYTHON_NUMPY" with mock.patch.object(cb_wrapper, 'is_python_script', return_value=True): actual = cb.check_for_python_embedding('FCST',{'fcst_name':'pyEmbed'}) assert actual == 'python_embedding' + with mock.patch.object(cb_wrapper, 'is_python_script', return_value=False): + actual = cb.check_for_python_embedding('FCST',{'fcst_name':'pyEmbed'}) + assert actual == 'pyEmbed' + # test field_info not set cb.c_dict['CURRENT_VAR_INFO'] = None actual = cb.set_current_field_config() diff --git a/internal/tests/pytests/wrappers/ensemble_stat/test_ensemble_stat_wrapper.py b/internal/tests/pytests/wrappers/ensemble_stat/test_ensemble_stat_wrapper.py index 3e714cc687..d767391ba0 100644 --- a/internal/tests/pytests/wrappers/ensemble_stat/test_ensemble_stat_wrapper.py +++ b/internal/tests/pytests/wrappers/ensemble_stat/test_ensemble_stat_wrapper.py @@ -4,9 +4,6 @@ import os -from datetime import datetime - - from metplus.wrappers.ensemble_stat_wrapper import EnsembleStatWrapper fcst_dir = '/some/path/fcst' @@ -27,7 +24,7 @@ run_times = ['2005080700', '2005080712'] -def set_minimum_config_settings(config, set_fields=True): +def set_minimum_config_settings(config, set_fields=True, set_obs=True): # set config variables to prevent command from running and bypass check # if input files actually exist config.set('config', 'DO_NOT_RUN_EXE', True) @@ -46,11 +43,12 @@ def set_minimum_config_settings(config, set_fields=True): config.set('config', 'ENSEMBLE_STAT_CONFIG_FILE', '{PARM_BASE}/met_config/EnsembleStatConfig_wrapped') config.set('config', 'FCST_ENSEMBLE_STAT_INPUT_DIR', fcst_dir) - config.set('config', 'OBS_ENSEMBLE_STAT_GRID_INPUT_DIR', obs_dir) config.set('config', 'FCST_ENSEMBLE_STAT_INPUT_TEMPLATE', '{init?fmt=%Y%m%d%H}/fcst_file_F{lead?fmt=%3H}') - config.set('config', 'OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE', - '{valid?fmt=%Y%m%d%H}/obs_file') + if set_obs: + config.set('config', 'OBS_ENSEMBLE_STAT_GRID_INPUT_DIR', obs_dir) + config.set('config', 'OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE', + '{valid?fmt=%Y%m%d%H}/obs_file') config.set('config', 'ENSEMBLE_STAT_OUTPUT_DIR', '{OUTPUT_BASE}/EnsembleStat/output') config.set('config', 'ENSEMBLE_STAT_OUTPUT_TEMPLATE', '{valid?fmt=%Y%m%d%H}') @@ -62,6 +60,74 @@ def set_minimum_config_settings(config, set_fields=True): config.set('config', 'OBS_VAR1_LEVELS', obs_level) +@pytest.mark.parametrize( + 'allow_missing, optional_input, missing, run, thresh, errors', [ + (True, None, 3, 8, 0.4, 0), + (True, None, 3, 8, 0.7, 1), + (False, None, 3, 8, 0.7, 3), + (True, 'obs_grid', 4, 8, 0.4, 0), + (True, 'obs_grid', 4, 8, 0.7, 1), + (False, 'obs_grid', 4, 8, 0.7, 4), + (True, 'point_grid', 4, 8, 0.4, 0), + (True, 'point_grid', 4, 8, 0.7, 1), + (False, 'point_grid', 4, 8, 0.7, 4), + (True, 'ens_mean', 4, 8, 0.4, 0), + (True, 'ens_mean', 4, 8, 0.7, 1), + (False, 'ens_mean', 4, 8, 0.7, 4), + (True, 'ctrl', 4, 8, 0.4, 0), + (True, 'ctrl', 4, 8, 0.7, 1), + (False, 'ctrl', 4, 8, 0.7, 4), + # still errors if more members than n_members found + (True, 'low_n_member', 8, 8, 0.7, 6), + (False, 'low_n_member', 8, 8, 0.7, 8), + ] +) +@pytest.mark.wrapper_b +def test_ensemble_stat_missing_inputs(metplus_config, get_test_data_dir, allow_missing, + optional_input, missing, run, thresh, errors): + config = metplus_config + set_minimum_config_settings(config, set_obs=False) + config.set('config', 'INPUT_MUST_EXIST', True) + config.set('config', 'ENSEMBLE_STAT_ALLOW_MISSING_INPUTS', allow_missing) + config.set('config', 'ENSEMBLE_STAT_INPUT_THRESH', thresh) + n_members = 4 if optional_input == 'low_n_member' else 6 + config.set('config', 'ENSEMBLE_STAT_N_MEMBERS', n_members) + config.set('config', 'INIT_BEG', '2009123106') + config.set('config', 'INIT_END', '2010010100') + config.set('config', 'INIT_INCREMENT', '6H') + config.set('config', 'LEAD_SEQ', '24H, 48H') + config.set('config', 'FCST_ENSEMBLE_STAT_INPUT_DIR', get_test_data_dir('ens')) + config.set('config', 'FCST_ENSEMBLE_STAT_INPUT_TEMPLATE', + '{init?fmt=%Y%m%d%H}/arw-*-gep?/d01_{init?fmt=%Y%m%d%H}_{lead?fmt=%3H}00.grib') + + if optional_input == 'obs_grid': + prefix = 'OBS_ENSEMBLE_STAT_GRID' + elif optional_input == 'point_grid': + prefix = 'OBS_ENSEMBLE_STAT_POINT' + elif optional_input == 'ens_mean': + prefix = 'ENSEMBLE_STAT_ENS_MEAN' + elif optional_input == 'ctrl': + prefix = 'ENSEMBLE_STAT_CTRL' + else: + prefix = None + + if prefix: + config.set('config', f'{prefix}_INPUT_DIR', get_test_data_dir('obs')) + config.set('config', f'{prefix}_INPUT_TEMPLATE', '{valid?fmt=%Y%m%d%H}_obs_file') + + wrapper = EnsembleStatWrapper(config) + assert wrapper.isOK + + all_cmds = wrapper.run_all_times() + for cmd, _ in all_cmds: + print(cmd) + + print(f'missing: {wrapper.missing_input_count} / {wrapper.run_count}, errors: {wrapper.errors}') + assert wrapper.missing_input_count == missing + assert wrapper.run_count == run + assert wrapper.errors == errors + + @pytest.mark.parametrize( 'config_overrides, expected_filename', [ # 0 - set forecast level diff --git a/internal/tests/pytests/wrappers/extract_tiles/test_extract_tiles.py b/internal/tests/pytests/wrappers/extract_tiles/test_extract_tiles.py index 9183821f1e..7af643d713 100644 --- a/internal/tests/pytests/wrappers/extract_tiles/test_extract_tiles.py +++ b/internal/tests/pytests/wrappers/extract_tiles/test_extract_tiles.py @@ -7,6 +7,7 @@ from metplus.wrappers.extract_tiles_wrapper import ExtractTilesWrapper + def extract_tiles_wrapper(metplus_config): config = metplus_config config.set('config', 'PROCESS_LIST', 'ExtractTiles') @@ -22,10 +23,6 @@ def extract_tiles_wrapper(metplus_config): config.set('config', 'EXTRACT_TILES_LAT_ADJ', '15') config.set('config', 'EXTRACT_TILES_LON_ADJ', '15') config.set('config', 'EXTRACT_TILES_FILTER_OPTS', '-basin ML') - config.set('config', 'FCST_EXTRACT_TILES_INPUT_TEMPLATE', - 'gfs_4_{init?fmt=%Y%m%d}_{init?fmt=%H}00_{lead?fmt=%HHH}.grb2') - config.set('config', 'OBS_EXTRACT_TILES_INPUT_TEMPLATE', - 'gfs_4_{valid?fmt=%Y%m%d}_{valid?fmt=%H}00_000.grb2') config.set('config', 'EXTRACT_TILES_GRID_INPUT_DIR', '{INPUT_BASE}/cyclone_track_feature/reduced_model_data') config.set('config', 'EXTRACT_TILES_PAIRS_INPUT_DIR', diff --git a/internal/tests/pytests/wrappers/gen_ens_prod/test_gen_ens_prod_wrapper.py b/internal/tests/pytests/wrappers/gen_ens_prod/test_gen_ens_prod_wrapper.py index 678753d59c..3c771f499f 100644 --- a/internal/tests/pytests/wrappers/gen_ens_prod/test_gen_ens_prod_wrapper.py +++ b/internal/tests/pytests/wrappers/gen_ens_prod/test_gen_ens_prod_wrapper.py @@ -14,7 +14,7 @@ run_times = ['2009123112', '2009123118'] -def set_minimum_config_settings(config): +def set_minimum_config_settings(config, set_ctrl=True): # set config variables to prevent command from running and bypass check # if input files actually exist config.set('config', 'DO_NOT_RUN_EXE', True) @@ -34,8 +34,9 @@ def set_minimum_config_settings(config): config.set('config', 'GEN_ENS_PROD_INPUT_TEMPLATE', '{init?fmt=%Y%m%d%H}/*gep*/d01_{init?fmt=%Y%m%d%H}_{lead?fmt=%3H}00.grib') - config.set('config', 'GEN_ENS_PROD_CTRL_INPUT_TEMPLATE', - '{init?fmt=%Y%m%d%H}/arw-tom-gep3/d01_{init?fmt=%Y%m%d%H}_{lead?fmt=%3H}00.grib') + if set_ctrl: + config.set('config', 'GEN_ENS_PROD_CTRL_INPUT_TEMPLATE', + '{init?fmt=%Y%m%d%H}/arw-tom-gep3/d01_{init?fmt=%Y%m%d%H}_{lead?fmt=%3H}00.grib') config.set('config', 'GEN_ENS_PROD_OUTPUT_DIR', '{OUTPUT_BASE}/GenEnsProd/output') config.set('config', 'GEN_ENS_PROD_OUTPUT_TEMPLATE', @@ -55,6 +56,59 @@ def handle_input_dir(config): return input_dir +@pytest.mark.parametrize( + 'allow_missing, optional_input, missing, run, thresh, errors', [ + (True, None, 3, 8, 0.4, 0), + (True, None, 3, 8, 0.7, 1), + (False, None, 3, 8, 0.7, 3), + (True, 'ctrl', 4, 8, 0.4, 0), + (True, 'ctrl', 4, 8, 0.7, 1), + (False, 'ctrl', 4, 8, 0.7, 4), + # still errors if more members than n_members found + (True, 'low_n_member', 8, 8, 0.7, 6), + (False, 'low_n_member', 8, 8, 0.7, 8), + ] +) +@pytest.mark.wrapper +def test_gen_ens_prod_missing_inputs(metplus_config, get_test_data_dir, allow_missing, + optional_input, missing, run, thresh, errors): + config = metplus_config + set_minimum_config_settings(config, set_ctrl=False) + config.set('config', 'INPUT_MUST_EXIST', True) + config.set('config', 'GEN_ENS_PROD_ALLOW_MISSING_INPUTS', allow_missing) + config.set('config', 'GEN_ENS_PROD_INPUT_THRESH', thresh) + n_members = 4 if optional_input == 'low_n_member' else 6 + config.set('config', 'GEN_ENS_PROD_N_MEMBERS', n_members) + config.set('config', 'INIT_BEG', '2009123106') + config.set('config', 'INIT_END', '2010010100') + config.set('config', 'INIT_INCREMENT', '6H') + config.set('config', 'LEAD_SEQ', '24H, 48H') + config.set('config', 'GEN_ENS_PROD_INPUT_DIR', get_test_data_dir('ens')) + config.set('config', 'GEN_ENS_PROD_INPUT_TEMPLATE', + '{init?fmt=%Y%m%d%H}/arw-*-gep?/d01_{init?fmt=%Y%m%d%H}_{lead?fmt=%3H}00.grib') + + if optional_input == 'ctrl': + prefix = 'GEN_ENS_PROD_CTRL' + else: + prefix = None + + if prefix: + config.set('config', f'{prefix}_INPUT_DIR', get_test_data_dir('obs')) + config.set('config', f'{prefix}_INPUT_TEMPLATE', '{valid?fmt=%Y%m%d%H}_obs_file') + + wrapper = GenEnsProdWrapper(config) + assert wrapper.isOK + + all_cmds = wrapper.run_all_times() + for cmd, _ in all_cmds: + print(cmd) + + print(f'missing: {wrapper.missing_input_count} / {wrapper.run_count}, errors: {wrapper.errors}') + assert wrapper.missing_input_count == missing + assert wrapper.run_count == run + assert wrapper.errors == errors + + @pytest.mark.parametrize( 'config_overrides, env_var_values', [ # 0 diff --git a/internal/tests/pytests/wrappers/gen_vx_mask/test_gen_vx_mask.py b/internal/tests/pytests/wrappers/gen_vx_mask/test_gen_vx_mask.py index f4973a55b7..3c745d4c64 100644 --- a/internal/tests/pytests/wrappers/gen_vx_mask/test_gen_vx_mask.py +++ b/internal/tests/pytests/wrappers/gen_vx_mask/test_gen_vx_mask.py @@ -21,6 +21,50 @@ def gen_vx_mask_wrapper(metplus_config): return GenVxMaskWrapper(config) +@pytest.mark.parametrize( + 'missing, run, thresh, errors, allow_missing', [ + (6, 12, 0.5, 0, True), + (6, 12, 0.6, 1, True), + (6, 12, 0.5, 6, False), + ] +) +@pytest.mark.wrapper +def test_gen_vx_mask_missing_inputs(metplus_config, get_test_data_dir, missing, + run, thresh, errors, allow_missing): + config = metplus_config + config.set('config', 'DO_NOT_RUN_EXE', True) + config.set('config', 'INPUT_MUST_EXIST', True) + config.set('config', 'GEN_VX_MASK_ALLOW_MISSING_INPUTS', allow_missing) + config.set('config', 'GEN_VX_MASK_INPUT_THRESH', thresh) + config.set('config', 'LOOP_BY', 'INIT') + config.set('config', 'INIT_TIME_FMT', '%Y%m%d%H') + config.set('config', 'INIT_BEG', '2017051001') + config.set('config', 'INIT_END', '2017051003') + config.set('config', 'INIT_INCREMENT', '2H') + config.set('config', 'LEAD_SEQ', '1,2,3,6,9,12') + config.set('config', 'GEN_VX_MASK_OPTIONS', "-type lat -thresh 'ge30&&le50', -type lon -thresh 'le-70&&ge-130' -intersection -name lat_lon_mask") + config.set('config', 'GEN_VX_MASK_INPUT_DIR', get_test_data_dir('fcst')) + config.set('config', 'GEN_VX_MASK_INPUT_TEMPLATE', + '{init?fmt=%Y%m%d}/{init?fmt=%Y%m%d_i%H}_f{lead?fmt=%3H}_HRRRTLE_PHPT.grb2') + + config.set('config', 'GEN_VX_MASK_INPUT_MASK_DIR', get_test_data_dir('obs')) + config.set('config', 'GEN_VX_MASK_INPUT_MASK_TEMPLATE', + '{valid?fmt=%Y%m%d}/qpe_{valid?fmt=%Y%m%d%H}_A06.nc,{valid?fmt=%Y%m%d}/qpe_{valid?fmt=%Y%m%d%H}_A06.nc') + config.set('config', 'GEN_VX_MASK_OUTPUT_TEMPLATE', '{OUTPUT_BASE}/GenVxMask/test.nc') + + wrapper = GenVxMaskWrapper(config) + assert wrapper.isOK + + all_cmds = wrapper.run_all_times() + for cmd, _ in all_cmds: + print(cmd) + + print(f'missing: {wrapper.missing_input_count} / {wrapper.run_count}, errors: {wrapper.errors}') + assert wrapper.missing_input_count == missing + assert wrapper.run_count == run + assert wrapper.errors == errors + + @pytest.mark.wrapper def test_run_gen_vx_mask_once(metplus_config): input_dict = {'valid': datetime.datetime.strptime("201802010000",'%Y%m%d%H%M'), diff --git a/internal/tests/pytests/wrappers/grid_diag/test_grid_diag.py b/internal/tests/pytests/wrappers/grid_diag/test_grid_diag.py index 40c6e2eee2..9971071229 100644 --- a/internal/tests/pytests/wrappers/grid_diag/test_grid_diag.py +++ b/internal/tests/pytests/wrappers/grid_diag/test_grid_diag.py @@ -58,6 +58,49 @@ def set_minimum_config_settings(config): config.set('config', 'BOTH_VAR2_OPTIONS', data_options_2) +@pytest.mark.parametrize( + 'missing, run, thresh, errors, allow_missing, runtime_freq', [ + (0, 1, 0.5, 0, True, 'RUN_ONCE'), + (0, 1, 0.5, 0, False, 'RUN_ONCE'), + (0, 2, 0.5, 0, True, 'RUN_ONCE_PER_INIT_OR_VALID'), + (0, 2, 0.5, 0, False, 'RUN_ONCE_PER_INIT_OR_VALID'), + (2, 7, 1.0, 1, True, 'RUN_ONCE_PER_LEAD'), + (2, 7, 1.0, 2, False, 'RUN_ONCE_PER_LEAD'), + (8, 14, 1.0, 1, True, 'RUN_ONCE_FOR_EACH'), + (8, 14, 1.0, 8, False, 'RUN_ONCE_FOR_EACH'), + ] +) +@pytest.mark.wrapper +def test_grid_diag_missing_inputs(metplus_config, get_test_data_dir, + missing, run, thresh, errors, allow_missing, + runtime_freq): + config = metplus_config + set_minimum_config_settings(config) + config.set('config', 'INPUT_MUST_EXIST', True) + config.set('config', 'GRID_DIAG_ALLOW_MISSING_INPUTS', allow_missing) + config.set('config', 'GRID_DIAG_INPUT_THRESH', thresh) + config.set('config', 'GRID_DIAG_RUNTIME_FREQ', runtime_freq) + config.set('config', 'INIT_BEG', '2017051001') + config.set('config', 'INIT_END', '2017051003') + config.set('config', 'INIT_INCREMENT', '2H') + config.set('config', 'LEAD_SEQ', '1,2,3,6,9,12,15') + config.set('config', 'GRID_DIAG_INPUT_DIR', get_test_data_dir('fcst')) + config.set('config', 'GRID_DIAG_INPUT_TEMPLATE', + '{init?fmt=%Y%m%d}/{init?fmt=%Y%m%d_i%H}_f{lead?fmt=%3H}_HRRRTLE_PHPT.grb2') + + wrapper = GridDiagWrapper(config) + assert wrapper.isOK + + all_cmds = wrapper.run_all_times() + for cmd, _ in all_cmds: + print(cmd) + + print(f'missing: {wrapper.missing_input_count} / {wrapper.run_count}, errors: {wrapper.errors}') + assert wrapper.missing_input_count == missing + assert wrapper.run_count == run + assert wrapper.errors == errors + + @pytest.mark.parametrize( 'time_info, expected_subset', [ # all files @@ -141,29 +184,31 @@ def test_get_all_files_and_subset(metplus_config, time_info, expected_subset): ('20141101093015', '20141101093015', '000'), ('20141101093015', '20141102093015', '024')]: filename = f'init_{init}_valid_{valid}_lead_{lead}.nc' - expected_files.append(os.path.join(input_dir, - filename)) + expected_files.append(os.path.join(input_dir, filename)) wrapper = GridDiagWrapper(config) - assert(wrapper.get_all_files()) + wrapper.c_dict['ALL_FILES'] = wrapper.get_all_files() # convert list of lists into a single list to compare to expected results actual_files = [item['input0'] for item in wrapper.c_dict['ALL_FILES']] actual_files = [item for sub in actual_files for item in sub] - assert(actual_files == expected_files) + assert actual_files == expected_files file_list_dict = wrapper.subset_input_files(time_info) assert file_list_dict - with open(file_list_dict['input0'], 'r') as file_handle: - file_list = file_handle.readlines() + if len(expected_subset) == 1: + file_list = [file_list_dict['input0']] + else: + with open(file_list_dict['input0'], 'r') as file_handle: + file_list = file_handle.readlines() - file_list = file_list[1:] - assert(len(file_list) == len(expected_subset)) + file_list = file_list[1:] + assert len(file_list) == len(expected_subset) for actual_file, expected_file in zip(file_list, expected_subset): actual_file = actual_file.strip() - assert(os.path.basename(actual_file) == expected_file) + assert os.path.basename(actual_file) == expected_file @pytest.mark.parametrize( diff --git a/internal/tests/pytests/wrappers/grid_stat/test_grid_stat_wrapper.py b/internal/tests/pytests/wrappers/grid_stat/test_grid_stat_wrapper.py index 15e4b0ce23..40413a3db9 100644 --- a/internal/tests/pytests/wrappers/grid_stat/test_grid_stat_wrapper.py +++ b/internal/tests/pytests/wrappers/grid_stat/test_grid_stat_wrapper.py @@ -56,6 +56,55 @@ def set_minimum_config_settings(config): config.set('config', 'BOTH_VAR1_THRESH', both_thresh) +@pytest.mark.parametrize( + 'once_per_field, missing, run, thresh, errors, allow_missing', [ + (False, 6, 12, 0.5, 0, True), + (False, 6, 12, 0.6, 1, True), + (True, 12, 24, 0.5, 0, True), + (True, 12, 24, 0.6, 1, True), + (False, 6, 12, 0.5, 6, False), + (True, 12, 24, 0.5, 12, False), + ] +) +@pytest.mark.wrapper_b +def test_grid_stat_missing_inputs(metplus_config, get_test_data_dir, + once_per_field, missing, run, thresh, errors, + allow_missing): + config = metplus_config + set_minimum_config_settings(config) + config.set('config', 'INPUT_MUST_EXIST', True) + config.set('config', 'GRID_STAT_ALLOW_MISSING_INPUTS', allow_missing) + config.set('config', 'GRID_STAT_INPUT_THRESH', thresh) + config.set('config', 'INIT_BEG', '2017051001') + config.set('config', 'INIT_END', '2017051003') + config.set('config', 'INIT_INCREMENT', '2H') + config.set('config', 'LEAD_SEQ', '1,2,3,6,9,12') + config.set('config', 'FCST_GRID_STAT_INPUT_DIR', get_test_data_dir('fcst')) + config.set('config', 'OBS_GRID_STAT_INPUT_DIR', get_test_data_dir('obs')) + config.set('config', 'FCST_GRID_STAT_INPUT_TEMPLATE', + '{init?fmt=%Y%m%d}/{init?fmt=%Y%m%d_i%H}_f{lead?fmt=%3H}_HRRRTLE_PHPT.grb2') + config.set('config', 'OBS_GRID_STAT_INPUT_TEMPLATE', + '{valid?fmt=%Y%m%d}/qpe_{valid?fmt=%Y%m%d%H}_A06.nc') + # add 2nd set of fields to test ONCE_PER_FIELD + config.set('config', 'FCST_VAR2_NAME', fcst_name) + config.set('config', 'FCST_VAR2_LEVELS', fcst_level) + config.set('config', 'OBS_VAR2_NAME', obs_name) + config.set('config', 'OBS_VAR2_LEVELS', obs_level) + config.set('config', 'GRID_STAT_ONCE_PER_FIELD', once_per_field) + + wrapper = GridStatWrapper(config) + assert wrapper.isOK + + all_cmds = wrapper.run_all_times() + for cmd, _ in all_cmds: + print(cmd) + + print(f'missing: {wrapper.missing_input_count} / {wrapper.run_count}, errors: {wrapper.errors}') + assert wrapper.missing_input_count == missing + assert wrapper.run_count == run + assert wrapper.errors == errors + + @pytest.mark.parametrize( 'config_overrides, expected_values', [ # 0 generic FCST is prob diff --git a/internal/tests/pytests/wrappers/ioda2nc/test_ioda2nc_wrapper.py b/internal/tests/pytests/wrappers/ioda2nc/test_ioda2nc_wrapper.py index 70b4936428..0a114624af 100644 --- a/internal/tests/pytests/wrappers/ioda2nc/test_ioda2nc_wrapper.py +++ b/internal/tests/pytests/wrappers/ioda2nc/test_ioda2nc_wrapper.py @@ -30,6 +30,55 @@ def set_minimum_config_settings(config): 'ioda.NC001007.{valid?fmt=%Y%m%d%H}.summary.nc') +@pytest.mark.parametrize( + 'missing, run, thresh, errors, allow_missing, runtime_freq', [ + (16, 24, 0.3, 0, True, 'RUN_ONCE_FOR_EACH'), + (16, 24, 0.7, 1, True, 'RUN_ONCE_FOR_EACH'), + (16, 24, 0.3, 16, False, 'RUN_ONCE_FOR_EACH'), + (2, 4, 0.4, 0, True, 'RUN_ONCE_PER_INIT_OR_VALID'), + (2, 4, 0.6, 1, True, 'RUN_ONCE_PER_INIT_OR_VALID'), + (2, 4, 0.6, 2, False, 'RUN_ONCE_PER_INIT_OR_VALID'), + (2, 5, 0.4, 0, True, 'RUN_ONCE_PER_LEAD'), + (2, 5, 0.7, 1, True, 'RUN_ONCE_PER_LEAD'), + (2, 5, 0.4, 2, False, 'RUN_ONCE_PER_LEAD'), + (0, 1, 0.4, 0, True, 'RUN_ONCE'), + (0, 1, 0.4, 0, False, 'RUN_ONCE'), + ] +) +@pytest.mark.wrapper +def test_ioda2nc_missing_inputs(metplus_config, get_test_data_dir, missing, + run, thresh, errors, allow_missing, runtime_freq): + config = metplus_config + config.set('config', 'DO_NOT_RUN_EXE', True) + config.set('config', 'INPUT_MUST_EXIST', True) + config.set('config', 'IODA2NC_ALLOW_MISSING_INPUTS', allow_missing) + config.set('config', 'IODA2NC_INPUT_THRESH', thresh) + config.set('config', 'IODA2NC_RUNTIME_FREQ', runtime_freq) + config.set('config', 'LOOP_BY', 'INIT') + config.set('config', 'INIT_TIME_FMT', '%Y%m%d%H') + config.set('config', 'INIT_LIST', '2017051001, 2017051003, 2017051201, 2017051203') + if runtime_freq == 'RUN_ONCE_PER_LEAD': + config.set('config', 'LEAD_SEQ', '6,9,12,15,18') + else: + config.set('config', 'LEAD_SEQ', '1,2,3,6,9,12') + config.set('config', 'IODA2NC_INPUT_DIR', get_test_data_dir('obs')) + config.set('config', 'IODA2NC_INPUT_TEMPLATE', + '{valid?fmt=%Y%m%d}/qpe_{valid?fmt=%Y%m%d%H}_A06.nc') + config.set('config', 'IODA2NC_OUTPUT_TEMPLATE', '{OUTPUT_BASE}/IODA2NC/output/test.nc') + + wrapper = IODA2NCWrapper(config) + assert wrapper.isOK + + all_cmds = wrapper.run_all_times() + for cmd, _ in all_cmds: + print(cmd) + + print(f'missing: {wrapper.missing_input_count} / {wrapper.run_count}, errors: {wrapper.errors}') + assert wrapper.missing_input_count == missing + assert wrapper.run_count == run + assert wrapper.errors == errors + + @pytest.mark.parametrize( 'config_overrides, env_var_values, extra_args', [ # 0 diff --git a/internal/tests/pytests/wrappers/mode/test_mode_wrapper.py b/internal/tests/pytests/wrappers/mode/test_mode_wrapper.py index 285abab96c..f510d524f2 100644 --- a/internal/tests/pytests/wrappers/mode/test_mode_wrapper.py +++ b/internal/tests/pytests/wrappers/mode/test_mode_wrapper.py @@ -58,6 +58,45 @@ def set_minimum_config_settings(config): config.set('config', 'OBS_VAR1_LEVELS', obs_level) +@pytest.mark.parametrize( + 'missing, run, thresh, errors, allow_missing', [ + (6, 12, 0.5, 0, True), + (6, 12, 0.6, 1, True), + (6, 12, 0.5, 6, False), + ] +) +@pytest.mark.wrapper_a +def test_mode_missing_inputs(metplus_config, get_test_data_dir, + missing, run, thresh, errors, allow_missing): + config = metplus_config + set_minimum_config_settings(config) + config.set('config', 'INPUT_MUST_EXIST', True) + config.set('config', 'MODE_ALLOW_MISSING_INPUTS', allow_missing) + config.set('config', 'MODE_INPUT_THRESH', thresh) + config.set('config', 'INIT_BEG', '2017051001') + config.set('config', 'INIT_END', '2017051003') + config.set('config', 'INIT_INCREMENT', '2H') + config.set('config', 'LEAD_SEQ', '1,2,3,6,9,12') + config.set('config', 'FCST_MODE_INPUT_DIR', get_test_data_dir('fcst')) + config.set('config', 'OBS_MODE_INPUT_DIR', get_test_data_dir('obs')) + config.set('config', 'FCST_MODE_INPUT_TEMPLATE', + '{init?fmt=%Y%m%d}/{init?fmt=%Y%m%d_i%H}_f{lead?fmt=%3H}_HRRRTLE_PHPT.grb2') + config.set('config', 'OBS_MODE_INPUT_TEMPLATE', + '{valid?fmt=%Y%m%d}/qpe_{valid?fmt=%Y%m%d%H}_A06.nc') + + wrapper = MODEWrapper(config) + assert wrapper.isOK + + all_cmds = wrapper.run_all_times() + for cmd, _ in all_cmds: + print(cmd) + + print(f'missing: {wrapper.missing_input_count} / {wrapper.run_count}, errors: {wrapper.errors}') + assert wrapper.missing_input_count == missing + assert wrapper.run_count == run + assert wrapper.errors == errors + + @pytest.mark.parametrize( 'config_overrides, env_var_values', [ ({'MODEL': 'my_model'}, diff --git a/internal/tests/pytests/wrappers/mtd/test_mtd_wrapper.py b/internal/tests/pytests/wrappers/mtd/test_mtd_wrapper.py index 5c54a41ef7..c9267afc76 100644 --- a/internal/tests/pytests/wrappers/mtd/test_mtd_wrapper.py +++ b/internal/tests/pytests/wrappers/mtd/test_mtd_wrapper.py @@ -21,13 +21,6 @@ f'level="{obs_level_no_quotes}"; cat_thresh=[ gt12.7 ]; }};') -def get_test_data_dir(subdir): - internal_tests_dir = os.path.abspath( - os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir) - ) - return os.path.join(internal_tests_dir, 'data', subdir) - - def mtd_wrapper(metplus_config, config_overrides): """! Returns a default MTDWrapper with /path/to entries in the metplus_system.conf and metplus_runtime.conf configuration @@ -47,7 +40,7 @@ def mtd_wrapper(metplus_config, config_overrides): return MTDWrapper(config) -def set_minimum_config_settings(config): +def set_minimum_config_settings(config, set_inputs=True): # set config variables to prevent command from running and bypass check # if input files actually exist config.set('config', 'DO_NOT_RUN_EXE', True) @@ -63,12 +56,13 @@ def set_minimum_config_settings(config): config.set('config', 'LEAD_SEQ', '6H, 9H, 12H') config.set('config', 'MTD_CONFIG_FILE', '{PARM_BASE}/met_config/MTDConfig_wrapped') - config.set('config', 'FCST_MTD_INPUT_DIR', fcst_dir) - config.set('config', 'OBS_MTD_INPUT_DIR', obs_dir) - config.set('config', 'FCST_MTD_INPUT_TEMPLATE', - '{init?fmt=%Y%m%d%H}/fcst_file_F{lead?fmt=%3H}') - config.set('config', 'OBS_MTD_INPUT_TEMPLATE', - '{valid?fmt=%Y%m%d%H}/obs_file') + if set_inputs: + config.set('config', 'FCST_MTD_INPUT_DIR', fcst_dir) + config.set('config', 'OBS_MTD_INPUT_DIR', obs_dir) + config.set('config', 'FCST_MTD_INPUT_TEMPLATE', + '{init?fmt=%Y%m%d%H}/fcst_file_F{lead?fmt=%3H}') + config.set('config', 'OBS_MTD_INPUT_TEMPLATE', + '{valid?fmt=%Y%m%d%H}/obs_file') config.set('config', 'MTD_OUTPUT_DIR', '{OUTPUT_BASE}/MTD/output') config.set('config', 'MTD_OUTPUT_TEMPLATE', '{valid?fmt=%Y%m%d%H}') @@ -81,6 +75,59 @@ def set_minimum_config_settings(config): config.set('config', 'OBS_VAR1_THRESH', obs_thresh) +@pytest.mark.parametrize( + 'missing, run, thresh, errors, allow_missing, inputs', [ + (1, 3, 0.3, 0, True, 'CHOCOLATE'), + (1, 3, 0.3, 0, True, 'BOTH'), + (1, 3, 0.8, 1, True, 'BOTH'), + (1, 3, 0.8, 1, False, 'BOTH'), + (1, 3, 0.3, 0, True, 'FCST'), + (1, 3, 0.8, 1, True, 'FCST'), + (1, 3, 0.8, 1, False, 'FCST'), + (1, 3, 0.3, 0, True, 'OBS'), + (1, 3, 0.8, 1, True, 'OBS'), + (1, 3, 0.8, 1, False, 'OBS'), + ] +) +@pytest.mark.wrapper_a +def test_mtd_missing_inputs(metplus_config, get_test_data_dir, + missing, run, thresh, errors, allow_missing, inputs): + config = metplus_config + set_minimum_config_settings(config, set_inputs=False) + config.set('config', 'INPUT_MUST_EXIST', True) + config.set('config', 'MTD_ALLOW_MISSING_INPUTS', allow_missing) + config.set('config', 'MTD_INPUT_THRESH', thresh) + config.set('config', 'INIT_LIST', '2017051001, 2017051003, 2017051303') + config.set('config', 'LEAD_SEQ', '1,2,3,6,9,12') + if inputs in ('BOTH', 'FCST'): + config.set('config', 'FCST_MTD_INPUT_DIR', get_test_data_dir('fcst')) + config.set('config', 'FCST_MTD_INPUT_TEMPLATE', + '{init?fmt=%Y%m%d}/{init?fmt=%Y%m%d_i%H}_f{lead?fmt=%3H}_HRRRTLE_PHPT.grb2') + if inputs in ('BOTH', 'OBS'): + config.set('config', 'OBS_MTD_INPUT_DIR', get_test_data_dir('obs')) + config.set('config', 'OBS_MTD_INPUT_TEMPLATE', + '{valid?fmt=%Y%m%d}/qpe_{valid?fmt=%Y%m%d%H}_A06.nc') + if inputs != 'BOTH': + config.set('config', 'MTD_SINGLE_RUN', True) + config.set('config', 'MTD_SINGLE_DATA_SRC', inputs) + + wrapper = MTDWrapper(config) + if inputs == 'CHOCOLATE': + assert not wrapper.isOK + return + + assert wrapper.isOK + + all_cmds = wrapper.run_all_times() + for cmd, _ in all_cmds: + print(cmd) + + print(f'missing: {wrapper.missing_input_count} / {wrapper.run_count}, errors: {wrapper.errors}') + assert wrapper.missing_input_count == missing + assert wrapper.run_count == run + assert wrapper.errors == errors + + @pytest.mark.parametrize( 'config_overrides, env_var_values', [ ({'MODEL': 'my_model'}, @@ -209,7 +256,7 @@ def test_mode_single_field(metplus_config, config_overrides, env_var_values): @pytest.mark.wrapper -def test_mtd_by_init_all_found(metplus_config): +def test_mtd_by_init_all_found(metplus_config, get_test_data_dir): obs_data_dir = get_test_data_dir('obs') fcst_data_dir = get_test_data_dir('fcst') overrides = { @@ -247,7 +294,7 @@ def test_mtd_by_init_all_found(metplus_config): @pytest.mark.wrapper -def test_mtd_by_valid_all_found(metplus_config): +def test_mtd_by_valid_all_found(metplus_config, get_test_data_dir): obs_data_dir = get_test_data_dir('obs') fcst_data_dir = get_test_data_dir('fcst') overrides = { @@ -285,7 +332,7 @@ def test_mtd_by_valid_all_found(metplus_config): @pytest.mark.wrapper -def test_mtd_by_init_miss_fcst(metplus_config): +def test_mtd_by_init_miss_fcst(metplus_config, get_test_data_dir): obs_data_dir = get_test_data_dir('obs') fcst_data_dir = get_test_data_dir('fcst') overrides = { @@ -323,7 +370,7 @@ def test_mtd_by_init_miss_fcst(metplus_config): @pytest.mark.wrapper -def test_mtd_by_init_miss_both(metplus_config): +def test_mtd_by_init_miss_both(metplus_config, get_test_data_dir): obs_data_dir = get_test_data_dir('obs') fcst_data_dir = get_test_data_dir('fcst') overrides = { @@ -359,7 +406,7 @@ def test_mtd_by_init_miss_both(metplus_config): @pytest.mark.wrapper -def test_mtd_single(metplus_config): +def test_mtd_single(metplus_config, get_test_data_dir): fcst_data_dir = get_test_data_dir('fcst') overrides = { 'LEAD_SEQ': '1, 2, 3', diff --git a/internal/tests/pytests/wrappers/pb2nc/test_pb2nc_wrapper.py b/internal/tests/pytests/wrappers/pb2nc/test_pb2nc_wrapper.py index 64bea4e074..4c596f8761 100644 --- a/internal/tests/pytests/wrappers/pb2nc/test_pb2nc_wrapper.py +++ b/internal/tests/pytests/wrappers/pb2nc/test_pb2nc_wrapper.py @@ -10,6 +10,9 @@ from metplus.util import time_util from metplus.util import do_string_sub +valid_beg = '20141031_18' +valid_end = '20141031_23' + def pb2nc_wrapper(metplus_config): """! Returns a default PB2NCWrapper with /path/to entries in the @@ -23,6 +26,55 @@ def pb2nc_wrapper(metplus_config): return PB2NCWrapper(config) +@pytest.mark.parametrize( + 'missing, run, thresh, errors, allow_missing, runtime_freq', [ + (16, 24, 0.3, 0, True, 'RUN_ONCE_FOR_EACH'), + (16, 24, 0.7, 1, True, 'RUN_ONCE_FOR_EACH'), + (16, 24, 0.3, 16, False, 'RUN_ONCE_FOR_EACH'), + (2, 4, 0.4, 0, True, 'RUN_ONCE_PER_INIT_OR_VALID'), + (2, 4, 0.6, 1, True, 'RUN_ONCE_PER_INIT_OR_VALID'), + (2, 4, 0.6, 2, False, 'RUN_ONCE_PER_INIT_OR_VALID'), + (2, 5, 0.4, 0, True, 'RUN_ONCE_PER_LEAD'), + (2, 5, 0.7, 1, True, 'RUN_ONCE_PER_LEAD'), + (2, 5, 0.4, 2, False, 'RUN_ONCE_PER_LEAD'), + (0, 1, 0.4, 0, True, 'RUN_ONCE'), + (0, 1, 0.4, 0, False, 'RUN_ONCE'), + ] +) +@pytest.mark.wrapper +def test_pb2nc_missing_inputs(metplus_config, get_test_data_dir, missing, + run, thresh, errors, allow_missing, runtime_freq): + config = metplus_config + config.set('config', 'DO_NOT_RUN_EXE', True) + config.set('config', 'INPUT_MUST_EXIST', True) + config.set('config', 'PB2NC_ALLOW_MISSING_INPUTS', allow_missing) + config.set('config', 'PB2NC_INPUT_THRESH', thresh) + config.set('config', 'PB2NC_RUNTIME_FREQ', runtime_freq) + config.set('config', 'LOOP_BY', 'INIT') + config.set('config', 'INIT_TIME_FMT', '%Y%m%d%H') + config.set('config', 'INIT_LIST', '2017051001, 2017051003, 2017051201, 2017051203') + if runtime_freq == 'RUN_ONCE_PER_LEAD': + config.set('config', 'LEAD_SEQ', '6,9,12,15,18') + else: + config.set('config', 'LEAD_SEQ', '1,2,3,6,9,12') + config.set('config', 'PB2NC_INPUT_DIR', get_test_data_dir('obs')) + config.set('config', 'PB2NC_INPUT_TEMPLATE', + '{valid?fmt=%Y%m%d}/qpe_{valid?fmt=%Y%m%d%H}_A06.nc') + config.set('config', 'PB2NC_OUTPUT_TEMPLATE', '{OUTPUT_BASE}/PB2NC/output/test.nc') + + wrapper = PB2NCWrapper(config) + assert wrapper.isOK + + all_cmds = wrapper.run_all_times() + for cmd, _ in all_cmds: + print(cmd) + + print(f'missing: {wrapper.missing_input_count} / {wrapper.run_count}, errors: {wrapper.errors}') + assert wrapper.missing_input_count == missing + assert wrapper.run_count == run + assert wrapper.errors == errors + + # --------------------- # test_get_command # test that command is generated correctly @@ -30,7 +82,6 @@ def pb2nc_wrapper(metplus_config): @pytest.mark.parametrize( # list of input files 'infiles', [ - [], ['file1'], ['file1', 'file2'], ['file1', 'file2', 'file3'], @@ -92,15 +143,16 @@ def test_find_input_files(metplus_config, offsets, offset_to_find): create_fullpath = os.path.join(fake_input_dir, create_file) open(create_fullpath, 'a').close() - # unset offset in time dictionary so it will be computed del input_dict['offset'] # set offset list pb.c_dict['OFFSETS'] = offsets + pb.c_dict['ALL_FILES'] = pb.get_all_files_for_each(input_dict) + # look for input files based on offset list - result = pb.find_input_files(input_dict) + result = pb.find_input_files() # check if correct offset file was found, if None expected, check against None if offset_to_find is None: @@ -226,12 +278,14 @@ def test_find_input_files(metplus_config, offsets, offset_to_find): 'vld_thresh = 0.1;}')}), ({'PB2NC_OBS_BUFR_MAP': '{key="POB"; val="PRES"; },{key="QOB"; val="SPFH";}', }, {'METPLUS_OBS_BUFR_MAP': 'obs_bufr_map = [{key="POB"; val="PRES"; }, {key="QOB"; val="SPFH";}];'}), + ({'PB2NC_VALID_BEGIN': valid_beg}, {}), + ({'PB2NC_VALID_END': valid_end}, {}), + ({'PB2NC_VALID_BEGIN': valid_beg, 'PB2NC_VALID_END': valid_end}, {}), ] ) @pytest.mark.wrapper -def test_pb2nc_all_fields(metplus_config, config_overrides, - env_var_values): +def test_pb2nc_all_fields(metplus_config, config_overrides, env_var_values): input_dir = '/some/input/dir' config = metplus_config @@ -249,7 +303,6 @@ def test_pb2nc_all_fields(metplus_config, config_overrides, config.set('config', 'VALID_INCREMENT', '12H') config.set('config', 'LEAD_SEQ', '0') config.set('config', 'PB2NC_OFFSETS', '12') - config.set('config', 'LOOP_ORDER', 'processes') config.set('config', 'PB2NC_CONFIG_FILE', '{PARM_BASE}/met_config/PB2NCConfig_wrapped') @@ -271,14 +324,21 @@ def test_pb2nc_all_fields(metplus_config, config_overrides, verbosity = f"-v {wrapper.c_dict['VERBOSITY']}" config_file = wrapper.c_dict.get('CONFIG_FILE') out_dir = wrapper.c_dict.get('OUTPUT_DIR') + + valid_args = '' + if 'PB2NC_VALID_BEGIN' in config_overrides: + valid_args += f' -valid_beg {valid_beg}' + if 'PB2NC_VALID_END' in config_overrides: + valid_args += f' -valid_end {valid_end}' + expected_cmds = [(f"{app_path} {verbosity} " f"{input_dir}/ndas.t00z.prepbufr.tm12.20070401.nr " f"{out_dir}/2007033112.nc " - f"{config_file}"), + f"{config_file}{valid_args}"), (f"{app_path} {verbosity} " f"{input_dir}/ndas.t12z.prepbufr.tm12.20070401.nr " f"{out_dir}/2007040100.nc " - f"{config_file}"), + f"{config_file}{valid_args}"), ] all_cmds = wrapper.run_all_times() diff --git a/internal/tests/pytests/wrappers/plot_data_plane/test_plot_data_plane_wrapper.py b/internal/tests/pytests/wrappers/plot_data_plane/test_plot_data_plane_wrapper.py new file mode 100644 index 0000000000..2102e9aa8a --- /dev/null +++ b/internal/tests/pytests/wrappers/plot_data_plane/test_plot_data_plane_wrapper.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 + +import pytest + +import os + +from metplus.wrappers.plot_data_plane_wrapper import PlotDataPlaneWrapper + +obs_dir = '/some/path/obs' +input_template_one = ( + 'pb2nc/ndas.{valid?fmt=%Y%m%d}.t{valid?fmt=%H}z.prepbufr.tm00.nc' +) +input_template_two = ( + 'pb2nc/ndas.{valid?fmt=%Y%m%d}.t{valid?fmt=%H}z.prepbufr.tm00.nc,' + 'ascii2nc/trmm_{valid?fmt=%Y%m%d%H}_3hr.nc' +) + +grid_dir = '/some/path/grid' +grid_template = 'nam_{init?fmt=%Y%m%d%H}_F{lead?fmt=%3H}.grib2' + +output_dir = '{OUTPUT_BASE}/plot_point_obs' +output_template = 'nam_and_ndas.{valid?fmt=%Y%m%d}.t{valid?fmt=%H}z.prepbufr_CONFIG.ps' + +title = 'NAM 2012040900 F12 vs NDAS 500mb RH and TRMM 3h > 0' + +point_data = ['{msg_typ = "ADPSFC";obs_gc = 61;obs_thresh = > 0.0;' + 'fill_color = [0,0,255];}', + '{msg_typ = "ADPSFC";obs_var = "RH";' + 'fill_color = [100,100,100];}'] +point_data_input = ', '.join(point_data) +point_data_format = f"[{point_data_input}];" + +time_fmt = '%Y%m%d%H' +run_times = ['2012040912', '2012041000'] + + +@pytest.mark.parametrize( + 'missing, run, thresh, errors, allow_missing', [ + (6, 12, 0.5, 0, True), + (6, 12, 0.6, 1, True), + (6, 12, 0.5, 6, False), + ] +) +@pytest.mark.wrapper_c +def test_plot_data_plane_missing_inputs(metplus_config, get_test_data_dir, + missing, run, thresh, errors, + allow_missing): + config = metplus_config + config.set('config', 'DO_NOT_RUN_EXE', True) + config.set('config', 'INPUT_MUST_EXIST', True) + + config.set('config', 'PROCESS_LIST', 'PlotDataPlane') + config.set('config', 'PLOT_DATA_PLANE_ALLOW_MISSING_INPUTS', allow_missing) + config.set('config', 'PLOT_DATA_PLANE_INPUT_THRESH', thresh) + config.set('config', 'LOOP_BY', 'INIT') + config.set('config', 'INIT_TIME_FMT', '%Y%m%d%H') + config.set('config', 'INIT_BEG', '2017051001') + config.set('config', 'INIT_END', '2017051003') + config.set('config', 'INIT_INCREMENT', '2H') + config.set('config', 'LEAD_SEQ', '1,2,3,6,9,12') + config.set('config', 'PLOT_DATA_PLANE_INPUT_DIR', get_test_data_dir('fcst')) + config.set('config', 'PLOT_DATA_PLANE_INPUT_TEMPLATE', + '{init?fmt=%Y%m%d}/{init?fmt=%Y%m%d_i%H}_f{lead?fmt=%3H}_HRRRTLE_PHPT.grb2') + config.set('config', 'PLOT_DATA_PLANE_OUTPUT_TEMPLATE', + '{OUTPUT_BASE}/{init?fmt=%Y%m%d_i%H}_f{lead?fmt=%3H}_HRRRTLE_PHPT.ps') + config.set('config', 'PLOT_DATA_PLANE_FIELD_NAME', 'APCP_12') + + wrapper = PlotDataPlaneWrapper(config) + assert wrapper.isOK + + all_cmds = wrapper.run_all_times() + for cmd, _ in all_cmds: + print(cmd) + + print(f'missing: {wrapper.missing_input_count} / {wrapper.run_count}, errors: {wrapper.errors}') + assert wrapper.missing_input_count == missing + assert wrapper.run_count == run + assert wrapper.errors == errors diff --git a/internal/tests/pytests/wrappers/plot_point_obs/test_plot_point_obs_wrapper.py b/internal/tests/pytests/wrappers/plot_point_obs/test_plot_point_obs_wrapper.py index 1e416cc6c6..e69a324418 100644 --- a/internal/tests/pytests/wrappers/plot_point_obs/test_plot_point_obs_wrapper.py +++ b/internal/tests/pytests/wrappers/plot_point_obs/test_plot_point_obs_wrapper.py @@ -4,9 +4,6 @@ import os -from datetime import datetime - - from metplus.wrappers.plot_point_obs_wrapper import PlotPointObsWrapper obs_dir = '/some/path/obs' @@ -57,6 +54,45 @@ def set_minimum_config_settings(config): config.set('config', 'PLOT_POINT_OBS_OUTPUT_TEMPLATE', output_template) +@pytest.mark.parametrize( + 'missing, run, thresh, errors, allow_missing', [ + (6, 12, 0.5, 0, True), + (6, 12, 0.6, 1, True), + (6, 12, 0.5, 6, False), + ] +) +@pytest.mark.wrapper_c +def test_plot_point_obs_missing_inputs(metplus_config, get_test_data_dir, + missing, run, thresh, errors, + allow_missing): + config = metplus_config + set_minimum_config_settings(config) + config.set('config', 'INPUT_MUST_EXIST', True) + config.set('config', 'PLOT_POINT_OBS_ALLOW_MISSING_INPUTS', allow_missing) + config.set('config', 'PLOT_POINT_OBS_INPUT_THRESH', thresh) + config.set('config', 'LOOP_BY', 'INIT') + config.set('config', 'INIT_TIME_FMT', '%Y%m%d%H') + config.set('config', 'INIT_BEG', '2017051001') + config.set('config', 'INIT_END', '2017051003') + config.set('config', 'INIT_INCREMENT', '2H') + config.set('config', 'LEAD_SEQ', '1,2,3,6,9,12') + config.set('config', 'PLOT_POINT_OBS_INPUT_DIR', get_test_data_dir('fcst')) + config.set('config', 'PLOT_POINT_OBS_INPUT_TEMPLATE', + '{init?fmt=%Y%m%d}/{init?fmt=%Y%m%d_i%H}_f{lead?fmt=%3H}_HRRRTLE_PHPT.grb2') + + wrapper = PlotPointObsWrapper(config) + assert wrapper.isOK + + all_cmds = wrapper.run_all_times() + for cmd, _ in all_cmds: + print(cmd) + + print(f'missing: {wrapper.missing_input_count} / {wrapper.run_count}, errors: {wrapper.errors}') + assert wrapper.missing_input_count == missing + assert wrapper.run_count == run + assert wrapper.errors == errors + + @pytest.mark.parametrize( 'config_overrides, env_var_values', [ # 0: no additional settings diff --git a/internal/tests/pytests/wrappers/point2grid/test_point2grid.py b/internal/tests/pytests/wrappers/point2grid/test_point2grid.py index 03184b80bf..bd4b0149d7 100644 --- a/internal/tests/pytests/wrappers/point2grid/test_point2grid.py +++ b/internal/tests/pytests/wrappers/point2grid/test_point2grid.py @@ -41,6 +41,43 @@ def set_minimum_config_settings(config): config.set('config', 'POINT2GRID_INPUT_FIELD', input_name) +@pytest.mark.parametrize( + 'missing, run, thresh, errors, allow_missing', [ + (6, 12, 0.5, 0, True), + (6, 12, 0.6, 1, True), + (6, 12, 0.5, 6, False), + ] +) +@pytest.mark.wrapper +def test_point2grid_missing_inputs(metplus_config, get_test_data_dir, + missing, run, thresh, errors, + allow_missing): + config = metplus_config + set_minimum_config_settings(config) + config.set('config', 'INPUT_MUST_EXIST', True) + config.set('config', 'POINT2GRID_ALLOW_MISSING_INPUTS', allow_missing) + config.set('config', 'POINT2GRID_INPUT_THRESH', thresh) + config.set('config', 'INIT_BEG', '2017051001') + config.set('config', 'INIT_END', '2017051003') + config.set('config', 'INIT_INCREMENT', '2H') + config.set('config', 'LEAD_SEQ', '1,2,3,6,9,12') + config.set('config', 'POINT2GRID_INPUT_DIR', get_test_data_dir('fcst')) + config.set('config', 'POINT2GRID_INPUT_TEMPLATE', + '{init?fmt=%Y%m%d}/{init?fmt=%Y%m%d_i%H}_f{lead?fmt=%3H}_HRRRTLE_PHPT.grb2') + + wrapper = Point2GridWrapper(config) + assert wrapper.isOK + + all_cmds = wrapper.run_all_times() + for cmd, _ in all_cmds: + print(cmd) + + print(f'missing: {wrapper.missing_input_count} / {wrapper.run_count}, errors: {wrapper.errors}') + assert wrapper.missing_input_count == missing + assert wrapper.run_count == run + assert wrapper.errors == errors + + @pytest.mark.parametrize( 'config_overrides, optional_args', [ ({}, {}), diff --git a/internal/tests/pytests/wrappers/point_stat/test_point_stat_wrapper.py b/internal/tests/pytests/wrappers/point_stat/test_point_stat_wrapper.py index a12441f85a..0d337cb22c 100755 --- a/internal/tests/pytests/wrappers/point_stat/test_point_stat_wrapper.py +++ b/internal/tests/pytests/wrappers/point_stat/test_point_stat_wrapper.py @@ -10,6 +10,11 @@ fcst_dir = '/some/path/fcst' obs_dir = '/some/path/obs' +fcst_name = 'APCP' +fcst_level = 'A03' +obs_name = 'APCP_03' +obs_level = '"(*,*)"' + inits = ['2005080700', '2005080712'] time_fmt = '%Y%m%d%H' lead_hour = 12 @@ -49,6 +54,59 @@ def set_minimum_config_settings(config): config.set('config', 'POINT_STAT_OUTPUT_TEMPLATE', '{valid?fmt=%Y%m%d%H}') +@pytest.mark.parametrize( + 'once_per_field, missing, run, thresh, errors, allow_missing', [ + (False, 6, 12, 0.5, 0, True), + (False, 6, 12, 0.6, 1, True), + (True, 12, 24, 0.5, 0, True), + (True, 12, 24, 0.6, 1, True), + (False, 6, 12, 0.5, 6, False), + (True, 12, 24, 0.5, 12, False), + ] +) +@pytest.mark.wrapper_a +def test_point_stat_missing_inputs(metplus_config, get_test_data_dir, + once_per_field, missing, run, thresh, errors, + allow_missing): + config = metplus_config + set_minimum_config_settings(config) + config.set('config', 'INPUT_MUST_EXIST', True) + config.set('config', 'POINT_STAT_ALLOW_MISSING_INPUTS', allow_missing) + config.set('config', 'POINT_STAT_INPUT_THRESH', thresh) + config.set('config', 'INIT_BEG', '2017051001') + config.set('config', 'INIT_END', '2017051003') + config.set('config', 'INIT_INCREMENT', '2H') + config.set('config', 'LEAD_SEQ', '1,2,3,6,9,12') + config.set('config', 'FCST_POINT_STAT_INPUT_DIR', get_test_data_dir('fcst')) + config.set('config', 'OBS_POINT_STAT_INPUT_DIR', get_test_data_dir('obs')) + config.set('config', 'FCST_POINT_STAT_INPUT_TEMPLATE', + '{init?fmt=%Y%m%d}/{init?fmt=%Y%m%d_i%H}_f{lead?fmt=%3H}_HRRRTLE_PHPT.grb2') + config.set('config', 'OBS_POINT_STAT_INPUT_TEMPLATE', + '{valid?fmt=%Y%m%d}/qpe_{valid?fmt=%Y%m%d%H}_A06.nc') + # add 2 sets of fields to test ONCE_PER_FIELD + config.set('config', 'FCST_VAR1_NAME', fcst_name) + config.set('config', 'FCST_VAR1_LEVELS', fcst_level) + config.set('config', 'OBS_VAR1_NAME', obs_name) + config.set('config', 'OBS_VAR1_LEVELS', obs_level) + config.set('config', 'FCST_VAR2_NAME', fcst_name) + config.set('config', 'FCST_VAR2_LEVELS', fcst_level) + config.set('config', 'OBS_VAR2_NAME', obs_name) + config.set('config', 'OBS_VAR2_LEVELS', obs_level) + config.set('config', 'POINT_STAT_ONCE_PER_FIELD', once_per_field) + + wrapper = PointStatWrapper(config) + assert wrapper.isOK + + all_cmds = wrapper.run_all_times() + for cmd, _ in all_cmds: + print(cmd) + + print(f'missing: {wrapper.missing_input_count} / {wrapper.run_count}, errors: {wrapper.errors}') + assert wrapper.missing_input_count == missing + assert wrapper.run_count == run + assert wrapper.errors == errors + + @pytest.mark.wrapper_a def test_met_dictionary_in_var_options(metplus_config): config = metplus_config diff --git a/internal/tests/pytests/wrappers/regrid_data_plane/test_regrid_data_plane.py b/internal/tests/pytests/wrappers/regrid_data_plane/test_regrid_data_plane.py index 0ec4edbfb0..54c4b19ef8 100644 --- a/internal/tests/pytests/wrappers/regrid_data_plane/test_regrid_data_plane.py +++ b/internal/tests/pytests/wrappers/regrid_data_plane/test_regrid_data_plane.py @@ -8,6 +8,11 @@ from metplus.wrappers.regrid_data_plane_wrapper import RegridDataPlaneWrapper from metplus.util import time_util +fcst_name = 'APCP' +fcst_level = 'A03' +obs_name = 'APCP_03' +obs_level = '"(*,*)"' + def rdp_wrapper(metplus_config): """! Returns a default RegridDataPlane with /path/to entries in the @@ -20,6 +25,68 @@ def rdp_wrapper(metplus_config): return RegridDataPlaneWrapper(config) +@pytest.mark.parametrize( + 'once_per_field, missing, run, thresh, errors, allow_missing', [ + (False, 10, 24, 0.5, 0, True), + (False, 10, 24, 0.6, 1, True), + (True, 10, 24, 0.5, 0, True), + (True, 10, 24, 0.6, 1, True), + (False, 10, 24, 0.5, 10, False), + (True, 10, 24, 0.5, 10, False), + ] +) +@pytest.mark.wrapper +def test_regrid_data_plane_missing_inputs(metplus_config, get_test_data_dir, + once_per_field, missing, run, thresh, errors, + allow_missing): + config = metplus_config + + config.set('config', 'INPUT_MUST_EXIST', True) + config.set('config', 'REGRID_DATA_PLANE_ALLOW_MISSING_INPUTS', allow_missing) + config.set('config', 'REGRID_DATA_PLANE_INPUT_THRESH', thresh) + config.set('config', 'LOOP_BY', 'INIT') + config.set('config', 'INIT_TIME_FMT', '%Y%m%d%H') + config.set('config', 'INIT_BEG', '2017051001') + config.set('config', 'INIT_END', '2017051003') + config.set('config', 'INIT_INCREMENT', '2H') + config.set('config', 'LEAD_SEQ', '1,2,3,6,9,12') + config.set('config', 'FCST_REGRID_DATA_PLANE_RUN', True) + config.set('config', 'OBS_REGRID_DATA_PLANE_RUN', True) + config.set('config', 'FCST_REGRID_DATA_PLANE_INPUT_DIR', get_test_data_dir('fcst')) + config.set('config', 'OBS_REGRID_DATA_PLANE_INPUT_DIR', get_test_data_dir('obs')) + config.set('config', 'FCST_REGRID_DATA_PLANE_INPUT_TEMPLATE', + '{init?fmt=%Y%m%d}/{init?fmt=%Y%m%d_i%H}_f{lead?fmt=%3H}_HRRRTLE_PHPT.grb2') + config.set('config', 'OBS_REGRID_DATA_PLANE_INPUT_TEMPLATE', + '{valid?fmt=%Y%m%d}/qpe_{valid?fmt=%Y%m%d%H}_A06.nc') + + config.set('config', 'FCST_REGRID_DATA_PLANE_OUTPUT_TEMPLATE', + '{OUTPUT_BASE}/{init?fmt=%Y%m%d_i%H}_f{lead?fmt=%3H}_HRRRTLE_PHPT.grb2') + config.set('config', 'OBS_REGRID_DATA_PLANE_OUTPUT_TEMPLATE', + '{OUTPUT_BASE}/qpe_{valid?fmt=%Y%m%d%H}_A06.nc') + # add 2nd set of fields to test ONCE_PER_FIELD + config.set('config', 'FCST_VAR1_NAME', fcst_name) + config.set('config', 'FCST_VAR1_LEVELS', fcst_level) + config.set('config', 'OBS_VAR1_NAME', obs_name) + config.set('config', 'OBS_VAR1_LEVELS', obs_level) + config.set('config', 'FCST_VAR2_NAME', fcst_name) + config.set('config', 'FCST_VAR2_LEVELS', fcst_level) + config.set('config', 'OBS_VAR2_NAME', obs_name) + config.set('config', 'OBS_VAR2_LEVELS', obs_level) + config.set('config', 'REGRID_DATA_PLANE_ONCE_PER_FIELD', once_per_field) + + wrapper = RegridDataPlaneWrapper(config) + assert wrapper.isOK + + all_cmds = wrapper.run_all_times() + for cmd, _ in all_cmds: + print(cmd) + + print(f'missing: {wrapper.missing_input_count} / {wrapper.run_count}, errors: {wrapper.errors}') + assert wrapper.missing_input_count == missing + assert wrapper.run_count == run + assert wrapper.errors == errors + + # field info is the input dictionary with name and level info to parse # expected_arg is the argument that should be set by the function # note: did not include OBS because they are handled the same way as FCST diff --git a/internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py b/internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py index 679f888f8f..5589b054a1 100644 --- a/internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py +++ b/internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py @@ -79,7 +79,6 @@ def set_minimum_config_settings(config): config.set('config', 'INIT_END', run_times[-1]) config.set('config', 'INIT_INCREMENT', '12H') config.set('config', 'LEAD_SEQ', '12H') - config.set('config', 'LOOP_ORDER', 'processes') config.set('config', 'SERIES_ANALYSIS_RUNTIME_FREQ', 'RUN_ONCE_PER_INIT_OR_VALID') config.set('config', 'SERIES_ANALYSIS_CONFIG_FILE', @@ -87,11 +86,11 @@ def set_minimum_config_settings(config): config.set('config', 'FCST_SERIES_ANALYSIS_INPUT_DIR', fcst_dir) config.set('config', 'OBS_SERIES_ANALYSIS_INPUT_DIR', obs_dir) config.set('config', 'FCST_SERIES_ANALYSIS_INPUT_TEMPLATE', - '{init?fmt=%Y%m%d%H}/fcst_file_F{lead?fmt=%3H}') + '{init?fmt=%Y%m%d%H}/fcst_file_F{lead?fmt=%3H},{init?fmt=%Y%m%d%H}/fcst_file_F{lead?fmt=%3H}') config.set('config', 'OBS_SERIES_ANALYSIS_INPUT_TEMPLATE', - '{valid?fmt=%Y%m%d%H}/obs_file') + '{valid?fmt=%Y%m%d%H}/obs_file,{valid?fmt=%Y%m%d%H}/obs_file') config.set('config', 'SERIES_ANALYSIS_OUTPUT_DIR', - '{OUTPUT_BASE}/GridStat/output') + '{OUTPUT_BASE}/SeriesAnalysis/output') config.set('config', 'SERIES_ANALYSIS_OUTPUT_TEMPLATE', '{init?fmt=%Y%m%d%H}') @@ -103,6 +102,52 @@ def set_minimum_config_settings(config): config.set('config', 'SERIES_ANALYSIS_STAT_LIST', stat_list) +@pytest.mark.parametrize( + 'missing, run, thresh, errors, allow_missing, runtime_freq', [ + (0, 1, 0.5, 0, True, 'RUN_ONCE'), + (0, 1, 0.5, 0, False, 'RUN_ONCE'), + (0, 2, 0.5, 0, True, 'RUN_ONCE_PER_INIT_OR_VALID'), + (0, 2, 0.5, 0, False, 'RUN_ONCE_PER_INIT_OR_VALID'), + (2, 7, 1.0, 1, True, 'RUN_ONCE_PER_LEAD'), + (2, 7, 1.0, 2, False, 'RUN_ONCE_PER_LEAD'), + (8, 14, 1.0, 1, True, 'RUN_ONCE_FOR_EACH'), + (8, 14, 1.0, 8, False, 'RUN_ONCE_FOR_EACH'), + ] +) +@pytest.mark.wrapper_a +def test_series_analysis_missing_inputs(metplus_config, get_test_data_dir, + missing, run, thresh, errors, allow_missing, + runtime_freq): + config = metplus_config + set_minimum_config_settings(config) + config.set('config', 'INPUT_MUST_EXIST', True) + config.set('config', 'SERIES_ANALYSIS_ALLOW_MISSING_INPUTS', allow_missing) + config.set('config', 'SERIES_ANALYSIS_INPUT_THRESH', thresh) + config.set('config', 'SERIES_ANALYSIS_RUNTIME_FREQ', runtime_freq) + config.set('config', 'INIT_BEG', '2017051001') + config.set('config', 'INIT_END', '2017051003') + config.set('config', 'INIT_INCREMENT', '2H') + config.set('config', 'LEAD_SEQ', '1,2,3,6,9,12,15') + config.set('config', 'FCST_SERIES_ANALYSIS_INPUT_DIR', get_test_data_dir('fcst')) + config.set('config', 'FCST_SERIES_ANALYSIS_INPUT_TEMPLATE', + '{init?fmt=%Y%m%d}/{init?fmt=%Y%m%d_i%H}_f{lead?fmt=%3H}_HRRRTLE_PHPT.grb2') + config.set('config', 'OBS_SERIES_ANALYSIS_INPUT_DIR', get_test_data_dir('obs')) + config.set('config', 'OBS_SERIES_ANALYSIS_INPUT_TEMPLATE', + '{valid?fmt=%Y%m%d}/qpe_{valid?fmt=%Y%m%d%H}_A06.nc') + + wrapper = SeriesAnalysisWrapper(config) + assert wrapper.isOK + + all_cmds = wrapper.run_all_times() + for cmd, _ in all_cmds: + print(cmd) + + print(f'missing: {wrapper.missing_input_count} / {wrapper.run_count}, errors: {wrapper.errors}') + assert wrapper.missing_input_count == missing + assert wrapper.run_count == run + assert wrapper.errors == errors + + @pytest.mark.parametrize( 'config_overrides, env_var_values', [ ({'SERIES_ANALYSIS_REGRID_TO_GRID': 'FCST', }, @@ -352,9 +397,9 @@ def set_minimum_config_settings(config): 'SERIES_ANALYSIS_MASK_POLY': 'MET_BASE/poly/EAST.poly', }, {'METPLUS_MASK_DICT': 'mask = {grid = "FULL";poly = "MET_BASE/poly/EAST.poly";}'}), - # check tags are resolved and animation config works + # check animation config works ({ - 'FCST_VAR1_LEVELS': 'A0{init?fmt=3}', + 'FCST_VAR1_LEVELS': 'A03', 'SERIES_ANALYSIS_GENERATE_PLOTS': 'True', 'SERIES_ANALYSIS_GENERATE_ANIMATIONS': 'True', 'CONVERT_EXE': 'animation_exe' @@ -362,7 +407,7 @@ def set_minimum_config_settings(config): {},), # check 'BOTH_*' and '*INPUT_FILE_LIST' config ({'SERIES_ANALYSIS_REGRID_TO_GRID': 'FCST', - 'BOTH_SERIES_ANALYSIS_INPUT_TEMPLATE': 'True', + 'BOTH_SERIES_ANALYSIS_INPUT_TEMPLATE': 'True,True', }, {'METPLUS_REGRID_DICT': 'regrid = {to_grid = FCST;}'}), # TODO: Fix these tests to include file list paths @@ -596,10 +641,8 @@ def test_get_all_files_and_subset(metplus_config, time_info, expect_fcst_subset, wrapper.c_dict['TC_STAT_INPUT_DIR'] = stat_input_dir wrapper.c_dict['TC_STAT_INPUT_TEMPLATE'] = stat_input_template - fcst_input_dir = os.path.join(tile_input_dir, - 'fcst') - obs_input_dir = os.path.join(tile_input_dir, - 'obs') + fcst_input_dir = os.path.join(tile_input_dir, 'fcst') + obs_input_dir = os.path.join(tile_input_dir, 'obs') wrapper.c_dict['FCST_INPUT_DIR'] = fcst_input_dir wrapper.c_dict['OBS_INPUT_DIR'] = obs_input_dir @@ -609,7 +652,7 @@ def test_get_all_files_and_subset(metplus_config, time_info, expect_fcst_subset, else: wrapper.c_dict['RUN_ONCE_PER_STORM_ID'] = True - assert wrapper.get_all_files() + wrapper.c_dict['ALL_FILES'] = wrapper.get_all_files() print(f"ALL FILES: {wrapper.c_dict['ALL_FILES']}") expected_fcst = [ 'fcst/20141214_00/ML1201072014/FCST_TILE_F000_gfs_4_20141214_0000_000.nc', diff --git a/internal/tests/pytests/wrappers/tc_diag/test_tc_diag_wrapper.py b/internal/tests/pytests/wrappers/tc_diag/test_tc_diag_wrapper.py index 2a0954cb2b..a2c2635aec 100644 --- a/internal/tests/pytests/wrappers/tc_diag/test_tc_diag_wrapper.py +++ b/internal/tests/pytests/wrappers/tc_diag/test_tc_diag_wrapper.py @@ -3,7 +3,6 @@ import pytest import os -from datetime import datetime from metplus.wrappers.tc_diag_wrapper import TCDiagWrapper @@ -48,7 +47,7 @@ def set_minimum_config_settings(config): config.set('config', 'INIT_INCREMENT', '6H') config.set('config', 'TC_DIAG_CONFIG_FILE', '{PARM_BASE}/met_config/TCDiagConfig_wrapped') - config.set('config', 'TC_DIAG_DECK_TEMPLATE', deck_template) + config.set('config', 'TC_DIAG_DECK_INPUT_TEMPLATE', deck_template) config.set('config', 'TC_DIAG_INPUT1_TEMPLATE', input_template) config.set('config', 'TC_DIAG_INPUT1_DOMAIN', input_domain) config.set('config', 'TC_DIAG_INPUT1_TECH_ID_LIST', input_tech_id_list) @@ -62,6 +61,44 @@ def set_minimum_config_settings(config): config.set('config', 'BOTH_VAR2_LEVELS', 'P1000, P900, P800, P700, P500, P100') + +@pytest.mark.parametrize( + 'missing, run, thresh, errors, allow_missing', [ + (1, 3, 0.5, 0, True), + (1, 3, 0.7, 1, True), + (1, 3, 0.5, 7, False), + ] +) +@pytest.mark.wrapper +def test_tc_diag_missing_inputs(metplus_config, get_test_data_dir, + missing, run, thresh, errors, allow_missing): + config = metplus_config + set_minimum_config_settings(config) + config.set('config', 'INPUT_MUST_EXIST', True) + config.set('config', 'TC_DIAG_ALLOW_MISSING_INPUTS', allow_missing) + config.set('config', 'TC_DIAG_INPUT_THRESH', thresh) + config.set('config', 'INIT_BEG', '2017051001') + config.set('config', 'INIT_END', '2017051005') + config.set('config', 'INIT_INCREMENT', '2H') + config.set('config', 'LEAD_SEQ', '1,2,3,6,9,12') + config.set('config', 'TC_DIAG_DECK_INPUT_DIR', get_test_data_dir('obs')) + config.set('config', 'TC_DIAG_DECK_INPUT_TEMPLATE', '{init?fmt=%Y%m%d}/qpe_{init?fmt=%Y%m%d%H?shift=3H}_A06.nc') + config.set('config', 'TC_DIAG_INPUT1_DIR', get_test_data_dir('fcst')) + config.set('config', 'TC_DIAG_INPUT1_TEMPLATE', '{init?fmt=%Y%m%d}/{init?fmt=%Y%m%d_i%H}_f{lead?fmt=%3H}_HRRRTLE_PHPT.grb2') + + wrapper = TCDiagWrapper(config) + assert wrapper.isOK + + all_cmds = wrapper.run_all_times() + for cmd, _ in all_cmds: + print(cmd) + + print(f'missing: {wrapper.missing_input_count} / {wrapper.run_count}, errors: {wrapper.errors}') + assert wrapper.missing_input_count == missing + assert wrapper.run_count == run + assert wrapper.errors == errors + + @pytest.mark.parametrize( 'config_overrides, env_var_values', [ ({}, {}), diff --git a/internal/tests/pytests/wrappers/tc_gen/test_tc_gen_wrapper.py b/internal/tests/pytests/wrappers/tc_gen/test_tc_gen_wrapper.py index 610b5573a6..c6fe70f2a7 100644 --- a/internal/tests/pytests/wrappers/tc_gen/test_tc_gen_wrapper.py +++ b/internal/tests/pytests/wrappers/tc_gen/test_tc_gen_wrapper.py @@ -7,6 +7,62 @@ from metplus.wrappers.tc_gen_wrapper import TCGenWrapper +@pytest.mark.parametrize( + 'allow_missing, optional_input, missing, run, thresh, errors', [ + (True, None, [0, 0, 1], [1, 1, 1], 0.0, [0, 0, 0]), + (False, None, [0, 0, 1], [1, 1, 1], 0.0, [0, 0, 1]), + (True, 'genesis', [0, 1, 1], [1, 1, 1], 0.0, [0, 0, 0]), + (False, 'genesis', [0, 1, 1], [1, 1, 1], 0.0, [0, 1, 1]), + (True, 'edeck', [0, 1, 1], [1, 1, 1], 0.0, [0, 0, 0]), + (False, 'edeck', [0, 1, 1], [1, 1, 1], 0.0, [0, 1, 1]), + (True, 'shape', [0, 1, 1], [1, 1, 1], 0.0, [0, 0, 0]), + (False, 'shape', [0, 1, 1], [1, 1, 1], 0.0, [0, 1, 1]), + ] +) +@pytest.mark.wrapper_a +def test_tc_gen_missing_inputs(metplus_config, get_test_data_dir, allow_missing, + optional_input, missing, run, thresh, errors): + init_times = ('2016', '2018', '2020') + for index, init_time in enumerate(init_times): + config = metplus_config + config.set('config', 'DO_NOT_RUN_EXE', True) + config.set('config', 'INPUT_MUST_EXIST', True) + config.set('config', 'TC_GEN_ALLOW_MISSING_INPUTS', allow_missing) + config.set('config', 'TC_GEN_INPUT_THRESH', thresh) + config.set('config', 'LOOP_BY', 'INIT') + config.set('config', 'INIT_TIME_FMT', '%Y') + config.set('config', 'INIT_BEG', init_time) + config.set('config', 'INIT_END', init_time) + config.set('config', 'TC_GEN_TRACK_INPUT_DIR', get_test_data_dir('tc_gen/track')) + config.set('config', 'TC_GEN_TRACK_INPUT_TEMPLATE', 'track_fake_{init?fmt=%Y}*') + config.set('config', 'TC_GEN_OUTPUT_TEMPLATE', '{OUTPUT_BASE}/output.nc') + + if optional_input == 'genesis': + prefix = 'TC_GEN_GENESIS' + elif optional_input == 'edeck': + prefix = 'TC_GEN_EDECK' + elif optional_input == 'shape': + prefix = 'TC_GEN_SHAPE' + else: + prefix = None + + if prefix: + config.set('config', f'{prefix}_INPUT_DIR', get_test_data_dir(f'tc_gen/{optional_input}')) + config.set('config', f'{prefix}_INPUT_TEMPLATE', optional_input + '_fake_{init?fmt=%Y}*') + + wrapper = TCGenWrapper(config) + assert wrapper.isOK + + all_cmds = wrapper.run_all_times() + for cmd, _ in all_cmds: + print(cmd) + + print(f'missing: {wrapper.missing_input_count} / {wrapper.run_count}, errors: {wrapper.errors}') + assert wrapper.missing_input_count == missing[index] + assert wrapper.run_count == run[index] + assert wrapper.errors == errors[index] + + @pytest.mark.parametrize( 'config_overrides, env_var_values', [ @@ -286,7 +342,7 @@ ] ) @pytest.mark.wrapper_a -def test_tc_gen(metplus_config, config_overrides, env_var_values): +def test_tc_gen(metplus_config, get_test_data_dir, config_overrides, env_var_values): # expected number of 2016 files (including file_list line) expected_genesis_count = 7 expected_track_count = expected_genesis_count @@ -295,10 +351,7 @@ def test_tc_gen(metplus_config, config_overrides, env_var_values): config = metplus_config - test_data_dir = os.path.join(config.getdir('METPLUS_BASE'), - 'internal', 'tests', - 'data', - 'tc_gen') + test_data_dir = get_test_data_dir('tc_gen') track_dir = os.path.join(test_data_dir, 'track') genesis_dir = os.path.join(test_data_dir, 'genesis') edeck_dir = os.path.join(test_data_dir, 'edeck') diff --git a/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py b/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py index c19aa85daa..354394da65 100644 --- a/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py +++ b/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py @@ -18,11 +18,6 @@ run_times = ['2014121318'] -def get_data_dir(config): - return os.path.join(config.getdir('METPLUS_BASE'), - 'internal', 'tests', 'data', 'tc_pairs') - - def set_minimum_config_settings(config, loop_by='INIT'): # set config variables to prevent command from running and bypass check # if input files actually exist @@ -136,7 +131,7 @@ def test_parse_storm_id(metplus_config, storm_id, basin, cyclone): ] ) @pytest.mark.wrapper -def test_get_bdeck(metplus_config, basin, cyclone, expected_files, +def test_get_bdeck(metplus_config, get_test_data_dir, basin, cyclone, expected_files, expected_wildcard): """! Checks that the correct list of empty test files are found and the correct boolean to signify if wildcards were used for different @@ -147,7 +142,7 @@ def test_get_bdeck(metplus_config, basin, cyclone, expected_files, set_minimum_config_settings(config) - test_data_dir = get_data_dir(config) + test_data_dir = get_test_data_dir('tc_pairs') bdeck_dir = os.path.join(test_data_dir, 'bdeck') config.set('config', 'TC_PAIRS_BDECK_INPUT_DIR', bdeck_dir) @@ -268,7 +263,7 @@ def test_get_basin_cyclone_from_bdeck_error(metplus_config): ] ) @pytest.mark.wrapper -def test_tc_pairs_storm_id_lists(metplus_config, config_overrides, +def test_tc_pairs_storm_id_lists(metplus_config, get_test_data_dir, config_overrides, storm_type, values_to_check, reformat): config = metplus_config @@ -279,7 +274,7 @@ def test_tc_pairs_storm_id_lists(metplus_config, config_overrides, config.set('config', 'INIT_BEG', '2019') config.set('config', 'INIT_END', '2019') - test_data_dir = get_data_dir(config) + test_data_dir = get_test_data_dir('tc_pairs') bdeck_dir = os.path.join(test_data_dir, 'bdeck') edeck_dir = os.path.join(test_data_dir, 'edeck') @@ -614,14 +609,14 @@ def test_tc_pairs_storm_id_lists(metplus_config, config_overrides, ] ) @pytest.mark.wrapper -def test_tc_pairs_run(metplus_config, loop_by, config_overrides, +def test_tc_pairs_run(metplus_config, get_test_data_dir, loop_by, config_overrides, env_var_values): config = metplus_config remove_beg = remove_end = remove_match_points = False set_minimum_config_settings(config, loop_by) - test_data_dir = get_data_dir(config) + test_data_dir = get_test_data_dir('tc_pairs') bdeck_dir = os.path.join(test_data_dir, 'bdeck') adeck_dir = os.path.join(test_data_dir, 'adeck') @@ -738,13 +733,13 @@ def test_tc_pairs_run(metplus_config, loop_by, config_overrides, ] ) @pytest.mark.wrapper -def test_tc_pairs_read_all_files(metplus_config, loop_by, config_overrides, +def test_tc_pairs_read_all_files(metplus_config, get_test_data_dir, loop_by, config_overrides, env_var_values): config = metplus_config set_minimum_config_settings(config, loop_by) - test_data_dir = get_data_dir(config) + test_data_dir = get_test_data_dir('tc_pairs') bdeck_dir = os.path.join(test_data_dir, 'bdeck') adeck_dir = os.path.join(test_data_dir, 'adeck') diff --git a/internal/tests/pytests/wrappers/tcrmw/test_tcrmw_wrapper.py b/internal/tests/pytests/wrappers/tcrmw/test_tcrmw_wrapper.py index a1bfe04064..3556a709ed 100644 --- a/internal/tests/pytests/wrappers/tcrmw/test_tcrmw_wrapper.py +++ b/internal/tests/pytests/wrappers/tcrmw/test_tcrmw_wrapper.py @@ -25,11 +25,6 @@ ) -def get_data_dir(config): - return os.path.join(config.getdir('METPLUS_BASE'), - 'internal', 'tests', 'data', 'tc_pairs') - - def set_minimum_config_settings(config): # set config variables to prevent command from running and bypass check # if input files actually exist @@ -47,8 +42,7 @@ def set_minimum_config_settings(config): '{PARM_BASE}/met_config/TCRMWConfig_wrapped') config.set('config', 'TC_RMW_DECK_TEMPLATE', deck_template) config.set('config', 'TC_RMW_INPUT_TEMPLATE', input_template) - config.set('config', 'TC_RMW_OUTPUT_DIR', - '{OUTPUT_BASE}/TCRMW/output') + config.set('config', 'TC_RMW_OUTPUT_DIR', '{OUTPUT_BASE}/TCRMW/output') config.set('config', 'TC_RMW_OUTPUT_TEMPLATE', output_template) config.set('config', 'BOTH_VAR1_NAME', 'PRMSL') @@ -134,13 +128,13 @@ def set_minimum_config_settings(config): ] ) @pytest.mark.wrapper -def test_tc_rmw_run(metplus_config, config_overrides, +def test_tc_rmw_run(metplus_config, get_test_data_dir, config_overrides, env_var_values): config = metplus_config set_minimum_config_settings(config) - test_data_dir = get_data_dir(config) + test_data_dir = get_test_data_dir('tc_pairs') deck_dir = os.path.join(test_data_dir, 'bdeck') config.set('config', 'TC_RMW_DECK_INPUT_DIR', deck_dir) diff --git a/internal/tests/pytests/wrappers/user_script/test_user_script.py b/internal/tests/pytests/wrappers/user_script/test_user_script.py index 060f8f2a32..64e0b765d4 100644 --- a/internal/tests/pytests/wrappers/user_script/test_user_script.py +++ b/internal/tests/pytests/wrappers/user_script/test_user_script.py @@ -5,7 +5,6 @@ import re from datetime import datetime - from metplus.wrappers.user_script_wrapper import UserScriptWrapper diff --git a/internal/tests/pytests/wrappers/wavelet_stat/test_wavelet_stat.py b/internal/tests/pytests/wrappers/wavelet_stat/test_wavelet_stat.py index d944846bc6..5b98680136 100644 --- a/internal/tests/pytests/wrappers/wavelet_stat/test_wavelet_stat.py +++ b/internal/tests/pytests/wrappers/wavelet_stat/test_wavelet_stat.py @@ -55,6 +55,56 @@ def set_minimum_config_settings(config): config.set('config', 'BOTH_VAR1_THRESH', both_thresh) +@pytest.mark.parametrize( + 'once_per_field, missing, run, thresh, errors, allow_missing', [ + (False, 6, 12, 0.5, 0, True), + (False, 6, 12, 0.6, 1, True), + (True, 12, 24, 0.5, 0, True), + (True, 12, 24, 0.6, 1, True), + (False, 6, 12, 0.5, 6, False), + (True, 12, 24, 0.5, 12, False), + ] +) +@pytest.mark.wrapper_b +def test_wavelet_stat_missing_inputs(metplus_config, get_test_data_dir, + once_per_field, missing, run, thresh, errors, + allow_missing): + config = metplus_config + set_minimum_config_settings(config) + config.set('config', 'INPUT_MUST_EXIST', True) + config.set('config', 'WAVELET_STAT_ALLOW_MISSING_INPUTS', allow_missing) + config.set('config', 'WAVELET_STAT_INPUT_THRESH', thresh) + config.set('config', 'INIT_BEG', '2017051001') + config.set('config', 'INIT_END', '2017051003') + config.set('config', 'INIT_INCREMENT', '2H') + config.set('config', 'LEAD_SEQ', '1,2,3,6,9,12') + config.set('config', 'FCST_WAVELET_STAT_INPUT_DIR', get_test_data_dir('fcst')) + config.set('config', 'OBS_WAVELET_STAT_INPUT_DIR', get_test_data_dir('obs')) + config.set('config', 'FCST_WAVELET_STAT_INPUT_TEMPLATE', + '{init?fmt=%Y%m%d}/{init?fmt=%Y%m%d_i%H}_f{lead?fmt=%3H}_HRRRTLE_PHPT.grb2') + config.set('config', 'OBS_WAVELET_STAT_INPUT_TEMPLATE', + '{valid?fmt=%Y%m%d}/qpe_{valid?fmt=%Y%m%d%H}_A06.nc') + # add 2nd set of fields to test ONCE_PER_FIELD + config.set('config', 'FCST_VAR2_NAME', fcst_name) + config.set('config', 'FCST_VAR2_LEVELS', fcst_level) + config.set('config', 'OBS_VAR2_NAME', obs_name) + config.set('config', 'OBS_VAR2_LEVELS', obs_level) + config.set('config', 'BOTH_VAR2_THRESH', both_thresh) + config.set('config', 'WAVELET_STAT_ONCE_PER_FIELD', once_per_field) + + wrapper = WaveletStatWrapper(config) + assert wrapper.isOK + + all_cmds = wrapper.run_all_times() + for cmd, _ in all_cmds: + print(cmd) + + print(f'missing: {wrapper.missing_input_count} / {wrapper.run_count}, errors: {wrapper.errors}') + assert wrapper.missing_input_count == missing + assert wrapper.run_count == run + assert wrapper.errors == errors + + @pytest.mark.parametrize( 'config_overrides, expected_values', [ # 0 generic FCST is prob diff --git a/metplus/VERSION b/metplus/VERSION index 23df389ae6..c1929f62be 100644 --- a/metplus/VERSION +++ b/metplus/VERSION @@ -1 +1 @@ -6.0.0-beta3-dev +6.0.0-beta4-dev diff --git a/metplus/util/constants.py b/metplus/util/constants.py index 38f133a0e0..72a316281f 100644 --- a/metplus/util/constants.py +++ b/metplus/util/constants.py @@ -84,6 +84,15 @@ 'CyclonePlotter', ) +# wrappers that takes multiple inputs via Python Embedding +# used to check if file_type is set properly to note Python Embedding is used +MULTIPLE_INPUT_WRAPPERS = ( + 'EnsembleStat', + 'MTD', + 'SeriesAnalysis', + 'GenEnsProd', +) + # configuration variables that are specific to a given run # these are copied from [config] to [runtime] at the # end of the run so they will not be read if the final diff --git a/metplus/util/run_util.py b/metplus/util/run_util.py index be0d5fd867..cb61e05aaa 100644 --- a/metplus/util/run_util.py +++ b/metplus/util/run_util.py @@ -142,7 +142,7 @@ def pre_run_setup(config_inputs): logger.info(f"Log file: {log_file}") logger.info(f"METplus Base: {config.getdir('METPLUS_BASE')}") logger.info(f"Final Conf: {config.getstr('config', 'METPLUS_CONF')}") - config_list = config.getstr('config', 'CONFIG_INPUT').split(',') + config_list = config.getraw('config', 'CONFIG_INPUT').split(',') for config_item in config_list: logger.info(f"Config Input: {config_item}") diff --git a/metplus/wrappers/ascii2nc_wrapper.py b/metplus/wrappers/ascii2nc_wrapper.py index ab079b1328..c4c9d3e803 100755 --- a/metplus/wrappers/ascii2nc_wrapper.py +++ b/metplus/wrappers/ascii2nc_wrapper.py @@ -108,21 +108,13 @@ def create_c_dict(self): ) c_dict[f'OBS_FILE_WINDOW_{edge}'] = file_window - + # skip RuntimeFreq input file logic - remove once integrated + c_dict['FIND_FILES'] = False return c_dict def get_command(self): cmd = self.app_path - # don't run if no input or output files were found - if not self.infiles: - self.log_error("No input files were found") - return - - if self.outfile == "": - self.log_error("No output file specified") - return - # add input files for infile in self.infiles: cmd += ' ' + infile @@ -145,15 +137,15 @@ def find_input_files(self, time_info): filename = do_string_sub(self.c_dict['OBS_INPUT_TEMPLATE'], **time_info) self.infiles.append(filename) - return self.infiles + return True # get list of files even if only one is found (return_list=True) obs_path = self.find_obs(time_info, return_list=True) if obs_path is None: - return None + return False self.infiles.extend(obs_path) - return self.infiles + return True def set_command_line_arguments(self, time_info): # add input data format if set @@ -162,8 +154,7 @@ def set_command_line_arguments(self, time_info): # add config file - passing through do_string_sub to get custom string if set if self.c_dict['CONFIG_FILE']: - config_file = do_string_sub(self.c_dict['CONFIG_FILE'], - **time_info) + config_file = do_string_sub(self.c_dict['CONFIG_FILE'], **time_info) self.args.append(f" -config {config_file}") # add mask grid if set diff --git a/metplus/wrappers/command_builder.py b/metplus/wrappers/command_builder.py index eaeb81205c..8886408857 100755 --- a/metplus/wrappers/command_builder.py +++ b/metplus/wrappers/command_builder.py @@ -15,7 +15,7 @@ from abc import ABCMeta from inspect import getframeinfo, stack -from ..util.constants import PYTHON_EMBEDDING_TYPES, COMPRESSION_EXTENSIONS +from ..util.constants import PYTHON_EMBEDDING_TYPES, COMPRESSION_EXTENSIONS, MULTIPLE_INPUT_WRAPPERS from ..util import getlist, preprocess_file, loop_over_times_and_call from ..util import do_string_sub, ti_calculate, get_seconds_from_string from ..util import get_time_from_file, shift_time_seconds, seconds_to_met_time @@ -198,12 +198,16 @@ def create_c_dict(self): return c_dict - def clear(self): + def clear(self, clear_input_files=True): """!Unset class variables to prepare for next run time + + @param clear_input_files If True, clear self.infiles, otherwise do not. + Defaults to True. """ self.args = [] self.input_dir = "" - self.infiles = [] + if clear_input_files: + self.infiles = [] self.outdir = "" self.outfile = "" self.param = "" @@ -435,6 +439,7 @@ def find_obs_offset(self, time_info, mandatory=True, return_list=False): # errors when searching through offset list is_mandatory = mandatory if offsets == [0] else False + self.c_dict['SUPRESS_WARNINGS'] = True for offset in offsets: time_info['offset_hours'] = offset time_info = ti_calculate(time_info) @@ -443,8 +448,11 @@ def find_obs_offset(self, time_info, mandatory=True, return_list=False): return_list=return_list) if obs_path is not None: + self.c_dict['SUPRESS_WARNINGS'] = False return obs_path, time_info + self.c_dict['SUPRESS_WARNINGS'] = False + # if no files are found return None # if offsets are specified, log error with list offsets used log_message = "Could not find observation file" @@ -453,8 +461,10 @@ def find_obs_offset(self, time_info, mandatory=True, return_list=False): f"{','.join([str(offset) for offset in offsets])}") # if mandatory, report error, otherwise report warning - if mandatory: - self.log_error(log_message) + if mandatory and not self.c_dict.get('ALLOW_MISSING_INPUTS', False): + # don't call log_error to increment error count because + # error should already be reported + self.logger.error(log_message) else: self.logger.warning(log_message) @@ -559,8 +569,13 @@ def _find_exact_file(self, level, data_type, time_info, mandatory=True, if not check_file_list: msg = f"Could not find any {data_type}INPUT files" # warn instead of error if it is not mandatory to find files - if not mandatory or not self.c_dict.get('MANDATORY', True): - self.logger.warning(msg) + if (not mandatory + or not self.c_dict.get('MANDATORY', True) + or self.c_dict.get('ALLOW_MISSING_INPUTS', False)): + if self.c_dict.get('SUPRESS_WARNINGS', False): + self.logger.debug(msg) + else: + self.logger.warning(msg) else: self.log_error(msg) @@ -661,8 +676,15 @@ def _check_that_files_exist(self, check_file_list, data_type, allow_dir, if not processed_path: msg = (f"Could not find {data_type}INPUT file {file_path} " f"using template {template}") - if not mandatory or not self.c_dict.get('MANDATORY', True): - self.logger.warning(msg) + if (not mandatory + or not self.c_dict.get('MANDATORY', True) + or self.c_dict.get('ALLOW_MISSING_INPUTS', False)): + + if self.c_dict.get('SUPRESS_WARNINGS', False): + self.logger.debug(msg) + else: + self.logger.warning(msg) + if self.c_dict.get(f'{data_type}FILL_MISSING'): found_file_list.append(f'MISSING{file_path}') continue @@ -705,8 +727,15 @@ def _find_file_in_window(self, data_type, time_info, mandatory=True, if not closest_files: msg = (f"Could not find {data_type}INPUT files under {data_dir} within range " f"[{valid_range_lower},{valid_range_upper}] using template {template}") - if not mandatory: - self.logger.warning(msg) + if (not mandatory + or not self.c_dict.get('MANDATORY', True) + or self.c_dict.get('ALLOW_MISSING_INPUTS', False)): + + if self.c_dict.get('SUPRESS_WARNINGS', False): + self.logger.debug(msg) + else: + self.logger.warning(msg) + else: self.log_error(msg) @@ -832,9 +861,14 @@ def find_input_files_ensemble(self, time_info, fill_missing=True): return True # get list of ensemble files to process - input_files = self.find_model(time_info, return_list=True) + input_files = self.find_model(time_info, return_list=True, mandatory=False) if not input_files: - self.log_error("Could not find any input files") + msg = "Could not find any input files" + if (not self.c_dict.get('MANDATORY', True) + or self.c_dict.get('ALLOW_MISSING_INPUTS', False)): + self.logger.warning(msg) + else: + self.log_error(msg) return False # if control file is requested, remove it from input list @@ -997,8 +1031,7 @@ def find_and_check_output_file(self, time_info=None, prefix = self.get_output_prefix(time_info, set_env_vars=False) prefix = f'{self.app_name}_{prefix}' if prefix else self.app_name search_string = f'{prefix}_{lead}L_{valid}V*' - search_path = os.path.join(output_path, - search_string) + search_path = os.path.join(output_path, search_string) if skip_if_output_exists: self.logger.debug("Looking for existing data that matches: " f"{search_path}") @@ -1102,35 +1135,45 @@ def set_current_field_config(self, field_info=None): field_info[name] if name in field_info else '') def check_for_python_embedding(self, input_type, var_info): - """!Check if field name of given input type is a python script. If it is not, return the field name. - If it is, check if the input datatype is a valid Python Embedding string, set the c_dict item - that sets the file_type in the MET config file accordingly, and set the output string to 'python_embedding. - Used to set up Python Embedding input for MET tools that support multiple input files, such as MTD, EnsembleStat, - and SeriesAnalysis. - Args: - @param input_type type of field input, i.e. FCST, OBS, ENS, POINT_OBS, GRID_OBS, or BOTH - @param var_info dictionary item containing field information for the current *_VAR_* configs being handled - @returns field name if not a python script, 'python_embedding' if it is, and None if configuration is invalid""" + """!Check if field name of given input type is a python script. + If it is not, return the field name. If it is, return 'python_embedding' + and set file_type in the MET config to a PYTHON keyword if it is not + already set. + @param input_type type of field input, e.g. FCST, OBS, ENS, POINT_OBS, + GRID_OBS, or BOTH + @param var_info dictionary item containing field information for the + current *_VAR_* configs being handled + @returns field name if not a python script, 'python_embedding' if it is + """ var_input_type = input_type.lower() if input_type != 'BOTH' else 'fcst' - # reset file type to empty string to handle if python embedding is used for one field but not for the next - self.c_dict[f'{input_type}_FILE_TYPE'] = '' - if not is_python_script(var_info[f"{var_input_type}_name"]): # if not a python script, return var name return var_info[f"{var_input_type}_name"] - # if it is a python script, set file extension to show that and make sure *_INPUT_DATATYPE is a valid PYTHON_* string + # if it is a python script, set file extension to show that and + # make sure *_INPUT_DATATYPE is a valid PYTHON_* string file_ext = 'python_embedding' + + # skip check of _INPUT_DATATYPE if _FILE_TYPE is already set + # or if wrapper does not support multiple inputs + if (self.env_var_dict.get(f'METPLUS_{input_type}_FILE_TYPE') + or get_wrapper_name(self.app_name) not in MULTIPLE_INPUT_WRAPPERS): + return file_ext + data_type = self.c_dict.get(f'{input_type}_INPUT_DATATYPE', '') + # error and return None if wrapper takes multiple inputs for Python + # Embedding but file_type has not been specified to note that if data_type not in PYTHON_EMBEDDING_TYPES: - self.log_error(f"{input_type}_{self.app_name.upper()}_INPUT_DATATYPE ({data_type}) must be set to a valid Python Embedding type " - f"if supplying a Python script as the {input_type}_VAR_NAME. Valid options: " - f"{','.join(PYTHON_EMBEDDING_TYPES)}") - return None + self.logger.warning( + f"{input_type}_{self.app_name.upper()}_FILE_TYPE must be set " + "when passing a Python Embedding script to a tool that takes " + "multiple inputs. Using PYTHON_NUMPY" + ) + data_type = 'PYTHON_NUMPY' - # set file type string to be set in MET config file to specify Python Embedding is being used for this dataset + # set file type string to be set in MET config file to specify + # Python Embedding is being used for this dataset file_type = f"file_type = {data_type};" - self.c_dict[f'{input_type}_FILE_TYPE'] = file_type self.env_var_dict[f'METPLUS_{input_type}_FILE_TYPE'] = file_type return file_ext @@ -1412,25 +1455,27 @@ def handle_climo_dict(self): output_dict=self.env_var_dict): self.errors += 1 - def get_wrapper_or_generic_config(self, generic_config_name): + def get_wrapper_or_generic_config(self, generic_name, var_type='str'): """! Check for config variable with _ prepended first. If set use that value. If not, check for config without prefix. - @param generic_config_name name of variable to read from config + @param generic_name name of variable to read from config + @param var_type type of variable to read, e.g. str, bool, int, or float. + Default is str. @returns value if set or empty string if not """ - wrapper_config_name = f'{self.app_name.upper()}_{generic_config_name}' - value = self.config.getstr_nocheck('config', - wrapper_config_name, - '') - - # if wrapper specific variable not set, check for generic - if not value: - value = self.config.getstr_nocheck('config', - generic_config_name, - '') - - return value + name = self.config.get_mp_config_name( + [f'{self.app_name}_{generic_name}'.upper(), generic_name.upper()] + ) + if not name: + return '' + if var_type == 'bool': + return self.config.getbool('config', name) + if var_type == 'float': + return self.config.getfloat('config', name) + if var_type == 'int': + return self.config.getint('config', name) + return self.config.getstr('config', name) def format_field(self, data_type, field_string, is_list=True): """! Set {data_type}_FIELD c_dict value to the formatted field string diff --git a/metplus/wrappers/compare_gridded_wrapper.py b/metplus/wrappers/compare_gridded_wrapper.py index 648166aec1..82a7a2f7e1 100755 --- a/metplus/wrappers/compare_gridded_wrapper.py +++ b/metplus/wrappers/compare_gridded_wrapper.py @@ -43,12 +43,10 @@ def create_c_dict(self): which config variables are used in the wrapper""" c_dict = super().create_c_dict() - self.add_met_config(name='model', - data_type='string', + self.add_met_config(name='model', data_type='string', metplus_configs=['MODEL']) - self.add_met_config(name='obtype', - data_type='string', + self.add_met_config(name='obtype', data_type='string', metplus_configs=['OBTYPE']) # read probabilistic variables for FCST and OBS fields @@ -94,11 +92,10 @@ def run_at_time_once(self, time_info): @param time_info dictionary containing timing information """ var_list = sub_var_list(self.c_dict['VAR_LIST_TEMP'], time_info) - if not var_list and not self.c_dict.get('VAR_LIST_OPTIONAL', False): - self.log_error('No input fields were specified. You must set ' - f'[FCST/OBS]_VAR_[NAME/LEVELS].') - return None + self.log_error('No input fields were specified.' + ' [FCST/OBS]_VAR_NAME must be set.') + return if self.c_dict.get('ONCE_PER_FIELD', False): # loop over all fields and levels (and probability thresholds) and @@ -107,6 +104,10 @@ def run_at_time_once(self, time_info): self.clear() self.c_dict['CURRENT_VAR_INFO'] = var_info add_field_info_to_time_info(time_info, var_info) + self.run_count += 1 + if not self.find_input_files(time_info): + self.missing_input_count += 1 + continue self.run_at_time_one_field(time_info, var_info) else: # loop over all variables and all them to the field list, @@ -116,33 +117,58 @@ def run_at_time_once(self, time_info): add_field_info_to_time_info(time_info, var_list[0]) self.clear() + self.run_count += 1 + if not self.find_input_files(time_info): + self.missing_input_count += 1 + return self.run_at_time_all_fields(time_info) - def run_at_time_one_field(self, time_info, var_info): - """! Build MET command for a single field for a given - init/valid time and forecast lead combination - Args: - @param time_info dictionary containing timing information - @param var_info object containing variable information - """ - - # get model to compare, return None if not found + def find_input_files(self, time_info): + # get model from first var to compare model_path = self.find_model(time_info, mandatory=True, return_list=True) - if model_path is None: - return + if not model_path: + return False + + # if there is more than 1 file, create file list file + if len(model_path) > 1: + list_filename = (f"{time_info['init_fmt']}_" + f"{time_info['lead_hours']}_" + f"{self.app_name}_fcst.txt") + model_path = self.write_list_file(list_filename, model_path) + else: + model_path = model_path[0] + + self.infiles.append(model_path) - self.infiles.extend(model_path) - # get observation to compare, return None if not found + # get observation to from first var compare obs_path, time_info = self.find_obs_offset(time_info, mandatory=True, return_list=True) if obs_path is None: - return + return False + + # if there is more than 1 file, create file list file + if len(obs_path) > 1: + list_filename = (f"{time_info['init_fmt']}_" + f"{time_info['lead_hours']}_" + f"{self.app_name}_obs.txt") + obs_path = self.write_list_file(list_filename, obs_path) + else: + obs_path = obs_path[0] + + self.infiles.append(obs_path) - self.infiles.extend(obs_path) + return True + def run_at_time_one_field(self, time_info, var_info): + """! Build MET command for a single field for a given + init/valid time and forecast lead combination + Args: + @param time_info dictionary containing timing information + @param var_info object containing variable information + """ # get field info field a single field to pass to the MET config file fcst_field_list = self.format_field_info(var_info=var_info, data_type='FCST') @@ -169,85 +195,60 @@ def run_at_time_all_fields(self, time_info): """ var_list = sub_var_list(self.c_dict['VAR_LIST_TEMP'], time_info) - # get model from first var to compare - model_path = self.find_model(time_info, - mandatory=True, - return_list=True) - if not model_path: - return - - # if there is more than 1 file, create file list file - if len(model_path) > 1: - list_filename = (f"{time_info['init_fmt']}_" - f"{time_info['lead_hours']}_" - f"{self.app_name}_fcst.txt") - model_path = self.write_list_file(list_filename, model_path) - else: - model_path = model_path[0] - - self.infiles.append(model_path) + # set field info + fcst_field = self.get_all_field_info(var_list, 'FCST') + obs_field = self.get_all_field_info(var_list, 'OBS') - # get observation to from first var compare - obs_path, time_info = self.find_obs_offset(time_info, - mandatory=True, - return_list=True) - if obs_path is None: + if not fcst_field or not obs_field: + self.log_error("Could not build field info for fcst or obs") return - # if there is more than 1 file, create file list file - if len(obs_path) > 1: - list_filename = (f"{time_info['init_fmt']}_" - f"{time_info['lead_hours']}_" - f"{self.app_name}_obs.txt") - obs_path = self.write_list_file(list_filename, obs_path) - else: - obs_path = obs_path[0] - - self.infiles.append(obs_path) - - fcst_field_list = [] - obs_field_list = [] - for var_info in var_list: - next_fcst = self.get_field_info(v_level=var_info['fcst_level'], - v_thresh=var_info['fcst_thresh'], - v_name=var_info['fcst_name'], - v_extra=var_info['fcst_extra'], - d_type='FCST') - - next_obs = self.get_field_info(v_level=var_info['obs_level'], - v_thresh=var_info['obs_thresh'], - v_name=var_info['obs_name'], - v_extra=var_info['obs_extra'], - d_type='OBS') - - if next_fcst is None or next_obs is None: - return - - fcst_field_list.extend(next_fcst) - obs_field_list.extend(next_obs) - - fcst_field = ','.join(fcst_field_list) - obs_field = ','.join(obs_field_list) - self.format_field('FCST', fcst_field) self.format_field('OBS', obs_field) self.process_fields(time_info) + def get_all_field_info(self, var_list, data_type): + """!Get field info based on data type""" + + field_list = [] + for var_info in var_list: + type_lower = data_type.lower() + level = var_info[f'{type_lower}_level'] + thresh = var_info[f'{type_lower}_thresh'] + name = var_info[f'{type_lower}_name'] + extra = var_info[f'{type_lower}_extra'] + + # check if python embedding is used and set up correctly + # set env var for file type if it is used + py_embed_ok = self.check_for_python_embedding(data_type, var_info) + if not py_embed_ok: + return '' + + next_field = self.get_field_info(v_level=level, + v_thresh=thresh, + v_name=name, + v_extra=extra, + d_type=data_type) + if next_field is None: + return '' + + field_list.extend(next_field) + + return ','.join(field_list) + def process_fields(self, time_info): """! Set and print environment variables, then build/run MET command @param time_info dictionary with time information """ # set config file since command is reset after each run - self.param = do_string_sub(self.c_dict['CONFIG_FILE'], - **time_info) + self.param = do_string_sub(self.c_dict['CONFIG_FILE'], **time_info) self.set_current_field_config() # set up output dir with time info - if not self.find_and_check_output_file(time_info, - is_directory=True): + if not self.find_and_check_output_file(time_info, is_directory=True): return # set command line arguments @@ -274,8 +275,7 @@ def get_command(self): @return Returns a MET command with arguments that you can run """ if self.app_path is None: - self.log_error('No app path specified. ' - 'You must use a subclass') + self.log_error('No app path specified. You must use a subclass') return None cmd = '{} -v {} '.format(self.app_path, self.c_dict['VERBOSITY']) diff --git a/metplus/wrappers/ensemble_stat_wrapper.py b/metplus/wrappers/ensemble_stat_wrapper.py index 1de97bf155..d5fa2542e5 100755 --- a/metplus/wrappers/ensemble_stat_wrapper.py +++ b/metplus/wrappers/ensemble_stat_wrapper.py @@ -132,13 +132,11 @@ def create_c_dict(self): ) c_dict['OBS_POINT_INPUT_DATATYPE'] = ( - self.config.getraw('config', - 'OBS_ENSEMBLE_STAT_INPUT_POINT_DATATYPE') + self.config.getraw('config', 'OBS_ENSEMBLE_STAT_INPUT_POINT_DATATYPE') ) c_dict['OBS_GRID_INPUT_DATATYPE'] = ( - self.config.getraw('config', - 'OBS_ENSEMBLE_STAT_INPUT_GRID_DATATYPE') + self.config.getraw('config', 'OBS_ENSEMBLE_STAT_INPUT_GRID_DATATYPE') ) # check if more than 1 obs datatype is set to python embedding, @@ -164,9 +162,6 @@ def create_c_dict(self): # allow multiple files in CommandBuilder.find_data logic c_dict['ALLOW_MULTIPLE_FILES'] = True - # not all input files are mandatory to be found - c_dict['MANDATORY'] = False - # fill inputs that are not found with fake path to note it is missing c_dict['FCST_FILL_MISSING'] = True @@ -384,7 +379,8 @@ def create_c_dict(self): self.config, met_tool=self.app_name ) - + # skip RuntimeFreq input file logic - remove once integrated + c_dict['FIND_FILES'] = False return c_dict def get_command(self): @@ -396,43 +392,14 @@ def get_command(self): f" {' '.join(self.infiles)} {self.param}" f" {' '.join(self.args)} -outdir {self.outdir}") - def run_at_time_all_fields(self, time_info): - """! Runs the MET application for a given time and forecast lead combination - Args: - @param time_info dictionary containing timing information - """ + def find_input_files(self, time_info): # get ensemble model files # do not fill file list with missing if ens_member_ids is used fill_missing = not self.env_var_dict.get('METPLUS_ENS_MEMBER_IDS') if not self.find_input_files_ensemble(time_info, fill_missing=fill_missing): - return + return False - if not self.set_command_line_arguments(time_info): - return - - # parse optional var list for FCST and/or OBS fields - var_list = sub_var_list(self.c_dict['VAR_LIST_TEMP'], time_info) - - # set field info - fcst_field = self.get_all_field_info(var_list, 'FCST') - obs_field = self.get_all_field_info(var_list, 'OBS') - - if not fcst_field and not obs_field: - self.log_error("Could not build field info for fcst or obs") - return - - self.format_field('FCST', fcst_field) - self.format_field('OBS', obs_field) - - self.process_fields(time_info) - - def set_command_line_arguments(self, time_info): - """! Set all arguments for plot_point_obs command. - - @param time_info dictionary containing timing information - @returns False if files could not be found, True on success - """ # get point observation file if requested if self.c_dict['OBS_POINT_INPUT_TEMPLATE']: point_obs_files = self.find_data(time_info, data_type='OBS_POINT', @@ -464,35 +431,6 @@ def set_command_line_arguments(self, time_info): return True - def get_all_field_info(self, var_list, data_type): - """!Get field info based on data type""" - - field_list = [] - for var_info in var_list: - type_lower = data_type.lower() - level = var_info[f'{type_lower}_level'] - thresh = var_info[f'{type_lower}_thresh'] - name = var_info[f'{type_lower}_name'] - extra = var_info[f'{type_lower}_extra'] - - # check if python embedding is used and set up correctly - # set env var for file type if it is used - py_embed_ok = self.check_for_python_embedding(data_type, var_info) - if not py_embed_ok: - return '' - - next_field = self.get_field_info(v_level=level, - v_thresh=thresh, - v_name=name, - v_extra=extra, - d_type=data_type) - if next_field is None: - return '' - - field_list.extend(next_field) - - return ','.join(field_list) - def set_environment_variables(self, time_info): self.add_env_var("MET_OBS_ERROR_TABLE", self.c_dict.get('MET_OBS_ERR_TABLE', '')) @@ -506,12 +444,10 @@ def process_fields(self, time_info): @param obs_field field information formatted for MET config file """ # set config file since command is reset after each run - self.param = do_string_sub(self.c_dict['CONFIG_FILE'], - **time_info) + self.param = do_string_sub(self.c_dict['CONFIG_FILE'], **time_info) # set up output dir with time info - if not self.find_and_check_output_file(time_info, - is_directory=True): + if not self.find_and_check_output_file(time_info, is_directory=True): return # set environment variables that are passed to the MET config diff --git a/metplus/wrappers/extract_tiles_wrapper.py b/metplus/wrappers/extract_tiles_wrapper.py index 246e5af688..fc091a4e5b 100755 --- a/metplus/wrappers/extract_tiles_wrapper.py +++ b/metplus/wrappers/extract_tiles_wrapper.py @@ -76,9 +76,7 @@ def create_c_dict(self): ) c_dict['TC_STAT_INPUT_TEMPLATE'] = ( - self.config.getraw('filename_templates', - 'EXTRACT_TILES_TC_STAT_INPUT_TEMPLATE', - '') + self.config.getraw('config', 'EXTRACT_TILES_TC_STAT_INPUT_TEMPLATE') ) # get MTD data dir/template to read c_dict['MTD_INPUT_DIR'] = ( @@ -86,9 +84,7 @@ def create_c_dict(self): ) c_dict['MTD_INPUT_TEMPLATE'] = ( - self.config.getraw('filename_templates', - 'EXTRACT_TILES_MTD_INPUT_TEMPLATE', - '') + self.config.getraw('config', 'EXTRACT_TILES_MTD_INPUT_TEMPLATE') ) # determine which location input to use: TCStat or MTD @@ -133,8 +129,7 @@ def create_c_dict(self): local_name = f'{data_type}_{put}_TEMPLATE' config_name = f'{data_type}_{et_upper}_{put}_TEMPLATE' c_dict[local_name] = ( - self.config.getraw('filename_templates', - config_name) + self.config.getraw('config', config_name) ) if not c_dict[local_name]: self.log_error(f"{config_name} must be set.") @@ -157,6 +152,10 @@ def create_c_dict(self): c_dict['VAR_LIST_TEMP'] = parse_var_list(self.config, met_tool=self.app_name) + # skip RuntimeFreq input file logic - remove once integrated + c_dict['FIND_FILES'] = False + # force error if inputs are missing + c_dict['ALLOW_MISSING_INPUTS'] = False return c_dict def regrid_data_plane_init(self): @@ -324,8 +323,7 @@ def get_location_input_file(self, time_info, input_type): """ input_path = os.path.join(self.c_dict[f'{input_type}_INPUT_DIR'], self.c_dict[f'{input_type}_INPUT_TEMPLATE']) - input_path = do_string_sub(input_path, - **time_info) + input_path = do_string_sub(input_path, **time_info) self.logger.debug(f"Looking for {input_type} file: {input_path}") if not os.path.exists(input_path): @@ -356,8 +354,7 @@ def call_regrid_data_plane(self, time_info, track_data, input_type): self.regrid_data_plane.c_dict['VAR_LIST'] = var_list for data_type in ['FCST', 'OBS']: - grid = self.get_grid(data_type, track_data[data_type], - input_type) + grid = self.get_grid(data_type, track_data[data_type], input_type) self.regrid_data_plane.c_dict['VERIFICATION_GRID'] = grid @@ -413,14 +410,12 @@ def set_time_info_from_track_data(storm_data, storm_id=None): input_dict = {} # read forecast lead from LEAD (TC_STAT) or FCST_LEAD (MTD) - lead = storm_data.get('LEAD', - storm_data.get('FCST_LEAD')) + lead = storm_data.get('LEAD', storm_data.get('FCST_LEAD')) if lead: input_dict['lead_hours'] = lead[:-4] # read valid time from VALID (TC_STAT) or FCST_VALID (MTD) - valid = storm_data.get('VALID', - storm_data.get('FCST_VALID')) + valid = storm_data.get('VALID', storm_data.get('FCST_VALID')) if valid: valid_dt = datetime.strptime(valid, '%Y%m%d_%H%M%S') input_dict['valid'] = valid_dt diff --git a/metplus/wrappers/gempak_to_cf_wrapper.py b/metplus/wrappers/gempak_to_cf_wrapper.py index f0ddd57a6b..6c6108dac8 100755 --- a/metplus/wrappers/gempak_to_cf_wrapper.py +++ b/metplus/wrappers/gempak_to_cf_wrapper.py @@ -43,14 +43,14 @@ def create_c_dict(self): c_dict['INPUT_DATATYPE'] = 'GEMPAK' c_dict['INPUT_DIR'] = self.config.getdir('GEMPAKTOCF_INPUT_DIR', '') c_dict['INPUT_TEMPLATE'] = ( - self.config.getraw('filename_templates', - 'GEMPAKTOCF_INPUT_TEMPLATE') + self.config.getraw('config', 'GEMPAKTOCF_INPUT_TEMPLATE') ) c_dict['OUTPUT_DIR'] = self.config.getdir('GEMPAKTOCF_OUTPUT_DIR', '') c_dict['OUTPUT_TEMPLATE'] = ( - self.config.getraw('filename_templates', - 'GEMPAKTOCF_OUTPUT_TEMPLATE') + self.config.getraw('config', 'GEMPAKTOCF_OUTPUT_TEMPLATE') ) + # skip RuntimeFreq input file logic - remove once integrated + c_dict['FIND_FILES'] = False return c_dict def get_command(self): @@ -72,16 +72,13 @@ def get_command(self): def run_at_time_once(self, time_info): """! Runs the MET application for a given time and forecast lead combination - Args: - @param time_info dictionary containing timing information + + @param time_info dictionary containing timing information """ - infile = do_string_sub(self.c_dict['INPUT_TEMPLATE'], - **time_info) - infile = os.path.join(self.c_dict.get('INPUT_DIR', ''), - infile) + infile = do_string_sub(self.c_dict['INPUT_TEMPLATE'], **time_info) + infile = os.path.join(self.c_dict.get('INPUT_DIR', ''), infile) self.infiles.append(infile) - # set environment variables self.set_environment_variables(time_info) if not self.find_and_check_output_file(time_info): diff --git a/metplus/wrappers/gen_ens_prod_wrapper.py b/metplus/wrappers/gen_ens_prod_wrapper.py index 26e4cd6590..c4d4ee05b2 100755 --- a/metplus/wrappers/gen_ens_prod_wrapper.py +++ b/metplus/wrappers/gen_ens_prod_wrapper.py @@ -87,9 +87,6 @@ def create_c_dict(self): self.log_error('GEN_ENS_PROD_INPUT_TEMPLATE or ' 'GEN_ENS_PROD_INPUT_FILE_LIST must be set') - # not all input files are mandatory to be found - c_dict['MANDATORY'] = False - # fill inputs that are not found with fake path to note it is missing c_dict['FCST_FILL_MISSING'] = True @@ -219,7 +216,8 @@ def create_c_dict(self): data_type='string') c_dict['ALLOW_MULTIPLE_FILES'] = True - + # skip RuntimeFreq input file logic - remove once integrated + c_dict['FIND_FILES'] = False return c_dict def run_at_time_once(self, time_info): @@ -234,10 +232,9 @@ def run_at_time_once(self, time_info): if not self.find_field_info(time_info): return False - # do not fill file list with missing if ens_member_ids is used - fill_missing = not self.env_var_dict.get('METPLUS_ENS_MEMBER_IDS') - if not self.find_input_files_ensemble(time_info, - fill_missing=fill_missing): + self.run_count += 1 + if not self.find_input_files(time_info): + self.missing_input_count += 1 return False if not self.find_and_check_output_file(time_info): @@ -248,6 +245,14 @@ def run_at_time_once(self, time_info): return self.build() + def find_input_files(self, time_info): + # do not fill file list with missing if ens_member_ids is used + fill_missing = not self.env_var_dict.get('METPLUS_ENS_MEMBER_IDS') + if not self.find_input_files_ensemble(time_info, + fill_missing=fill_missing): + return False + return True + def find_field_info(self, time_info): """! parse var list for ENS fields diff --git a/metplus/wrappers/gen_vx_mask_wrapper.py b/metplus/wrappers/gen_vx_mask_wrapper.py index 8eb4a21ab3..68aa313746 100755 --- a/metplus/wrappers/gen_vx_mask_wrapper.py +++ b/metplus/wrappers/gen_vx_mask_wrapper.py @@ -41,13 +41,11 @@ def create_c_dict(self): c_dict['ALLOW_MULTIPLE_FILES'] = False # input and output files - c_dict['INPUT_DIR'] = self.config.getdir('GEN_VX_MASK_INPUT_DIR', - '') + c_dict['INPUT_DIR'] = self.config.getdir('GEN_VX_MASK_INPUT_DIR', '') c_dict['INPUT_TEMPLATE'] = self.config.getraw('config', 'GEN_VX_MASK_INPUT_TEMPLATE') - c_dict['OUTPUT_DIR'] = self.config.getdir('GEN_VX_MASK_OUTPUT_DIR', - '') + c_dict['OUTPUT_DIR'] = self.config.getdir('GEN_VX_MASK_OUTPUT_DIR', '') c_dict['OUTPUT_TEMPLATE'] = self.config.getraw('config', 'GEN_VX_MASK_OUTPUT_TEMPLATE') @@ -95,7 +93,8 @@ def create_c_dict(self): # use the same file windows for input and mask files c_dict['MASK_FILE_WINDOW_BEGIN'] = c_dict['FILE_WINDOW_BEGIN'] c_dict['MASK_FILE_WINDOW_END'] = c_dict['FILE_WINDOW_END'] - + # skip RuntimeFreq input file logic - remove once integrated + c_dict['FIND_FILES'] = False return c_dict def get_command(self): @@ -137,16 +136,17 @@ def run_at_time_once(self, time_info): self.set_environment_variables(time_info) # loop over mask templates and command line args, + self.run_count += 1 temp_file = '' for index, (mask_template, cmd_args) in enumerate(zip(self.c_dict['MASK_INPUT_TEMPLATES'], self.c_dict['COMMAND_OPTIONS'])): # set mask input template and command line arguments self.c_dict['MASK_INPUT_TEMPLATE'] = mask_template - self.args = do_string_sub(cmd_args, - **time_info) + self.args = do_string_sub(cmd_args, **time_info) if not self.find_input_files(time_info, temp_file): + self.missing_input_count += 1 return # break out of loop if this is the last iteration to @@ -173,10 +173,11 @@ def run_at_time_once(self, time_info): def find_input_files(self, time_info, temp_file): """!Handle setting of input file list. - Args: - @param time_info time dictionary for current runtime - @param temp_file path to temporary file used for previous run or empty string on first iteration - @returns True if successfully found all inputs, False if not + + @param time_info time dictionary for current runtime + @param temp_file path to temporary file used for previous run or + empty string on first iteration + @returns True if successfully found all inputs, False if not """ # clear out input file list @@ -195,8 +196,7 @@ def find_input_files(self, time_info, temp_file): input_path = temp_file # find mask file, using MASK_INPUT_TEMPLATE - mask_file = self.find_data(time_info, - data_type='MASK') + mask_file = self.find_data(time_info, data_type='MASK') if not mask_file: return False diff --git a/metplus/wrappers/gfdl_tracker_wrapper.py b/metplus/wrappers/gfdl_tracker_wrapper.py index 1e686109d8..5526d61a9d 100755 --- a/metplus/wrappers/gfdl_tracker_wrapper.py +++ b/metplus/wrappers/gfdl_tracker_wrapper.py @@ -228,7 +228,10 @@ def create_c_dict(self): if not c_dict['OUTPUT_DIR']: self.log_error('GFDL_TRACKER_OUTPUT_DIR must be set') - + # skip RuntimeFreq input file logic - remove once integrated + c_dict['FIND_FILES'] = False + # force error if inputs are missing + c_dict['ALLOW_MISSING_INPUTS'] = False return c_dict def _read_gfdl_config_variables(self, c_dict): diff --git a/metplus/wrappers/grid_diag_wrapper.py b/metplus/wrappers/grid_diag_wrapper.py index 7219aa5ec2..c95ab75d9a 100755 --- a/metplus/wrappers/grid_diag_wrapper.py +++ b/metplus/wrappers/grid_diag_wrapper.py @@ -12,9 +12,8 @@ import os -from ..util import time_util -from . import RuntimeFreqWrapper from ..util import do_string_sub, parse_var_list, sub_var_list +from . import RuntimeFreqWrapper '''!@namespace GridDiagWrapper @brief Wraps the Grid-Diag tool @@ -60,8 +59,7 @@ def create_c_dict(self): # get the MET config file path or use default c_dict['CONFIG_FILE'] = self.get_config_file('GridDiagConfig_wrapped') - c_dict['INPUT_DIR'] = self.config.getdir('GRID_DIAG_INPUT_DIR', '') - self.get_input_templates(c_dict) + self.get_input_templates_multiple(c_dict) # error if no input templates are set if not c_dict['TEMPLATE_DICT']: @@ -205,8 +203,7 @@ def set_command_line_arguments(self, time_info): @param time_info dictionary containing time information """ - config_file = do_string_sub(self.c_dict['CONFIG_FILE'], - **time_info) + config_file = do_string_sub(self.c_dict['CONFIG_FILE'], **time_info) self.args.append(f"-config {config_file}") def get_files_from_time(self, time_info): @@ -220,12 +217,21 @@ def get_files_from_time(self, time_info): @returns dictionary containing time_info dict and any relevant files with a key representing a description of that file """ - file_dict = super().get_files_from_time(time_info) - input_files = self.get_input_files(time_info) + input_files, offset_time_info = self.get_input_files(time_info) if input_files is None: return None + file_dict = {'time_info': time_info.copy()} for key, value in input_files.items(): file_dict[key] = value return file_dict + + def _update_list_with_new_files(self, time_info, list_to_update): + new_files = self.get_files_from_time(time_info) + if not new_files: + return + if isinstance(new_files, list): + list_to_update.extend(new_files) + else: + list_to_update.append(new_files) diff --git a/metplus/wrappers/grid_stat_wrapper.py b/metplus/wrappers/grid_stat_wrapper.py index bc60e2b7e4..afc32ccbbd 100755 --- a/metplus/wrappers/grid_stat_wrapper.py +++ b/metplus/wrappers/grid_stat_wrapper.py @@ -131,26 +131,14 @@ def create_c_dict(self): # get the MET config file path or use default c_dict['CONFIG_FILE'] = self.get_config_file('GridStatConfig_wrapped') - c_dict['OBS_INPUT_DIR'] = \ - self.config.getdir('OBS_GRID_STAT_INPUT_DIR', '') - c_dict['OBS_INPUT_TEMPLATE'] = \ - self.config.getraw('filename_templates', - 'OBS_GRID_STAT_INPUT_TEMPLATE') - if not c_dict['OBS_INPUT_TEMPLATE']: - self.log_error("OBS_GRID_STAT_INPUT_TEMPLATE required to run") + self.get_input_templates(c_dict, { + 'FCST': {'prefix': 'FCST_GRID_STAT', 'required': True}, + 'OBS': {'prefix': 'OBS_GRID_STAT', 'required': True}, + }) c_dict['OBS_INPUT_DATATYPE'] = \ self.config.getstr('config', 'OBS_GRID_STAT_INPUT_DATATYPE', '') - c_dict['FCST_INPUT_DIR'] = \ - self.config.getdir('FCST_GRID_STAT_INPUT_DIR', '') - c_dict['FCST_INPUT_TEMPLATE'] = \ - self.config.getraw('filename_templates', - 'FCST_GRID_STAT_INPUT_TEMPLATE') - - if not c_dict['FCST_INPUT_TEMPLATE']: - self.log_error("FCST_GRID_STAT_INPUT_TEMPLATE required to run") - c_dict['FCST_INPUT_DATATYPE'] = \ self.config.getstr('config', 'FCST_GRID_STAT_INPUT_DATATYPE', '') @@ -272,5 +260,6 @@ def create_c_dict(self): self.add_met_config(name='seeps_p1_thresh', data_type='string', extra_args={'remove_quotes': True}) - + # skip RuntimeFreq input file logic - remove once integrated + c_dict['FIND_FILES'] = False return c_dict diff --git a/metplus/wrappers/ioda2nc_wrapper.py b/metplus/wrappers/ioda2nc_wrapper.py index 323fbbe0cd..8016fc6da4 100755 --- a/metplus/wrappers/ioda2nc_wrapper.py +++ b/metplus/wrappers/ioda2nc_wrapper.py @@ -58,12 +58,10 @@ def create_c_dict(self): # file I/O c_dict['ALLOW_MULTIPLE_FILES'] = True - c_dict['OBS_INPUT_DIR'] = self.config.getdir('IODA2NC_INPUT_DIR', '') - c_dict['OBS_INPUT_TEMPLATE'] = ( - self.config.getraw('config', 'IODA2NC_INPUT_TEMPLATE') - ) - if not c_dict['OBS_INPUT_TEMPLATE']: - self.log_error("IODA2NC_INPUT_TEMPLATE required to run") + + self.get_input_templates(c_dict, { + 'OBS': {'prefix': 'IODA2NC', 'required': True}, + }) # handle input file window variables self.handle_file_window_variables(c_dict, data_types=['OBS']) @@ -118,13 +116,16 @@ def find_input_files(self, time_info): @param time_info dictionary containing timing information @returns List of files that were found or None if no files were found """ - # get list of files even if only one is found (return_list=True) - obs_path = self.find_obs(time_info, return_list=True) - if obs_path is None: + if not self.c_dict.get('ALL_FILES'): + return None + + input_files = self.c_dict['ALL_FILES'][0].get('OBS', []) + if not input_files: return None - self.infiles.extend(obs_path) - return self.infiles + self.logger.debug(f"Adding input: {' and '.join(input_files)}") + self.infiles.extend(input_files) + return self.c_dict['ALL_FILES'][0].get('time_info') def set_command_line_arguments(self, time_info): """! Set all arguments for ioda2nc command. diff --git a/metplus/wrappers/met_db_load_wrapper.py b/metplus/wrappers/met_db_load_wrapper.py index a837647f22..5c28dac84e 100755 --- a/metplus/wrappers/met_db_load_wrapper.py +++ b/metplus/wrappers/met_db_load_wrapper.py @@ -146,7 +146,7 @@ def get_stat_directories(self, input_paths): """! Traverse through files under input path and find all directories that contain .stat, .tcst, mode*.txt, and mtd*.txt files. - @param input_path top level directory to search + @param input_paths top level directory to search @returns list of unique directories that contain stat files """ stat_dirs = set() diff --git a/metplus/wrappers/mode_wrapper.py b/metplus/wrappers/mode_wrapper.py index ab856a1432..36a83c3941 100755 --- a/metplus/wrappers/mode_wrapper.py +++ b/metplus/wrappers/mode_wrapper.py @@ -134,9 +134,9 @@ def __init__(self, config, instance=None): self.app_name) super().__init__(config, instance=instance) - def add_merge_config_file(self, time_info): + def set_command_line_arguments(self, time_info): """!If merge config file is defined, add it to the command""" - if self.c_dict['MERGE_CONFIG_FILE'] != '': + if self.c_dict['MERGE_CONFIG_FILE']: merge_config_file = do_string_sub(self.c_dict['MERGE_CONFIG_FILE'], **time_info) self.args.append('-config_merge {}'.format(merge_config_file)) @@ -444,39 +444,16 @@ def create_c_dict(self): self.add_met_config(name='multivar_intensity_flag', data_type='list', extra_args={'remove_quotes': True, 'uppercase': True}) - + # skip RuntimeFreq input file logic - remove once integrated + c_dict['FIND_FILES'] = False return c_dict def run_at_time_one_field(self, time_info, var_info): - """! Runs mode instances for a given time and forecast lead combination - Overrides run_at_time_one_field function in compare_gridded_wrapper.py - Args: - @param time_info dictionary containing timing information - @param var_info object containing variable information - """ - # get model to compare - model_path = self.find_model(time_info) - if model_path is None: - return - - # get observation to compare - obs_path = self.find_obs(time_info) - if obs_path is None: - return - - # loop over all variables and levels (and probability thresholds) and - # call the app for each - self.process_fields_one_thresh(time_info, var_info, model_path, - obs_path) - - def process_fields_one_thresh(self, time_info, var_info, model_path, - obs_path): - """! For each threshold, set up environment variables and run mode + """! Runs mode once for each fcst/obs threshold. + Overrides run_at_time_one_field function in compare_gridded_wrapper.py @param time_info dictionary containing timing information @param var_info object containing variable information - @param model_path forecast file - @param obs_path observation file """ # if no thresholds are specified, run once fcst_thresh_list = [] @@ -506,20 +483,11 @@ def process_fields_one_thresh(self, time_info, var_info, model_path, # loop through fields and call MODE for fcst_field, obs_field in zip(fcst_field_list, obs_field_list): - self.clear() - self.format_field('FCST', - fcst_field, - is_list=False) - self.format_field('OBS', - obs_field, - is_list=False) - - self.param = do_string_sub(self.c_dict['CONFIG_FILE'], - **time_info) - - self.infiles.append(model_path) - self.infiles.append(obs_path) - self.add_merge_config_file(time_info) + self.clear(clear_input_files=False) + self.format_field('FCST', fcst_field, is_list=False) + self.format_field('OBS', obs_field, is_list=False) + self.param = do_string_sub(self.c_dict['CONFIG_FILE'], **time_info) + self.set_command_line_arguments(time_info) self.set_current_field_config(var_info) self.set_environment_variables(time_info) if not self.find_and_check_output_file(time_info, diff --git a/metplus/wrappers/mtd_wrapper.py b/metplus/wrappers/mtd_wrapper.py index aa3a46dab5..c2c19a1e3c 100755 --- a/metplus/wrappers/mtd_wrapper.py +++ b/metplus/wrappers/mtd_wrapper.py @@ -12,10 +12,7 @@ import os -from ..util import get_lead_sequence, sub_var_list -from ..util import ti_calculate, getlist -from ..util import do_string_sub, skip_time -from ..util import parse_var_list, add_field_info_to_time_info +from ..util import get_lead_sequence, ti_calculate, do_string_sub, parse_var_list from . import CompareGriddedWrapper @@ -63,8 +60,6 @@ def __init__(self, config, instance=None): self.app_path = os.path.join(config.getdir('MET_BIN_DIR', ''), self.app_name) super().__init__(config, instance=instance) - self.fcst_file = None - self.obs_file = None def create_c_dict(self): c_dict = super().create_c_dict() @@ -90,18 +85,31 @@ def create_c_dict(self): # new method of reading/setting MET config values self.add_met_config(name='min_volume', data_type='int') + input_info = { + 'FCST': {'prefix': 'FCST_MTD', 'required': True}, + 'OBS': {'prefix': 'OBS_MTD', 'required': True}, + } + c_dict['SINGLE_RUN'] = ( self.config.getbool('config', 'MTD_SINGLE_RUN', False) ) if c_dict['SINGLE_RUN']: - c_dict['SINGLE_DATA_SRC'] = ( - self.config.getstr('config', 'MTD_SINGLE_DATA_SRC', '') - ) - if not c_dict['SINGLE_DATA_SRC']: + single_src = self.config.getraw('config', 'MTD_SINGLE_DATA_SRC') + c_dict['SINGLE_DATA_SRC'] = single_src + if not single_src: self.log_error('Must set MTD_SINGLE_DATA_SRC if ' 'MTD_SINGLE_RUN is True') + elif single_src not in ('FCST', 'OBS'): + self.log_error('MTD_SINGLE_DATA_SRC must be FCST or OBS.' + f' It is set to {single_src}') + + # do not read input templates for other data source if single mode + if single_src == 'FCST': + del input_info['OBS'] + else: + del input_info['FCST'] - self.get_input_templates(c_dict) + self.get_input_templates(c_dict, input_info) # if single run for OBS, read OBS values into FCST keys read_type = 'FCST' @@ -180,12 +188,15 @@ def run_at_time_once(self, time_info): outfile = f"{time_fmt}_mtd_{dt.lower()}_{file_ext}.txt" inputs[data_type] = self.write_list_file(outfile, file_list) - if not inputs: - self.log_error('Input files not found') - continue - if len(inputs) < 2 and not self.c_dict['SINGLE_RUN']: - self.log_error('Could not find all required inputs files') + if not inputs or (len(inputs) < 2 and not self.c_dict['SINGLE_RUN']): + self.missing_input_count += 1 + msg = 'Could not find all required inputs files' + if self.c_dict['ALLOW_MISSING_INPUTS']: + self.logger.warning(msg) + else: + self.log_error(msg) continue + arg_dict = { 'obs_path': inputs.get('OBS'), 'model_path': inputs.get('FCST'), @@ -286,21 +297,16 @@ def process_fields_one_thresh(self, first_valid_time_info, var_info, is_directory=True): return - fcst_file = model_path if self.c_dict['SINGLE_RUN']: if self.c_dict.get('SINGLE_DATA_SRC') == 'OBS': - fcst_file = obs_path + self.infiles.append(obs_path) + else: + self.infiles.append(model_path) else: - self.obs_file = obs_path + self.infiles.extend([model_path, obs_path]) - self.fcst_file = fcst_file self.build() - def clear(self): - super().clear() - self.fcst_file = None - self.obs_file = None - def get_command(self): """! Builds the command to run the MET application @rtype string @@ -312,10 +318,9 @@ def get_command(self): cmd += a + " " if self.c_dict['SINGLE_RUN']: - cmd += '-single ' + self.fcst_file + ' ' + cmd += f'-single {self.infiles[0]} ' else: - cmd += '-fcst ' + self.fcst_file + ' ' - cmd += '-obs ' + self.obs_file + ' ' + cmd += f'-fcst {self.infiles[0]} -obs {self.infiles[1]} ' cmd += '-config ' + self.param + ' ' @@ -323,88 +328,3 @@ def get_command(self): cmd += '-outdir {}'.format(self.outdir) return cmd - - def get_input_templates(self, c_dict): - input_types = ['FCST', 'OBS'] - if c_dict.get('SINGLE_RUN', False): - input_types = [c_dict['SINGLE_DATA_SRC']] - - app = self.app_name.upper() - template_dict = {} - for in_type in input_types: - template_path = ( - self.config.getraw('config', - f'{in_type}_{app}_INPUT_FILE_LIST') - ) - if template_path: - c_dict['EXPLICIT_FILE_LIST'] = True - else: - in_dir = self.config.getdir(f'{in_type}_{app}_INPUT_DIR', '') - templates = getlist( - self.config.getraw('config', - f'{in_type}_{app}_INPUT_TEMPLATE') - ) - template_list = [os.path.join(in_dir, template) - for template in templates] - template_path = ','.join(template_list) - - template_dict[in_type] = template_path - - c_dict['TEMPLATE_DICT'] = template_dict - - def get_files_from_time(self, time_info): - """! Create dictionary containing time information (key time_info) and - any relevant files for that runtime. The parent implementation of - this function creates a dictionary and adds the time_info to it. - This wrapper gets all files for the current runtime and adds it to - the dictionary with keys 'FCST' and 'OBS' - - @param time_info dictionary containing time information - @returns dictionary containing time_info dict and any relevant - files with a key representing a description of that file - """ - if self.c_dict.get('ONCE_PER_FIELD', False): - var_list = sub_var_list(self.c_dict.get('VAR_LIST_TEMP'), time_info) - else: - var_list = [None] - - # create a dictionary for each field (var) with time_info and files - file_dict_list = [] - for var_info in var_list: - file_dict = {'var_info': var_info} - if var_info: - add_field_info_to_time_info(time_info, var_info) - - input_files = self.get_input_files(time_info, fill_missing=True) - # only add all input files if none are missing - no_missing = True - if input_files: - for key, value in input_files.items(): - if 'missing' in value: - no_missing = False - file_dict[key] = value - if no_missing: - file_dict_list.append(file_dict) - - return file_dict_list - - def _update_list_with_new_files(self, time_info, list_to_update): - new_files = self.get_files_from_time(time_info) - if not new_files: - return - - # if list to update is empty, copy new items into list - if not list_to_update: - for new_file in new_files: - list_to_update.append(new_file.copy()) - return - - # if list to update is not empty, add new files to each file list, - # make sure new files correspond to the correct field (var) - assert len(list_to_update) == len(new_files) - for new_file, existing_item in zip(new_files, list_to_update): - assert new_file['var_info'] == existing_item['var_info'] - for key, value in new_file.items(): - if key == 'var_info': - continue - existing_item[key].extend(value) diff --git a/metplus/wrappers/pb2nc_wrapper.py b/metplus/wrappers/pb2nc_wrapper.py index 479c567aed..f17ab2c183 100755 --- a/metplus/wrappers/pb2nc_wrapper.py +++ b/metplus/wrappers/pb2nc_wrapper.py @@ -11,11 +11,10 @@ """ import os -import re -from ..util import getlistint, skip_time, get_lead_sequence -from ..util import ti_calculate +from ..util import getlistint from ..util import do_string_sub +from ..util import add_field_info_to_time_info from . import LoopTimesWrapper @@ -82,28 +81,18 @@ def create_c_dict(self): 'PB2NC_OFFSETS', '0')) - # Directories - # these are optional because users can specify full file path - # in template instead - c_dict['OBS_INPUT_DIR'] = self.config.getdir('PB2NC_INPUT_DIR', '') - c_dict['OUTPUT_DIR'] = self.config.getdir('PB2NC_OUTPUT_DIR', '') - - # filename templates, exit if not set - c_dict['OBS_INPUT_TEMPLATE'] = ( - self.config.getraw('filename_templates', - 'PB2NC_INPUT_TEMPLATE') - ) - if not c_dict['OBS_INPUT_TEMPLATE']: - self.log_error('Must set PB2NC_INPUT_TEMPLATE in config file') + self.get_input_templates(c_dict, { + 'OBS': {'prefix': 'PB2NC', 'required': True}, + }) - c_dict['OUTPUT_TEMPLATE'] = self.config.getraw('filename_templates', + c_dict['OUTPUT_DIR'] = self.config.getdir('PB2NC_OUTPUT_DIR', '') + c_dict['OUTPUT_TEMPLATE'] = self.config.getraw('config', 'PB2NC_OUTPUT_TEMPLATE') if not c_dict['OUTPUT_TEMPLATE']: self.log_error('Must set PB2NC_OUTPUT_TEMPLATE in config file') c_dict['OBS_INPUT_DATATYPE'] = ( - self.config.getstr('config', - 'PB2NC_INPUT_DATATYPE', '') + self.config.getraw('config', 'PB2NC_INPUT_DATATYPE', '') ) # get the MET config file path or use default @@ -117,8 +106,7 @@ def create_c_dict(self): self.handle_mask(single_value=True) - self.add_met_config(name='obs_bufr_var', - data_type='list', + self.add_met_config(name='obs_bufr_var', data_type='list', metplus_configs=['PB2NC_OBS_BUFR_VAR_LIST', 'PB2NC_OBS_BUFR_VAR'], extra_args={'allow_empty': True}) @@ -127,11 +115,10 @@ def create_c_dict(self): self.handle_file_window_variables(c_dict, data_types=['OBS']) - c_dict['VALID_BEGIN_TEMPLATE'] = \ - self.config.getraw('config', 'PB2NC_VALID_BEGIN', '') - - c_dict['VALID_END_TEMPLATE'] = \ - self.config.getraw('config', 'PB2NC_VALID_END', '') + c_dict['VALID_BEGIN_TEMPLATE'] = self.config.getraw('config', + 'PB2NC_VALID_BEGIN') + c_dict['VALID_END_TEMPLATE'] = self.config.getraw('config', + 'PB2NC_VALID_END') c_dict['ALLOW_MULTIPLE_FILES'] = True @@ -143,59 +130,50 @@ def create_c_dict(self): # get level_range beg and end self.add_met_config_window('level_range') - self.add_met_config(name='level_category', - data_type='list', + self.add_met_config(name='level_category', data_type='list', metplus_configs=['PB2NC_LEVEL_CATEGORY'], extra_args={'remove_quotes': True}) - self.add_met_config(name='quality_mark_thresh', - data_type='int', + self.add_met_config(name='quality_mark_thresh', data_type='int', metplus_configs=['PB2NC_QUALITY_MARK_THRESH']) - self.add_met_config(name='obs_bufr_map', - data_type='list', + self.add_met_config(name='obs_bufr_map', data_type='list', extra_args={'remove_quotes': True}) return c_dict - def find_input_files(self, input_dict): + def find_input_files(self): """!Find prepbufr data to convert. - @param input_dict dictionary containing some time information - @returns time info if files are found, None otherwise + @returns time info if files are found, None otherwise """ + if not self.c_dict.get('ALL_FILES'): + return None - infiles, time_info = self.find_obs_offset(input_dict, - mandatory=True, - return_list=True) - - # if file is found, return timing info dict so - # output template can use offset value - if infiles is None: + input_files = self.c_dict['ALL_FILES'][0].get('OBS', []) + if not input_files: return None - self.logger.debug(f"Adding input: {' and '.join(infiles)}") - self.infiles.extend(infiles) - return time_info + self.logger.debug(f"Adding input: {' and '.join(input_files)}") + self.infiles.extend(input_files) + return self.c_dict['ALL_FILES'][0].get('time_info') def set_valid_window_variables(self, time_info): begin_template = self.c_dict['VALID_BEGIN_TEMPLATE'] end_template = self.c_dict['VALID_END_TEMPLATE'] if begin_template: - self.c_dict['VALID_WINDOW_BEGIN'] = \ - do_string_sub(begin_template, - **time_info) + self.c_dict['VALID_WINDOW_BEGIN'] = do_string_sub(begin_template, + **time_info) if end_template: - self.c_dict['VALID_WINDOW_END'] = \ - do_string_sub(end_template, - **time_info) + self.c_dict['VALID_WINDOW_END'] = do_string_sub(end_template, + **time_info) def run_at_time_once(self, input_dict): """!Find files needed to run pb2nc and run if found""" # look for input files to process - time_info = self.find_input_files(input_dict) + time_info = self.find_input_files() # if no files were found, don't run pb2nc if time_info is None: @@ -227,12 +205,6 @@ def get_command(self): for arg in self.args: cmd += f' {arg}' - # if multiple input files, add first now, then add rest with - # -pbfile argument - if not self.infiles: - self.log_error("No input files found") - return None - cmd += f" {self.infiles[0]}" out_path = self.get_output_path() @@ -240,6 +212,7 @@ def get_command(self): cmd += f" {self.c_dict['CONFIG_FILE']}" + # add additional input files with -pbfile argument if len(self.infiles) > 1: for infile in self.infiles[1:]: cmd += f" -pbfile {infile}" diff --git a/metplus/wrappers/pcp_combine_wrapper.py b/metplus/wrappers/pcp_combine_wrapper.py index 77b5b0b823..d4617e3053 100755 --- a/metplus/wrappers/pcp_combine_wrapper.py +++ b/metplus/wrappers/pcp_combine_wrapper.py @@ -228,7 +228,8 @@ def set_fcst_or_obs_dict_items(self, d_type, c_dict): self.log_error(f'{d_type}_PCP_COMBINE_INPUT_LEVELS list ' 'should be either empty or the same length as ' f'{d_type}_PCP_COMBINE_INPUT_ACCUMS list.') - + # skip RuntimeFreq input file logic - remove once integrated + c_dict['FIND_FILES'] = False return c_dict def run_at_time_once(self, time_info): @@ -495,12 +496,18 @@ def setup_add_method(self, time_info, lookback, data_src): # create list of tuples for input levels and optional field names self._build_input_accum_list(data_src, time_info) + self.run_count += 1 files_found = self.get_accumulation(time_info, lookback, data_src) if not files_found: - self.log_error( + self.missing_input_count += 1 + msg = ( f'Could not find files to build accumulation in ' f"{self.c_dict[f'{data_src}_INPUT_DIR']} using template " f"{self.c_dict[f'{data_src}_INPUT_TEMPLATE']}") + if self.c_dict['ALLOW_MISSING_INPUTS']: + self.logger.warning(msg) + else: + self.log_error(msg) return False return files_found @@ -533,10 +540,12 @@ def setup_derive_method(self, time_info, lookback, data_src): name=accum_dict['name'], level=accum_dict['level'], extra=accum_dict['extra']) + self.run_count += 1 input_files = self.find_data(time_info, data_type=data_src, return_list=True) if not input_files: + self.missing_input_count += 1 return None files_found = [] @@ -546,15 +555,21 @@ def setup_derive_method(self, time_info, lookback, data_src): files_found.append((input_file, field_info)) else: + self.run_count += 1 files_found = self.get_accumulation(time_info, lookback, data_src, field_info_after_file=False) if not files_found: - self.log_error( + self.missing_input_count += 1 + msg = ( f'Could not find files to build accumulation in ' f"{self.c_dict[f'{data_src}_INPUT_DIR']} using template " f"{self.c_dict[f'{data_src}_INPUT_TEMPLATE']}") + if self.c_dict['ALLOW_MISSING_INPUTS']: + self.logger.warning(msg) + else: + self.log_error(msg) return None # set -field name and level from first file field info diff --git a/metplus/wrappers/plot_data_plane_wrapper.py b/metplus/wrappers/plot_data_plane_wrapper.py index b0efc87980..362488e975 100755 --- a/metplus/wrappers/plot_data_plane_wrapper.py +++ b/metplus/wrappers/plot_data_plane_wrapper.py @@ -97,7 +97,8 @@ def create_c_dict(self): if c_dict['CONVERT_TO_IMAGE'] and not c_dict['CONVERT_EXE']: self.log_error("[exe] CONVERT must be set correctly if " "PLOT_DATA_PLANE_CONVERT_TO_IMAGE is True") - + # skip RuntimeFreq input file logic - remove once integrated + c_dict['FIND_FILES'] = False return c_dict def get_command(self): @@ -111,27 +112,14 @@ def get_command(self): return cmd def run_at_time_once(self, time_info): - """! Process runtime and try to build command to run ascii2nc - Args: - @param time_info dictionary containing timing information + """! Process runtime and try to build command to run plot_data_plane. + Calls parent run_at_time_once (RuntimeFreq) then optionally converts + PS output to PNG if requested. + + @param time_info dictionary containing timing information """ self.clear() - - # get input files - if not self.find_input_files(time_info): - return False - - # get output path - if not self.find_and_check_output_file(time_info): - return False - - # get other configurations for command - self.set_command_line_arguments(time_info) - - # set environment variables if using config file - self.set_environment_variables(time_info) - - if not self.build(): + if not super().run_at_time_once(time_info): return False if self.c_dict['CONVERT_TO_IMAGE']: @@ -144,14 +132,14 @@ def find_input_files(self, time_info): # just pass value to input file list if 'PYTHON' in self.c_dict['INPUT_TEMPLATE']: self.infiles.append(self.c_dict['INPUT_TEMPLATE']) - return self.infiles + return True file_path = self.find_data(time_info, return_list=False) if not file_path: - return None + return False self.infiles.append(file_path) - return self.infiles + return True def set_command_line_arguments(self, time_info): field_name = do_string_sub(self.c_dict['FIELD_NAME'], diff --git a/metplus/wrappers/plot_point_obs_wrapper.py b/metplus/wrappers/plot_point_obs_wrapper.py index bf6f2798f2..8373b1b0c0 100755 --- a/metplus/wrappers/plot_point_obs_wrapper.py +++ b/metplus/wrappers/plot_point_obs_wrapper.py @@ -175,6 +175,8 @@ def create_c_dict(self): extra_args={'remove_quotes': True}) c_dict['ALLOW_MULTIPLE_FILES'] = True + # skip RuntimeFreq input file logic - remove once integrated + c_dict['FIND_FILES'] = False return c_dict def get_command(self): @@ -192,7 +194,7 @@ def find_input_files(self, time_info): return_list=True, mandatory=True) if input_files is None: - return None + return False self.infiles.extend(input_files) @@ -202,17 +204,17 @@ def find_input_files(self, time_info): data_type='GRID', return_list=True) if not grid_file: - return None + return False if len(grid_file) > 1: self.log_error('More than one file found from ' 'PLOT_POINT_OBS_GRID_INPUT_TEMPLATE: ' f'{grid_file.split(",")}') - return None + return False self.c_dict['GRID_INPUT_PATH'] = grid_file[0] - return self.infiles + return True def set_command_line_arguments(self, time_info): """! Set all arguments for plot_point_obs command. diff --git a/metplus/wrappers/point2grid_wrapper.py b/metplus/wrappers/point2grid_wrapper.py index e4cb356cd0..15bbbd1374 100755 --- a/metplus/wrappers/point2grid_wrapper.py +++ b/metplus/wrappers/point2grid_wrapper.py @@ -12,8 +12,6 @@ import os -from ..util import get_lead_sequence -from ..util import ti_calculate from ..util import do_string_sub from ..util import remove_quotes from . import LoopTimesWrapper @@ -111,7 +109,8 @@ def create_c_dict(self): c_dict['VLD_THRESH'] = self.config.getstr('config', 'POINT2GRID_VLD_THRESH', '') - + # skip RuntimeFreq input file logic - remove once integrated + c_dict['FIND_FILES'] = False return c_dict def get_command(self): @@ -147,28 +146,6 @@ def get_command(self): cmd += ' -v ' + self.c_dict['VERBOSITY'] return cmd - def run_at_time_once(self, time_info): - """! Process runtime and try to build command to run point2grid - Args: - @param time_info dictionary containing timing information - """ - # get input files - if self.find_input_files(time_info) is None: - return - - # get output path - if not self.find_and_check_output_file(time_info): - return - - # get other configurations for command - self.set_command_line_arguments(time_info) - - # set environment variables if using config file - self.set_environment_variables(time_info) - - # build command and run - self.build() - def find_input_files(self, time_info): """!Find input file and mask file and add them to the list of input files. Args: @@ -179,14 +156,14 @@ def find_input_files(self, time_info): # calling find_obs because we set OBS_ variables in c_dict for the input data input_path = self.find_obs(time_info) if input_path is None: - return None + return False self.infiles.append(input_path) self.c_dict['GRID'] = do_string_sub(self.c_dict['GRID_TEMPLATE'], **time_info) - return self.infiles + return True def set_command_line_arguments(self, time_info): """!Set command line arguments from c_dict diff --git a/metplus/wrappers/point_stat_wrapper.py b/metplus/wrappers/point_stat_wrapper.py index d8809c7225..e88dd80d9b 100755 --- a/metplus/wrappers/point_stat_wrapper.py +++ b/metplus/wrappers/point_stat_wrapper.py @@ -291,7 +291,8 @@ def create_c_dict(self): if not c_dict['OUTPUT_DIR']: self.log_error('Must set POINT_STAT_OUTPUT_DIR in config file') - + # skip RuntimeFreq input file logic - remove once integrated + c_dict['FIND_FILES'] = False return c_dict def set_command_line_arguments(self, time_info): diff --git a/metplus/wrappers/py_embed_ingest_wrapper.py b/metplus/wrappers/py_embed_ingest_wrapper.py index 0f6d069175..2f86694e33 100755 --- a/metplus/wrappers/py_embed_ingest_wrapper.py +++ b/metplus/wrappers/py_embed_ingest_wrapper.py @@ -112,6 +112,7 @@ def create_c_dict(self): RegridDataPlaneWrapper(self.config, instance=instance) ) + c_dict['FIND_FILES'] = False return c_dict def get_ingest_items(self, item_type, index, ingest_script_addons): @@ -135,8 +136,7 @@ def run_at_time_once(self, time_info): index = ingester['index'] # get grid information to project output data - output_grid = do_string_sub(ingester['output_grid'], - **time_info) + output_grid = do_string_sub(ingester['output_grid'], **time_info) rdp.clear() # get output file path @@ -148,8 +148,7 @@ def run_at_time_once(self, time_info): rdp.infiles.append(f"PYTHON_{ingester['input_type']}") for script_raw in ingester['scripts']: - script = do_string_sub(script_raw, - **time_info) + script = do_string_sub(script_raw, **time_info) rdp.infiles.append(f'-field \'name="{script}\";\'') diff --git a/metplus/wrappers/regrid_data_plane_wrapper.py b/metplus/wrappers/regrid_data_plane_wrapper.py index 7761d600e1..5ae51fd08b 100755 --- a/metplus/wrappers/regrid_data_plane_wrapper.py +++ b/metplus/wrappers/regrid_data_plane_wrapper.py @@ -160,7 +160,8 @@ def create_c_dict(self): if 'RegridDataPlane' in get_process_list(self.config): if not c_dict['VERIFICATION_GRID']: self.log_error("REGRID_DATA_PLANE_VERIF_GRID must be set.") - + # skip RuntimeFreq input file logic - remove once integrated + c_dict['FIND_FILES'] = False return c_dict def handle_output_file(self, time_info, field_info, data_type): @@ -319,7 +320,9 @@ def run_at_time_once(self, time_info): return False add_field_info_to_time_info(time_info, var_list[0]) + self.run_count += 1 if not self.find_input_files(time_info, data_type): + self.missing_input_count += 1 return False # set environment variables @@ -344,17 +347,16 @@ def find_input_files(self, time_info, data_type): """ input_path = self.find_data(time_info, data_type=data_type) if not input_path: - return None + return False self.infiles.append(input_path) - verif_grid = do_string_sub(self.c_dict['VERIFICATION_GRID'], - **time_info) + grid = do_string_sub(self.c_dict['VERIFICATION_GRID'], **time_info) # put quotes around verification grid in case it is a grid description - self.infiles.append(f'"{verif_grid}"') + self.infiles.append(f'"{grid}"') - return self.infiles + return True def set_command_line_arguments(self): """!Returns False if command should not be run""" diff --git a/metplus/wrappers/runtime_freq_wrapper.py b/metplus/wrappers/runtime_freq_wrapper.py index 7b195a8de7..03a6f1a31e 100755 --- a/metplus/wrappers/runtime_freq_wrapper.py +++ b/metplus/wrappers/runtime_freq_wrapper.py @@ -11,14 +11,13 @@ """ import os -from datetime import datetime from ..util import time_util -from . import CommandBuilder -from ..util import do_string_sub -from ..util import log_runtime_banner, get_lead_sequence, is_loop_by_init +from ..util import log_runtime_banner, get_lead_sequence from ..util import skip_time, getlist, get_start_and_end_times, get_time_prefix from ..util import time_generator, add_to_time_input +from ..util import sub_var_list, add_field_info_to_time_info +from . import CommandBuilder '''!@namespace RuntimeFreqWrapper @brief Parent class for wrappers that run over a grouping of times @@ -38,6 +37,8 @@ class RuntimeFreqWrapper(CommandBuilder): def __init__(self, config, instance=None): super().__init__(config, instance=instance) + self.run_count = 0 + self.missing_input_count = 0 def create_c_dict(self): c_dict = super().create_c_dict() @@ -59,6 +60,13 @@ def create_c_dict(self): ) self.validate_runtime_freq(c_dict) + # check if missing inputs are allowed and threshold of missing inputs + name = 'ALLOW_MISSING_INPUTS' + c_dict[name] = self.get_wrapper_or_generic_config(name, 'bool') + if c_dict[name]: + name = 'INPUT_THRESH' + c_dict[name] = self.get_wrapper_or_generic_config(name, 'float') + return c_dict def validate_runtime_freq(self, c_dict): @@ -103,28 +111,62 @@ def validate_runtime_freq(self, c_dict): self.log_error(err_msg) return - def get_input_templates(self, c_dict): + def get_input_templates(self, c_dict, input_info=None): + """!Read input templates from config. + """ + template_dict = {} + if not input_info: + return + + for label, info in input_info.items(): + prefix = info.get('prefix') + required = info.get('required', True) + + template = self.config.getraw('config', f'{prefix}_INPUT_FILE_LIST') + if template: + c_dict['EXPLICIT_FILE_LIST'] = True + else: + input_dir = self.config.getdir(f'{prefix}_INPUT_DIR', '') + c_dict[f'{label}_INPUT_DIR'] = input_dir + templates = getlist( + self.config.getraw('config', f'{prefix}_INPUT_TEMPLATE') + ) + template_list = [os.path.join(input_dir, template) + for template in templates] + template = ','.join(template_list) + c_dict[f'{label}_INPUT_TEMPLATE'] = template + if not c_dict[f'{label}_INPUT_TEMPLATE']: + if required: + self.log_error(f'{prefix}_INPUT_TEMPLATE required to run') + continue + + template_dict[label] = (template, True) + + c_dict['TEMPLATE_DICT'] = template_dict + + def get_input_templates_multiple(self, c_dict): + """!Read input templates from config. Use this function when a given + input template may have multiple items separated by comma that need to + be handled separately. For example, GridDiag's input templates + correspond to each field specified in the MET config file. For example, + UserScript may call a script that needs to read multiple groups of files + separately. + + @param c_dict config dictionary to set INPUT_TEMPLATES + """ app_upper = self.app_name.upper() template_dict = {} + # read and set input directory + c_dict['INPUT_DIR'] = self.config.getdir(f'{app_upper}_INPUT_DIR', '') + input_templates = getlist( - self.config.getraw('config', - f'{app_upper}_INPUT_TEMPLATE', - '') + self.config.getraw('config', f'{app_upper}_INPUT_TEMPLATE') ) input_template_labels = getlist( - self.config.getraw('config', - f'{app_upper}_INPUT_TEMPLATE_LABELS', - '') + self.config.getraw('config', f'{app_upper}_INPUT_TEMPLATE_LABELS') ) - # cannot have more labels than templates specified - if len(input_template_labels) > len(input_templates): - self.log_error('Cannot supply more labels than templates. ' - f'{app_upper}_INPUT_TEMPLATE_LABELS length must be ' - f'less than {app_upper}_INPUT_TEMPLATES length.') - return - for idx, template in enumerate(input_templates): # if fewer labels than templates, fill in labels with input{idx} if len(input_template_labels) <= idx: @@ -132,7 +174,7 @@ def get_input_templates(self, c_dict): else: label = input_template_labels[idx] - template_dict[label] = template + template_dict[label] = (template, False) c_dict['TEMPLATE_DICT'] = template_dict @@ -149,6 +191,18 @@ def run_all_times(self): self.run_all_times_custom(custom_string) + # if missing inputs are allowed, check threshold to report error + if self.c_dict['ALLOW_MISSING_INPUTS']: + success_rate = (1 - (self.missing_input_count / self.run_count)) * 100 + allowed_rate = self.c_dict['INPUT_THRESH'] * 100 + if success_rate < allowed_rate: + self.log_error( + f'{success_rate}% of {wrapper_instance_name} runs had all ' + f'required inputs. Must have {allowed_rate}% to prevent error. ' + f'{self.missing_input_count} out of {self.run_count} runs ' + 'had missing inputs.' + ) + return self.all_commands def run_all_times_custom(self, custom): @@ -171,8 +225,7 @@ def run_once(self, custom): # create input dictionary and set clock time, instance, and custom time_input = {} add_to_time_input(time_input, - clock_time=self.config.getstr('config', - 'CLOCK_TIME'), + clock_time=self.config.getstr('config', 'CLOCK_TIME'), instance=self.instance, custom=custom) @@ -190,9 +243,9 @@ def run_once(self, custom): time_info = time_util.ti_calculate(time_input) - if not self.get_all_files(custom): - self.log_error("A problem occurred trying to obtain input files") - return None + self.c_dict['ALL_FILES'] = self.get_all_files(custom) + if not self._check_input_files(): + return False self.clear() return self.run_at_time_once(time_info) @@ -220,6 +273,8 @@ def run_once_per_init_or_valid(self, custom): time_info = time_util.ti_calculate(time_input) self.c_dict['ALL_FILES'] = self.get_all_files_from_leads(time_info) + if not self._check_input_files(): + continue self.clear() if not self.run_at_time_once(time_info): @@ -251,6 +306,8 @@ def run_once_per_lead(self, custom): time_info = time_util.ti_calculate(time_input) self.c_dict['ALL_FILES'] = self.get_all_files_for_lead(time_info) + if not self._check_input_files(): + continue self.clear() if not self.run_at_time_once(time_info): @@ -301,11 +358,9 @@ def run_at_time(self, input_dict): self.logger.debug('Skipping run time') continue - # since run_all_times was not called (LOOP_BY=times) then - # get files for current run time - all_files = [] - self._update_list_with_new_files(time_info, all_files) - self.c_dict['ALL_FILES'] = all_files + self.c_dict['ALL_FILES'] = self.get_all_files_for_each(time_info) + if not self._check_input_files(): + continue # Run for given init/valid time and forecast lead combination self.clear() @@ -325,7 +380,11 @@ def run_at_time_once(self, time_info): False if something went wrong """ # get input files + if not self.c_dict.get('TEMPLATE_DICT'): + self.run_count += 1 if not self.find_input_files(time_info): + if not self.c_dict.get('TEMPLATE_DICT'): + self.missing_input_count += 1 return False # get output path @@ -356,7 +415,7 @@ def get_all_files(self, custom=None): # loop over all init/valid times for time_input in time_generator(self.config): if time_input is None: - return False + return [] add_to_time_input(time_input, instance=self.instance, @@ -365,10 +424,20 @@ def get_all_files(self, custom=None): lead_files = self.get_all_files_from_leads(time_input) all_files.extend(lead_files) - if not all_files: - return False + return all_files - self.c_dict['ALL_FILES'] = all_files + def _check_input_files(self): + if self.c_dict['ALL_FILES'] is True: + return True + self.run_count += 1 + if not self.c_dict['ALL_FILES'] and self.app_name != 'user_script': + self.missing_input_count += 1 + msg = 'A problem occurred trying to obtain input files' + if self.c_dict['ALLOW_MISSING_INPUTS']: + self.logger.warning(msg) + else: + self.log_error(msg) + return False return True def get_all_files_from_leads(self, time_input): @@ -419,24 +488,76 @@ def get_all_files_for_lead(self, time_input): return new_files - @staticmethod - def get_files_from_time(time_info): + def get_all_files_for_each(self, time_info): + if not self.c_dict.get('FIND_FILES', True): + return True + + all_files = [] + self._update_list_with_new_files(time_info, all_files) + return all_files + + def get_files_from_time(self, time_info): """! Create dictionary containing time information (key time_info) and - any relevant files for that runtime. + any relevant files for that runtime. The parent implementation of + this function creates a dictionary and adds the time_info to it. + This wrapper gets all files for the current runtime and adds it to + the dictionary with keys 'FCST' and 'OBS' + @param time_info dictionary containing time information - @returns list of dict containing time_info dict and any relevant + @returns dictionary containing time_info dict and any relevant files with a key representing a description of that file """ - return {'time_info': time_info.copy()} + if self.c_dict.get('ONCE_PER_FIELD', False): + var_list = sub_var_list(self.c_dict.get('VAR_LIST_TEMP'), time_info) + else: + var_list = [None] + + # create a dictionary for each field (var) with time_info and files + file_dict_list = [] + for var_info in var_list: + file_dict = {'var_info': var_info} + if var_info: + add_field_info_to_time_info(time_info, var_info) + + input_files, offset_time_info = ( + self.get_input_files(time_info, fill_missing=True) + ) + file_dict['time_info'] = offset_time_info.copy() + # only add all input files if none are missing + no_missing = True + if input_files: + for key, value in input_files.items(): + if 'missing' in value: + no_missing = False + file_dict[key] = value + if no_missing: + file_dict_list.append(file_dict) + + return file_dict_list def _update_list_with_new_files(self, time_info, list_to_update): new_files = self.get_files_from_time(time_info) if not new_files: return - if isinstance(new_files, list): - list_to_update.extend(new_files) - else: - list_to_update.append(new_files) + + if not isinstance(new_files, list): + new_files = [new_files] + + # if list to update is empty, copy new items into list + if not list_to_update: + for new_file in new_files: + list_to_update.append(new_file.copy()) + return + + # if list to update is not empty, add new files to each file list, + # make sure new files correspond to the correct field (var) + assert len(list_to_update) == len(new_files) + for new_file, existing_item in zip(new_files, list_to_update): + assert new_file.get('var_info') == existing_item.get('var_info') + for key, value in new_file.items(): + if key == 'var_info' or key == 'time_info': + continue + existing_item[key].extend(value) @staticmethod def compare_time_info(runtime, filetime): @@ -482,21 +603,29 @@ def get_input_files(self, time_info, fill_missing=False): """ all_input_files = {} if not self.c_dict.get('TEMPLATE_DICT'): - return None + return None, time_info - for label, input_template in self.c_dict['TEMPLATE_DICT'].items(): + offset_time_info = time_info + for label, (template, required) in self.c_dict['TEMPLATE_DICT'].items(): data_type = '' template_key = 'INPUT_TEMPLATE' if label in ('FCST', 'OBS'): data_type = label template_key = f'{label}_{template_key}' - self.c_dict[template_key] = input_template + self.c_dict[template_key] = template # if fill missing is true, data is not mandatory to find - mandatory = not fill_missing - input_files = self.find_data(time_info, data_type=data_type, - return_list=True, - mandatory=mandatory) + mandatory = required and not fill_missing + if label == 'OBS': + input_files, offset_time_info = ( + self.find_obs_offset(time_info, mandatory=mandatory, + return_list=True) + ) + else: + input_files = self.find_data(time_info, data_type=data_type, + return_list=True, + mandatory=mandatory) + if not input_files: if not fill_missing: continue @@ -508,21 +637,27 @@ def get_input_files(self, time_info, fill_missing=False): # return None if no matching input files were found if not all_input_files: - return None + return None, None - return all_input_files + return all_input_files, offset_time_info - def subset_input_files(self, time_info, output_dir=None, leads=None): + def subset_input_files(self, time_info, output_dir=None, leads=None, + force_list=False): """! Obtain a subset of input files from the c_dict ALL_FILES based on the time information for the current run. @param time_info dictionary containing time information + @param output_dir (optional) directory to write file list files. + If no directory is provided, files are written to staging dir + @param leads (optional) list of forecast leads to consider + @param force_list (optional) boolean - if True, write a file list + text file even only 1 file was found. Defaults to False. @returns dictionary with keys of the input identifier and the value is the path to a ascii file containing the list of files or None if could not find any files """ all_input_files = {} - if not self.c_dict.get('ALL_FILES'): + if not self.c_dict.get('ALL_FILES') or self.c_dict.get('ALL_FILES') is True: return all_input_files if leads is None: @@ -561,6 +696,10 @@ def subset_input_files(self, time_info, output_dir=None, leads=None): # loop over all inputs and write a file list file for each list_file_dict = {} for identifier, input_files in all_input_files.items(): + if len(input_files) == 1 and not force_list: + list_file_dict[identifier] = input_files[0] + continue + list_file_name = self.get_list_file_name(time_info, identifier) list_file_path = self.write_list_file(list_file_name, input_files, diff --git a/metplus/wrappers/series_analysis_wrapper.py b/metplus/wrappers/series_analysis_wrapper.py index 14c0c96df0..633efdce76 100755 --- a/metplus/wrappers/series_analysis_wrapper.py +++ b/metplus/wrappers/series_analysis_wrapper.py @@ -202,9 +202,6 @@ def create_c_dict(self): '') ) - # initialize list path to None for each type - c_dict[f'{data_type}_LIST_PATH'] = None - # read and set file type env var for FCST and OBS if data_type == 'BOTH': continue @@ -396,12 +393,6 @@ def _plot_data_plane_init(self): instance=instance) return pdp_wrapper - def clear(self): - """! Call parent's clear function and clear additional values """ - super().clear() - for data_type in ('FCST', 'OBS', 'BOTH'): - self.c_dict[f'{data_type}_LIST_PATH'] = None - def run_all_times(self): """! Process all run times defined for this wrapper """ super().run_all_times() @@ -460,6 +451,8 @@ def run_once_per_lead(self, custom): self.c_dict['ALL_FILES'] = ( self.get_all_files_for_leads(input_dict, lead_group[1]) ) + if not self._check_input_files(): + continue # if only 1 forecast lead is being processed, set it in time dict if len(lead_group[1]) == 1: @@ -504,7 +497,11 @@ def run_at_time_once(self, time_info, lead_group=None): lead_group) ) if not fcst_path or not obs_path: - self.log_error('No ASCII file lists were created. Skipping.') + msg = 'No ASCII file lists were created. Skipping.' + if self.c_dict['ALLOW_MISSING_INPUTS']: + self.logger.warning(msg) + else: + self.log_error(msg) continue # Build up the arguments to and then run the MET tool series_analysis. @@ -575,7 +572,7 @@ def get_files_from_time(self, time_info): for storm_id in storm_list: time_info['storm_id'] = storm_id - file_dict = super().get_files_from_time(time_info) + file_dict = {'time_info': time_info.copy()} if self.c_dict['USING_BOTH']: fcst_files = self.find_input_files(time_info, 'BOTH') obs_files = fcst_files @@ -670,7 +667,11 @@ def _get_fcst_and_obs_path(self, time_info, storm_id, lead_group): **time_info) self.logger.debug(f"Explicit BOTH file list file: {both_path}") if not os.path.exists(both_path): - self.log_error(f'Could not find file: {both_path}') + msg = f'Could not find file: {both_path}' + if self.c_dict['ALLOW_MISSING_INPUTS']: + self.logger.warning(msg) + else: + self.log_error(msg) return None, None return both_path, both_path @@ -679,14 +680,24 @@ def _get_fcst_and_obs_path(self, time_info, storm_id, lead_group): **time_info) self.logger.debug(f"Explicit FCST file list file: {fcst_path}") if not os.path.exists(fcst_path): - self.log_error(f'Could not find forecast file: {fcst_path}') + msg = f'Could not find forecast file: {fcst_path}' + if self.c_dict['ALLOW_MISSING_INPUTS']: + self.logger.warning(msg) + else: + self.log_error(msg) + fcst_path = None obs_path = do_string_sub(self.c_dict['OBS_INPUT_FILE_LIST'], **time_info) self.logger.debug(f"Explicit OBS file list file: {obs_path}") if not os.path.exists(obs_path): - self.log_error(f'Could not find observation file: {obs_path}') + msg = f'Could not find observation file: {obs_path}' + if self.c_dict['ALLOW_MISSING_INPUTS']: + self.logger.warning(msg) + else: + self.log_error(msg) + obs_path = None return fcst_path, obs_path @@ -695,7 +706,8 @@ def _get_fcst_and_obs_path(self, time_info, storm_id, lead_group): list_file_dict = self.subset_input_files(time_info, output_dir=output_dir, - leads=leads) + leads=leads, + force_list=True) if not list_file_dict: return None, None @@ -784,10 +796,9 @@ def build_and_run_series_request(self, time_info, fcst_path, obs_path): # build the command and run series_analysis for each variable for var_info in self.c_dict['VAR_LIST']: if self.c_dict['USING_BOTH']: - self.c_dict['BOTH_LIST_PATH'] = fcst_path + self.infiles.append(fcst_path) else: - self.c_dict['FCST_LIST_PATH'] = fcst_path - self.c_dict['OBS_LIST_PATH'] = obs_path + self.infiles.extend([fcst_path, obs_path]) add_field_info_to_time_info(time_info, var_info) @@ -825,8 +836,7 @@ def set_command_line_arguments(self, time_info): # add config file - passing through do_string_sub # to get custom string if set - config_file = do_string_sub(self.c_dict['CONFIG_FILE'], - **time_info) + config_file = do_string_sub(self.c_dict['CONFIG_FILE'], **time_info) self.args.append(f" -config {config_file}") def get_command(self): @@ -837,10 +847,10 @@ def get_command(self): cmd = self.app_path if self.c_dict['USING_BOTH']: - cmd += f" -both {self.c_dict['BOTH_LIST_PATH']}" + cmd += f" -both {self.infiles[0]}" else: - cmd += f" -fcst {self.c_dict['FCST_LIST_PATH']}" - cmd += f" -obs {self.c_dict['OBS_LIST_PATH']}" + cmd += f" -fcst {self.infiles[0]}" + cmd += f" -obs {self.infiles[1]}" # add output path cmd += f' -out {self.get_output_path()}' @@ -876,8 +886,7 @@ def _generate_plots(self, fcst_path, time_info, storm_id): # get the output directory where the series_analysis output # was written. Plots will be written to the same directory - plot_input = do_string_sub(output_template, - **time_info) + plot_input = do_string_sub(output_template, **time_info) # Get the number of forecast tile files and the name of the # first and last in the list to be used in the -title @@ -968,8 +977,11 @@ def get_fcst_file_info(self, fcst_path): be parsed, return (None, None, None) """ # read the file but skip the first line because it contains 'file_list' - with open(fcst_path, 'r') as file_handle: - files_of_interest = file_handle.readlines() + try: + with open(fcst_path, 'r') as file_handle: + files_of_interest = file_handle.readlines() + except FileNotFoundError: + return None, None, None if len(files_of_interest) < 2: self.log_error(f"No files found in file list: {fcst_path}") @@ -1141,8 +1153,11 @@ def _get_times_from_file_list(file_path, templates): @param templates list of filename templates to use to parse time info out of file paths found in file_path file """ - with open(file_path, 'r') as file_handle: - file_list = file_handle.read().splitlines()[1:] + try: + with open(file_path, 'r') as file_handle: + file_list = file_handle.read().splitlines()[1:] + except FileNotFoundError: + return for file_name in file_list: found = False @@ -1154,3 +1169,12 @@ def _get_times_from_file_list(file_path, templates): if not found: continue yield file_time_info + + def _update_list_with_new_files(self, time_info, list_to_update): + new_files = self.get_files_from_time(time_info) + if not new_files: + return + if isinstance(new_files, list): + list_to_update.extend(new_files) + else: + list_to_update.append(new_files) diff --git a/metplus/wrappers/stat_analysis_wrapper.py b/metplus/wrappers/stat_analysis_wrapper.py index 7011ef77ef..4f7525fcff 100755 --- a/metplus/wrappers/stat_analysis_wrapper.py +++ b/metplus/wrappers/stat_analysis_wrapper.py @@ -203,6 +203,9 @@ def create_c_dict(self): data_type='float', metplus_configs=['STAT_ANALYSIS_HSS_EC_VALUE']) + # force error if inputs are missing + c_dict['ALLOW_MISSING_INPUTS'] = False + return self._c_dict_error_check(c_dict, all_field_lists_empty) def validate_runtime_freq(self, c_dict): @@ -962,8 +965,7 @@ def _get_lookin_dir(self, dir_path, config_dict): @returns string of the filled directory from dir_path """ stringsub_dict = self._build_stringsub_dict(config_dict) - dir_path_filled = do_string_sub(dir_path, - **stringsub_dict) + dir_path_filled = do_string_sub(dir_path, **stringsub_dict) all_paths = [] for one_path in dir_path_filled.split(','): @@ -1124,8 +1126,7 @@ def _process_job_args(self, job_type, job, model_info, output_template, stringsub_dict) ) - output_file = os.path.join(self.c_dict['OUTPUT_DIR'], - output_filename) + output_file = os.path.join(self.c_dict['OUTPUT_DIR'], output_filename) # substitute output filename in JOBS line job = job.replace(f'[{job_type}_file]', output_file) diff --git a/metplus/wrappers/tc_diag_wrapper.py b/metplus/wrappers/tc_diag_wrapper.py index 554151a0f9..b70392b599 100755 --- a/metplus/wrappers/tc_diag_wrapper.py +++ b/metplus/wrappers/tc_diag_wrapper.py @@ -86,9 +86,6 @@ def create_c_dict(self): c_dict['VERBOSITY']) c_dict['ALLOW_MULTIPLE_FILES'] = True - # skip RuntimeFreq wrapper logic to find files - c_dict['FIND_FILES'] = False - # get command line arguments domain and tech id list for -data self._read_data_inputs(c_dict) @@ -96,15 +93,13 @@ def create_c_dict(self): c_dict['DECK_INPUT_DIR'] = self.config.getdir('TC_DIAG_DECK_INPUT_DIR', '') c_dict['DECK_INPUT_TEMPLATE'] = ( - self.config.getraw('config', - 'TC_DIAG_DECK_TEMPLATE') + self.config.getraw('config', 'TC_DIAG_DECK_INPUT_TEMPLATE') ) # get output dir/template c_dict['OUTPUT_DIR'] = self.config.getdir('TC_DIAG_OUTPUT_DIR', '') c_dict['OUTPUT_TEMPLATE'] = ( - self.config.getraw('config', - 'TC_DIAG_OUTPUT_TEMPLATE') + self.config.getraw('config', 'TC_DIAG_OUTPUT_TEMPLATE') ) # get the MET config file path or use default @@ -223,6 +218,9 @@ def create_c_dict(self): self.add_met_config(name='output_base_format', data_type='string') + # skip RuntimeFreq input file logic - remove once integrated + c_dict['FIND_FILES'] = False + return c_dict def _read_data_inputs(self, c_dict): @@ -311,7 +309,9 @@ def run_at_time_once(self, time_info): time_info = time_util.ti_calculate(time_info) # get input files + self.run_count += 1 if not self.find_input_files(time_info): + self.missing_input_count += 1 return # get output path @@ -389,7 +389,11 @@ def _find_data_inputs(self, data_dict, lead_seq, time_info, deck_file): self.logger.debug(f"Explicit file list file: {input_file_list}") list_file = do_string_sub(input_file_list, **time_info) if not os.path.exists(list_file): - self.log_error(f'Could not find file list: {list_file}') + msg = f'Could not find file list: {list_file}' + if self.c_dict['ALLOW_MISSING_INPUTS']: + self.logger.warning(msg) + else: + self.log_error(msg) return False else: # set c_dict variables that are used in find_data function diff --git a/metplus/wrappers/tc_gen_wrapper.py b/metplus/wrappers/tc_gen_wrapper.py index 6f3a9a9f70..aa7ab2d558 100755 --- a/metplus/wrappers/tc_gen_wrapper.py +++ b/metplus/wrappers/tc_gen_wrapper.py @@ -106,8 +106,7 @@ def create_c_dict(self): app_name_upper = self.app_name.upper() c_dict['VERBOSITY'] = ( - self.config.getstr('config', - f'LOG_{app_name_upper}_VERBOSITY', + self.config.getstr('config', f'LOG_{app_name_upper}_VERBOSITY', c_dict['VERBOSITY']) ) c_dict['ALLOW_MULTIPLE_FILES'] = True @@ -280,7 +279,8 @@ def create_c_dict(self): metplus_configs=['TC_GEN_GENESIS_MATCH_POINT_TO_TRACK'] ) self.add_met_config_window('genesis_match_window') - + # skip RuntimeFreq input file logic - remove once integrated + c_dict['FIND_FILES'] = False return c_dict def handle_filter(self): @@ -323,28 +323,6 @@ def get_command(self): return cmd - def run_at_time_once(self, time_info): - """! Process runtime and try to build command to run ascii2nc - Args: - @param time_info dictionary containing timing information - """ - # get input files - if not self.find_input_files(time_info): - return - - # get output path - if not self.find_and_check_output_file(time_info): - return - - # get other configurations for command - self.set_command_line_arguments(time_info) - - # set environment variables if using config file - self.set_environment_variables(time_info) - - # build command and run - self.build() - def find_input_files(self, time_info): """!Get track and genesis files and set c_dict items. Also format forecast lead sequence to be read by the MET configuration file and diff --git a/metplus/wrappers/tc_pairs_wrapper.py b/metplus/wrappers/tc_pairs_wrapper.py index f1b726e3f7..c5db08df23 100755 --- a/metplus/wrappers/tc_pairs_wrapper.py +++ b/metplus/wrappers/tc_pairs_wrapper.py @@ -284,7 +284,10 @@ def create_c_dict(self): c_dict['GET_EDECK'] = True if c_dict['EDECK_TEMPLATE'] else False self.handle_description() - + # skip RuntimeFreq input file logic - remove once integrated + c_dict['FIND_FILES'] = False + # force error if inputs are missing + c_dict['ALLOW_MISSING_INPUTS'] = False return c_dict def validate_runtime_freq(self, c_dict): @@ -639,8 +642,8 @@ def process_data(self, basin, cyclone, time_info): edeck_list = self.find_a_or_e_deck_files('E', time_storm_info) if not adeck_list and not edeck_list: - self.log_error('Could not find any corresponding ' - 'ADECK or EDECK files') + msg = 'Could not find any corresponding ADECK or EDECK files' + self.log_error(msg) continue # reformat extra tropical cyclone files if necessary @@ -657,6 +660,7 @@ def process_data(self, basin, cyclone, time_info): # find -diag file if requested if not self._get_diag_file(time_storm_info): + self.missing_input_count += 1 return [] # change wildcard basin/cyclone to 'all' for output filename @@ -976,7 +980,8 @@ def _get_diag_file(self, time_info): all_files.extend(filepaths) if not all_files: - self.log_error('Could not get -diag files') + msg = 'Could not get -diag files' + self.log_error(msg) return False # remove duplicate files diff --git a/metplus/wrappers/tc_stat_wrapper.py b/metplus/wrappers/tc_stat_wrapper.py index acd3857a9a..503ce76962 100755 --- a/metplus/wrappers/tc_stat_wrapper.py +++ b/metplus/wrappers/tc_stat_wrapper.py @@ -181,7 +181,10 @@ def create_c_dict(self): c_dict['CONFIG_FILE'] = self.get_config_file('TCStatConfig_wrapped') self.set_met_config_for_environment_variables() - + # skip RuntimeFreq input file logic + c_dict['FIND_FILES'] = False + # force error if inputs are missing + c_dict['ALLOW_MISSING_INPUTS'] = False return c_dict def set_met_config_for_environment_variables(self): diff --git a/metplus/wrappers/tcrmw_wrapper.py b/metplus/wrappers/tcrmw_wrapper.py index 72618fef7d..86760d574b 100755 --- a/metplus/wrappers/tcrmw_wrapper.py +++ b/metplus/wrappers/tcrmw_wrapper.py @@ -143,7 +143,8 @@ def create_c_dict(self): met_tool=self.app_name) if not c_dict['VAR_LIST_TEMP']: self.log_error("Could not get field information from config.") - + # skip RuntimeFreq input file logic - remove once integrated + c_dict['FIND_FILES'] = False return c_dict def get_command(self): @@ -186,7 +187,7 @@ def find_input_files(self, time_info): # get deck file deck_file = self.find_data(time_info, data_type='DECK') if not deck_file: - return None + return False self.c_dict['DECK_FILE'] = deck_file @@ -199,8 +200,12 @@ def find_input_files(self, time_info): list_file = do_string_sub(self.c_dict['INPUT_FILE_LIST'], **time_info) if not os.path.exists(list_file): - self.log_error(f'Could not find file list: {list_file}') - return None + msg = f'Could not find file list: {list_file}' + if self.c_dict['ALLOW_MISSING_INPUTS']: + self.logger.warning(msg) + else: + self.log_error(msg) + return False else: all_input_files = [] @@ -219,7 +224,7 @@ def find_input_files(self, time_info): all_input_files.extend(input_files) if not all_input_files: - return None + return False # create an ascii file with a list of the input files list_file = f"{os.path.basename(deck_file)}_data_files.txt" @@ -228,11 +233,11 @@ def find_input_files(self, time_info): self.infiles.append(list_file) if not self._set_data_field(time_info): - return None + return False self._set_lead_list(time_info, lead_seq) - return self.infiles + return True def _set_data_field(self, time_info): """!Get list of fields from config to process. Build list of field info diff --git a/metplus/wrappers/user_script_wrapper.py b/metplus/wrappers/user_script_wrapper.py index dae9bdf0ae..cf393ae638 100755 --- a/metplus/wrappers/user_script_wrapper.py +++ b/metplus/wrappers/user_script_wrapper.py @@ -10,13 +10,11 @@ Condition codes: 0 for success, 1 for failure """ -import os -from datetime import datetime - from ..util import time_util -from . import RuntimeFreqWrapper from ..util import do_string_sub +from . import RuntimeFreqWrapper + '''!@namespace UserScriptWrapper @brief Parent class for wrappers that run over a grouping of times @endcode @@ -42,10 +40,7 @@ def create_c_dict(self): self.log_error("Must supply a command to run with " "USER_SCRIPT_COMMAND") - c_dict['INPUT_DIR'] = self.config.getraw('config', - 'USER_SCRIPT_INPUT_DIR', - '') - self.get_input_templates(c_dict) + self.get_input_templates_multiple(c_dict) c_dict['ALLOW_MULTIPLE_FILES'] = True c_dict['IS_MET_CMD'] = False @@ -74,15 +69,15 @@ def run_at_time_once(self, time_info): # create file list text files for the current run time criteria # set c_dict to the input file dict to set as environment vars - self.c_dict['INPUT_LIST_DICT'] = self.subset_input_files(time_info) + self.c_dict['INPUT_LIST_DICT'] = ( + self.subset_input_files(time_info, force_list=True) + ) self.set_environment_variables(time_info) # substitute values from dictionary into command - self.c_dict['COMMAND'] = ( - do_string_sub(self.c_dict['COMMAND_TEMPLATE'], - **time_info) - ) + self.c_dict['COMMAND'] = do_string_sub(self.c_dict['COMMAND_TEMPLATE'], + **time_info) return self.build() @@ -97,9 +92,9 @@ def get_files_from_time(self, time_info): @returns dictionary containing time_info dict and any relevant files with a key representing a description of that file """ - file_dict = super().get_files_from_time(time_info) + file_dict = {'time_info': time_info.copy()} - input_files = self.get_input_files(time_info, fill_missing=True) + input_files, _ = self.get_input_files(time_info, fill_missing=True) if input_files is None: return file_dict @@ -128,5 +123,7 @@ def get_all_files(self, custom=None): @returns True """ - super().get_all_files(custom) - return True + all_files = super().get_all_files(custom) + if not all_files: + return True + return all_files diff --git a/metplus/wrappers/wavelet_stat_wrapper.py b/metplus/wrappers/wavelet_stat_wrapper.py index 2e824b4de9..2c56c7408e 100755 --- a/metplus/wrappers/wavelet_stat_wrapper.py +++ b/metplus/wrappers/wavelet_stat_wrapper.py @@ -176,5 +176,6 @@ def create_c_dict(self): }) self.add_met_config(name='output_prefix', data_type='string') - + # skip RuntimeFreq input file logic - remove once integrated + c_dict['FIND_FILES'] = False return c_dict diff --git a/parm/metplus_config/defaults.conf b/parm/metplus_config/defaults.conf index ba67ccd233..4d27e017a9 100644 --- a/parm/metplus_config/defaults.conf +++ b/parm/metplus_config/defaults.conf @@ -65,6 +65,10 @@ GFDL_TRACKER_EXEC = /path/to/standalone_gfdl-vortextracker_v3.9a/trk_exec # that value will be used instead of the value set in this file. # # * SCRUB_STAGING_DIR removes intermediate files generated by a METplus run # # Set to False to preserve these files # +# * ALLOW_MISSING_INPUTS determines if an error should be reported if input # +# files are not found, or if a warning should be reported instead # +# * INPUT_THRESH specifies the percentage (0-1) of inputs that must be # +# found to prevent an error (only used if ALLOW_MISSING_INPUTS=True) # ############################################################################### PROCESS_LIST = Usage @@ -73,6 +77,9 @@ OMP_NUM_THREADS = 1 SCRUB_STAGING_DIR = True +ALLOW_MISSING_INPUTS = False +INPUT_THRESH = 0.0 + ############################################################################### # Log File Information (Where to write logs files) # diff --git a/parm/use_cases/met_tool_wrapper/TCDiag/TCDiag.conf b/parm/use_cases/met_tool_wrapper/TCDiag/TCDiag.conf index 37f191c712..fc4d27a554 100644 --- a/parm/use_cases/met_tool_wrapper/TCDiag/TCDiag.conf +++ b/parm/use_cases/met_tool_wrapper/TCDiag/TCDiag.conf @@ -38,7 +38,7 @@ LEAD_SEQ = 0, 6, 12 ### TC_DIAG_DECK_INPUT_DIR = {INPUT_BASE}/met_test/new/tc_data/adeck -TC_DIAG_DECK_TEMPLATE = subset.aal03{date?fmt=%Y}.dat +TC_DIAG_DECK_INPUT_TEMPLATE = subset.aal03{date?fmt=%Y}.dat TC_DIAG_INPUT1_DIR = {INPUT_BASE}/met_test/new/model_data/grib2/gfs TC_DIAG_INPUT1_TEMPLATE = subset.gfs.t12z.pgrb2.0p50.f* diff --git a/parm/use_cases/model_applications/short_range/MODEMultivar_fcstHRRR_obsMRMS_HRRRanl.conf b/parm/use_cases/model_applications/short_range/MODEMultivar_fcstHRRR_obsMRMS_HRRRanl.conf index c466aec372..1f3764074d 100644 --- a/parm/use_cases/model_applications/short_range/MODEMultivar_fcstHRRR_obsMRMS_HRRRanl.conf +++ b/parm/use_cases/model_applications/short_range/MODEMultivar_fcstHRRR_obsMRMS_HRRRanl.conf @@ -31,7 +31,7 @@ FCST_MODE_INPUT_DIR = {INPUT_BASE}/model_applications/short_range/MODEMultivar_f FCST_MODE_INPUT_TEMPLATE = hrrr.t{init?fmt=%H}z.wrfprsf{lead?fmt=%H}.sub.grib2,hrrr.t{init?fmt=%H}z.wrfprsf{lead?fmt=%H}.sub.grib2,hrrr.t{init?fmt=%H}z.wrfprsf{lead?fmt=%H}.sub.grib2 OBS_MODE_INPUT_DIR = {INPUT_BASE}/model_applications/short_range/MODEMultivar_fcstHRRR_obsMRMS_HRRRanl -OBS_MODE_INPUT_TEMPLATE = PrecipFlag_00.00_{valid?fmt=%Y%m%d}-{valid?fmt=%2H}0000.sub.grib2,hrrr.t{valid?fmt=%H}z.wrfprsf00.sub.grib2,hrrr.t{valid?fmt=%H}z.wrfprsf00.sub.grib2 +OBS_MODE_INPUT_TEMPLATE = PrecipFlag_00.00_{valid?fmt=%Y%m%d}-{valid?fmt=%H}0000.sub.grib2,hrrr.t{valid?fmt=%H}z.wrfprsf00.sub.grib2,hrrr.t{valid?fmt=%H}z.wrfprsf00.sub.grib2 MODE_OUTPUT_DIR = {OUTPUT_BASE}/mode MODE_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/f{lead?fmt=%2H}