From 368ac7863ff6e92aa0be75d8c57e620d00fb29ee Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 24 Aug 2022 13:26:27 -0600 Subject: [PATCH 1/9] Per dtcenter/METplus-Internal#32, update logic to compute LOG_TIMESTAMP from LOG_TIMESTAMP_TEMPLATE and save it in the METplusConfig so it can be referenced by other variables including METPLUS_CONF (final conf file). Removed logic to handle if LOG_METPLUS is unset that uses an old config variable that is not documented or used anywhere --- metplus/util/config_metplus.py | 33 +++++---------------------------- 1 file changed, 5 insertions(+), 28 deletions(-) diff --git a/metplus/util/config_metplus.py b/metplus/util/config_metplus.py index e407774400..1e3ef9a962 100644 --- a/metplus/util/config_metplus.py +++ b/metplus/util/config_metplus.py @@ -294,6 +294,10 @@ def _set_logvars(config, logger=None): log_filenametimestamp = date_t.strftime(log_timestamp_template) + # Adding LOG_TIMESTAMP to the final configuration file. + logger.info('Adding LOG_TIMESTAMP=%s' % repr(log_filenametimestamp)) + config.set('config', 'LOG_TIMESTAMP', log_filenametimestamp) + log_dir = config.getdir('LOG_DIR') # NOTE: LOG_METPLUS or metpluslog is meant to include the absolute path @@ -303,6 +307,7 @@ def _set_logvars(config, logger=None): # if LOG_METPLUS = unset in the conf file, means NO logging. # Also, assUmes the user has included the intended path in LOG_METPLUS. user_defined_log_file = None + metpluslog = '' if config.has_option('config', 'LOG_METPLUS'): user_defined_log_file = True # strinterp will set metpluslog to '' if LOG_METPLUS = is unset. @@ -317,34 +322,6 @@ def _set_logvars(config, logger=None): if metpluslog: if os.path.basename(metpluslog) == metpluslog: metpluslog = os.path.join(log_dir, metpluslog) - else: - # No LOG_METPLUS in conf file, so let the code try to set it, - # if the user defined the variable LOG_FILENAME_TEMPLATE. - # LOG_FILENAME_TEMPLATE is an 'unpublished' variable - no one knows - # about it unless you are reading this. Why does this exist ? - # It was from my first cycle implementation. I did not want to pull - # it out, in case the group wanted a stand alone metplus log filename - # template variable. - - # If metpluslog_filename includes a path, python joins it intelligently - # Set the metplus log filename. - # strinterp will set metpluslog_filename to '' if template is empty - if config.has_option('config', 'LOG_FILENAME_TEMPLATE'): - metpluslog_filename = config.strinterp( - 'config', - '{LOG_FILENAME_TEMPLATE}', - LOG_TIMESTAMP_TEMPLATE=log_filenametimestamp - ) - else: - metpluslog_filename = '' - if metpluslog_filename: - metpluslog = os.path.join(log_dir, metpluslog_filename) - else: - metpluslog = '' - - # Adding LOG_TIMESTAMP to the final configuration file. - logger.info('Adding LOG_TIMESTAMP=%s' % repr(log_filenametimestamp)) - config.set('config', 'LOG_TIMESTAMP', log_filenametimestamp) # Setting LOG_METPLUS in the configuration object # At this point LOG_METPLUS will have a value or '' the empty string. From 2b144c40af3c2ff281513e89a73d1a4ac9dcbe53 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 24 Aug 2022 13:29:14 -0600 Subject: [PATCH 2/9] Per dtcenter/METplus-Internal#32, changed default values for METPLUS_CONF to include the log timestamp and changed LOG_METPLUS to use LOG_TIMESTAMP instead of LOG_TIMESTAMP_TEMPLATE -- in metplus_config, the formatted log timestamp is substituted in the read of the LOG_METPLUS variable, but it would not be formatted properly in any other use. Formatting the log timestamp and saving it as LOG_TIMESTAMP will make it available to use in other places --- parm/metplus_config/defaults.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/metplus_config/defaults.conf b/parm/metplus_config/defaults.conf index 2122f41e34..1f8d03b2f5 100644 --- a/parm/metplus_config/defaults.conf +++ b/parm/metplus_config/defaults.conf @@ -32,7 +32,7 @@ INPUT_BASE = /path/to OUTPUT_BASE = /path/to -METPLUS_CONF = {OUTPUT_BASE}/metplus_final.conf +METPLUS_CONF = {OUTPUT_BASE}/metplus_final.conf.{LOG_TIMESTAMP} TMP_DIR = {OUTPUT_BASE}/tmp STAGING_DIR = {OUTPUT_BASE}/stage @@ -75,7 +75,7 @@ OMP_NUM_THREADS = 1 ############################################################################### -LOG_METPLUS = {LOG_DIR}/metplus.log.{LOG_TIMESTAMP_TEMPLATE} +LOG_METPLUS = {LOG_DIR}/metplus.log.{LOG_TIMESTAMP} LOG_DIR = {OUTPUT_BASE}/logs From f4d4de04f9b5583e373427e26750657b02bda6b6 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 24 Aug 2022 13:31:55 -0600 Subject: [PATCH 3/9] changed logic to skip metplus final conf so that it will be skipped if there is a timestamp included at the end of the filename --- metplus/util/diff_util.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/metplus/util/diff_util.py b/metplus/util/diff_util.py index 009939333b..d6625454cd 100644 --- a/metplus/util/diff_util.py +++ b/metplus/util/diff_util.py @@ -78,8 +78,8 @@ def compare_dir(dir_a, dir_b, debug=False, save_diff=False): if not os.path.isfile(filepath_a): continue - # skip metplus_final.conf - if filepath_a.endswith('metplus_final.conf'): + # skip final conf file + if 'metplus_final.conf' in os.path.basename(filepath_a): continue filepath_b = filepath_a.replace(dir_a, dir_b) @@ -109,8 +109,8 @@ def compare_dir(dir_a, dir_b, debug=False, save_diff=False): for filename in files: filepath_b = os.path.join(root, filename) - # skip metplus_final.conf - if filepath_b.endswith('metplus_final.conf'): + # skip final conf file + if 'metplus_final.conf' in os.path.basename(filepath_b): continue filepath_a = filepath_b.replace(dir_b, dir_a) From 0bf8dcbd6dba75a463a2afd3d68a29f840ea3f53 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 24 Aug 2022 14:10:39 -0600 Subject: [PATCH 4/9] per dtcenter/METplus-Internal#32, added default config variable FILE_LISTS_DIR that includes the log timestamp to prevent issues with multiple runs but still allow users to change the default to remove the timestamp if they desire --- metplus/wrappers/command_builder.py | 5 ++--- parm/metplus_config/defaults.conf | 1 + 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/metplus/wrappers/command_builder.py b/metplus/wrappers/command_builder.py index 4148725b14..88f00db0ee 100755 --- a/metplus/wrappers/command_builder.py +++ b/metplus/wrappers/command_builder.py @@ -901,12 +901,11 @@ def write_list_file(self, filename, file_list, output_dir=None): @param filename name of ascii file to write @param file_list list of files to write to ascii file @param output_dir (Optional) directory to write files. If None, - ascii files are written to {STAGING_DIR}/file_lists + ascii files are written to {FILE_LIST_DIR} @returns path to output file """ if output_dir is None: - list_dir = os.path.join(self.config.getdir('STAGING_DIR'), - 'file_lists') + list_dir = self.config.getdir('FILE_LISTS_DIR') else: list_dir = output_dir diff --git a/parm/metplus_config/defaults.conf b/parm/metplus_config/defaults.conf index 1f8d03b2f5..d6bb256d39 100644 --- a/parm/metplus_config/defaults.conf +++ b/parm/metplus_config/defaults.conf @@ -36,6 +36,7 @@ METPLUS_CONF = {OUTPUT_BASE}/metplus_final.conf.{LOG_TIMESTAMP} TMP_DIR = {OUTPUT_BASE}/tmp STAGING_DIR = {OUTPUT_BASE}/stage +FILE_LISTS_DIR = {STAGING_DIR}/file_lists.{LOG_TIMESTAMP} ############################################################################### From eff624d2c0c0f650fe7504d5b37a9d97cd8b6335 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 24 Aug 2022 14:11:35 -0600 Subject: [PATCH 5/9] added default overrides for METPLUS_CONF and FILE_LISTS_DIR for the config files that are used to run pytest unit tests and GitHub Actions automated test suite --- .github/parm/test_settings.conf | 2 ++ internal_tests/pytests/minimum_pytest.conf | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/parm/test_settings.conf b/.github/parm/test_settings.conf index 01cbc7f58d..956c7e48aa 100644 --- a/.github/parm/test_settings.conf +++ b/.github/parm/test_settings.conf @@ -8,6 +8,8 @@ LOG_INFO_LINE_FORMAT = {LOG_LINE_FORMAT} LOG_METPLUS = {LOG_DIR}/metplus.log LOG_TIMESTAMP_TEMPLATE = +METPLUS_CONF = {OUTPUT_BASE}/metplus_final.conf +FILE_LISTS_DIR = {STAGING_DIR}/file_lists # also set path to GempakToCF.jar for GEMPAK use cases GEMPAKTOCF_JAR = /data/input/GempakToCF.jar diff --git a/internal_tests/pytests/minimum_pytest.conf b/internal_tests/pytests/minimum_pytest.conf index d7bf430143..5a68934956 100644 --- a/internal_tests/pytests/minimum_pytest.conf +++ b/internal_tests/pytests/minimum_pytest.conf @@ -1,10 +1,9 @@ -[dir] +[config] INPUT_BASE = {ENV[METPLUS_TEST_INPUT_BASE]} OUTPUT_BASE = {ENV[METPLUS_TEST_OUTPUT_BASE]} MET_INSTALL_DIR = {ENV[METPLUS_TEST_MET_INSTALL_DIR]} TMP_DIR = {ENV[METPLUS_TEST_TMP_DIR]} -[config] LOG_LEVEL = DEBUG LOG_MET_OUTPUT_TO_METPLUS = no LOG_LINE_FORMAT = (%(filename)s) %(levelname)s: %(message)s @@ -14,3 +13,5 @@ LOG_INFO_LINE_FORMAT = {LOG_LINE_FORMAT} LOG_METPLUS = {LOG_DIR}/metplus.log LOG_TIMESTAMP_TEMPLATE = +METPLUS_CONF = {OUTPUT_BASE}/metplus_final.conf +FILE_LISTS_DIR = {STAGING_DIR}/file_lists \ No newline at end of file From ab458fcd58583462874d29d0e0aab4fe841aab48 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 8 Sep 2022 11:24:02 -0600 Subject: [PATCH 6/9] per dtcenter/METplus-Internal#32, move SCRUB_STAGING_DIR to defaults.conf and change default value to True --- metplus/util/met_util.py | 6 ++++-- parm/metplus_config/defaults.conf | 6 ++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/metplus/util/met_util.py b/metplus/util/met_util.py index 56668be919..c11d8bfa42 100644 --- a/metplus/util/met_util.py +++ b/metplus/util/met_util.py @@ -179,10 +179,12 @@ def run_metplus(config, process_list): def post_run_cleanup(config, app_name, total_errors): logger = config.logger # scrub staging directory if requested - if config.getbool('config', 'SCRUB_STAGING_DIR', False) and\ - os.path.exists(config.getdir('STAGING_DIR')): + if (config.getbool('config', 'SCRUB_STAGING_DIR') and + os.path.exists(config.getdir('STAGING_DIR'))): staging_dir = config.getdir('STAGING_DIR') logger.info("Scrubbing staging dir: %s", staging_dir) + logger.info('Set SCRUB_STAGING_DIR to False to preserve ' + 'intermediate files.') shutil.rmtree(staging_dir) # save log file path and clock time before writing final conf file diff --git a/parm/metplus_config/defaults.conf b/parm/metplus_config/defaults.conf index d6bb256d39..45ea9fb548 100644 --- a/parm/metplus_config/defaults.conf +++ b/parm/metplus_config/defaults.conf @@ -56,12 +56,15 @@ GEMPAKTOCF_JAR = GempakToCF.jar GFDL_TRACKER_EXEC = /path/to/standalone_gfdl-vortextracker_v3.9a/trk_exec + ############################################################################### # Runtime Configuration # # * OMP_NUM_THREADS sets an environment variable of the same name that # # determines the number of threads to use in the MET executables. If the # # environment variable is already set in the user's environment, then # # that value will be used instead of the value set in this file. # +# * SCRUB_STAGING_DIR removes intermediate files generated by a METplus run # +# Set to False to preserve these files # ############################################################################### @@ -71,6 +74,9 @@ PROCESS_LIST = Usage OMP_NUM_THREADS = 1 +SCRUB_STAGING_DIR = True + + ############################################################################### # Log File Information (Where to write logs files) # ############################################################################### From dc1ede45c0dd077a3fd523d3eb63c01e71f47cb7 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 8 Sep 2022 11:24:23 -0600 Subject: [PATCH 7/9] per dtcenter/METplus-Internal#32, added documentation for changes --- docs/Users_Guide/glossary.rst | 41 ++++++++++++++++++++---- docs/Users_Guide/systemconfiguration.rst | 17 ++++++++-- 2 files changed, 50 insertions(+), 8 deletions(-) diff --git a/docs/Users_Guide/glossary.rst b/docs/Users_Guide/glossary.rst index 052c4f30d9..b2a4501c91 100644 --- a/docs/Users_Guide/glossary.rst +++ b/docs/Users_Guide/glossary.rst @@ -2336,12 +2336,15 @@ METplus Configuration Glossary | *Used by:* All LOG_METPLUS - Control the filename of the METplus log file. Control the timestamp appended to the filename with LOG_TIMESTAMP_TEMPLATE. To turn OFF all logging, do not set this option. + Path to the METplus log file. Control the timestamp appended to the + filename with :term:`LOG_TIMESTAMP_TEMPLATE`. + Set this variable to an empty string to turn off all logging. | *Used by:* All LOG_MET_OUTPUT_TO_METPLUS - Control whether logging output from the MET tools is sent to the METplus log file, or individual log files for each MET tool. + Control whether logging output from each executable is sent to the METplus + log file or individual log files. | *Used by:* All @@ -2351,7 +2354,16 @@ METplus Configuration Glossary | *Used by:* All LOG_TIMESTAMP_TEMPLATE - Set the timestamp template for the METplus log file. Use Python strftime directives, e.g.%Y%m%d for YYYYMMDD. + Set the timestamp template used to set :term:`LOG_TIMESTAMP`. + Use only Python strftime directives, e.g. %Y%m%d for YYYYMMDD. + See also :term:`LOG_TIMESTAMP_USE_DATATIME`. + + | *Used by:* All + + LOG_TIMESTAMP + Automatically set by METplus based on the values set for + :term:`LOG_TIMESTAMP_TEMPLATE` and :term:`LOG_TIMESTAMP_USE_DATATIME`. + Setting this in a configuration file will have no effect. | *Used by:* All @@ -2387,7 +2399,11 @@ METplus Configuration Glossary | *Used by:* All METPLUS_CONF - Provide the absolute path to the METplus final configuration file. This file will contain every configuration option and value used when METplus was run. + Path to the final METplus configuration file. This file will contain every + configuration option and value used when METplus was run, including any + default values that were used. By default the filename includes the + :term:`LOG_TIMESTAMP` so the final conf file and the corresponding log + file can be reviewed. | *Used by:* All @@ -3642,7 +3658,9 @@ METplus Configuration Glossary | *Used by:* TCMPRPlotter SCRUB_STAGING_DIR - Remove staging directory after METplus has completed running if set to True. Set to False to preserve data for subsequent runs. + If True, remove staging directory after METplus has completed running. + Set to False to preserve data for subsequent runs or debugging purposes. + Defaults to True. | *Used by:* All @@ -3780,7 +3798,18 @@ METplus Configuration Glossary .. warning:: **DEPRECATED:** Please use :term:`INIT_BEG` or :term:`VALID_BEG` instead. STAGING_DIR - Directory to uncompress or convert data into for use in METplus. + Directory to store intermediate files such as data files that were + automatically uncompressed or converted. + Also includes :term:`FILE_LISTS_DIR` by default. + + | *Used by:* All + + FILE_LISTS_DIR + Directory to store text files generated by METplus that contain a list of + input file paths to pass in a MET executable that allows multiple input + files. By default this directory is found under the :term:`STAGING_DIR` + and contains the :term:`LOG_TIMESTAMP` to easily identify which file lists + were generated from a METplus run. | *Used by:* All diff --git a/docs/Users_Guide/systemconfiguration.rst b/docs/Users_Guide/systemconfiguration.rst index 35ac0a4ee2..4f2903f238 100644 --- a/docs/Users_Guide/systemconfiguration.rst +++ b/docs/Users_Guide/systemconfiguration.rst @@ -14,6 +14,9 @@ Config Best Practices / Recommendations encouraged to run with DEBUG when getting started with METplus or when investigating unexpected behavior. +* Set :term:`SCRUB_STAGING_DIR` to False to preserve intermediate files to + help with debugging issues. + * Review the log files to verify that all of the processes ran cleanly. Some log output will be written to the screen, but the log files contain more information, such as log output from the MET tools. @@ -182,10 +185,11 @@ If a value set in the final conf differs from what was set in a configuration file passed to run_metplus.py, there is a good chance that this variable is set in another configuration file that was passed in afterwards. -The default value is a file called metplus_final.conf that is written in the +The default value is a file called metplus_final.conf followed by the +log timestamp (see :term:`LOG_TIMESTAMP`) that is written in the :ref:`OUTPUT_BASE` directory:: - METPLUS_CONF = {OUTPUT_BASE}/metplus_final.conf + METPLUS_CONF = {OUTPUT_BASE}/metplus_final.conf.{LOG_TIMESTAMP} This value is rarely changed, but it can be if desired. @@ -217,6 +221,15 @@ By default this is a directory called **stage** inside the This value is rarely changed, but it can be if desired. +See also :term:`FILE_LISTS_DIR`. + +SCRUB_STAGING_DIR +^^^^^^^^^^^^^^^^^ + +True or False variable to determine if the :term:`STAGING_DIR` should be +removed after the METplus has finished running. + + OMP_NUM_THREADS ^^^^^^^^^^^^^^^ From d1262f4cd6b23acec216c63b4f8be23f0a6bd5ab Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 8 Sep 2022 11:33:10 -0600 Subject: [PATCH 8/9] rearrange docs for clarity --- docs/Users_Guide/systemconfiguration.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/Users_Guide/systemconfiguration.rst b/docs/Users_Guide/systemconfiguration.rst index 4f2903f238..e9e3868f2d 100644 --- a/docs/Users_Guide/systemconfiguration.rst +++ b/docs/Users_Guide/systemconfiguration.rst @@ -212,7 +212,7 @@ wrapper scripts. Files are written to this directory to prevent corrupting input data directories in case something goes wrong. File list ASCII files that contain a list of file paths to pass into MET tools such as MODE-TimeDomain or SeriesAnalysis are also written to this -directory. +directory. See :term:`FILE_LISTS_DIR` for more information. By default this is a directory called **stage** inside the :ref:`OUTPUT_BASE` directory:: @@ -221,8 +221,6 @@ By default this is a directory called **stage** inside the This value is rarely changed, but it can be if desired. -See also :term:`FILE_LISTS_DIR`. - SCRUB_STAGING_DIR ^^^^^^^^^^^^^^^^^ From d61a09e46d76b3c241bcbe591ee4eee88e057304 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 8 Sep 2022 14:09:46 -0600 Subject: [PATCH 9/9] override SCRUB_STAGING_DIR for tests so there will be no differences and the files will be available if needed to debug --- .github/parm/test_settings.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/parm/test_settings.conf b/.github/parm/test_settings.conf index 956c7e48aa..9927f8f7e1 100644 --- a/.github/parm/test_settings.conf +++ b/.github/parm/test_settings.conf @@ -11,6 +11,8 @@ LOG_TIMESTAMP_TEMPLATE = METPLUS_CONF = {OUTPUT_BASE}/metplus_final.conf FILE_LISTS_DIR = {STAGING_DIR}/file_lists +SCRUB_STAGING_DIR=False + # also set path to GempakToCF.jar for GEMPAK use cases GEMPAKTOCF_JAR = /data/input/GempakToCF.jar