diff --git a/.github/parm/test_settings.conf b/.github/parm/test_settings.conf index 956c7e48aa..9927f8f7e1 100644 --- a/.github/parm/test_settings.conf +++ b/.github/parm/test_settings.conf @@ -11,6 +11,8 @@ LOG_TIMESTAMP_TEMPLATE = METPLUS_CONF = {OUTPUT_BASE}/metplus_final.conf FILE_LISTS_DIR = {STAGING_DIR}/file_lists +SCRUB_STAGING_DIR=False + # also set path to GempakToCF.jar for GEMPAK use cases GEMPAKTOCF_JAR = /data/input/GempakToCF.jar diff --git a/docs/Users_Guide/glossary.rst b/docs/Users_Guide/glossary.rst index 052c4f30d9..b2a4501c91 100644 --- a/docs/Users_Guide/glossary.rst +++ b/docs/Users_Guide/glossary.rst @@ -2336,12 +2336,15 @@ METplus Configuration Glossary | *Used by:* All LOG_METPLUS - Control the filename of the METplus log file. Control the timestamp appended to the filename with LOG_TIMESTAMP_TEMPLATE. To turn OFF all logging, do not set this option. + Path to the METplus log file. Control the timestamp appended to the + filename with :term:`LOG_TIMESTAMP_TEMPLATE`. + Set this variable to an empty string to turn off all logging. | *Used by:* All LOG_MET_OUTPUT_TO_METPLUS - Control whether logging output from the MET tools is sent to the METplus log file, or individual log files for each MET tool. + Control whether logging output from each executable is sent to the METplus + log file or individual log files. | *Used by:* All @@ -2351,7 +2354,16 @@ METplus Configuration Glossary | *Used by:* All LOG_TIMESTAMP_TEMPLATE - Set the timestamp template for the METplus log file. Use Python strftime directives, e.g.%Y%m%d for YYYYMMDD. + Set the timestamp template used to set :term:`LOG_TIMESTAMP`. + Use only Python strftime directives, e.g. %Y%m%d for YYYYMMDD. + See also :term:`LOG_TIMESTAMP_USE_DATATIME`. + + | *Used by:* All + + LOG_TIMESTAMP + Automatically set by METplus based on the values set for + :term:`LOG_TIMESTAMP_TEMPLATE` and :term:`LOG_TIMESTAMP_USE_DATATIME`. + Setting this in a configuration file will have no effect. | *Used by:* All @@ -2387,7 +2399,11 @@ METplus Configuration Glossary | *Used by:* All METPLUS_CONF - Provide the absolute path to the METplus final configuration file. This file will contain every configuration option and value used when METplus was run. + Path to the final METplus configuration file. This file will contain every + configuration option and value used when METplus was run, including any + default values that were used. By default the filename includes the + :term:`LOG_TIMESTAMP` so the final conf file and the corresponding log + file can be reviewed. | *Used by:* All @@ -3642,7 +3658,9 @@ METplus Configuration Glossary | *Used by:* TCMPRPlotter SCRUB_STAGING_DIR - Remove staging directory after METplus has completed running if set to True. Set to False to preserve data for subsequent runs. + If True, remove staging directory after METplus has completed running. + Set to False to preserve data for subsequent runs or debugging purposes. + Defaults to True. | *Used by:* All @@ -3780,7 +3798,18 @@ METplus Configuration Glossary .. warning:: **DEPRECATED:** Please use :term:`INIT_BEG` or :term:`VALID_BEG` instead. STAGING_DIR - Directory to uncompress or convert data into for use in METplus. + Directory to store intermediate files such as data files that were + automatically uncompressed or converted. + Also includes :term:`FILE_LISTS_DIR` by default. + + | *Used by:* All + + FILE_LISTS_DIR + Directory to store text files generated by METplus that contain a list of + input file paths to pass in a MET executable that allows multiple input + files. By default this directory is found under the :term:`STAGING_DIR` + and contains the :term:`LOG_TIMESTAMP` to easily identify which file lists + were generated from a METplus run. | *Used by:* All diff --git a/docs/Users_Guide/systemconfiguration.rst b/docs/Users_Guide/systemconfiguration.rst index 35ac0a4ee2..e9e3868f2d 100644 --- a/docs/Users_Guide/systemconfiguration.rst +++ b/docs/Users_Guide/systemconfiguration.rst @@ -14,6 +14,9 @@ Config Best Practices / Recommendations encouraged to run with DEBUG when getting started with METplus or when investigating unexpected behavior. +* Set :term:`SCRUB_STAGING_DIR` to False to preserve intermediate files to + help with debugging issues. + * Review the log files to verify that all of the processes ran cleanly. Some log output will be written to the screen, but the log files contain more information, such as log output from the MET tools. @@ -182,10 +185,11 @@ If a value set in the final conf differs from what was set in a configuration file passed to run_metplus.py, there is a good chance that this variable is set in another configuration file that was passed in afterwards. -The default value is a file called metplus_final.conf that is written in the +The default value is a file called metplus_final.conf followed by the +log timestamp (see :term:`LOG_TIMESTAMP`) that is written in the :ref:`OUTPUT_BASE` directory:: - METPLUS_CONF = {OUTPUT_BASE}/metplus_final.conf + METPLUS_CONF = {OUTPUT_BASE}/metplus_final.conf.{LOG_TIMESTAMP} This value is rarely changed, but it can be if desired. @@ -208,7 +212,7 @@ wrapper scripts. Files are written to this directory to prevent corrupting input data directories in case something goes wrong. File list ASCII files that contain a list of file paths to pass into MET tools such as MODE-TimeDomain or SeriesAnalysis are also written to this -directory. +directory. See :term:`FILE_LISTS_DIR` for more information. By default this is a directory called **stage** inside the :ref:`OUTPUT_BASE` directory:: @@ -217,6 +221,13 @@ By default this is a directory called **stage** inside the This value is rarely changed, but it can be if desired. +SCRUB_STAGING_DIR +^^^^^^^^^^^^^^^^^ + +True or False variable to determine if the :term:`STAGING_DIR` should be +removed after the METplus has finished running. + + OMP_NUM_THREADS ^^^^^^^^^^^^^^^ diff --git a/metplus/util/met_util.py b/metplus/util/met_util.py index 56668be919..c11d8bfa42 100644 --- a/metplus/util/met_util.py +++ b/metplus/util/met_util.py @@ -179,10 +179,12 @@ def run_metplus(config, process_list): def post_run_cleanup(config, app_name, total_errors): logger = config.logger # scrub staging directory if requested - if config.getbool('config', 'SCRUB_STAGING_DIR', False) and\ - os.path.exists(config.getdir('STAGING_DIR')): + if (config.getbool('config', 'SCRUB_STAGING_DIR') and + os.path.exists(config.getdir('STAGING_DIR'))): staging_dir = config.getdir('STAGING_DIR') logger.info("Scrubbing staging dir: %s", staging_dir) + logger.info('Set SCRUB_STAGING_DIR to False to preserve ' + 'intermediate files.') shutil.rmtree(staging_dir) # save log file path and clock time before writing final conf file diff --git a/parm/metplus_config/defaults.conf b/parm/metplus_config/defaults.conf index d6bb256d39..45ea9fb548 100644 --- a/parm/metplus_config/defaults.conf +++ b/parm/metplus_config/defaults.conf @@ -56,12 +56,15 @@ GEMPAKTOCF_JAR = GempakToCF.jar GFDL_TRACKER_EXEC = /path/to/standalone_gfdl-vortextracker_v3.9a/trk_exec + ############################################################################### # Runtime Configuration # # * OMP_NUM_THREADS sets an environment variable of the same name that # # determines the number of threads to use in the MET executables. If the # # environment variable is already set in the user's environment, then # # that value will be used instead of the value set in this file. # +# * SCRUB_STAGING_DIR removes intermediate files generated by a METplus run # +# Set to False to preserve these files # ############################################################################### @@ -71,6 +74,9 @@ PROCESS_LIST = Usage OMP_NUM_THREADS = 1 +SCRUB_STAGING_DIR = True + + ############################################################################### # Log File Information (Where to write logs files) # ###############################################################################