From 2c4603d53e804c407425c04ad59c53a1e35a264f Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 4 Dec 2024 14:00:07 -0700 Subject: [PATCH 1/6] exclude metplus/scripts dir from PyCharm project to prevent incorrect duplicate code warnings with files that are sym linked --- .idea/METplus.iml | 1 + 1 file changed, 1 insertion(+) diff --git a/.idea/METplus.iml b/.idea/METplus.iml index 54d4a167a..1644746a0 100644 --- a/.idea/METplus.iml +++ b/.idea/METplus.iml @@ -3,6 +3,7 @@ + From 42355aa70c1b6424a1fb4d45030bdccb007073ed Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Fri, 6 Dec 2024 12:23:14 -0700 Subject: [PATCH 2/6] to expand on #2772, updating instructions to include note to set tmp directory for apptainer to prevent issues pulling large images that require a lot of temp space --- docs/Users_Guide/getting_started.rst | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/Users_Guide/getting_started.rst b/docs/Users_Guide/getting_started.rst index 258fe225a..812669e80 100644 --- a/docs/Users_Guide/getting_started.rst +++ b/docs/Users_Guide/getting_started.rst @@ -463,10 +463,22 @@ METplus in Apptainer Apptainer (formerly Singularity) can also be used to run METplus. Images can be pulled from DockerHub using Apptainer commands. -If running on the NCAR Casper cluster, be sure to first load the Apptainer module via:: +If running on the NCAR Casper cluster, +be sure to first load the Apptainer module via:: module load apptainer +It is recommended to set the **APPTAINER_TMPDIR** environment variable to a +directory where the user has write permissions to prevent issues with temporary +files exceeding the size of the /tmp directory. +See the +`Apptainer documentation `_ +for more information. + +Example:: + + export APPTAINER_TMP_DIR=/d1/${USER}/apptainer_tmp + Navigate to a working directory and pull an image from DockerHub, e.g.:: apptainer pull docker://dtcenter/metplus:5.1-latest From 043dd614b1dab546c44fa52fd412df49075893db Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Fri, 6 Dec 2024 12:38:46 -0700 Subject: [PATCH 3/6] per #511, add unit test for expected behavior to support comma-separated lists in a command line single config override that should fail until fix is made --- internal/tests/pytests/run_metplus/test_run_metplus.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/internal/tests/pytests/run_metplus/test_run_metplus.py b/internal/tests/pytests/run_metplus/test_run_metplus.py index 6567e2d8e..1f7532012 100644 --- a/internal/tests/pytests/run_metplus/test_run_metplus.py +++ b/internal/tests/pytests/run_metplus/test_run_metplus.py @@ -20,6 +20,12 @@ NEW_OUTPUT_BASE = os.path.join(TEST_OUTPUT_DIR, 'run_metplus') OUTPUT_BASE_OVERRIDE = f"config.OUTPUT_BASE={NEW_OUTPUT_BASE}" +# test that a list of values specified in a command line config override +# will no longer cause an error in the run. The list can be specified without +# quotes if no spaces are present or with quotes if spaces are present +LIST_CONFIG_OVERRIDE_1 = 'config.LEAD_SEQ=3H,6H' +LIST_CONFIG_OVERRIDE_2 = 'config.LEAD_SEQ="3H, 6H"' + @pytest.mark.run_metplus def test_run_metplus_exists(): """! Check that run_metplus.py script exists """ @@ -31,6 +37,8 @@ def test_run_metplus_exists(): ([RUN_METPLUS], 2), ([RUN_METPLUS, EXAMPLE_CONF], 2), ([RUN_METPLUS, EXAMPLE_CONF, MINIMUM_CONF, OUTPUT_BASE_OVERRIDE], 0), + ([RUN_METPLUS, EXAMPLE_CONF, MINIMUM_CONF, LIST_CONFIG_OVERRIDE_1], 0), + ([RUN_METPLUS, EXAMPLE_CONF, MINIMUM_CONF, LIST_CONFIG_OVERRIDE_2], 0), ] ) @pytest.mark.run_metplus From f7536dbdc20246de39a651afeec3fe38dd31be87 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Fri, 6 Dec 2024 12:42:19 -0700 Subject: [PATCH 4/6] add a test to ensure that the -c argument is properly ignored since it is can be used in old use cases that were created when the argument was required --- internal/tests/pytests/run_metplus/test_run_metplus.py | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/tests/pytests/run_metplus/test_run_metplus.py b/internal/tests/pytests/run_metplus/test_run_metplus.py index 1f7532012..eced4bc11 100644 --- a/internal/tests/pytests/run_metplus/test_run_metplus.py +++ b/internal/tests/pytests/run_metplus/test_run_metplus.py @@ -37,6 +37,7 @@ def test_run_metplus_exists(): ([RUN_METPLUS], 2), ([RUN_METPLUS, EXAMPLE_CONF], 2), ([RUN_METPLUS, EXAMPLE_CONF, MINIMUM_CONF, OUTPUT_BASE_OVERRIDE], 0), + ([RUN_METPLUS, '-c', EXAMPLE_CONF, MINIMUM_CONF, OUTPUT_BASE_OVERRIDE], 0), ([RUN_METPLUS, EXAMPLE_CONF, MINIMUM_CONF, LIST_CONFIG_OVERRIDE_1], 0), ([RUN_METPLUS, EXAMPLE_CONF, MINIMUM_CONF, LIST_CONFIG_OVERRIDE_2], 0), ] From c1d8a11a86ce6671c857b0db5de87099791cd2a5 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Fri, 6 Dec 2024 12:44:32 -0700 Subject: [PATCH 5/6] per #511, add support for command line single config overrides to include values that are lists. Simplify logic to parse arguments to strip out -c/--config/-config arguments and skip check/error if argument is invalid because it is already handled in the metplus_config setup step that parses the arguments --- ush/run_metplus.py | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/ush/run_metplus.py b/ush/run_metplus.py index b7a99274a..b26698853 100755 --- a/ush/run_metplus.py +++ b/ush/run_metplus.py @@ -78,24 +78,9 @@ def get_config_inputs_from_command_line(): if any(arg in sys.argv for arg in help_args): usage() - # pull out command line arguments - config_inputs = [] - for arg in sys.argv[1:]: - if arg.startswith('-'): - # ignore -c and --config since they are now optional - if arg == '-c' or arg == '--config' or arg == '-config': - continue - - # error/exit if an argument that is not supported was used - print('ERROR: Invalid argument: %s.' % arg) - usage() - - # split up comma separated lists into individual items - # and add each to list of arguments - # NOTE: to support lists in a config variable override, - # this logic will have to be enhanced - # i.e. config.PROCESS_LIST=PCPCombine,GridStat - config_inputs.extend(arg.split(',')) + # pull out command line arguments, removing deprecated config arguments + config_args = ('-c', '--config', '-config') + config_inputs = [arg for arg in sys.argv[1:] if arg not in config_args] # if no valid config_inputs were found, print usage and exit if not config_inputs: From c881623b08805783d4016f63a2003b66393d6bc2 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Fri, 6 Dec 2024 12:49:07 -0700 Subject: [PATCH 6/6] added unit test to ensure that an invalid command line argument causes the appropriate failure from run_metplus.py --- internal/tests/pytests/run_metplus/test_run_metplus.py | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/tests/pytests/run_metplus/test_run_metplus.py b/internal/tests/pytests/run_metplus/test_run_metplus.py index eced4bc11..a550d8cd7 100644 --- a/internal/tests/pytests/run_metplus/test_run_metplus.py +++ b/internal/tests/pytests/run_metplus/test_run_metplus.py @@ -40,6 +40,7 @@ def test_run_metplus_exists(): ([RUN_METPLUS, '-c', EXAMPLE_CONF, MINIMUM_CONF, OUTPUT_BASE_OVERRIDE], 0), ([RUN_METPLUS, EXAMPLE_CONF, MINIMUM_CONF, LIST_CONFIG_OVERRIDE_1], 0), ([RUN_METPLUS, EXAMPLE_CONF, MINIMUM_CONF, LIST_CONFIG_OVERRIDE_2], 0), + ([RUN_METPLUS, EXAMPLE_CONF, MINIMUM_CONF, '--fake-arg'], 1), ] ) @pytest.mark.run_metplus