From 572b54178c8b03a9137bc3292b8e430feacda5d0 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Wed, 12 Apr 2023 01:00:52 +0000 Subject: [PATCH 01/24] Move first unittest to shared location. --- tests/test_python/__init__.py | 0 .../test_python/test_generate_FV3LAM_wflow.py | 79 +++++++++++++++++++ ush/generate_FV3LAM_wflow.py | 49 ------------ 3 files changed, 79 insertions(+), 49 deletions(-) create mode 100644 tests/test_python/__init__.py create mode 100644 tests/test_python/test_generate_FV3LAM_wflow.py diff --git a/tests/test_python/__init__.py b/tests/test_python/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/test_python/test_generate_FV3LAM_wflow.py b/tests/test_python/test_generate_FV3LAM_wflow.py new file mode 100644 index 0000000000..7894538bd3 --- /dev/null +++ b/tests/test_python/test_generate_FV3LAM_wflow.py @@ -0,0 +1,79 @@ +""" Defines an integration test for generate_FV3LAM_wflow script in the +ush directory """ + +#pylint: disable=invalid-name, no-self-use +import os +import sys +import unittest +from multiprocessing import Process + +from python_utils import ( + load_config_file, + update_dict, + cp_vrfy, + run_command, + define_macos_utilities, + cfg_to_yaml_str, + set_env_var, + get_env_var, +) + +from generate_FV3LAM_wflow import generate_FV3LAM_wflow + +class Testing(unittest.TestCase): + """ Class to run the tests. """ + def test_generate_FV3LAM_wflow(self): + + """ Test that a community and nco sample config can successfully + lead to the creation of an experiment directory. No jobs are + submitted. """ + + # run workflows in separate process to avoid conflict between community and nco settings + def run_workflow(USHdir, logfile): + p = Process(target=generate_FV3LAM_wflow, args=(USHdir, logfile)) + p.start() + p.join() + exit_code = p.exitcode + if exit_code != 0: + sys.exit(exit_code) + + test_dir = os.path.dirname(os.path.abspath(__file__)) + USHdir = os.path.join(test_dir, "..", "..", "ush") + logfile = "log.generate_FV3LAM_wflow" + sed = get_env_var("SED") + + # community test case + cp_vrfy(f"{USHdir}/config.community.yaml", f"{USHdir}/config.yaml") + run_command( + f"""{sed} -i 's/MACHINE: hera/MACHINE: linux/g' {USHdir}/config.yaml""" + ) + run_workflow(USHdir, logfile) + + # nco test case + nco_test_config = load_config_file(f"{USHdir}/config.nco.yaml") + # Since we don't have a pre-gen grid dir on a generic linux + # platform, turn the make_* tasks on for this test. + cfg_updates = { + "user": { + "MACHINE": "linux", + }, + "rocoto": { + "tasks": { + "taskgroups": \ + """'{{ ["parm/wflow/prep.yaml", + "parm/wflow/coldstart.yaml", + "parm/wflow/post.yaml"]|include }}'""" + }, + }, + } + update_dict(cfg_updates, nco_test_config) + + with open(f"{USHdir}/config.yaml", "w") as cfg_file: + cfg_file.write(cfg_to_yaml_str(nco_test_config)) + + run_workflow(USHdir, logfile) + + def setUp(self): + define_macos_utilities() + set_env_var("DEBUG", False) + set_env_var("VERBOSE", False) diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index 3e3d689dc7..efe4bcc1fa 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -788,52 +788,3 @@ def setup_logging(logfile: str = "log.generate_FV3LAM_wflow", debug: bool = Fals ) -class Testing(unittest.TestCase): - def test_generate_FV3LAM_wflow(self): - - # run workflows in separate process to avoid conflict between community and nco settings - def run_workflow(USHdir, logfile): - p = Process(target=generate_FV3LAM_wflow, args=(USHdir, logfile)) - p.start() - p.join() - exit_code = p.exitcode - if exit_code != 0: - sys.exit(exit_code) - - USHdir = os.path.dirname(os.path.abspath(__file__)) - logfile = "log.generate_FV3LAM_wflow" - SED = get_env_var("SED") - - # community test case - cp_vrfy(f"{USHdir}/config.community.yaml", f"{USHdir}/config.yaml") - run_command( - f"""{SED} -i 's/MACHINE: hera/MACHINE: linux/g' {USHdir}/config.yaml""" - ) - run_workflow(USHdir, logfile) - - # nco test case - nco_test_config = load_config_file(f"{USHdir}/config.nco.yaml") - # Since we don't have a pre-gen grid dir on a generic linux - # platform, turn the make_* tasks on for this test. - cfg_updates = { - "user": { - "MACHINE": "linux", - }, - "rocoto": { - "tasks": { - "taskgroups": \ - '\'{{ ["parm/wflow/prep.yaml","parm/wflow/coldstart.yaml", "parm/wflow/post.yaml"]|include }}\'' - }, - }, - } - update_dict(cfg_updates, nco_test_config) - - with open(f"{USHdir}/config.yaml", "w") as cfg_file: - cfg_file.write(cfg_to_yaml_str(nco_test_config)) - - run_workflow(USHdir, logfile) - - def setUp(self): - define_macos_utilities() - set_env_var("DEBUG", False) - set_env_var("VERBOSE", False) From a3fc81e0d79e9ab3686b76a2f98722fbbe045ea9 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Wed, 12 Apr 2023 01:22:26 +0000 Subject: [PATCH 02/24] Adding another batch. --- tests/test_python/test_check_ruc_lsm.py | 27 +++++++++++ .../test_python}/test_retrieve_data.py | 0 .../test_set_gridparams_GFDLgrid.py | 35 +++++++++++++++ .../test_set_predef_grid_params.py | 45 +++++++++++++++++++ ush/check_ruc_lsm.py | 14 ------ ush/set_gridparams_GFDLgrid.py | 26 ----------- ush/set_predef_grid_params.py | 35 --------------- 7 files changed, 107 insertions(+), 75 deletions(-) create mode 100644 tests/test_python/test_check_ruc_lsm.py rename {ush => tests/test_python}/test_retrieve_data.py (100%) create mode 100644 tests/test_python/test_set_gridparams_GFDLgrid.py create mode 100644 tests/test_python/test_set_predef_grid_params.py diff --git a/tests/test_python/test_check_ruc_lsm.py b/tests/test_python/test_check_ruc_lsm.py new file mode 100644 index 0000000000..7d340e0b0f --- /dev/null +++ b/tests/test_python/test_check_ruc_lsm.py @@ -0,0 +1,27 @@ +""" Test for the define_ruc_lsm script. """ + +#pylint: disable=invalid-name + +import os +import unittest + +from python_utils import set_env_var + +from check_ruc_lsm import check_ruc_lsm + +class Testing(unittest.TestCase): + """ Define the tests""" + + def test_check_ruc_lsm(self): + """" Read in a CCPP suite definition file and check that it is + using RUC LSM as part of the suite. """ + test_dir = os.path.dirname(os.path.abspath(__file__)) + USHdir = os.path.join(test_dir, "..", "..", "ush") + self.assertTrue( + check_ruc_lsm( + ccpp_phys_suite_fp=f"{USHdir}{os.sep}test_data{os.sep}suite_FV3_GSD_SAR.xml" + ) + ) + + def setUp(self): + set_env_var("DEBUG", True) diff --git a/ush/test_retrieve_data.py b/tests/test_python/test_retrieve_data.py similarity index 100% rename from ush/test_retrieve_data.py rename to tests/test_python/test_retrieve_data.py diff --git a/tests/test_python/test_set_gridparams_GFDLgrid.py b/tests/test_python/test_set_gridparams_GFDLgrid.py new file mode 100644 index 0000000000..469c95a96a --- /dev/null +++ b/tests/test_python/test_set_gridparams_GFDLgrid.py @@ -0,0 +1,35 @@ +""" Defines the tests for set_gridparams_GFDLgrid """ + +#pylint: disable=invalid-name + +import unittest + +from set_gridparams_GFDLgrid import set_gridparams_GFDLgrid + +class Testing(unittest.TestCase): + """ Define the tests """ + + def test_set_gridparams_GFDLgrid(self): + + """ Test that grid parameters are set as expected, and that a + list is returned with expected values. Some work here is needed + to understand why we have a list with these magic numbers.""" + grid_params = set_gridparams_GFDLgrid( + lon_of_t6_ctr=-97.5, + lat_of_t6_ctr=38.5, + res_of_t6g=96, + stretch_factor=1.4, + refine_ratio_t6g_to_t7g=3, + istart_of_t7_on_t6g=13, + iend_of_t7_on_t6g=84, + jstart_of_t7_on_t6g=17, + jend_of_t7_on_t6g=80, + run_envir="community", + verbose=True, + nh4=4, + ) + + self.assertEqual( + list(grid_params.values()), + [-97.5, 38.5, 216, 192, 6, 1.4, 21, 172, 29, 164], + ) diff --git a/tests/test_python/test_set_predef_grid_params.py b/tests/test_python/test_set_predef_grid_params.py new file mode 100644 index 0000000000..b0b859e9b9 --- /dev/null +++ b/tests/test_python/test_set_predef_grid_params.py @@ -0,0 +1,45 @@ +""" Defines tests for setting the predefined grid parameters. """ + +import os +import unittest + +from set_predef_grid_params import set_predef_grid_params + +class Testing(unittest.TestCase): + + "Define the tests" + + def test_set_predef_grid_params(self): + """ Check that the method called updates or sets specific entries + in the parameter dict.""" + test_dir = os.path.dirname(os.path.abspath(__file__)) + ushdir = os.path.join(test_dir, "..", "..", "ush") + fcst_config = dict( + PREDEF_GRID_NAME="RRFS_CONUS_3km", + QUILTING=False, + DT_ATMOS=36, + LAYOUT_X=18, + LAYOUT_Y=36, + BLOCKSIZE=28, + ) + params_dict = set_predef_grid_params( + ushdir, + fcst_config["PREDEF_GRID_NAME"], + fcst_config["QUILTING"], + ) + self.assertEqual(params_dict["GRID_GEN_METHOD"], "ESGgrid") + self.assertEqual(params_dict["ESGgrid_LON_CTR"], -97.5) + fcst_config = dict( + PREDEF_GRID_NAME="RRFS_CONUS_3km", + QUILTING=True, + DT_ATMOS=36, + LAYOUT_X=18, + LAYOUT_Y=36, + BLOCKSIZE=28, + ) + params_dict = set_predef_grid_params( + ushdir, + fcst_config["PREDEF_GRID_NAME"], + fcst_config["QUILTING"], + ) + self.assertEqual(params_dict["WRTCMP_nx"], 1799) diff --git a/ush/check_ruc_lsm.py b/ush/check_ruc_lsm.py index 3afe1f8264..6e2364ea22 100644 --- a/ush/check_ruc_lsm.py +++ b/ush/check_ruc_lsm.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 import os -import unittest from python_utils import ( set_env_var, @@ -26,16 +25,3 @@ def check_ruc_lsm(ccpp_phys_suite_fp): tree = load_xml_file(ccpp_phys_suite_fp) has_ruc = has_tag_with_value(tree, "scheme", "lsm_ruc") return has_ruc - - -class Testing(unittest.TestCase): - def test_check_ruc_lsm(self): - USHdir = os.path.dirname(os.path.abspath(__file__)) - self.assertTrue( - check_ruc_lsm( - ccpp_phys_suite_fp=f"{USHdir}{os.sep}test_data{os.sep}suite_FV3_GSD_SAR.xml" - ) - ) - - def setUp(self): - set_env_var("DEBUG", True) diff --git a/ush/set_gridparams_GFDLgrid.py b/ush/set_gridparams_GFDLgrid.py index a87079a899..5968c5cd06 100644 --- a/ush/set_gridparams_GFDLgrid.py +++ b/ush/set_gridparams_GFDLgrid.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 import os -import unittest from python_utils import ( import_vars, @@ -478,28 +477,3 @@ def set_gridparams_GFDLgrid( "JEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG": jend_of_t7_with_halo_on_t6sg, } - -class Testing(unittest.TestCase): - def test_set_gridparams_GFDLgrid(self): - grid_params = set_gridparams_GFDLgrid( - lon_of_t6_ctr=-97.5, - lat_of_t6_ctr=38.5, - res_of_t6g=96, - stretch_factor=1.4, - refine_ratio_t6g_to_t7g=3, - istart_of_t7_on_t6g=13, - iend_of_t7_on_t6g=84, - jstart_of_t7_on_t6g=17, - jend_of_t7_on_t6g=80, - run_envir="community", - verbose=True, - nh4=4, - ) - - self.assertEqual( - list(grid_params.values()), - [-97.5, 38.5, 216, 192, 6, 1.4, 21, 172, 29, 164], - ) - - def setUp(self): - pass diff --git a/ush/set_predef_grid_params.py b/ush/set_predef_grid_params.py index b5761992b7..d7219c416c 100644 --- a/ush/set_predef_grid_params.py +++ b/ush/set_predef_grid_params.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -import unittest import os from textwrap import dedent @@ -39,37 +38,3 @@ def set_predef_grid_params(USHdir, grid_name, quilting): params_dict = flatten_dict(params_dict) return params_dict - - -class Testing(unittest.TestCase): - def test_set_predef_grid_params(self): - ushdir = os.path.dirname(os.path.abspath(__file__)) - fcst_config = dict( - PREDEF_GRID_NAME="RRFS_CONUS_3km", - QUILTING=False, - DT_ATMOS=36, - LAYOUT_X=18, - LAYOUT_Y=36, - BLOCKSIZE=28, - ) - params_dict = set_predef_grid_params( - ushdir, - fcst_config["PREDEF_GRID_NAME"], - fcst_config["QUILTING"], - ) - self.assertEqual(params_dict["GRID_GEN_METHOD"], "ESGgrid") - self.assertEqual(params_dict["ESGgrid_LON_CTR"], -97.5) - fcst_config = dict( - PREDEF_GRID_NAME="RRFS_CONUS_3km", - QUILTING=True, - DT_ATMOS=36, - LAYOUT_X=18, - LAYOUT_Y=36, - BLOCKSIZE=28, - ) - params_dict = set_predef_grid_params( - ushdir, - fcst_config["PREDEF_GRID_NAME"], - fcst_config["QUILTING"], - ) - self.assertEqual(params_dict["WRTCMP_nx"], 1799) From f8b84c2a6a391e858f376dcbe8a4794d41d235c2 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Wed, 12 Apr 2023 01:51:12 +0000 Subject: [PATCH 03/24] Moving another batch. --- .../test_python/test_get_crontab_contents.py | 18 ++++++ .../test_set_FV3nml_ens_stoch_seeds.py | 62 +++++++++++++++++++ .../test_set_FV3nml_sfc_climo_filenames.py | 38 ++++++++++++ .../test_set_gridparams_ESGgrid.py | 52 ++++++++++++++++ ush/generate_FV3LAM_wflow.py | 1 - ush/get_crontab_contents.py | 12 ---- ush/mrms_pull_topofhour.py | 6 -- ush/set_FV3nml_ens_stoch_seeds.py | 44 ------------- ush/set_FV3nml_sfc_climo_filenames.py | 28 --------- ush/set_gridparams_ESGgrid.py | 41 ------------ 10 files changed, 170 insertions(+), 132 deletions(-) create mode 100644 tests/test_python/test_get_crontab_contents.py create mode 100644 tests/test_python/test_set_FV3nml_ens_stoch_seeds.py create mode 100644 tests/test_python/test_set_FV3nml_sfc_climo_filenames.py create mode 100644 tests/test_python/test_set_gridparams_ESGgrid.py diff --git a/tests/test_python/test_get_crontab_contents.py b/tests/test_python/test_get_crontab_contents.py new file mode 100644 index 0000000000..7998550f2f --- /dev/null +++ b/tests/test_python/test_get_crontab_contents.py @@ -0,0 +1,18 @@ +""" Tests for get_crontab_contents.py""" + +import unittest + +from python_utils import define_macos_utilities, set_env_var +from get_crontab_contents import get_crontab_contents + +class Testing(unittest.TestCase): + """ Define the tests""" + def test_get_crontab_contents(self): + """ Call the function and make sure it doesn't fail. """ + crontab_cmd, _ = get_crontab_contents(called_from_cron=True) + self.assertEqual(crontab_cmd, "crontab") + + def setUp(self): + define_macos_utilities() + set_env_var("DEBUG", False) + set_env_var("MACHINE", "HERA") diff --git a/tests/test_python/test_set_FV3nml_ens_stoch_seeds.py b/tests/test_python/test_set_FV3nml_ens_stoch_seeds.py new file mode 100644 index 0000000000..bf9d61e785 --- /dev/null +++ b/tests/test_python/test_set_FV3nml_ens_stoch_seeds.py @@ -0,0 +1,62 @@ +""" Tests for set_FV3nml_ens_stoch_seeds.py """ + +#pylint: disable=invalid-name + +from datetime import datetime +import os +import unittest + +from python_utils import ( + cd_vrfy, + cp_vrfy, + date_to_str, + define_macos_utilities, + mkdir_vrfy, + set_env_var, +) + +from set_FV3nml_ens_stoch_seeds import set_FV3nml_ens_stoch_seeds + +class Testing(unittest.TestCase): + """ Define the tests """ + def test_set_FV3nml_ens_stoch_seeds(self): + """ Call the function and make sure it doesn't fail""" + set_FV3nml_ens_stoch_seeds(cdate=self.cdate) + + def setUp(self): + define_macos_utilities() + set_env_var("DEBUG", True) + set_env_var("VERBOSE", True) + self.cdate = datetime(2021, 1, 1) + test_dir = os.path.dirname(os.path.abspath(__file__)) + USHdir = os.path.join(test_dir, "..", "..", "ush") + PARMdir = os.path.join(USHdir, "..", "parm") + EXPTDIR = os.path.join(USHdir, "test_data", "expt") + mkdir_vrfy("-p", EXPTDIR) + cp_vrfy( + os.path.join(PARMdir, "input.nml.FV3"), + os.path.join(EXPTDIR, "input.nml"), + ) + for i in range(2): + mkdir_vrfy( + "-p", + os.path.join( + EXPTDIR, + f"{date_to_str(self.cdate,format='%Y%m%d%H')}{os.sep}mem{i+1}", + ), + ) + + cd_vrfy( + f"{EXPTDIR}{os.sep}{date_to_str(self.cdate,format='%Y%m%d%H')}{os.sep}mem2" + ) + set_env_var("USHdir", USHdir) + set_env_var("ENSMEM_INDX", 2) + set_env_var("FV3_NML_FN", "input.nml") + set_env_var("FV3_NML_FP", os.path.join(EXPTDIR, "input.nml")) + set_env_var("DO_SHUM", True) + set_env_var("DO_SKEB", True) + set_env_var("DO_SPPT", True) + set_env_var("DO_SPP", True) + set_env_var("DO_LSM_SPP", True) + ISEED_SPP = [4, 5, 6, 7, 8] + set_env_var("ISEED_SPP", ISEED_SPP) diff --git a/tests/test_python/test_set_FV3nml_sfc_climo_filenames.py b/tests/test_python/test_set_FV3nml_sfc_climo_filenames.py new file mode 100644 index 0000000000..a93461545a --- /dev/null +++ b/tests/test_python/test_set_FV3nml_sfc_climo_filenames.py @@ -0,0 +1,38 @@ +""" Tests for set_FV3nml_sfc_climo_filenames.py """ + +#pylint: disable=invalid-name + +import os +import unittest + +from python_utils import set_env_var, mkdir_vrfy, cp_vrfy, define_macos_utilities +from set_FV3nml_sfc_climo_filenames import set_FV3nml_sfc_climo_filenames + +class Testing(unittest.TestCase): + """ Define the tests """ + def test_set_FV3nml_sfc_climo_filenames(self): #pylint: disable=no-self-use + """ Call the function and don't raise an Exception. """ + set_FV3nml_sfc_climo_filenames() + + def setUp(self): + define_macos_utilities() + set_env_var("DEBUG", True) + set_env_var("VERBOSE", True) + test_dir = os.path.dirname(os.path.abspath(__file__)) + USHdir = os.path.join(test_dir, "..", "..", "ush") + PARMdir = os.path.join(USHdir, "..", "parm") + EXPTDIR = os.path.join(USHdir, "test_data", "expt") + FIXlam = os.path.join(EXPTDIR, "fix_lam") + mkdir_vrfy("-p", FIXlam) + mkdir_vrfy("-p", EXPTDIR) + cp_vrfy( + os.path.join(PARMdir, "input.nml.FV3"), + os.path.join(EXPTDIR, "input.nml"), + ) + set_env_var("PARMdir", PARMdir) + set_env_var("EXPTDIR", EXPTDIR) + set_env_var("FIXlam", FIXlam) + set_env_var("DO_ENSEMBLE", False) + set_env_var("CRES", "C3357") + set_env_var("RUN_ENVIR", "nco") + set_env_var("FV3_NML_FP", os.path.join(EXPTDIR, "input.nml")) diff --git a/tests/test_python/test_set_gridparams_ESGgrid.py b/tests/test_python/test_set_gridparams_ESGgrid.py new file mode 100644 index 0000000000..de087081da --- /dev/null +++ b/tests/test_python/test_set_gridparams_ESGgrid.py @@ -0,0 +1,52 @@ +""" Test for the set_gridparams_ESGgrid.py script. """ + +#pylint: disable=invalid-name + +import unittest + +from python_utils import set_env_var +from set_gridparams_ESGgrid import set_gridparams_ESGgrid + +class Testing(unittest.TestCase): + """ Define the tests """ + + def test_set_gridparams_ESGgrid(self): + """ Test that when provided inputs, the expected output is + provided in a list. Some work is needed here to remove magic + from the numbers. """ + + grid_parms = set_gridparams_ESGgrid( + lon_ctr=-97.5, + lat_ctr=38.5, + nx=1748, + ny=1038, + pazi=0.0, + halo_width=6, + delx=3000.0, + dely=3000.0, + constants=dict( + RADIUS_EARTH=6371200.0, + DEGS_PER_RADIAN=57.29577951308232087679, + ), + ) + + self.assertEqual( + list(grid_parms.values()), + [ + -97.5, + 38.5, + 1748, + 1038, + 0.0, + 6, + 0.999, + 0.013489400626196555, + 0.013489400626196555, + -1760, + -1050, + ], + ) + + def setUp(self): + set_env_var("DEBUG", False) + set_env_var("VERBOSE", False) diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index efe4bcc1fa..508cb49a8c 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -3,7 +3,6 @@ import os import sys import subprocess -import unittest import logging from multiprocessing import Process from textwrap import dedent diff --git a/ush/get_crontab_contents.py b/ush/get_crontab_contents.py index 774d311ac0..acc222822a 100644 --- a/ush/get_crontab_contents.py +++ b/ush/get_crontab_contents.py @@ -2,7 +2,6 @@ import os import sys -import unittest import argparse from datetime import datetime from textwrap import dedent @@ -227,14 +226,3 @@ def parse_args(argv): args = parse_args(sys.argv[1:]) if args.delete: delete_crontab_line(args.called_from_cron) - - -class Testing(unittest.TestCase): - def test_get_crontab_contents(self): - crontab_cmd, crontab_contents = get_crontab_contents(called_from_cron=True) - self.assertEqual(crontab_cmd, "crontab") - - def setUp(self): - define_macos_utilities() - set_env_var("DEBUG", False) - set_env_var("MACHINE", "HERA") diff --git a/ush/mrms_pull_topofhour.py b/ush/mrms_pull_topofhour.py index bfca98fb4e..7fc98f42c6 100644 --- a/ush/mrms_pull_topofhour.py +++ b/ush/mrms_pull_topofhour.py @@ -3,7 +3,6 @@ import re, csv, glob import bisect import numpy as np -import unittest if __name__ == "__main__": # Copy and unzip MRMS files that are closest to top of hour @@ -162,8 +161,3 @@ + filename2 ) os.system("gunzip " + VALID_DIR + "/" + filename2) - -# dummy unittest -class Testing(unittest.TestCase): - def test_mrms_pull_topfhour(self): - pass diff --git a/ush/set_FV3nml_ens_stoch_seeds.py b/ush/set_FV3nml_ens_stoch_seeds.py index 08ed944f46..2898319eaa 100644 --- a/ush/set_FV3nml_ens_stoch_seeds.py +++ b/ush/set_FV3nml_ens_stoch_seeds.py @@ -3,7 +3,6 @@ import os import sys import argparse -import unittest from textwrap import dedent from datetime import datetime @@ -157,46 +156,3 @@ def parse_args(argv): cfg = flatten_dict(cfg) import_vars(dictionary=cfg) set_FV3nml_ens_stoch_seeds(str_to_type(args.cdate)) - - -class Testing(unittest.TestCase): - def test_set_FV3nml_ens_stoch_seeds(self): - set_FV3nml_ens_stoch_seeds(cdate=self.cdate) - - def setUp(self): - define_macos_utilities() - set_env_var("DEBUG", True) - set_env_var("VERBOSE", True) - self.cdate = datetime(2021, 1, 1) - USHdir = os.path.dirname(os.path.abspath(__file__)) - PARMdir = os.path.join(USHdir, "..", "parm") - EXPTDIR = os.path.join(USHdir, "test_data", "expt") - mkdir_vrfy("-p", EXPTDIR) - cp_vrfy( - os.path.join(PARMdir, "input.nml.FV3"), - os.path.join(EXPTDIR, "input.nml"), - ) - for i in range(2): - mkdir_vrfy( - "-p", - os.path.join( - EXPTDIR, - f"{date_to_str(self.cdate,format='%Y%m%d%H')}{os.sep}mem{i+1}", - ), - ) - - cd_vrfy( - f"{EXPTDIR}{os.sep}{date_to_str(self.cdate,format='%Y%m%d%H')}{os.sep}mem2" - ) - - set_env_var("USHdir", USHdir) - set_env_var("ENSMEM_INDX", 2) - set_env_var("FV3_NML_FN", "input.nml") - set_env_var("FV3_NML_FP", os.path.join(EXPTDIR, "input.nml")) - set_env_var("DO_SHUM", True) - set_env_var("DO_SKEB", True) - set_env_var("DO_SPPT", True) - set_env_var("DO_SPP", True) - set_env_var("DO_LSM_SPP", True) - ISEED_SPP = [4, 5, 6, 7, 8] - set_env_var("ISEED_SPP", ISEED_SPP) diff --git a/ush/set_FV3nml_sfc_climo_filenames.py b/ush/set_FV3nml_sfc_climo_filenames.py index b2fb7a7deb..cdc4ed1628 100644 --- a/ush/set_FV3nml_sfc_climo_filenames.py +++ b/ush/set_FV3nml_sfc_climo_filenames.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -import unittest import os import sys import argparse @@ -145,30 +144,3 @@ def parse_args(argv): cfg = flatten_dict(cfg) import_vars(dictionary=cfg) set_FV3nml_sfc_climo_filenames() - - -class Testing(unittest.TestCase): - def test_set_FV3nml_sfc_climo_filenames(self): - set_FV3nml_sfc_climo_filenames() - - def setUp(self): - define_macos_utilities() - set_env_var("DEBUG", True) - set_env_var("VERBOSE", True) - USHdir = os.path.dirname(os.path.abspath(__file__)) - PARMdir = os.path.join(USHdir, "..", "parm") - EXPTDIR = os.path.join(USHdir, "test_data", "expt") - FIXlam = os.path.join(EXPTDIR, "fix_lam") - mkdir_vrfy("-p", FIXlam) - mkdir_vrfy("-p", EXPTDIR) - cp_vrfy( - os.path.join(PARMdir, "input.nml.FV3"), - os.path.join(EXPTDIR, "input.nml"), - ) - set_env_var("PARMdir", PARMdir) - set_env_var("EXPTDIR", EXPTDIR) - set_env_var("FIXlam", FIXlam) - set_env_var("DO_ENSEMBLE", False) - set_env_var("CRES", "C3357") - set_env_var("RUN_ENVIR", "nco") - set_env_var("FV3_NML_FP", os.path.join(EXPTDIR, "input.nml")) diff --git a/ush/set_gridparams_ESGgrid.py b/ush/set_gridparams_ESGgrid.py index cf8ddb9ff8..abafd24ae4 100644 --- a/ush/set_gridparams_ESGgrid.py +++ b/ush/set_gridparams_ESGgrid.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 import os -import unittest from datetime import datetime, timedelta from python_utils import ( @@ -74,43 +73,3 @@ def set_gridparams_ESGgrid( "NEG_NX_OF_DOM_WITH_WIDE_HALO": int(-(nx + 2 * halo_width)), "NEG_NY_OF_DOM_WITH_WIDE_HALO": int(-(ny + 2 * halo_width)), } - - -class Testing(unittest.TestCase): - def test_set_gridparams_ESGgrid(self): - - grid_parms = set_gridparams_ESGgrid( - lon_ctr=-97.5, - lat_ctr=38.5, - nx=1748, - ny=1038, - pazi=0.0, - halo_width=6, - delx=3000.0, - dely=3000.0, - constants=dict( - RADIUS_EARTH=6371200.0, - DEGS_PER_RADIAN=57.29577951308232087679, - ), - ) - - self.assertEqual( - list(grid_parms.values()), - [ - -97.5, - 38.5, - 1748, - 1038, - 0.0, - 6, - 0.999, - 0.013489400626196555, - 0.013489400626196555, - -1760, - -1050, - ], - ) - - def setUp(self): - set_env_var("DEBUG", False) - set_env_var("VERBOSE", False) From 9a55a1a7b1c6841574e2a64191accf0bfe099434 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Wed, 12 Apr 2023 14:28:30 +0000 Subject: [PATCH 04/24] Moving another batch of tests. --- tests/test_python/test_calculate_cost.py | 17 +++++ .../test_create_model_configure_file.py | 63 +++++++++++++++++++ .../test_set_thompson_mp_fix_files.py | 21 +++++++ ush/calculate_cost.py | 9 --- ush/create_model_configure_file.py | 49 --------------- ush/create_nems_configure_file.py | 1 - ush/set_thompson_mp_fix_files.py | 12 ---- 7 files changed, 101 insertions(+), 71 deletions(-) create mode 100644 tests/test_python/test_calculate_cost.py create mode 100644 tests/test_python/test_create_model_configure_file.py create mode 100644 tests/test_python/test_set_thompson_mp_fix_files.py diff --git a/tests/test_python/test_calculate_cost.py b/tests/test_python/test_calculate_cost.py new file mode 100644 index 0000000000..c306fc9276 --- /dev/null +++ b/tests/test_python/test_calculate_cost.py @@ -0,0 +1,17 @@ +""" Tests for calculate_cost.py""" + +#pylint: disable=invalid-name +import os +import unittest + +from calculate_cost import calculate_cost + +class Testing(unittest.TestCase): + """ Define the tests""" + def test_calculate_cost(self): + """ Test that the function returns the expected value for a + given config file.""" + test_dir = os.path.dirname(os.path.abspath(__file__)) + USHdir = os.path.join(test_dir, "..", "..", "ush") + params = calculate_cost(os.path.join(USHdir, 'config.community.yaml')) + self.assertCountEqual(params, [180, 28689, 180, 28689]) diff --git a/tests/test_python/test_create_model_configure_file.py b/tests/test_python/test_create_model_configure_file.py new file mode 100644 index 0000000000..d98f2d56ca --- /dev/null +++ b/tests/test_python/test_create_model_configure_file.py @@ -0,0 +1,63 @@ +""" Tests for create_model_configure_file.py""" + +#pylint: disable=invalid-name +from datetime import datetime +import os +import unittest + +from python_utils import set_env_var + +from create_model_configure_file import create_model_configure_file + + +class Testing(unittest.TestCase): + """ Define tests""" + + def test_create_model_configure_file(self): + """ Test that the function returns True when configured with + valid input data. """ + path = os.path.join(os.getenv("USHdir"), "test_data") + self.assertTrue( + create_model_configure_file( + run_dir=path, + cdate=datetime(2021, 1, 1), + fcst_len_hrs=72, + fhrot=0, + sub_hourly_post=True, + dt_subhourly_post_mnts=4, + dt_atmos=1, + ) + ) + + def setUp(self): + test_dir = os.path.dirname(os.path.abspath(__file__)) + USHdir = os.path.join(test_dir, "..", "..", "ush") + PARMdir = os.path.join(USHdir, "..", "parm") + MODEL_CONFIG_FN = "model_configure" + MODEL_CONFIG_TMPL_FP = os.path.join(PARMdir, MODEL_CONFIG_FN) + + set_env_var("DEBUG", True) + set_env_var("VERBOSE", True) + set_env_var("QUILTING", True) + set_env_var("WRITE_DOPOST", True) + set_env_var("USHdir", USHdir) + set_env_var("MODEL_CONFIG_FN", MODEL_CONFIG_FN) + set_env_var("MODEL_CONFIG_TMPL_FP", MODEL_CONFIG_TMPL_FP) + set_env_var("FCST_LEN_HRS", 72) + set_env_var("FHROT", 0) + set_env_var("DT_ATMOS", 1) + set_env_var("RESTART_INTERVAL", 4) + + set_env_var("WRTCMP_write_groups", 1) + set_env_var("WRTCMP_write_tasks_per_group", 2) + set_env_var("WRTCMP_output_grid", "lambert_conformal") + set_env_var("WRTCMP_cen_lon", -97.5) + set_env_var("WRTCMP_cen_lat", 35.0) + set_env_var("WRTCMP_stdlat1", 35.0) + set_env_var("WRTCMP_stdlat2", 35.0) + set_env_var("WRTCMP_nx", 199) + set_env_var("WRTCMP_ny", 111) + set_env_var("WRTCMP_lon_lwr_left", -121.23349066) + set_env_var("WRTCMP_lat_lwr_left", 23.41731593) + set_env_var("WRTCMP_dx", 3000.0) + set_env_var("WRTCMP_dy", 3000.0) diff --git a/tests/test_python/test_set_thompson_mp_fix_files.py b/tests/test_python/test_set_thompson_mp_fix_files.py new file mode 100644 index 0000000000..25459d6c43 --- /dev/null +++ b/tests/test_python/test_set_thompson_mp_fix_files.py @@ -0,0 +1,21 @@ +""" Tests for set_thompson_mp_fix_files.py """ + +#pylint: disable=invalid-name +import os +import unittest + +from set_thompson_mp_fix_files import set_thompson_mp_fix_files + +class Testing(unittest.TestCase): + """ Define the tests""" + def test_set_thompson_mp_fix_files(self): + """ Test that when given a CCPP physics suite that uses Thompson + mp, the function returns True. """ + test_dir = os.path.dirname(os.path.abspath(__file__)) + USHdir = os.path.join(test_dir, "..", "..", "ush") + uses_thompson, _, _ = set_thompson_mp_fix_files( + os.path.join(f"{USHdir}", "test_data", "suite_FV3_GSD_SAR.xml"), + "Thompson_MP_MONTHLY_CLIMO.nc", + False, + ) + self.assertEqual(True, uses_thompson) diff --git a/ush/calculate_cost.py b/ush/calculate_cost.py index 731cce76f7..b5e952b252 100755 --- a/ush/calculate_cost.py +++ b/ush/calculate_cost.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 import os -import unittest import argparse from python_utils import ( @@ -98,11 +97,3 @@ def calculate_cost(config_fn): params = calculate_cost(args.cfg) print(" ".join(map(str, params))) - - -class Testing(unittest.TestCase): - def test_calculate_cost(self): - USHdir = os.path.dirname(os.path.abspath(__file__)) - params = calculate_cost(os.path.join(USHdir, 'config.community.yaml')) - self.assertCountEqual(params, [180, 28689, 180, 28689]) - diff --git a/ush/create_model_configure_file.py b/ush/create_model_configure_file.py index 47e1ba3daf..f835212e7d 100644 --- a/ush/create_model_configure_file.py +++ b/ush/create_model_configure_file.py @@ -3,7 +3,6 @@ import os import sys import argparse -import unittest from datetime import datetime from textwrap import dedent @@ -317,51 +316,3 @@ def parse_args(argv): dt_subhourly_post_mnts=str_to_type(args.dt_subhourly_post_mnts), dt_atmos=str_to_type(args.dt_atmos), ) - - -class Testing(unittest.TestCase): - def test_create_model_configure_file(self): - path = os.path.join(os.getenv("USHdir"), "test_data") - self.assertTrue( - create_model_configure_file( - run_dir=path, - cdate=datetime(2021, 1, 1), - fcst_len_hrs=72, - fhrot=0, - sub_hourly_post=True, - dt_subhourly_post_mnts=4, - dt_atmos=1, - ) - ) - - def setUp(self): - USHdir = os.path.dirname(os.path.abspath(__file__)) - PARMdir = os.path.join(USHdir, "..", "parm") - MODEL_CONFIG_FN = "model_configure" - MODEL_CONFIG_TMPL_FP = os.path.join(PARMdir, MODEL_CONFIG_FN) - - set_env_var("DEBUG", True) - set_env_var("VERBOSE", True) - set_env_var("QUILTING", True) - set_env_var("WRITE_DOPOST", True) - set_env_var("USHdir", USHdir) - set_env_var("MODEL_CONFIG_FN", MODEL_CONFIG_FN) - set_env_var("MODEL_CONFIG_TMPL_FP", MODEL_CONFIG_TMPL_FP) - set_env_var("FCST_LEN_HRS", 72) - set_env_var("FHROT", 0) - set_env_var("DT_ATMOS", 1) - set_env_var("RESTART_INTERVAL", 4) - - set_env_var("WRTCMP_write_groups", 1) - set_env_var("WRTCMP_write_tasks_per_group", 2) - set_env_var("WRTCMP_output_grid", "lambert_conformal") - set_env_var("WRTCMP_cen_lon", -97.5) - set_env_var("WRTCMP_cen_lat", 35.0) - set_env_var("WRTCMP_stdlat1", 35.0) - set_env_var("WRTCMP_stdlat2", 35.0) - set_env_var("WRTCMP_nx", 199) - set_env_var("WRTCMP_ny", 111) - set_env_var("WRTCMP_lon_lwr_left", -121.23349066) - set_env_var("WRTCMP_lat_lwr_left", 23.41731593) - set_env_var("WRTCMP_dx", 3000.0) - set_env_var("WRTCMP_dy", 3000.0) diff --git a/ush/create_nems_configure_file.py b/ush/create_nems_configure_file.py index f1837f54a2..9d4b9cd2de 100644 --- a/ush/create_nems_configure_file.py +++ b/ush/create_nems_configure_file.py @@ -3,7 +3,6 @@ import os import sys import argparse -import unittest from datetime import datetime from textwrap import dedent diff --git a/ush/set_thompson_mp_fix_files.py b/ush/set_thompson_mp_fix_files.py index 4b41c44310..8083044aac 100644 --- a/ush/set_thompson_mp_fix_files.py +++ b/ush/set_thompson_mp_fix_files.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 import os -import unittest from textwrap import dedent from python_utils import ( @@ -84,14 +83,3 @@ def set_thompson_mp_fix_files( mapping.append(f"{fix_file} | {fix_file}") return sdf_uses_thompson_mp, mapping, thompson_mp_fix_files - - -class Testing(unittest.TestCase): - def test_set_thompson_mp_fix_files(self): - USHdir = os.path.dirname(os.path.abspath(__file__)) - uses_thompson, _, _ = set_thompson_mp_fix_files( - f"{USHdir}{os.sep}test_data{os.sep}suite_FV3_GSD_SAR.xml", - "Thompson_MP_MONTHLY_CLIMO.nc", - False, - ) - self.assertEqual(True, uses_thompson) From 558d8642bd2cb2cb390110eaf3792899eb40a214 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Wed, 12 Apr 2023 17:52:49 +0000 Subject: [PATCH 05/24] Moving the last set of tests. --- .../test_create_diag_table_file.py | 31 ++++++++++++ tests/test_python/test_link_fix.py | 49 +++++++++++++++++++ tests/test_python/test_set_cycle_dates.py | 29 +++++++++++ tests/test_python/test_set_ozone_param.py | 45 +++++++++++++++++ ush/create_aqm_rc_file.py | 1 - ush/create_diag_table_file.py | 20 -------- ush/link_fix.py | 37 -------------- ush/set_cycle_dates.py | 21 -------- ush/set_ozone_param.py | 35 ------------- ush/update_input_nml.py | 1 - 10 files changed, 154 insertions(+), 115 deletions(-) create mode 100644 tests/test_python/test_create_diag_table_file.py create mode 100644 tests/test_python/test_link_fix.py create mode 100644 tests/test_python/test_set_cycle_dates.py create mode 100644 tests/test_python/test_set_ozone_param.py diff --git a/tests/test_python/test_create_diag_table_file.py b/tests/test_python/test_create_diag_table_file.py new file mode 100644 index 0000000000..f7559536f2 --- /dev/null +++ b/tests/test_python/test_create_diag_table_file.py @@ -0,0 +1,31 @@ +""" Tests for create_diag_table_file.py """ + +#pylint: disable=invalid-name +import os +import unittest + +from python_utils import set_env_var + +from create_diag_table_file import create_diag_table_file + +class Testing(unittest.TestCase): + """ Define the tests """ + def test_create_diag_table_file(self): + """ Test that when called with user config settings, the + function returns True """ + path = os.path.join(os.getenv("USHdir"), "test_data") + self.assertTrue(create_diag_table_file(run_dir=path)) + + def setUp(self): + test_dir = os.path.dirname(os.path.abspath(__file__)) + USHdir = os.path.join(test_dir, "..", "..", "ush") + PARMdir = os.path.join(USHdir, "..", "parm") + diag_table_fn = "diag_table" + diag_table_tmpl_fp = os.path.join(PARMdir, f"{diag_table_fn}.FV3_GFS_v15p2") + set_env_var("DEBUG", True) + set_env_var("VERBOSE", True) + set_env_var("USHdir", USHdir) + set_env_var("DIAG_TABLE_FN", diag_table_fn) + set_env_var("DIAG_TABLE_TMPL_FP", diag_table_tmpl_fp) + set_env_var("CRES", "C48") + set_env_var("CDATE", "2021010106") diff --git a/tests/test_python/test_link_fix.py b/tests/test_python/test_link_fix.py new file mode 100644 index 0000000000..89a13c1101 --- /dev/null +++ b/tests/test_python/test_link_fix.py @@ -0,0 +1,49 @@ +""" Test for link_fix.py""" + +#pylint: disable=invalid-name +import os +import unittest + +from python_utils import mkdir_vrfy, define_macos_utilities + +from link_fix import link_fix + +class Testing(unittest.TestCase): + """ Define the tests. """ + def test_link_fix(self): + + """ Test that link_fix returns the expected value for the given + input configuration """ + res = link_fix( + verbose=True, + file_group="grid", + source_dir=self.task_dir, + target_dir=self.FIXlam, + ccpp_phys_suite=self.cfg["CCPP_PHYS_SUITE"], + constants=self.cfg["constants"], + dot_or_uscore=self.cfg["DOT_OR_USCORE"], + nhw=self.cfg["NHW"], + run_task=False, + sfc_climo_fields=["foo", "bar"], + ) + self.assertTrue(res == "3357") + + def setUp(self): + define_macos_utilities() + test_dir = os.path.dirname(os.path.abspath(__file__)) + test_data_dir = os.path.join(os.path.dirname(test_dir), "test_data") + self.FIXlam = os.path.join(test_data_dir, "expt", "fix_lam") + self.task_dir = os.path.join(test_data_dir, "RRFS_CONUS_3km") + mkdir_vrfy("-p", self.FIXlam) + + self.cfg = { + "DOT_OR_USCORE": "_", + "NHW": 6, + "CCPP_PHYS_SUITE": "FV3_GSD_SAR", + "constants": { + "NH0": 0, + "NH4": 4, + "NH3": 3, + "TILE_RGNL": 7, + }, + } diff --git a/tests/test_python/test_set_cycle_dates.py b/tests/test_python/test_set_cycle_dates.py new file mode 100644 index 0000000000..eb76f579c6 --- /dev/null +++ b/tests/test_python/test_set_cycle_dates.py @@ -0,0 +1,29 @@ +""" Test set_cycle_dates.py """ + +from datetime import datetime +import unittest + +from set_cycle_dates import set_cycle_dates + +class Testing(unittest.TestCase): + """ Define the tests""" + def test_set_cycle_dates(self): + + """ Test that the proper list of dates are produced given the + intput data""" + cdates = set_cycle_dates( + date_start=datetime(2022, 1, 1, 6), + date_end=datetime(2022, 1, 2, 12), + incr_cycl_freq=6, + ) + self.assertEqual( + cdates, + [ + "2022010106", + "2022010112", + "2022010118", + "2022010200", + "2022010206", + "2022010212", + ], + ) diff --git a/tests/test_python/test_set_ozone_param.py b/tests/test_python/test_set_ozone_param.py new file mode 100644 index 0000000000..1d0e6d6aa7 --- /dev/null +++ b/tests/test_python/test_set_ozone_param.py @@ -0,0 +1,45 @@ +""" Tests for set_ozone_param.py """ + +#pylint: disable=invalid-name + +import os +import unittest + +from set_ozone_param import set_ozone_param + +class Testing(unittest.TestCase): + """ Define the tests """ + def test_set_ozone_param(self): + """ Test that when the CCPP phyiscs suite XML is provided that + activates ozone, the expected ozone parameter is returned""" + test_dir = os.path.dirname(os.path.abspath(__file__)) + USHdir = os.path.join(test_dir, "..", "..", "ush") + ozone_param, _, _ = set_ozone_param( + os.path.join(USHdir, "test_data", "suite_FV3_GSD_SAR.xml"), + self.CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING, + ) + self.assertEqual("ozphys_2015", ozone_param) + + def setUp(self): + self.CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING = [ + "aerosol.dat | global_climaeropac_global.txt", + "co2historicaldata_2010.txt | fix_co2_proj/global_co2historicaldata_2010.txt", + "co2historicaldata_2011.txt | fix_co2_proj/global_co2historicaldata_2011.txt", + "co2historicaldata_2012.txt | fix_co2_proj/global_co2historicaldata_2012.txt", + "co2historicaldata_2013.txt | fix_co2_proj/global_co2historicaldata_2013.txt", + "co2historicaldata_2014.txt | fix_co2_proj/global_co2historicaldata_2014.txt", + "co2historicaldata_2015.txt | fix_co2_proj/global_co2historicaldata_2015.txt", + "co2historicaldata_2016.txt | fix_co2_proj/global_co2historicaldata_2016.txt", + "co2historicaldata_2017.txt | fix_co2_proj/global_co2historicaldata_2017.txt", + "co2historicaldata_2018.txt | fix_co2_proj/global_co2historicaldata_2018.txt", + "co2historicaldata_2019.txt | fix_co2_proj/global_co2historicaldata_2019.txt", + "co2historicaldata_2020.txt | fix_co2_proj/global_co2historicaldata_2020.txt", + "co2historicaldata_2021.txt | fix_co2_proj/global_co2historicaldata_2021.txt", + "co2historicaldata_glob.txt | global_co2historicaldata_glob.txt", + "co2monthlycyc.txt | co2monthlycyc.txt", + "global_h2oprdlos.f77 | global_h2o_pltc.f77", + "global_zorclim.1x1.grb | global_zorclim.1x1.grb", + "sfc_emissivity_idx.txt | global_sfc_emissivity_idx.txt", + "solarconstant_noaa_an.txt | global_solarconstant_noaa_an.txt", + "global_o3prdlos.f77 | ozprdlos_2015_new_sbuvO3_tclm15_nuchem.f77", + ] diff --git a/ush/create_aqm_rc_file.py b/ush/create_aqm_rc_file.py index 926278de7c..fa8960f7db 100644 --- a/ush/create_aqm_rc_file.py +++ b/ush/create_aqm_rc_file.py @@ -3,7 +3,6 @@ import os import sys import argparse -import unittest from datetime import datetime from textwrap import dedent diff --git a/ush/create_diag_table_file.py b/ush/create_diag_table_file.py index 15a42729f4..845ce2dc77 100644 --- a/ush/create_diag_table_file.py +++ b/ush/create_diag_table_file.py @@ -3,7 +3,6 @@ import os import sys import argparse -import unittest from textwrap import dedent from python_utils import ( @@ -121,22 +120,3 @@ def parse_args(argv): cfg = flatten_dict(cfg) import_vars(dictionary=cfg) create_diag_table_file(args.run_dir) - - -class Testing(unittest.TestCase): - def test_create_diag_table_file(self): - path = os.path.join(os.getenv("USHdir"), "test_data") - self.assertTrue(create_diag_table_file(run_dir=path)) - - def setUp(self): - USHdir = os.path.dirname(os.path.abspath(__file__)) - PARMdir = os.path.join(USHdir, "..", "parm") - DIAG_TABLE_FN = "diag_table" - DIAG_TABLE_TMPL_FP = os.path.join(PARMdir, f"{DIAG_TABLE_FN}.FV3_GFS_v15p2") - set_env_var("DEBUG", True) - set_env_var("VERBOSE", True) - set_env_var("USHdir", USHdir) - set_env_var("DIAG_TABLE_FN", DIAG_TABLE_FN) - set_env_var("DIAG_TABLE_TMPL_FP", DIAG_TABLE_TMPL_FP) - set_env_var("CRES", "C48") - set_env_var("CDATE", "2021010106") diff --git a/ush/link_fix.py b/ush/link_fix.py index 114d645f10..99156019bd 100755 --- a/ush/link_fix.py +++ b/ush/link_fix.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -import unittest import os import sys import argparse @@ -419,39 +418,3 @@ def parse_args(argv): run_task=True, sfc_climo_fields=cfg["fixed_files"]["SFC_CLIMO_FIELDS"], ) - - -class Testing(unittest.TestCase): - def test_link_fix(self): - res = link_fix( - verbose=True, - file_group="grid", - source_dir=self.task_dir, - target_dir=self.FIXlam, - ccpp_phys_suite=self.cfg["CCPP_PHYS_SUITE"], - constants=self.cfg["constants"], - dot_or_uscore=self.cfg["DOT_OR_USCORE"], - nhw=self.cfg["NHW"], - run_task=False, - sfc_climo_fields=["foo", "bar"], - ) - self.assertTrue(res == "3357") - - def setUp(self): - define_macos_utilities() - TEST_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "test_data") - self.FIXlam = os.path.join(TEST_DIR, "expt", "fix_lam") - self.task_dir = os.path.join(TEST_DIR, "RRFS_CONUS_3km") - mkdir_vrfy("-p", self.FIXlam) - - self.cfg = { - "DOT_OR_USCORE": "_", - "NHW": 6, - "CCPP_PHYS_SUITE": "FV3_GSD_SAR", - "constants": { - "NH0": 0, - "NH4": 4, - "NH3": 3, - "TILE_RGNL": 7, - }, - } diff --git a/ush/set_cycle_dates.py b/ush/set_cycle_dates.py index 069a3891b1..0c63a87e49 100644 --- a/ush/set_cycle_dates.py +++ b/ush/set_cycle_dates.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -import unittest from datetime import datetime, timedelta, date from python_utils import print_input_args, print_err_msg_exit @@ -33,23 +32,3 @@ def set_cycle_dates(date_start, date_end, incr_cycl_freq): all_cdates.append(cyc) cdate += freq_delta return all_cdates - - -class Testing(unittest.TestCase): - def test_set_cycle_dates(self): - cdates = set_cycle_dates( - date_start=datetime(2022, 1, 1, 6), - date_end=datetime(2022, 1, 2, 12), - incr_cycl_freq=6, - ) - self.assertEqual( - cdates, - [ - "2022010106", - "2022010112", - "2022010118", - "2022010200", - "2022010206", - "2022010212", - ], - ) diff --git a/ush/set_ozone_param.py b/ush/set_ozone_param.py index a82199d966..14a57b3fe9 100644 --- a/ush/set_ozone_param.py +++ b/ush/set_ozone_param.py @@ -2,7 +2,6 @@ import copy import os -import unittest from textwrap import dedent from python_utils import ( @@ -129,37 +128,3 @@ def set_ozone_param(ccpp_phys_suite_fp, link_mappings): ) return ozone_param, fixgsm_ozone_fn, ozone_link_mappings - - -class Testing(unittest.TestCase): - def test_set_ozone_param(self): - USHdir = os.path.dirname(os.path.abspath(__file__)) - ozone_param, _, _ = set_ozone_param( - f"{USHdir}{os.sep}test_data{os.sep}suite_FV3_GSD_SAR.xml", - self.CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING, - ) - self.assertEqual("ozphys_2015", ozone_param) - - def setUp(self): - self.CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING = [ - "aerosol.dat | global_climaeropac_global.txt", - "co2historicaldata_2010.txt | fix_co2_proj/global_co2historicaldata_2010.txt", - "co2historicaldata_2011.txt | fix_co2_proj/global_co2historicaldata_2011.txt", - "co2historicaldata_2012.txt | fix_co2_proj/global_co2historicaldata_2012.txt", - "co2historicaldata_2013.txt | fix_co2_proj/global_co2historicaldata_2013.txt", - "co2historicaldata_2014.txt | fix_co2_proj/global_co2historicaldata_2014.txt", - "co2historicaldata_2015.txt | fix_co2_proj/global_co2historicaldata_2015.txt", - "co2historicaldata_2016.txt | fix_co2_proj/global_co2historicaldata_2016.txt", - "co2historicaldata_2017.txt | fix_co2_proj/global_co2historicaldata_2017.txt", - "co2historicaldata_2018.txt | fix_co2_proj/global_co2historicaldata_2018.txt", - "co2historicaldata_2019.txt | fix_co2_proj/global_co2historicaldata_2019.txt", - "co2historicaldata_2020.txt | fix_co2_proj/global_co2historicaldata_2020.txt", - "co2historicaldata_2021.txt | fix_co2_proj/global_co2historicaldata_2021.txt", - "co2historicaldata_glob.txt | global_co2historicaldata_glob.txt", - "co2monthlycyc.txt | co2monthlycyc.txt", - "global_h2oprdlos.f77 | global_h2o_pltc.f77", - "global_zorclim.1x1.grb | global_zorclim.1x1.grb", - "sfc_emissivity_idx.txt | global_sfc_emissivity_idx.txt", - "solarconstant_noaa_an.txt | global_solarconstant_noaa_an.txt", - "global_o3prdlos.f77 | ozprdlos_2015_new_sbuvO3_tclm15_nuchem.f77", - ] diff --git a/ush/update_input_nml.py b/ush/update_input_nml.py index 703032a96c..0f10c675b2 100644 --- a/ush/update_input_nml.py +++ b/ush/update_input_nml.py @@ -3,7 +3,6 @@ import os import sys import argparse -import unittest import logging from textwrap import dedent From e5bbda3f0568a7442954121decc15931f7e9f588 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Wed, 12 Apr 2023 19:28:39 +0000 Subject: [PATCH 06/24] Use tempfile package for easy cleanup --- tests/test_python/test_link_fix.py | 23 ++++++++++--- .../test_set_FV3nml_ens_stoch_seeds.py | 32 +++++++++++-------- .../test_set_FV3nml_sfc_climo_filenames.py | 22 +++++++++++-- 3 files changed, 56 insertions(+), 21 deletions(-) diff --git a/tests/test_python/test_link_fix.py b/tests/test_python/test_link_fix.py index 89a13c1101..5dda28ccc0 100644 --- a/tests/test_python/test_link_fix.py +++ b/tests/test_python/test_link_fix.py @@ -2,16 +2,17 @@ #pylint: disable=invalid-name import os +import tempfile import unittest -from python_utils import mkdir_vrfy, define_macos_utilities +from python_utils import define_macos_utilities from link_fix import link_fix class Testing(unittest.TestCase): """ Define the tests. """ - def test_link_fix(self): + def test_link_fix(self): """ Test that link_fix returns the expected value for the given input configuration """ res = link_fix( @@ -31,10 +32,20 @@ def test_link_fix(self): def setUp(self): define_macos_utilities() test_dir = os.path.dirname(os.path.abspath(__file__)) - test_data_dir = os.path.join(os.path.dirname(test_dir), "test_data") - self.FIXlam = os.path.join(test_data_dir, "expt", "fix_lam") + ushdir = os.path.join(test_dir, "..", "..", "ush") + test_data_dir = os.path.join(ushdir, "test_data") + + # This is the known data location self.task_dir = os.path.join(test_data_dir, "RRFS_CONUS_3km") - mkdir_vrfy("-p", self.FIXlam) + + # Create a space to link that data into. It need not be in the + # same space. + self.tmp_dir = tempfile.TemporaryDirectory( + dir=os.path.abspath("."), + prefix="expt_fix_lam", + ) + self.FIXlam = self.tmp_dir.name + self.cfg = { "DOT_OR_USCORE": "_", @@ -47,3 +58,5 @@ def setUp(self): "TILE_RGNL": 7, }, } + def tearDown(self): + self.tmp_dir.cleanup() diff --git a/tests/test_python/test_set_FV3nml_ens_stoch_seeds.py b/tests/test_python/test_set_FV3nml_ens_stoch_seeds.py index bf9d61e785..97fc584bb0 100644 --- a/tests/test_python/test_set_FV3nml_ens_stoch_seeds.py +++ b/tests/test_python/test_set_FV3nml_ens_stoch_seeds.py @@ -4,6 +4,7 @@ from datetime import datetime import os +import tempfile import unittest from python_utils import ( @@ -31,24 +32,26 @@ def setUp(self): test_dir = os.path.dirname(os.path.abspath(__file__)) USHdir = os.path.join(test_dir, "..", "..", "ush") PARMdir = os.path.join(USHdir, "..", "parm") - EXPTDIR = os.path.join(USHdir, "test_data", "expt") - mkdir_vrfy("-p", EXPTDIR) + + # Create an temporary experiment directory + self.tmp_dir = tempfile.TemporaryDirectory( + dir=os.path.abspath("."), + prefix="expt", + ) + EXPTDIR = self.tmp_dir.name + cp_vrfy( os.path.join(PARMdir, "input.nml.FV3"), os.path.join(EXPTDIR, "input.nml"), ) - for i in range(2): - mkdir_vrfy( - "-p", - os.path.join( - EXPTDIR, - f"{date_to_str(self.cdate,format='%Y%m%d%H')}{os.sep}mem{i+1}", - ), - ) - cd_vrfy( - f"{EXPTDIR}{os.sep}{date_to_str(self.cdate,format='%Y%m%d%H')}{os.sep}mem2" - ) + # Put this in the tmp_dir structure so it gets cleaned up + mem_dir = os.path.join( + EXPTDIR, + f"{date_to_str(self.cdate,format='%Y%m%d%H')}", + f"mem2", + ) + mkdir_vrfy("-p", mem_dir) set_env_var("USHdir", USHdir) set_env_var("ENSMEM_INDX", 2) set_env_var("FV3_NML_FN", "input.nml") @@ -60,3 +63,6 @@ def setUp(self): set_env_var("DO_LSM_SPP", True) ISEED_SPP = [4, 5, 6, 7, 8] set_env_var("ISEED_SPP", ISEED_SPP) + + def tearDown(self): + self.tmp_dir.cleanup() diff --git a/tests/test_python/test_set_FV3nml_sfc_climo_filenames.py b/tests/test_python/test_set_FV3nml_sfc_climo_filenames.py index a93461545a..9e7ce01d64 100644 --- a/tests/test_python/test_set_FV3nml_sfc_climo_filenames.py +++ b/tests/test_python/test_set_FV3nml_sfc_climo_filenames.py @@ -3,9 +3,15 @@ #pylint: disable=invalid-name import os +import tempfile import unittest -from python_utils import set_env_var, mkdir_vrfy, cp_vrfy, define_macos_utilities +from python_utils import ( + cp_vrfy, + define_macos_utilities, + mkdir_vrfy, + set_env_var, + ) from set_FV3nml_sfc_climo_filenames import set_FV3nml_sfc_climo_filenames class Testing(unittest.TestCase): @@ -21,10 +27,17 @@ def setUp(self): test_dir = os.path.dirname(os.path.abspath(__file__)) USHdir = os.path.join(test_dir, "..", "..", "ush") PARMdir = os.path.join(USHdir, "..", "parm") - EXPTDIR = os.path.join(USHdir, "test_data", "expt") + + # Create a temporary experiment directory structure + here = os.getcwd() + self.tmp_dir = tempfile.TemporaryDirectory( + dir=os.path.abspath(here), + prefix="expt", + ) + EXPTDIR = self.tmp_dir.name FIXlam = os.path.join(EXPTDIR, "fix_lam") + mkdir_vrfy("-p", FIXlam) - mkdir_vrfy("-p", EXPTDIR) cp_vrfy( os.path.join(PARMdir, "input.nml.FV3"), os.path.join(EXPTDIR, "input.nml"), @@ -36,3 +49,6 @@ def setUp(self): set_env_var("CRES", "C3357") set_env_var("RUN_ENVIR", "nco") set_env_var("FV3_NML_FP", os.path.join(EXPTDIR, "input.nml")) + + def tearDown(self): + self.tmp_dir.cleanup() From 28f39eec731d5adce7bf74fa6fbba766d76af6ec Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Wed, 12 Apr 2023 19:36:04 +0000 Subject: [PATCH 07/24] Fix the relative paths after the test file moved. --- tests/test_python/test_retrieve_data.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test_python/test_retrieve_data.py b/tests/test_python/test_retrieve_data.py index 85e9b2860d..f1f05e44a5 100644 --- a/tests/test_python/test_retrieve_data.py +++ b/tests/test_python/test_retrieve_data.py @@ -29,7 +29,13 @@ class FunctionalTesting(unittest.TestCase): def setUp(self): self.path = os.path.dirname(__file__) - self.config = f"{self.path}/../parm/data_locations.yml" + self.config = os.path.join( + self.path, + "..", + "..", + "parm", + "data_locations.yml" + ) @unittest.skipIf(os.environ.get("CI") == "true", "Skipping HPSS tests") def test_fv3gfs_grib2_lbcs_from_hpss(self): From b2cb4155740b65e591df441f4fe10787b48b7675 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Wed, 12 Apr 2023 19:56:37 +0000 Subject: [PATCH 08/24] Missed a file. --- {ush/python_utils => tests/test_python}/test_python_utils.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {ush/python_utils => tests/test_python}/test_python_utils.py (100%) diff --git a/ush/python_utils/test_python_utils.py b/tests/test_python/test_python_utils.py similarity index 100% rename from ush/python_utils/test_python_utils.py rename to tests/test_python/test_python_utils.py From 505808ebfef770bcfbe97507ecbdf6f0db945406 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Wed, 12 Apr 2023 23:07:45 +0000 Subject: [PATCH 09/24] Linting and cleaning up a bit. --- tests/test_python/test_python_utils.py | 233 ++++++++++++++++++------- 1 file changed, 173 insertions(+), 60 deletions(-) diff --git a/tests/test_python/test_python_utils.py b/tests/test_python/test_python_utils.py index 69b6daf1a2..48520f139c 100644 --- a/tests/test_python/test_python_utils.py +++ b/tests/test_python/test_python_utils.py @@ -11,119 +11,232 @@ """ +#pylint: disable=invalid-name + import unittest import glob +import tempfile import os -from python_utils import * +import python_utils as util class Testing(unittest.TestCase): - def test_misc(self): - self.assertEqual(uppercase("upper"), "UPPER") - self.assertEqual(lowercase("LOWER"), "lower") - # regex in file - pattern = "^[ ]*(lsm_ruc)<\/scheme>[ ]*$" - FILE = f"{self.PATH}/../test_data/suite_FV3_GSD_SAR.xml" - match = find_pattern_in_file(pattern, FILE) + """ Define the tests""" + + def test_case_handlers(self): + """ Test that the case handling string manipulators work as + expected. """ + self.assertEqual(util.uppercase("upper"), "UPPER") + self.assertEqual(util.lowercase("LOWER"), "lower") + + def test_pattern_finding(self): + """ Test that find_pattern_in_file can work with a string or a + file path""" + + # Test given a file path + pattern = "^[ ]*(lsm_ruc)[ ]*$" + test_file = os.path.join( + self.ushdir, + "test_data", + "suite_FV3_GSD_SAR.xml", + ) + match = util.find_pattern_in_file(pattern, test_file) + self.assertEqual(("lsm_ruc",), match) + + # Test given a string + with open(test_file) as file_: + content = file_.read() + + util.find_pattern_in_str(pattern, content) self.assertEqual(("lsm_ruc",), match) - # regex in string - with open(FILE) as f: - content = f.read() - find_pattern_in_str(pattern, content) - self.assertEqual(("lsm_ruc",), match) def test_xml_parser(self): - FILE = f"{self.PATH}/../test_data/suite_FV3_GSD_SAR.xml" - tree = load_xml_file(FILE) - self.assertTrue(has_tag_with_value(tree, "scheme", "lsm_ruc")) + """ Given an input CCPP xml file, check that the XML is loaded + as expected, and has a tag.""" + test_file = os.path.join( + self.ushdir, + "test_data", + "suite_FV3_GSD_SAR.xml", + ) + tree = util.load_xml_file(test_file) + self.assertTrue(util.has_tag_with_value(tree, "scheme", "lsm_ruc")) def test_check_for_preexist_dir_file(self): - cmd_vrfy("mkdir -p test_data/dir") - self.assertTrue(os.path.exists("test_data/dir")) - check_for_preexist_dir_file("test_data/dir", "rename") - dirs = glob.glob("test_data/dir_*") + """ Test that when an existing directory should be renamed, it + still exists and that a new directory is made""" + + tmp_dir = tempfile.TemporaryDirectory( + dir=os.path.abspath("."), + prefix="preexist_space", + ) + + # Check cmd_vrfy works + existing_dir = os.path.join(tmp_dir, "dir") + util.cmd_vrfy(f"mkdir -p {existing_dir}") + self.assertTrue(os.path.exists("{existing_dir}")) + + # Given a preexisting directory, move it and test that they both + # exist. + util.check_for_preexist_dir_file(existing_dir, "rename") + dirs = glob.glob(f"{existing_dir}_*") self.assertEqual(len(dirs), 1) - rm_vrfy("-rf test_data/dir*") + + dirs = glob.glob(f"{existing_dir}*") + self.assertEqual(len(dirs), 2) + + # Clean up the older version, and test rm_vrfy + util.rm_vrfy(f"-rf {existing_dir}_*") + dirs = glob.glob(f"{existing_dir}_*") + self.assertEqual(len(dirs), 0) + tmp_dir.cleanup() def test_check_var_valid_value(self): - self.assertTrue(check_var_valid_value("rice", ["egg", "spam", "rice"])) + """ Test that a string is available in a given list. """ + self.assertTrue(util.check_var_valid_value("rice", ["egg", "spam", "rice"])) def test_filesys_cmds(self): - dPATH = f"{self.PATH}/test_data/dir" - mkdir_vrfy(dPATH) - self.assertTrue(os.path.exists(dPATH)) - cp_vrfy(f"{self.PATH}/misc.py", f"{dPATH}/miscs.py") - self.assertTrue(os.path.exists(f"{dPATH}/miscs.py")) - cmd_vrfy(f"rm -rf {dPATH}") - self.assertFalse(os.path.exists("tt.py")) + """ Test the functions that perform filesystem commands""" + + tmp_dir = tempfile.TemporaryDirectory( + dir=os.path.abspath("."), + prefix="filesys_space", + ) + testable_path = os.path.join( + tmp_dir, + "dir", + ) + + # Make sure a desired path is created + util.mkdir_vrfy(testable_path) + self.assertTrue(os.path.exists(testable_path)) + + # Make sure a file is copied + util.cp_vrfy(f"{self.ushdir}/python_utils/misc.py", f"{testable_path}/miscs.py") + self.assertTrue(os.path.exists(f"{testable_path}/miscs.py")) + + # Run a platform native command + util.cmd_vrfy(f"rm -rf {testable_path}") + + self.assertFalse(os.path.exists(testable_path)) + tmp_dir.cleanup() def test_run_command(self): - self.assertEqual(run_command("echo hello"), (0, "hello", "")) + """ Test the return of the run_command task is as expected.""" + self.assertEqual(util.run_command("echo hello"), (0, "hello", "")) def test_create_symlink_to_file(self): - TARGET = f"{self.PATH}/test_python_utils.py" - SYMLINK = f"{self.PATH}/test_data/test_python_utils.py" - create_symlink_to_file(TARGET, SYMLINK) + """ Test that a simlink is created as expected.""" + + target = f"{self.test_dir}/test_python_utils.py" + tmp_dir = tempfile.TemporaryDirectory( + dir=os.path.abspath("."), + prefix="simlink_space", + ) + + symlink = os.path.join( + tmp_dir, + "test_python_utils.py" + ) + util.create_symlink_to_file(target, symlink) + tmp_dir.cleanup() def test_define_macos_utilities(self): - set_env_var("MYVAR", "MYVAL") - val = os.getenv("MYVAR") - self.assertEqual(val, "MYVAL") + """ Test that environment setting and getting utils work. Also, + that environment contains macos utilities (arranged in setUP)""" + + util.set_env_var("MACOS_TEST_VAR", "MYVAL") + py_val = os.getenv("MACOS_TEST_VAR") + srw_val = util.get_env_var("MACOS_TEST_VAR") + + # Validate the real env was set, and that the srw_supported + # function retrieves the same value. + self.assertEqual(py_val, "MYVAL") + self.assertEqual(srw_val, "MYVAL") + self.assertEqual(os.getenv("SED"), "gsed" if os.uname() == "Darwin" else "sed") def test_print_input_args(self): + """ Test that print_input_args can count the args. """ valid_args = {"arg1": 1, "arg2": 2, "arg3": 3, "arg4": 4} - self.assertEqual(print_input_args(valid_args), 4) + self.assertEqual(util.print_input_args(valid_args), 4) def test_import_vars(self): + """ Test import/export vars.""" # test import - global MYVAR - set_env_var("MYVAR", "MYVAL") - env_vars = ["PWD", "MYVAR"] - import_vars(env_vars=env_vars) - self.assertEqual(os.path.realpath(PWD), os.path.realpath(os.getcwd())) - self.assertEqual(MYVAR, "MYVAL") + global IMPORT_TEST_VAR #pylint: disable=global-variable-undefined + + util.set_env_var("IMPORT_TEST_VAR", "MYVAL") + env_vars = ["PWD", "IMPORT_TEST_VAR"] + + # Makes all environment variables available in local scope for + # python + util.import_vars(env_vars=env_vars) + + # assuming all environments arlready have $PWD set + #pylint: disable= + self.assertEqual( + os.path.realpath(PWD), #pylint: disable=undefined-variable + os.path.realpath(os.getcwd()) + ) + self.assertEqual(IMPORT_TEST_VAR, "MYVAL") #pylint: disable=used-before-assignment + # test export - MYVAR = "MYNEWVAL" - self.assertEqual(os.environ["MYVAR"], "MYVAL") - export_vars(env_vars=env_vars) - self.assertEqual(os.environ["MYVAR"], "MYNEWVAL") + IMPORT_TEST_VAR = "MYNEWVAL" + self.assertEqual(os.environ["IMPORT_TEST_VAR"], "MYVAL") + util.export_vars(env_vars=env_vars) + self.assertEqual(os.environ["IMPORT_TEST_VAR"], "MYNEWVAL") + # test custom dictionary dictionary = {"Hello": "World!"} - import_vars(dictionary=dictionary) - self.assertEqual(Hello, "World!") - # test array + util.import_vars(dictionary=dictionary) + self.assertEqual(Hello, "World!") #pylint: disable=undefined-variable + + def test_str_to_list(self): + """ Test transforming a string formatted like a list into a + proper python list""" + # string has closing bracket shell_str = '("1" "2") \n' - v = str_to_list(shell_str) + v = util.str_to_list(shell_str) self.assertTrue(isinstance(v, list)) self.assertEqual(v, [1, 2]) + + # string does not have closing bracket shell_str = '( "1" "2" \n' - v = str_to_list(shell_str) + v = util.str_to_list(shell_str) self.assertFalse(isinstance(v, list)) + self.assertEqual(v, [1, 2]) def test_config_parser(self): + """ Test loading different config files """ cfg = {"HRS": ["1", "2"]} - shell_str = cfg_to_shell_str(cfg) + shell_str = util.cfg_to_shell_str(cfg) self.assertIn('HRS=( "1" "2" )\n', shell_str) # ini file - cfg = load_ini_config(f"{self.PATH}/test_data/Externals.cfg") + file_path = os.path.join( + self.ushdir, + "python_utils", + "test_data", + "Externals.cfg", + ) + cfg = util.load_ini_config(file_path) self.assertIn( - "regional_workflow", get_ini_value(cfg, "regional_workflow", "repo_url") + "regional_workflow", util.get_ini_value(cfg, "regional_workflow", "repo_url") ) def test_print_msg(self): - self.assertEqual(print_info_msg("Hello World!", verbose=False), False) + """ Test that a bool is returned from print_info_msg""" + self.assertEqual(util.print_info_msg("Hello World!", verbose=False), False) def setUp(self): """setUp is where we do preparation for running the unittests. If you need to download files for running test cases, prepare common stuff for all test cases etc, this is the best place to do it""" - define_macos_utilities() - set_env_var("DEBUG", "FALSE") - self.PATH = os.path.dirname(__file__) - + util.define_macos_utilities() + util.set_env_var("DEBUG", "FALSE") + self.test_dir = os.path.dirname(os.path.abspath(__file__)) + self.ushdir = os.path.join(self.test_dir, "..", "..", "ush") if __name__ == "__main__": unittest.main() From fa7f6b279a76d671013a9dc4d4c28b20c9b88dee Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Thu, 13 Apr 2023 00:03:13 +0000 Subject: [PATCH 10/24] Making the tests run. --- tests/test_python/test_python_utils.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/tests/test_python/test_python_utils.py b/tests/test_python/test_python_utils.py index 48520f139c..a0ffb3d841 100644 --- a/tests/test_python/test_python_utils.py +++ b/tests/test_python/test_python_utils.py @@ -72,9 +72,9 @@ def test_check_for_preexist_dir_file(self): ) # Check cmd_vrfy works - existing_dir = os.path.join(tmp_dir, "dir") + existing_dir = os.path.join(tmp_dir.name, "dir") util.cmd_vrfy(f"mkdir -p {existing_dir}") - self.assertTrue(os.path.exists("{existing_dir}")) + self.assertTrue(os.path.exists(existing_dir)) # Given a preexisting directory, move it and test that they both # exist. @@ -82,9 +82,6 @@ def test_check_for_preexist_dir_file(self): dirs = glob.glob(f"{existing_dir}_*") self.assertEqual(len(dirs), 1) - dirs = glob.glob(f"{existing_dir}*") - self.assertEqual(len(dirs), 2) - # Clean up the older version, and test rm_vrfy util.rm_vrfy(f"-rf {existing_dir}_*") dirs = glob.glob(f"{existing_dir}_*") @@ -103,7 +100,7 @@ def test_filesys_cmds(self): prefix="filesys_space", ) testable_path = os.path.join( - tmp_dir, + tmp_dir.name, "dir", ) @@ -135,7 +132,7 @@ def test_create_symlink_to_file(self): ) symlink = os.path.join( - tmp_dir, + tmp_dir.name, "test_python_utils.py" ) util.create_symlink_to_file(target, symlink) @@ -205,7 +202,6 @@ def test_str_to_list(self): shell_str = '( "1" "2" \n' v = util.str_to_list(shell_str) self.assertFalse(isinstance(v, list)) - self.assertEqual(v, [1, 2]) def test_config_parser(self): """ Test loading different config files """ From 9c6bcb8e2bced648d8d31a95d07205688b132adb Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Thu, 13 Apr 2023 00:05:16 +0000 Subject: [PATCH 11/24] Update GitHub Actions. Turn off the retrieve_data tests in the unit_tests. --- .github/workflows/python_func_tests.yaml | 6 ++---- .github/workflows/python_unittests.yaml | 9 ++++----- tests/test_python/test_retrieve_data.py | 1 + 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/python_func_tests.yaml b/.github/workflows/python_func_tests.yaml index f9faf765f2..b4b2725f14 100644 --- a/.github/workflows/python_func_tests.yaml +++ b/.github/workflows/python_func_tests.yaml @@ -30,11 +30,9 @@ jobs: run: | sudo apt-get update sudo apt-get install python3 python3-pip netcdf-bin - sudo pip3 install pyyaml jinja2 f90nml - sudo pip3 install numpy matplotlib basemap + sudo pip3 install pyyaml # Run python functional tests - name: Run python functional tests run: | - cd ush - python3 -m unittest -b test_retrieve_data.FunctionalTesting + python3 -m unittest -b tests/test_python/test_retrieve_data.py diff --git a/.github/workflows/python_unittests.yaml b/.github/workflows/python_unittests.yaml index 34eb54648c..4b4fd20723 100644 --- a/.github/workflows/python_unittests.yaml +++ b/.github/workflows/python_unittests.yaml @@ -10,6 +10,9 @@ on: - 'release/*' workflow_dispatch: +env: + UNIT_TEST: True + defaults: run: shell: bash @@ -35,10 +38,6 @@ jobs: - name: Run python unittests run: | ./manage_externals/checkout_externals ufs-weather-model - cd ush - python3 -m unittest -b python_utils/test_python_utils.py # exclude test_retrieve_data that is tested in functional test - files=$(find . -maxdepth 1 -name '*.py' -a ! -name 'test_retrieve_data.py' -exec basename {} \;) - files=$(echo $(echo "${files[@]}")) - python3 -m unittest -b ${files[@]} + python3 -m unittest -b tests/test_python/*.py diff --git a/tests/test_python/test_retrieve_data.py b/tests/test_python/test_retrieve_data.py index f1f05e44a5..962a0d7171 100644 --- a/tests/test_python/test_retrieve_data.py +++ b/tests/test_python/test_retrieve_data.py @@ -23,6 +23,7 @@ import retrieve_data +@unittest.skipIf(os.environ.get("UNIT_TEST") == "true", "Skipping functional tests") class FunctionalTesting(unittest.TestCase): """Test class for retrieve data""" From 878f93aefca7c161f3f0e35492d3dee8b9c9f97b Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Thu, 13 Apr 2023 00:06:11 +0000 Subject: [PATCH 12/24] Add Jenkins test to run the HPSS tests on jet/hera. --- .cicd/Jenkinsfile | 27 ++++++++++++++++++--------- .cicd/scripts/srw_unittest.sh | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 9 deletions(-) create mode 100755 .cicd/scripts/srw_unittest.sh diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile index 69b86926ac..da3af3002e 100644 --- a/.cicd/Jenkinsfile +++ b/.cicd/Jenkinsfile @@ -122,17 +122,26 @@ pipeline { } } - // Run the unified build script; if successful create a tarball of the build and upload to S3 - stage('Build') { - steps { - echo "Building SRW (${env.SRW_COMPILER}) on ${env.SRW_PLATFORM} (using ${env.WORKSPACE})" - sh 'bash --login "${WORKSPACE}/.cicd/scripts/srw_build.sh"' + parallel{ + // Run the unittest functional tests that require an HPC platform + stage('Functional Tests) { + steps { + echo "Running unittest on retrieve_data.py" + sh '${WORKSPACE}/.cicd/scripts/srw_unittest.sh"' + } } + // Run the unified build script; if successful create a tarball of the build and upload to S3 + stage('Build') { + steps { + echo "Building SRW (${env.SRW_COMPILER}) on ${env.SRW_PLATFORM} (using ${env.WORKSPACE})" + sh 'bash --login "${WORKSPACE}/.cicd/scripts/srw_build.sh"' + } - post { - success { - sh 'cd "${WORKSPACE}/${INSTALL_NAME}" && tar --create --gzip --verbose --file "${WORKSPACE}/${BUILD_NAME}.tgz" *' - s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: true, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.BUILD_NAME}.tgz", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false], [bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: true, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "build_${env.SRW_COMPILER}/srw_build-${env.SRW_PLATFORM}-${env.SRW_COMPILER}.txt", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] + post { + success { + sh 'cd "${WORKSPACE}/${INSTALL_NAME}" && tar --create --gzip --verbose --file "${WORKSPACE}/${BUILD_NAME}.tgz" *' + s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: true, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.BUILD_NAME}.tgz", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false], [bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: true, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "build_${env.SRW_COMPILER}/srw_build-${env.SRW_PLATFORM}-${env.SRW_COMPILER}.txt", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] + } } } } diff --git a/.cicd/scripts/srw_unittest.sh b/.cicd/scripts/srw_unittest.sh new file mode 100755 index 0000000000..7ab7ad9c12 --- /dev/null +++ b/.cicd/scripts/srw_unittest.sh @@ -0,0 +1,33 @@ +#!/user/bin/env bash +# +# A test script for running the SRW application unittest tests that +# should be tested on-prem. +# +set -e -u -x + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" > /dev/null 2>&1 +&& pwd)" + +# Get repository root from Jenkins WORKSPACE variable if set, otherwise, +# set relative to script directory. +declare workspace +if [[ -n "${WORKSPACE}" ]]; then + workspace="${WORKSPACE}" +else + workspace="$(cd -- "${script_dir}/../.." && pwd)" +fi + +# Only run this on machines with hpss access +hpss_machines=( jet hera ) +if [ $hpss_machines =~ ${SRW_PLATFORM} ] ; then + + module load hpss + module use ${workspace}/modulefiles + module load wflow_${SRW_PLATFORM} + + conda activate regional_workflow + + export PYTHONPATH=${workspace}/ush + python -m unittest $workspace/tests/test_python/test_retrieve_data.py + +fi From 4f35d17e4bb7a425231ef20d0c7202fea3440df8 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Thu, 13 Apr 2023 00:16:24 +0000 Subject: [PATCH 13/24] Linting. --- tests/test_python/test_retrieve_data.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_python/test_retrieve_data.py b/tests/test_python/test_retrieve_data.py index 962a0d7171..8d194d5195 100644 --- a/tests/test_python/test_retrieve_data.py +++ b/tests/test_python/test_retrieve_data.py @@ -110,7 +110,7 @@ def test_gdas_ics_from_aws(self): with tempfile.TemporaryDirectory(dir=".") as tmp_dir: os.chdir(tmp_dir) - out_path_tmpl = os.path.join(tmp_dir, f"mem{{mem:03d}}") + out_path_tmpl = os.path.join(tmp_dir, "mem{{mem:03d}}") # fmt: off args = [ @@ -146,7 +146,7 @@ def test_gefs_grib2_ics_from_aws(self): with tempfile.TemporaryDirectory(dir=".") as tmp_dir: os.chdir(tmp_dir) - out_path_tmpl = os.path.join(tmp_dir, f"mem{{mem:03d}}") + out_path_tmpl = os.path.join(tmp_dir, "mem{{mem:03d}}") # fmt: off args = [ @@ -269,7 +269,7 @@ def test_hrrr_lbcs_from_aws(self): with tempfile.TemporaryDirectory(dir=".") as tmp_dir: os.chdir(tmp_dir) - + # fmt: off args = [ '--file_set', 'fcst', From 500afce014146352499bc8544315a03251950c83 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Thu, 13 Apr 2023 00:16:44 +0000 Subject: [PATCH 14/24] Linting --- tests/test_python/test_set_FV3nml_ens_stoch_seeds.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_python/test_set_FV3nml_ens_stoch_seeds.py b/tests/test_python/test_set_FV3nml_ens_stoch_seeds.py index 97fc584bb0..139ff2f2f1 100644 --- a/tests/test_python/test_set_FV3nml_ens_stoch_seeds.py +++ b/tests/test_python/test_set_FV3nml_ens_stoch_seeds.py @@ -8,7 +8,6 @@ import unittest from python_utils import ( - cd_vrfy, cp_vrfy, date_to_str, define_macos_utilities, @@ -49,7 +48,7 @@ def setUp(self): mem_dir = os.path.join( EXPTDIR, f"{date_to_str(self.cdate,format='%Y%m%d%H')}", - f"mem2", + "mem2", ) mkdir_vrfy("-p", mem_dir) set_env_var("USHdir", USHdir) From e7855b2c7077c09b5db0aab2e1756c2b48a4ccae Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Thu, 13 Apr 2023 00:17:03 +0000 Subject: [PATCH 15/24] Adding a new github task to lint code that's already linted --- .github/workflows/python_linter.py | 36 ++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .github/workflows/python_linter.py diff --git a/.github/workflows/python_linter.py b/.github/workflows/python_linter.py new file mode 100644 index 0000000000..836ab5137d --- /dev/null +++ b/.github/workflows/python_linter.py @@ -0,0 +1,36 @@ +name: Python linting +on: + push: + branches: + - develop + - 'release/*' + pull_request: + branches: + - develop + - 'release/*' + workflow_dispatch: + +defaults: + run: + shell: bash +jobs: + + python_linter: + name: Python unittests + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Install dependencies + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install python3 python3-pip netcdf-bin + sudo pip3 install pylint + + # Run python unittests + - name: Lint the test directory + run: | + pylint --ignore-imports=yes tests/test_python/ From 52b82e8a1e9326ed76b7e1e0a483dcf375fc570c Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Thu, 13 Apr 2023 00:30:43 +0000 Subject: [PATCH 16/24] Tell the tests where the code is. --- .github/workflows/python_func_tests.yaml | 1 + .github/workflows/{python_linter.py => python_linter.yaml} | 1 + .github/workflows/python_unittests.yaml | 1 + 3 files changed, 3 insertions(+) rename .github/workflows/{python_linter.py => python_linter.yaml} (94%) diff --git a/.github/workflows/python_func_tests.yaml b/.github/workflows/python_func_tests.yaml index b4b2725f14..980b67a77c 100644 --- a/.github/workflows/python_func_tests.yaml +++ b/.github/workflows/python_func_tests.yaml @@ -35,4 +35,5 @@ jobs: # Run python functional tests - name: Run python functional tests run: | + export PYTHONPATH=${PWD}/ush python3 -m unittest -b tests/test_python/test_retrieve_data.py diff --git a/.github/workflows/python_linter.py b/.github/workflows/python_linter.yaml similarity index 94% rename from .github/workflows/python_linter.py rename to .github/workflows/python_linter.yaml index 836ab5137d..3616eddc14 100644 --- a/.github/workflows/python_linter.py +++ b/.github/workflows/python_linter.yaml @@ -33,4 +33,5 @@ # Run python unittests - name: Lint the test directory run: | + export PYTHONPATH=${PWD}/ush pylint --ignore-imports=yes tests/test_python/ diff --git a/.github/workflows/python_unittests.yaml b/.github/workflows/python_unittests.yaml index 4b4fd20723..0f213ff6bb 100644 --- a/.github/workflows/python_unittests.yaml +++ b/.github/workflows/python_unittests.yaml @@ -39,5 +39,6 @@ jobs: run: | ./manage_externals/checkout_externals ufs-weather-model # exclude test_retrieve_data that is tested in functional test + export PYTHONPATH=${PWD}/ush python3 -m unittest -b tests/test_python/*.py From 73d7b004c29025ec1a4abd0d4fe0b4edce82f4a1 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Thu, 13 Apr 2023 00:36:54 +0000 Subject: [PATCH 17/24] Make pylint consistent with version on Hera. --- .github/workflows/python_linter.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python_linter.yaml b/.github/workflows/python_linter.yaml index 3616eddc14..7b7cfed85b 100644 --- a/.github/workflows/python_linter.yaml +++ b/.github/workflows/python_linter.yaml @@ -28,7 +28,7 @@ jobs: run: | sudo apt-get update sudo apt-get install python3 python3-pip netcdf-bin - sudo pip3 install pylint + sudo pip3 install pylint==2.6 # Run python unittests - name: Lint the test directory From d020ff5d683dc854531487b6841671e6e913ea23 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Fri, 21 Apr 2023 18:30:24 +0000 Subject: [PATCH 18/24] Addressing Jenkins test failures. --- .cicd/Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile index da3af3002e..b761184e50 100644 --- a/.cicd/Jenkinsfile +++ b/.cicd/Jenkinsfile @@ -124,10 +124,10 @@ pipeline { parallel{ // Run the unittest functional tests that require an HPC platform - stage('Functional Tests) { + stage('Functional Tests') { steps { echo "Running unittest on retrieve_data.py" - sh '${WORKSPACE}/.cicd/scripts/srw_unittest.sh"' + sh 'bash --login "${WORKSPACE}/.cicd/scripts/srw_unittest.sh"' } } // Run the unified build script; if successful create a tarball of the build and upload to S3 From 744698c28a0c6fcfddc97e7e283918f80dbf0a22 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Mon, 24 Apr 2023 15:29:51 +0000 Subject: [PATCH 19/24] Remove parallel section. --- .cicd/Jenkinsfile | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile index b761184e50..eea2b48900 100644 --- a/.cicd/Jenkinsfile +++ b/.cicd/Jenkinsfile @@ -122,26 +122,24 @@ pipeline { } } - parallel{ - // Run the unittest functional tests that require an HPC platform - stage('Functional Tests') { - steps { - echo "Running unittest on retrieve_data.py" - sh 'bash --login "${WORKSPACE}/.cicd/scripts/srw_unittest.sh"' - } + // Run the unittest functional tests that require an HPC platform + stage('Functional Tests') { + steps { + echo "Running unittest on retrieve_data.py" + sh 'bash --login "${WORKSPACE}/.cicd/scripts/srw_unittest.sh"' + } + } + // Run the unified build script; if successful create a tarball of the build and upload to S3 + stage('Build') { + steps { + echo "Building SRW (${env.SRW_COMPILER}) on ${env.SRW_PLATFORM} (using ${env.WORKSPACE})" + sh 'bash --login "${WORKSPACE}/.cicd/scripts/srw_build.sh"' } - // Run the unified build script; if successful create a tarball of the build and upload to S3 - stage('Build') { - steps { - echo "Building SRW (${env.SRW_COMPILER}) on ${env.SRW_PLATFORM} (using ${env.WORKSPACE})" - sh 'bash --login "${WORKSPACE}/.cicd/scripts/srw_build.sh"' - } - post { - success { - sh 'cd "${WORKSPACE}/${INSTALL_NAME}" && tar --create --gzip --verbose --file "${WORKSPACE}/${BUILD_NAME}.tgz" *' - s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: true, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.BUILD_NAME}.tgz", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false], [bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: true, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "build_${env.SRW_COMPILER}/srw_build-${env.SRW_PLATFORM}-${env.SRW_COMPILER}.txt", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] - } + post { + success { + sh 'cd "${WORKSPACE}/${INSTALL_NAME}" && tar --create --gzip --verbose --file "${WORKSPACE}/${BUILD_NAME}.tgz" *' + s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: true, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.BUILD_NAME}.tgz", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false], [bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: true, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "build_${env.SRW_COMPILER}/srw_build-${env.SRW_PLATFORM}-${env.SRW_COMPILER}.txt", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] } } } From 8971e2293aff845fed64c0e695298678e432e22d Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Fri, 28 Apr 2023 15:06:17 -0600 Subject: [PATCH 20/24] Fix a wrapping issue. --- .cicd/scripts/srw_unittest.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.cicd/scripts/srw_unittest.sh b/.cicd/scripts/srw_unittest.sh index 7ab7ad9c12..1cb8c3a18c 100755 --- a/.cicd/scripts/srw_unittest.sh +++ b/.cicd/scripts/srw_unittest.sh @@ -5,8 +5,7 @@ # set -e -u -x -script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" > /dev/null 2>&1 -&& pwd)" +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" # Get repository root from Jenkins WORKSPACE variable if set, otherwise, # set relative to script directory. From 4850943813f1d906e4d2b9648c7ba08d019d047c Mon Sep 17 00:00:00 2001 From: Christina Holt <56881914+christinaholtNOAA@users.noreply.github.com> Date: Wed, 3 May 2023 08:13:48 -0600 Subject: [PATCH 21/24] Update .cicd/scripts/srw_unittest.sh Co-authored-by: Michael Lueken <63728921+MichaelLueken@users.noreply.github.com> --- .cicd/scripts/srw_unittest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cicd/scripts/srw_unittest.sh b/.cicd/scripts/srw_unittest.sh index 1cb8c3a18c..65ef135c95 100755 --- a/.cicd/scripts/srw_unittest.sh +++ b/.cicd/scripts/srw_unittest.sh @@ -1,4 +1,4 @@ -#!/user/bin/env bash +#!/usr/bin/env bash # # A test script for running the SRW application unittest tests that # should be tested on-prem. From cf4284244e24ae55f26219c3d09bf46bc3e3accc Mon Sep 17 00:00:00 2001 From: Christina Holt <56881914+christinaholtNOAA@users.noreply.github.com> Date: Wed, 3 May 2023 08:20:41 -0600 Subject: [PATCH 22/24] Update .cicd/scripts/srw_unittest.sh Co-authored-by: Michael Lueken <63728921+MichaelLueken@users.noreply.github.com> --- .cicd/scripts/srw_unittest.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.cicd/scripts/srw_unittest.sh b/.cicd/scripts/srw_unittest.sh index 65ef135c95..c767b7dbbd 100755 --- a/.cicd/scripts/srw_unittest.sh +++ b/.cicd/scripts/srw_unittest.sh @@ -20,11 +20,8 @@ fi hpss_machines=( jet hera ) if [ $hpss_machines =~ ${SRW_PLATFORM} ] ; then + source ${workspace}/ush/load_modules_wflow.sh ${SRW_PLATFORM} module load hpss - module use ${workspace}/modulefiles - module load wflow_${SRW_PLATFORM} - - conda activate regional_workflow export PYTHONPATH=${workspace}/ush python -m unittest $workspace/tests/test_python/test_retrieve_data.py From f5025471c72fdaba6236e29b9124b3bca6c640f2 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Wed, 3 May 2023 15:33:35 +0000 Subject: [PATCH 23/24] Make changes to script found in debugging. --- .cicd/scripts/srw_unittest.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.cicd/scripts/srw_unittest.sh b/.cicd/scripts/srw_unittest.sh index c767b7dbbd..75d18b56c6 100755 --- a/.cicd/scripts/srw_unittest.sh +++ b/.cicd/scripts/srw_unittest.sh @@ -10,15 +10,16 @@ script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd) # Get repository root from Jenkins WORKSPACE variable if set, otherwise, # set relative to script directory. declare workspace -if [[ -n "${WORKSPACE}" ]]; then +if [[ -n "${WORKSPACE:-}" ]]; then workspace="${WORKSPACE}" else workspace="$(cd -- "${script_dir}/../.." && pwd)" fi +cd $workspace # Only run this on machines with hpss access hpss_machines=( jet hera ) -if [ $hpss_machines =~ ${SRW_PLATFORM} ] ; then +if [[ ${hpss_machines[@]} =~ ${SRW_PLATFORM} ]] ; then source ${workspace}/ush/load_modules_wflow.sh ${SRW_PLATFORM} module load hpss From 23623c40304f6698d375673b8056d37b34d114c5 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Wed, 3 May 2023 18:09:49 +0000 Subject: [PATCH 24/24] Change the call to unittest here. --- .cicd/scripts/srw_unittest.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.cicd/scripts/srw_unittest.sh b/.cicd/scripts/srw_unittest.sh index 75d18b56c6..9424fd4f68 100755 --- a/.cicd/scripts/srw_unittest.sh +++ b/.cicd/scripts/srw_unittest.sh @@ -25,6 +25,9 @@ if [[ ${hpss_machines[@]} =~ ${SRW_PLATFORM} ]] ; then module load hpss export PYTHONPATH=${workspace}/ush - python -m unittest $workspace/tests/test_python/test_retrieve_data.py + + # Don't include an absolute path here because unittest get's confused + # on-prem where there may be a dot in the user path. + python -m unittest tests/test_python/test_retrieve_data.py fi