diff --git a/tests/test_next_workers.py b/tests/test_next_workers.py index 16ba6733..7521f58b 100644 --- a/tests/test_next_workers.py +++ b/tests/test_next_workers.py @@ -14,7 +14,11 @@ # limitations under the License. """Unit tests for nowcast.next_workers module. """ +import textwrap +from pathlib import Path + import arrow +import nemo_nowcast import pytest from nemo_nowcast import Message, NextWorker @@ -22,54 +26,88 @@ @pytest.fixture -def config(): - """Nowcast system config dict data structure; - a mock for :py:attr:`nemo_nowcast.config.Config._dict`. - """ - return { - "rivers": { - "stations": { - "Capilano": "08GA010", - "Englishman": "08HB002", - "Fraser": "08MF005", - } - }, - "observations": { - "ctd data": {"stations": ["SCVIP", "SEVIP", "USDDL"]}, - "ferry data": {"ferries": {"TWDP": {}}}, - "hadcp data": {}, - }, - "run types": { - "nowcast": {}, - "nowcast-dev": {}, - "nowcast-green": {}, - "nowcast-agrif": {}, - "forecast": {}, - "forecast2": {}, - }, - "run": { - "enabled hosts": { - "arbutus.cloud": { - "shared storage": False, - "make forcing links": True, - "run types": ["nowcast", "forecast", "forecast2", "nowcast-green"], - }, - "salish": { - "shared storage": True, - "make forcing links": True, - "run types": ["nowcast-dev"], - }, - "orcinus": { - "shared storage": False, - "make forcing links": False, - "run types": ["nowcast-agrif"], - }, - }, - "hindcast hosts": {"optimum-hindcast": {}}, - }, - "wave forecasts": {"host": "arbutus.cloud", "run when": "after nowcast-green"}, - "vhfr fvcom runs": {"host": "arbutus.cloud"}, - } +def config(base_config): + """:py:class:`nemo_nowcast.Config` instance from YAML fragment to use as config for unit tests.""" + config_file = Path(base_config.file) + with config_file.open("at") as f: + f.write( + textwrap.dedent( + """\ + rivers: + stations: + Capilano: 08GA010 + Englishman: 08HB002 + Fraser: 08MF005 + + observations: + ctd data: + stations: + - SCVIP + - SEVIP + - USDDL + ferry data: + ferries: + TWDP: + route name: Tsawwassen - Duke Point + hadcp data: + csv dir: observations/AISDATA/ + + run types: + nowcast: + config name: SalishSeaCast_Blue + nowcast-dev: + config name: SalishSeaCast_Blue + nowcast-green: + config name: SalishSeaCast + nowcast-agrif: + config name: SMELTAGRIF + forecast: + config name: SalishSeaCast_Blue + forecast2: + config name: SalishSeaCast_Blue + + run: + enabled hosts: + arbutus.cloud: + shared storage: False + make forcing links: True + run types: + nowcast: + run sets dir: SS-run-sets/v201905/nowcast-blue/ + forecast: + run sets dir: SS-run-sets/v201905/forecast/ + forecast2: + run sets dir: SS-run-sets/v201905/forecast2/ + nowcast-green: + run sets dir: SS-run-sets/v201905/nowcast-green/ + salish: + shared storage: True + make forcing links: True + run types: + nowcast-dev: + run sets dir: SS-run-sets/v201905/nowcast-dev/ + orcinus: + shared storage: False + make forcing links: True + run types: + nowcast-agrif: + results: nowcast-agrif/ + hindcast hosts: + optimum-hindcast: + queue info cmd: /usr/bin/qstat + + wave forecasts: + host: arbutus.cloud + run when: after nowcast-green + + vhfr fvcom runs: + host: arbutus.cloud + """ + ) + ) + config_ = nemo_nowcast.Config() + config_.load(config_file) + return config_ @pytest.fixture diff --git a/tests/workers/test_run_NEMO.py b/tests/workers/test_run_NEMO.py index b36168f8..e9f494ec 100644 --- a/tests/workers/test_run_NEMO.py +++ b/tests/workers/test_run_NEMO.py @@ -15,124 +15,126 @@ """Unit tests for Salish Sea NEMO nowcast run_NEMO worker. """ import subprocess +import textwrap from pathlib import Path from types import SimpleNamespace from unittest.mock import call, patch, Mock import arrow +import nemo_nowcast import pytest from nowcast.workers import run_NEMO @pytest.fixture -def config(scope="function"): - return { - "coordinates": "coordinates_seagrid_SalishSea.nc", - "run types": { - "nowcast": { - "config name": "SalishSea", - "coordinates": "coordinates_seagrid_SalishSea2.nc", - "bathymetry": "bathy_downonegrid2.nc", - "mesh_mask": "mesh_mask_downbyone2.nc", - "land processor elimination": "bathy_downonegrid2.csv", - "duration": 1, - "restart from": "nowcast", - }, - "nowcast-dev": { - "config name": "SalishSea", - "coordinates": "coordinates_seagrid_SalishSea201702.nc", - "bathymetry": "bathymetry_201702.nc", - "mesh_mask": "mesh_mask201702.nc", - "land processor elimination": "bathymetry_201702.csv", - "duration": 1, - "restart from": "nowcast-green", - }, - "nowcast-green": { - "config name": "SOG", - "coordinates": "coordinates_seagrid_SalishSea2.nc", - "bathymetry": "bathy_downonegrid2.nc", - "mesh_mask": "mesh_mask_downbyone2.nc", - "land processor elimination": "bathy_downonegrid2.csv", - "duration": 1, - "restart from": "nowcast-green", - }, - "forecast": { - "config name": "SalishSea", - "coordinates": "coordinates_seagrid_SalishSea2.nc", - "bathymetry": "bathy_downonegrid2.nc", - "mesh_mask": "mesh_mask_downbyone2.nc", - "land processor elimination": "bathy_downonegrid2.csv", - "duration": 1.5, - "restart from": "nowcast", - }, - "forecast2": { - "config name": "SalishSea", - "coordinates": "coordinates_seagrid_SalishSea2.nc", - "bathymetry": "bathy_downonegrid2.nc", - "mesh_mask": "mesh_mask_downbyone2.nc", - "land processor elimination": "bathy_downonegrid2.csv", - "duration": 1.25, - "restart from": "forecast", - }, - }, - "run": { - "enabled hosts": { - "arbutus.cloud": { - "mpi hosts file": "${HOME}/mpi_hosts", - "xios host": "192.168.238.14", - "run prep dir": "nowcast-sys/runs/", - "grid dir": "nowcast-sys/grid/", - "salishsea_cmd": "bin/salishsea", - "job exec cmd": "bash", - "run types": { - "nowcast": { - "run sets dir": "SS-run-sets/SalishSea/nemo3.6/nowcast-blue/", - "mpi decomposition": "9x19", - "results": "results/SalishSea/nowcast", - }, - "forecast": { - "run sets dir": "SS-run-sets/SalishSea/nemo3.6/forecast/", - "mpi decomposition": "9x19", - "results": "results/SalishSea/forecast/", - }, - "forecast2": { - "run sets dir": "SS-run-sets/SalishSea/nemo3.6/forecast2/", - "mpi decomposition": "9x19", - "results": "results/SalishSea/forecast2/", - }, - "nowcast-green": { - "run sets dir": "SS-run-sets/SalishSea/nemo3.6/nowcast-green/", - "mpi decomposition": "9x19", - "results": "results/SalishSea/nowcast-green/", - }, - }, - "forcing": { - "bottom friction mask": "/grid/jetty_mask_bathy201702.nc" - }, - }, - "salish-nowcast": { - "run prep dir": "nowcast-sys/runs/", - "grid dir": "nowcast-sys/grid/", - "salishsea_cmd": "bin/salishsea", - "job exec cmd": "qsub", - "email": "somebody@example.com", - "run types": { - "nowcast-dev": { - "run sets dir": "SS-run-sets/SalishSea/nemo3.6/nowcast-dev/", - "mpi decomposition": "1x7", - "walltime": "23:30:00", - "results": "results/SalishSea/nowcast-dev", - }, - "nowcast-green": {"results": "results/SalishSea/nowcast-dev"}, - }, - "forcing": { - "bottom friction mask": "/grid/jetty_mask_bathy201702.nc" - }, - }, - } - }, - } +def config(base_config): + """:py:class:`nemo_nowcast.Config` instance from YAML fragment to use as config for unit tests.""" + config_file = Path(base_config.file) + with config_file.open("at") as f: + f.write( + textwrap.dedent( + """\ + results archive: + nowcast: results/SalishSea/nowcast-blue.201905/ + nowcast-dev: results/SalishSea/nowcast-dev.201905/ + nowcast-green: results2/SalishSea/nowcast-green.201905/ + forecast: results/SalishSea/forecast.201905/ + forecast2: results/SalishSea/forecast2.201905/ + + run types: + nowcast: + config name: SalishSeaCast_Blue + coordinates: coordinates_seagrid_SalishSea201702.nc + bathymetry: bathymetry_201702.nc + mesh mask: mesh_mask201702.nc + land processor elimination: bathymetry_201702.csv + duration: 1 # day + restart from: nowcast + nowcast-dev: + config name: SalishSeaCast_Blue + coordinates: coordinates_seagrid_SalishSea201702.nc + bathymetry: bathymetry_201702.nc + mesh mask: mesh_mask201702.nc + land processor elimination: bathymetry_201702.csv + duration: 1 # day + restart from: nowcast-dev + nowcast-green: + config name: SalishSeaCast + coordinates: coordinates_seagrid_SalishSea201702.nc + bathymetry: bathymetry_201702.nc + mesh mask: mesh_mask201702.nc + land processor elimination: bathymetry_201702.csv + duration: 1 # day + restart from: nowcast-green + forecast: + config name: SalishSeaCast_Blue + coordinates: coordinates_seagrid_SalishSea201702.nc + bathymetry: bathymetry_201702.nc + mesh mask: mesh_mask201702.nc + land processor elimination: bathymetry_201702.csv + duration: 1.5 # days + restart from: nowcast + forecast2: + config name: SalishSeaCast_Blue + coordinates: coordinates_seagrid_SalishSea201702.nc + bathymetry: bathymetry_201702.nc + mesh mask: mesh_mask201702.nc + land processor elimination: bathymetry_201702.csv + duration: 1.25 # days + restart from: forecast + + run: + enabled hosts: + arbutus.cloud: + mpi hosts file: ${HOME}/mpi_hosts + xios host: 192.168.238.14 + run prep dir: nowcast-sys/runs/ + grid dir: nowcast-sys/grid/ + salishsea_cmd: bin/salishsea + job exec cmd: bash + run types: + nowcast: + run sets dir: SS-run-sets/v201905/nowcast-blue/ + mpi decomposition: 11x18 + results: results/SalishSea/nowcast/ + forecast: + run sets dir: SS-run-sets/v201905/forecast/ + mpi decomposition: 11x18 + results: results/SalishSea/forecast/ + forecast2: + run sets dir: SS-run-sets/v201905/forecast2/ + mpi decomposition: 11x18 + results: results/SalishSea/forecast2/ + nowcast-green: + run sets dir: SS-run-sets/v201905/nowcast-green/ + mpi decomposition: 11x18 + results: results/SalishSea/nowcast-green/ + forcing: + bottom friction mask: grid/jetty_mask_bathy201702.nc + + salish-nowcast: + run prep dir: nowcast-sys/runs/ + grid dir: nowcast-sys/grid/ + salishsea_cmd: bin/salishsea + job exec cmd: qsub + email: somebody@example.com + run types: + nowcast-dev: + run sets dir: SS-run-sets/SalishSea/nemo3.6/nowcast-dev/ + mpi decomposition: 1x7 + walltime: "23:30:00" + results: results/SalishSea/nowcast-dev/ + nowcast-green: + results: results/SalishSea/nowcast-dev/ + forcing: + bottom friction mask: grid/jetty_mask_bathy201702.nc + """ + ) + ) + config_ = nemo_nowcast.Config() + config_.load(config_file) + return config_ @pytest.fixture @@ -422,7 +424,11 @@ def test_config_missing_results_dir(self, config): dmy = run_date.format("DDMMMYY").lower() run_id = "{dmy}{run_type}".format(dmy=dmy, run_type="nowcast") host_config = config["run"]["enabled hosts"]["arbutus.cloud"] - p_config = patch.dict(host_config["run types"], nowcast={}) + p_config = patch.dict( + host_config["run types"]["nowcast"], + {"run sets dir": "foo", "mpi decomposition": "11x18"}, + clear=True, + ) with p_config: with patch("nowcast.workers.run_NEMO.logger", autospec=True): with pytest.raises(run_NEMO.WorkerError): @@ -433,11 +439,11 @@ def test_config_missing_results_dir(self, config): @pytest.mark.parametrize( "host_name, run_type, expected", [ - ("arbutus.cloud", "nowcast", "SalishSea"), - ("arbutus.cloud", "nowcast-green", "SOG"), - ("salish-nowcast", "nowcast-dev", "SalishSea"), - ("arbutus.cloud", "forecast", "SalishSea"), - ("arbutus.cloud", "forecast2", "SalishSea"), + ("arbutus.cloud", "nowcast", "SalishSeaCast_Blue"), + ("arbutus.cloud", "nowcast-green", "SalishSeaCast"), + ("salish-nowcast", "nowcast-dev", "SalishSeaCast_Blue"), + ("arbutus.cloud", "forecast", "SalishSeaCast_Blue"), + ("arbutus.cloud", "forecast2", "SalishSeaCast_Blue"), ], ) def test_config_name( @@ -491,11 +497,11 @@ def test_run_id( @pytest.mark.parametrize( "host_name, run_type, expected", [ - ("arbutus.cloud", "nowcast", "9x19"), - ("arbutus.cloud", "nowcast-green", "9x19"), + ("arbutus.cloud", "nowcast", "11x18"), + ("arbutus.cloud", "nowcast-green", "11x18"), ("salish-nowcast", "nowcast-dev", "1x7"), - ("arbutus.cloud", "forecast", "9x19"), - ("arbutus.cloud", "forecast2", "9x19"), + ("arbutus.cloud", "forecast", "11x18"), + ("arbutus.cloud", "forecast2", "11x18"), ], ) def test_mpi_decomposition( @@ -606,12 +612,12 @@ def test_runs_dir_path( ( "arbutus.cloud", "nowcast", - "nowcast-sys/grid/coordinates_seagrid_SalishSea2.nc", + "nowcast-sys/grid/coordinates_seagrid_SalishSea201702.nc", ), ( "arbutus.cloud", "nowcast-green", - "nowcast-sys/grid/coordinates_seagrid_SalishSea2.nc", + "nowcast-sys/grid/coordinates_seagrid_SalishSea201702.nc", ), ( "salish-nowcast", @@ -621,12 +627,12 @@ def test_runs_dir_path( ( "arbutus.cloud", "forecast", - "nowcast-sys/grid/coordinates_seagrid_SalishSea2.nc", + "nowcast-sys/grid/coordinates_seagrid_SalishSea201702.nc", ), ( "arbutus.cloud", "forecast2", - "nowcast-sys/grid/coordinates_seagrid_SalishSea2.nc", + "nowcast-sys/grid/coordinates_seagrid_SalishSea201702.nc", ), ], ) @@ -652,15 +658,15 @@ def test_grid_coordinates( @pytest.mark.parametrize( "host_name, run_type, expected", [ - ("arbutus.cloud", "nowcast", "nowcast-sys/grid/bathy_downonegrid2.nc"), + ("arbutus.cloud", "nowcast", "nowcast-sys/grid/bathymetry_201702.nc"), ( "arbutus.cloud", "nowcast-green", - "nowcast-sys/grid/bathy_downonegrid2.nc", + "nowcast-sys/grid/bathymetry_201702.nc", ), ("salish-nowcast", "nowcast-dev", "nowcast-sys/grid/bathymetry_201702.nc"), - ("arbutus.cloud", "forecast", "nowcast-sys/grid/bathy_downonegrid2.nc"), - ("arbutus.cloud", "forecast2", "nowcast-sys/grid/bathy_downonegrid2.nc"), + ("arbutus.cloud", "forecast", "nowcast-sys/grid/bathymetry_201702.nc"), + ("arbutus.cloud", "forecast2", "nowcast-sys/grid/bathymetry_201702.nc"), ], ) def test_grid_bathymetry( @@ -685,11 +691,11 @@ def test_grid_bathymetry( @pytest.mark.parametrize( "host_name, run_type, expected", [ - ("arbutus.cloud", "nowcast", "nowcast-sys/grid/bathy_downonegrid2.csv"), + ("arbutus.cloud", "nowcast", "nowcast-sys/grid/bathymetry_201702.csv"), ( "arbutus.cloud", "nowcast-green", - "nowcast-sys/grid/bathy_downonegrid2.csv", + "nowcast-sys/grid/bathymetry_201702.csv", ), ("salish-nowcast", "nowcast-dev", False), ], @@ -821,7 +827,7 @@ def test_bottom_friction_mask_link( run_desc = run_NEMO._run_description( run_date, run_type, run_id, 2160, host_name, config ) - expected = "/grid/jetty_mask_bathy201702.nc" + expected = "grid/jetty_mask_bathy201702.nc" assert run_desc["forcing"]["bfr_coef.nc"]["link to"] == expected @pytest.mark.parametrize( @@ -1057,12 +1063,12 @@ class TestBuildScript: @patch( "nowcast.workers.run_NEMO.nemo_cmd.prepare.get_n_processors", return_value=119 ) - def test_script_west_cloud(self, m_gnp, m_lrd, run_type, config, tmpdir): + def test_script_arbutus_cloud(self, m_gnp, m_lrd, run_type, config, tmpdir): tmp_run_dir = tmpdir.ensure_dir("tmp_run_dir") run_desc_file = tmpdir.ensure("13may17.yaml") host_config = config["run"]["enabled hosts"]["arbutus.cloud"] results_dir = tmpdir.ensure_dir(host_config["run types"][run_type]["results"]) - p_config = patch.dict(config, [("results archive", str(results_dir))]) + p_config = patch.dict(config["results archive"], {run_type: str(results_dir)}) m_lrd.return_value = { "run_id": "13may17nowcast", "MPI decomposition": "11x18", @@ -1132,7 +1138,9 @@ def test_script_salish(self, m_gnp, m_lrd, config, tmpdir): results_dir = tmpdir.ensure_dir( host_config["run types"]["nowcast-dev"]["results"] ) - p_config = patch.dict(config, [("results archive", str(results_dir))]) + p_config = patch.dict( + config["results archive"], {"nowcast-dev": str(results_dir)} + ) m_lrd.return_value = { "run_id": "13may17nowcast", "MPI decomposition": "1x7", diff --git a/tests/workers/test_run_NEMO_agrif.py b/tests/workers/test_run_NEMO_agrif.py index 70bc06b1..42d37897 100644 --- a/tests/workers/test_run_NEMO_agrif.py +++ b/tests/workers/test_run_NEMO_agrif.py @@ -14,6 +14,8 @@ # limitations under the License. """Unit tests for SalishSeaCast run_NEMO_agrif worker. """ +import textwrap +from pathlib import Path from types import SimpleNamespace from unittest.mock import call, Mock, patch @@ -25,6 +27,30 @@ from nowcast.workers import run_NEMO_agrif +@pytest.fixture +def config(base_config): + """:py:class:`nemo_nowcast.Config` instance from YAML fragment to use as config for unit tests.""" + config_file = Path(base_config.file) + with config_file.open("at") as f: + f.write( + textwrap.dedent( + """\ + run: + enabled hosts: + orcinus: + ssh key: SalishSeaNEMO-nowcast_id_rsa + scratch dir: scratch/nowcast-agrif + run prep dir: nowcast-agrif-sys/runs + salishsea cmd: .local/bin/salishsea + + """ + ) + ) + config_ = nemo_nowcast.Config() + config_.load(config_file) + return config_ + + @patch("nowcast.workers.run_NEMO_agrif.NowcastWorker", spec=True) class TestMain: """Unit tests for main() function.""" @@ -121,18 +147,6 @@ def test_failure(self, m_logger): class TestRunNEMO_AGRIF: """Unit test for run_NEMO_agrif() function.""" - config = { - "run": { - "enabled hosts": { - "orcinus": { - "ssh key": "SalishSeaNEMO-nowcast_id_rsa", - "scratch dir": "scratch/nowcast-agrif", - "salishsea cmd": "/home/dlatorne/.local/bin/salishsea", - } - } - } - } - def test_checklist( self, m_launch_run, @@ -141,11 +155,12 @@ def test_checklist( _get_prev_run_namelists_info, m_sftp, m_logger, + config, ): parsed_args = SimpleNamespace( host_name="orcinus", run_date=arrow.get("2018-04-30") ) - checklist = run_NEMO_agrif.run_NEMO_agrif(parsed_args, self.config) + checklist = run_NEMO_agrif.run_NEMO_agrif(parsed_args, config) expected = { "nowcast-agrif": { "host": "orcinus", @@ -165,18 +180,12 @@ class TestEditNamelistTimes: Unit tests for _edit_namelist_times() function. """ - config = { - "run": { - "enabled hosts": {"orcinus": {"run prep dir": "nowcast-agrif-sys/runs"}} - } - } - - def test_download_namelist_times(self, m_patch, m_logger): + def test_download_namelist_times(self, m_patch, m_logger, config): m_sftp_client = Mock(name="sftp_client") prev_run_namelists_info = SimpleNamespace(itend=2_363_040, rdt=40) run_date = arrow.get("2018-04-30") run_NEMO_agrif._edit_namelist_times( - m_sftp_client, "orcinus", prev_run_namelists_info, run_date, self.config + m_sftp_client, "orcinus", prev_run_namelists_info, run_date, config ) assert m_sftp_client.get.call_args_list == [ call( @@ -189,12 +198,12 @@ def test_download_namelist_times(self, m_patch, m_logger): ), ] - def test_patch_namelist_times(self, m_patch, m_logger): + def test_patch_namelist_times(self, m_patch, m_logger, config): m_sftp_client = Mock(name="sftp_client") prev_run_namelists_info = SimpleNamespace(itend=2_363_040, rdt=40) run_date = arrow.get("2018-04-30") run_NEMO_agrif._edit_namelist_times( - m_sftp_client, "orcinus", prev_run_namelists_info, run_date, self.config + m_sftp_client, "orcinus", prev_run_namelists_info, run_date, config ) assert m_patch.call_args_list == [ call( @@ -215,12 +224,12 @@ def test_patch_namelist_times(self, m_patch, m_logger): ), ] - def test_upload_namelist_times(self, m_patch, m_logger): + def test_upload_namelist_times(self, m_patch, m_logger, config): m_sftp_client = Mock(name="sftp_client") prev_run_namelists_info = SimpleNamespace(itend=2_363_040, rdt=40) run_date = arrow.get("2018-04-30") run_NEMO_agrif._edit_namelist_times( - m_sftp_client, "orcinus", prev_run_namelists_info, run_date, self.config + m_sftp_client, "orcinus", prev_run_namelists_info, run_date, config ) assert m_sftp_client.put.call_args_list == [ call( @@ -252,18 +261,7 @@ class TestEditRunDesc: Unit tests for __edit_run_desc() function. """ - config = { - "run": { - "enabled hosts": { - "orcinus": { - "scratch dir": "scratch/nowcast-agrif", - "run prep dir": "nowcast-agrif-sys/runs", - } - } - } - } - - def test_download_run_desc_template(self, m_safe_load, m_logger, tmpdir): + def test_download_run_desc_template(self, m_safe_load, m_logger, config, tmpdir): m_sftp_client = Mock(name="sftp_client") prev_run_namelists_info = SimpleNamespace(itend=2_363_040, rdt=40) setattr(prev_run_namelists_info, "1_rdt", 20) @@ -274,7 +272,7 @@ def test_download_run_desc_template(self, m_safe_load, m_logger, tmpdir): prev_run_namelists_info, "30apr18nowcast-agrif", arrow.get("2018-04-30"), - self.config, + config, yaml_tmpl=yaml_tmpl, ) m_sftp_client.get.assert_called_once_with( @@ -283,7 +281,7 @@ def test_download_run_desc_template(self, m_safe_load, m_logger, tmpdir): ) @patch("nowcast.workers.run_NEMO_agrif.yaml.safe_dump", autospec=True) - def test_edit_run_desc(self, m_safe_dump, m_safe_load, m_logger, tmpdir): + def test_edit_run_desc(self, m_safe_dump, m_safe_load, m_logger, config, tmpdir): m_sftp_client = Mock(name="sftp_client") prev_run_namelists_info = SimpleNamespace(itend=2_363_040, rdt=40) setattr(prev_run_namelists_info, "1_rdt", 20) @@ -295,7 +293,7 @@ def test_edit_run_desc(self, m_safe_dump, m_safe_load, m_logger, tmpdir): prev_run_namelists_info, "30apr18nowcast-agrif", arrow.get("2018-04-30"), - self.config, + config, yaml_tmpl=yaml_tmpl, ) m_safe_dump.assert_called_once_with( @@ -314,7 +312,7 @@ def test_edit_run_desc(self, m_safe_dump, m_safe_load, m_logger, tmpdir): default_flow_style=False, ) - def test_upload_run_desc(self, m_safe_load, m_logger, tmpdir): + def test_upload_run_desc(self, m_safe_load, m_logger, config, tmpdir): m_sftp_client = Mock(name="sftp_client") prev_run_namelists_info = SimpleNamespace(itend=2_363_040, rdt=40) setattr(prev_run_namelists_info, "1_rdt", 20) @@ -325,7 +323,7 @@ def test_upload_run_desc(self, m_safe_load, m_logger, tmpdir): prev_run_namelists_info, "30apr18nowcast-agrif", arrow.get("2018-04-30"), - self.config, + config, yaml_tmpl=yaml_tmpl, ) m_sftp_client.put.assert_called_once_with( @@ -341,19 +339,7 @@ class TestLaunchRun: Unit tests for _launch_run() function. """ - config = { - "run": { - "enabled hosts": { - "orcinus": { - "scratch dir": "scratch/nowcast-agrif", - "run prep dir": "nowcast-agrif-sys/runs", - "salishsea cmd": "/home/dlatorne/.local/bin/salishsea", - } - } - } - } - - def test_launch_run(self, m_ssh_exec_cmd, m_logger): + def test_launch_run(self, m_ssh_exec_cmd, m_logger, config): m_ssh_client = Mock(name="ssh_client") run_id = "30apr18nowcast-agrif" m_ssh_exec_cmd.return_value = ( @@ -363,11 +349,11 @@ def test_launch_run(self, m_ssh_exec_cmd, m_logger): f"salishsea_cmd.run INFO: 9332731.orca2.ibb\n\n" ) run_dir, job_id = run_NEMO_agrif._launch_run( - m_ssh_client, "orcinus", run_id, self.config + m_ssh_client, "orcinus", run_id, config ) m_ssh_exec_cmd.assert_called_once_with( m_ssh_client, - f"/home/dlatorne/.local/bin/salishsea run " + f".local/bin/salishsea run " f"nowcast-agrif-sys/runs/{run_id}.yaml " f"scratch/nowcast-agrif/30apr18 --debug", "orcinus", @@ -377,13 +363,13 @@ def test_launch_run(self, m_ssh_exec_cmd, m_logger): assert run_dir == expected assert job_id == "9332731.orca2.ibb" - def test_ssh_error(self, m_ssh_exec_cmd, m_logger): + def test_ssh_error(self, m_ssh_exec_cmd, m_logger, config): m_ssh_client = Mock(name="ssh_client") m_ssh_exec_cmd.side_effect = nowcast.ssh_sftp.SSHCommandError( "cmd", "stdout", "stderr" ) with pytest.raises(nemo_nowcast.WorkerError): run_NEMO_agrif._launch_run( - m_ssh_client, "orcinus", "30apr18nowcast-agrif", self.config + m_ssh_client, "orcinus", "30apr18nowcast-agrif", config ) m_logger.error.assert_called_once_with("stderr") diff --git a/tests/workers/test_split_results.py b/tests/workers/test_split_results.py index c1874438..09ca61c8 100644 --- a/tests/workers/test_split_results.py +++ b/tests/workers/test_split_results.py @@ -15,7 +15,8 @@ """Unit tests for Salish Sea NEMO nowcast split_results worker. """ import logging -import os +import textwrap +from pathlib import Path from types import SimpleNamespace import arrow @@ -25,6 +26,25 @@ from nowcast.workers import split_results +@pytest.fixture +def config(base_config): + """:py:class:`nemo_nowcast.Config` instance from YAML fragment to use as config for unit tests.""" + config_file = Path(base_config.file) + with config_file.open("at") as f: + f.write( + textwrap.dedent( + """\ + results archive: + hindcast: + localhost: results/SalishSea/hindcast.201905/ + """ + ) + ) + config_ = nemo_nowcast.Config() + config_.load(config_file) + return config_ + + @pytest.fixture def mock_worker(mock_nowcast_worker, monkeypatch): monkeypatch.setattr(split_results, "NowcastWorker", mock_nowcast_worker) @@ -121,12 +141,14 @@ def test_failure(self, caplog): class TestSplitResults: """Integration test for split_reults() function.""" - def test_checklist(self, caplog, tmp_path, monkeypatch): - run_type_results = tmp_path / "hindcast.201905" - run_type_results.mkdir() - config = { - "results archive": {"hindcast": {"localhost": os.fspath(run_type_results)}} - } + def test_checklist(self, config, caplog, tmp_path, monkeypatch): + run_type_results = tmp_path / Path( + config["results archive"]["hindcast"]["localhost"] + ) + run_type_results.mkdir(parents=True) + monkeypatch.setitem( + config["results archive"]["hindcast"], "localhost", run_type_results + ) results_dir = run_type_results / "01jan07" results_dir.mkdir() @@ -167,9 +189,7 @@ def mock_glob(path, pattern): / "01jan07" / "SalishSea_1h_20070101_20070101_grid_T.nc" ).exists() - assert ( - tmp_path / run_type_results / "02jan07" / "SalishSea_01587600_restart.nc" - ).exists() + assert (run_type_results / "02jan07" / "SalishSea_01587600_restart.nc").exists() class TestMkDestDir: diff --git a/tests/workers/test_watch_NEMO.py b/tests/workers/test_watch_NEMO.py index 461a64c9..e50949c4 100644 --- a/tests/workers/test_watch_NEMO.py +++ b/tests/workers/test_watch_NEMO.py @@ -15,17 +15,60 @@ """Unit tests for Salish Sea NEMO nowcast watch_NEMO worker. """ import subprocess +import textwrap from pathlib import Path from types import SimpleNamespace from unittest.mock import call, Mock, patch import arrow +import nemo_nowcast import pytest from nemo_nowcast import NowcastWorker, Message from nowcast.workers import watch_NEMO +@pytest.fixture +def config(base_config): + """:py:class:`nemo_nowcast.Config` instance from YAML fragment to use as config for unit tests.""" + config_file = Path(base_config.file) + with config_file.open("at") as f: + f.write( + textwrap.dedent( + """\ + run types: + nowcast: + duration: 1 # day + nowcast-dev: + duration: 1 # day + nowcast-green: + duration: 1 # day + forecast: + duration: 1.5 # days + forecast2: + duration: 1.25 # days + run: + enabled hosts: + arbutus.cloud: + run types: + nowcast: + results: results/SalishSea/nowcast/ + nowcast-green: + results: results/SalishSea/nowcast-green/ + nowcast-dev: + results: results/SalishSea/nowcast-dev/ + forecast: + results: results/SalishSea/forecast/ + forecast2: + results: results/SalishSea/forecast2/ + """ + ) + ) + config_ = nemo_nowcast.Config() + config_.load(config_file) + return config_ + + @patch("nowcast.workers.watch_NEMO.NowcastWorker", spec=True) class TestMain: """Unit tests for main() function.""" @@ -42,11 +85,6 @@ def test_init_cli(self, m_worker): watch_NEMO.main() m_worker().init_cli.assert_called_once_with() - def test_init_cli(self, m_worker): - m_worker().cli = Mock(name="cli") - watch_NEMO.main() - m_worker().init_cli.assert_called_once_with() - def test_add_host_name_arg(self, m_worker): m_worker().cli = Mock(name="cli") watch_NEMO.main() @@ -138,16 +176,6 @@ def test_failure(self, m_logger, run_type, host_name): class TestWatchNEMO: """Unit tests for watch_NEMO function.""" - config = { - "run types": { - "nowcast": {"duration": 1}, # day - "nowcast-dev": {"duration": 1}, # day - "nowcast-green": {"duration": 1}, # day - "forecast": {"duration": 1.5}, # day - "forecast2": {"duration": 1.25}, # day - } - } - def test_checklist( self, m_confirm, @@ -157,6 +185,7 @@ def test_checklist( m_logger, run_type, host_name, + config, ): m_nml_read.return_value = { "namrun": {"nn_it000": 1, "nn_itend": 2160, "nn_date0": 20_171_113}, @@ -178,7 +207,7 @@ def test_checklist( }, ), ) - checklist = watch_NEMO.watch_NEMO(parsed_args, self.config, tell_manager) + checklist = watch_NEMO.watch_NEMO(parsed_args, config, tell_manager) expected = { run_type: {"host": host_name, "run date": "2017-11-13", "completed": True} } @@ -193,6 +222,7 @@ def test_time_step_not_found( m_logger, run_type, host_name, + config, tmpdir, ): tmp_run_dir = tmpdir.ensure_dir("tmp_run_dir") @@ -217,7 +247,7 @@ def test_time_step_not_found( ), ) watch_NEMO.POLL_INTERVAL = 0 - watch_NEMO.watch_NEMO(parsed_args, self.config, tell_manager) + watch_NEMO.watch_NEMO(parsed_args, config, tell_manager) m_logger.info.assert_called_once_with( f"{run_type} on {host_name}: " f"time.step not found; continuing to watch..." @@ -232,6 +262,7 @@ def test_time_step_found( m_logger, run_type, host_name, + config, tmpdir, ): tmp_run_dir = tmpdir.ensure_dir("tmp_run_dir") @@ -263,7 +294,7 @@ def test_time_step_found( ), ) watch_NEMO.POLL_INTERVAL = 0 - watch_NEMO.watch_NEMO(parsed_args, self.config, tell_manager) + watch_NEMO.watch_NEMO(parsed_args, config, tell_manager) m_logger.info.assert_called_once_with( f"{run_type} on {host_name}: " f"timestep: 1081 = 2017-11-13 12:00:00 UTC, 50.0% complete" @@ -278,6 +309,7 @@ def test_confirm_run_success( m_logger, run_type, host_name, + config, ): m_nml_read.return_value = { "namrun": {"nn_it000": 1, "nn_itend": 2160, "nn_date0": 20_171_113}, @@ -299,7 +331,7 @@ def test_confirm_run_success( }, ), ) - checklist = watch_NEMO.watch_NEMO(parsed_args, self.config, tell_manager) + checklist = watch_NEMO.watch_NEMO(parsed_args, config, tell_manager) expected = { run_type: {"host": host_name, "run date": "2017-11-13", "completed": True} } @@ -310,7 +342,7 @@ def test_confirm_run_success( Path("tmp_run_dir"), 2160, 2160, - self.config, + config, ) @@ -388,22 +420,6 @@ def test_oserror(self, m_kill): class TestConfirmRunSuccess: """Unit tests for _confirm_run_success() function.""" - config = { - "run": { - "enabled hosts": { - "arbutus.cloud": { - "run types": { - "nowcast": {"results": "SalishSea/nowcast-blue/"}, - "nowcast-green": {"results": "SalishSea/nowcast-green/"}, - "nowcast-dev": {"results": "SalishSea/nowcast-dev/"}, - "forecast": {"results": "SalishSea/forecast/"}, - "forecast2": {"results": "SalishSea/forecast2/"}, - } - } - } - } - } - @pytest.mark.parametrize( "run_type, itend, restart_timestep", [ @@ -414,7 +430,9 @@ class TestConfirmRunSuccess: ("forecast2", 2700, 2160), ], ) - def test_run_succeeded(self, m_logger, run_type, itend, restart_timestep, tmpdir): + def test_run_succeeded( + self, m_logger, run_type, itend, restart_timestep, config, tmpdir + ): with patch("nowcast.workers.watch_NEMO.Path.exists") as m_exists: m_exists.side_effect = (True, False, True, True) with patch("nowcast.workers.watch_NEMO.Path.open") as m_open: @@ -426,7 +444,7 @@ def test_run_succeeded(self, m_logger, run_type, itend, restart_timestep, tmpdir Path("tmp_run_dir"), itend, restart_timestep, - self.config, + config, ) assert run_succeeded @@ -440,7 +458,7 @@ def test_run_succeeded(self, m_logger, run_type, itend, restart_timestep, tmpdir ("forecast2", 2700, 2160), ], ) - def test_no_results_dir(self, m_logger, run_type, itend, restart_timestep): + def test_no_results_dir(self, m_logger, run_type, itend, restart_timestep, config): with patch("nowcast.workers.watch_NEMO.Path.exists") as m_exists: m_exists.side_effect = (False, False, True, True) with patch("nowcast.workers.watch_NEMO.Path.open") as m_open: @@ -452,7 +470,7 @@ def test_no_results_dir(self, m_logger, run_type, itend, restart_timestep): Path("tmp_run_dir"), itend, restart_timestep, - self.config, + config, ) assert not run_succeeded @@ -467,7 +485,12 @@ def test_no_results_dir(self, m_logger, run_type, itend, restart_timestep): ], ) def test_output_abort_file_exists( - self, m_logger, run_type, itend, restart_timestep + self, + m_logger, + run_type, + itend, + restart_timestep, + config, ): with patch("nowcast.workers.watch_NEMO.Path.exists") as m_exists: m_exists.side_effect = (True, True, True, True) @@ -480,7 +503,7 @@ def test_output_abort_file_exists( Path("tmp_run_dir"), itend, restart_timestep, - self.config, + config, ) assert not run_succeeded @@ -494,7 +517,9 @@ def test_output_abort_file_exists( ("forecast2", 2700, 2160), ], ) - def test_no_time_step_file(self, m_logger, run_type, itend, restart_timestep): + def test_no_time_step_file( + self, m_logger, run_type, itend, restart_timestep, config + ): with patch("nowcast.workers.watch_NEMO.Path.exists") as m_exists: m_exists.side_effect = (True, False, True, True) with patch("nowcast.workers.watch_NEMO.Path.open") as m_open: @@ -506,7 +531,7 @@ def test_no_time_step_file(self, m_logger, run_type, itend, restart_timestep): Path("tmp_run_dir"), itend, restart_timestep, - self.config, + config, ) assert "time.step" in m_logger.critical.call_args_list[0][0][0] assert not run_succeeded @@ -521,7 +546,9 @@ def test_no_time_step_file(self, m_logger, run_type, itend, restart_timestep): ("forecast2", 2700, 2160), ], ) - def test_wrong_final_time_step(self, m_logger, run_type, itend, restart_timestep): + def test_wrong_final_time_step( + self, m_logger, run_type, itend, restart_timestep, config + ): with patch("nowcast.workers.watch_NEMO.Path.exists") as m_exists: m_exists.side_effect = (True, False, True, True) with patch("nowcast.workers.watch_NEMO.Path.open") as m_open: @@ -533,7 +560,7 @@ def test_wrong_final_time_step(self, m_logger, run_type, itend, restart_timestep Path("tmp_run_dir"), itend, restart_timestep, - self.config, + config, ) assert not run_succeeded @@ -547,7 +574,9 @@ def test_wrong_final_time_step(self, m_logger, run_type, itend, restart_timestep ("forecast2", 2700, 2160), ], ) - def test_no_physics_restart_file(self, m_logger, run_type, itend, restart_timestep): + def test_no_physics_restart_file( + self, m_logger, run_type, itend, restart_timestep, config + ): with patch("nowcast.workers.watch_NEMO.Path.exists") as m_exists: m_exists.side_effect = (True, False, False, True) with patch("nowcast.workers.watch_NEMO.Path.open") as m_open: @@ -559,13 +588,13 @@ def test_no_physics_restart_file(self, m_logger, run_type, itend, restart_timest Path("tmp_run_dir"), itend, restart_timestep, - self.config, + config, ) expected = f"SalishSea_{restart_timestep:08d}_restart.nc" assert expected in m_logger.critical.call_args[0][0] assert not run_succeeded - def test_no_tracers_restart_file(self, m_logger): + def test_no_tracers_restart_file(self, m_logger, config): restart_timestep = 2160 with patch("nowcast.workers.watch_NEMO.Path.exists") as m_exists: m_exists.side_effect = (True, False, True, False) @@ -578,7 +607,7 @@ def test_no_tracers_restart_file(self, m_logger): Path("tmp_run_dir"), 2160, restart_timestep, - self.config, + config, ) expected = f"SalishSea_{restart_timestep:08d}_restart_trc.nc" assert expected in m_logger.critical.call_args[0][0] @@ -594,7 +623,9 @@ def test_no_tracers_restart_file(self, m_logger): ("forecast2", 2700, 2160), ], ) - def test_no_ocean_output_file(self, m_logger, run_type, itend, restart_timestep): + def test_no_ocean_output_file( + self, m_logger, run_type, itend, restart_timestep, config + ): with patch("nowcast.workers.watch_NEMO.Path.exists") as m_exists: m_exists.side_effect = (True, False, True, True) with patch("nowcast.workers.watch_NEMO.Path.open") as m_open: @@ -606,7 +637,7 @@ def test_no_ocean_output_file(self, m_logger, run_type, itend, restart_timestep) Path("tmp_run_dir"), itend, restart_timestep, - self.config, + config, ) assert "ocean.output" in m_logger.critical.call_args_list[1][0][0] assert not run_succeeded @@ -621,7 +652,9 @@ def test_no_ocean_output_file(self, m_logger, run_type, itend, restart_timestep) ("forecast2", 2700, 2160), ], ) - def test_error_in_ocean_output(self, m_logger, run_type, itend, restart_timestep): + def test_error_in_ocean_output( + self, m_logger, run_type, itend, restart_timestep, config + ): with patch("nowcast.workers.watch_NEMO.Path.exists") as m_exists: m_exists.side_effect = (True, False, True, True) with patch("nowcast.workers.watch_NEMO.Path.open") as m_open: @@ -634,7 +667,7 @@ def test_error_in_ocean_output(self, m_logger, run_type, itend, restart_timestep Path("tmp_run_dir"), itend, restart_timestep, - self.config, + config, ) assert not run_succeeded @@ -648,7 +681,9 @@ def test_error_in_ocean_output(self, m_logger, run_type, itend, restart_timestep ("forecast2", 2700, 2160), ], ) - def test_no_solver_stat_file(self, m_logger, run_type, itend, restart_timestep): + def test_no_solver_stat_file( + self, m_logger, run_type, itend, restart_timestep, config + ): with patch("nowcast.workers.watch_NEMO.Path.exists") as m_exists: m_exists.side_effect = (True, False, True, True) with patch("nowcast.workers.watch_NEMO.Path.open") as m_open: @@ -660,7 +695,7 @@ def test_no_solver_stat_file(self, m_logger, run_type, itend, restart_timestep): Path("tmp_run_dir"), itend, restart_timestep, - self.config, + config, ) assert "solver.stat" in m_logger.critical.call_args_list[2][0][0] assert not run_succeeded @@ -675,7 +710,9 @@ def test_no_solver_stat_file(self, m_logger, run_type, itend, restart_timestep): ("forecast2", 2700, 2160), ], ) - def test_nan_in_solver_stat(self, m_logger, run_type, itend, restart_timestep): + def test_nan_in_solver_stat( + self, m_logger, run_type, itend, restart_timestep, config + ): with patch("nowcast.workers.watch_NEMO.Path.exists") as m_exists: m_exists.side_effect = (True, False, True, True) with patch("nowcast.workers.watch_NEMO.Path.open") as m_open: @@ -691,11 +728,11 @@ def test_nan_in_solver_stat(self, m_logger, run_type, itend, restart_timestep): Path("tmp_run_dir"), itend, restart_timestep, - self.config, + config, ) assert not run_succeeded - def test_nan_in_tracer_stat(self, m_logger): + def test_nan_in_tracer_stat(self, m_logger, config): with patch("nowcast.workers.watch_NEMO.Path.exists") as m_exists: m_exists.side_effect = (True, False, True, True) with patch("nowcast.workers.watch_NEMO.Path.open") as m_open: @@ -712,11 +749,11 @@ def test_nan_in_tracer_stat(self, m_logger): Path("tmp_run_dir"), 2160, 2160, - self.config, + config, ) assert not run_succeeded - def test_no_tracer_stat_file(self, m_logger): + def test_no_tracer_stat_file(self, m_logger, config): with patch("nowcast.workers.watch_NEMO.Path.exists") as m_exists: m_exists.side_effect = (True, False, True, True) with patch("nowcast.workers.watch_NEMO.Path.open") as m_open: @@ -729,7 +766,7 @@ def test_no_tracer_stat_file(self, m_logger): Path("tmp_run_dir"), 2160, 2160, - self.config, + config, ) assert "tracer.stat" in m_logger.critical.call_args_list[3][0][0] assert not run_succeeded diff --git a/tests/workers/test_watch_NEMO_agrif.py b/tests/workers/test_watch_NEMO_agrif.py index 60398cdf..27f37eb9 100644 --- a/tests/workers/test_watch_NEMO_agrif.py +++ b/tests/workers/test_watch_NEMO_agrif.py @@ -14,6 +14,7 @@ # limitations under the License. """Unit tests for SalishSeaCast watch_NEMO_agrif worker. """ +import textwrap from pathlib import Path from types import SimpleNamespace from unittest.mock import patch, Mock @@ -26,6 +27,27 @@ from nowcast.workers import watch_NEMO_agrif +@pytest.fixture +def config(base_config): + """:py:class:`nemo_nowcast.Config` instance from YAML fragment to use as config for unit tests.""" + config_file = Path(base_config.file) + with config_file.open("at") as f: + f.write( + textwrap.dedent( + """\ + run: + enabled hosts: + orcinus: + ssh key: SalishSeaNEMO-nowcast_id_rsa, + scratch dir: scratch/nowcast-agrif, + """ + ) + ) + config_ = nemo_nowcast.Config() + config_.load(config_file) + return config_ + + @patch("nowcast.workers.watch_NEMO_agrif.NowcastWorker", spec=True) class TestMain: """Unit tests for main() function.""" @@ -126,18 +148,9 @@ def test_checklist( m_get_run_id, m_sftp, m_logger, + config, ): parsed_args = SimpleNamespace(host_name="orcinus", job_id="9305855.orca2.ibb") - config = { - "run": { - "enabled hosts": { - "orcinus": { - "ssh key": "SalishSeaNEMO-nowcast_id_rsa", - "scratch dir": "scratch/nowcast-agrif", - } - } - } - } checklist = watch_NEMO_agrif.watch_NEMO_agrif(parsed_args, config) expected = { "nowcast-agrif": {