From 9f165c870a040efbe8324c49bc1cd80b1d26098f Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 4 Dec 2024 22:16:43 +0000 Subject: [PATCH 01/34] Add requests library --- recipe/meta.json | 4 +++- recipe/meta.yaml | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/recipe/meta.json b/recipe/meta.json index 8cfa7a58f..cdccd2d82 100644 --- a/recipe/meta.json +++ b/recipe/meta.json @@ -22,6 +22,7 @@ "pytest-xdist =3.6.*", "python >=3.9,<3.13", "pyyaml =6.0.*", + "requests =2.32.*", "setuptools" ], "run": [ @@ -31,7 +32,8 @@ "jsonschema >=4.18,<4.24", "lxml =5.3.*", "python >=3.9,<3.13", - "pyyaml =6.0.*" + "pyyaml =6.0.*", + "requests =2.32.*" ] }, "version": "2.5.0" diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 9a9dfd0d9..83e95675d 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -21,6 +21,7 @@ requirements: - lxml 5.3.* - python >=3.9,<3.13 - pyyaml 6.0.* + - requests 2.32.* test: requires: - black 24.8.* From 840a0212e9720d8aada267593ea0212adde5a1ff Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Thu, 5 Dec 2024 00:32:33 +0000 Subject: [PATCH 02/34] Work on _check_destination_paths() --- src/uwtools/cli.py | 3 ++- src/uwtools/fs.py | 35 +++++++++++++++++++++++++---------- src/uwtools/tests/test_fs.py | 18 +++++++++--------- 3 files changed, 36 insertions(+), 20 deletions(-) diff --git a/src/uwtools/cli.py b/src/uwtools/cli.py index e268a2e9b..1bf358e1f 100644 --- a/src/uwtools/cli.py +++ b/src/uwtools/cli.py @@ -94,7 +94,8 @@ def main() -> None: modes = {**tools, **drivers} sys.exit(0 if modes[args[STR.mode]](args) else 1) except UWError as e: - log.error(str(e)) + for line in str(e).split("\n"): + log.error(line) sys.exit(1) diff --git a/src/uwtools/fs.py b/src/uwtools/fs.py index 46bf68244..3df81d09f 100644 --- a/src/uwtools/fs.py +++ b/src/uwtools/fs.py @@ -6,6 +6,7 @@ from abc import ABC, abstractmethod from pathlib import Path from typing import Optional, Union +from urllib.parse import urlparse from iotaa import dryrun, tasks @@ -57,20 +58,34 @@ def __init__( self._config = yaml_config.data self._set_config_block() self._validate() - self._check_paths() + self._check_destination_paths() - def _check_paths(self) -> None: + def _check_destination_paths(self) -> None: """ - Check that all paths are absolute if no target directory is specified. + Check that destination paths are valid. - :parm paths: The paths to check. - :raises: UWConfigError if no target directory is specified and a relative path is. + :raises: UWConfigError when a bad path is detected. """ - if not self._target_dir: - errmsg = "Relative path '%s' requires the target directory to be specified" - for dst in self._dst_paths: - if not Path(dst).is_absolute(): - raise UWConfigError(errmsg % dst) + msg = [ + "Path '%s' invalid when target directory is specified", + "Non-filesystem destination path '%s' not currently supported", + "Relative path '%s' requires target directory to be specified", + ] + errors = [] + report = lambda dst, i: errors.append(msg[i] % dst) + for dst in self._dst_paths: + scheme = urlparse(dst).scheme + if self._target_dir: + if scheme: + report(dst, 0) + else: + if scheme and scheme != "file": + report(dst, 1) + else: + if not Path(dst).is_absolute(): + report(dst, 2) + if errors: + raise UWConfigError("\n".join(errors)) def _set_config_block(self) -> None: """ diff --git a/src/uwtools/tests/test_fs.py b/src/uwtools/tests/test_fs.py index 6e54dd07d..4a62d95e6 100644 --- a/src/uwtools/tests/test_fs.py +++ b/src/uwtools/tests/test_fs.py @@ -49,7 +49,7 @@ def _schema(self): @mark.parametrize("source", ("dict", "file")) -def test_Copier(assets, source): +def test_fs_Copier(assets, source): dstdir, cfgdict, cfgfile = assets config = cfgdict if source == "dict" else cfgfile assert not (dstdir / "foo").exists() @@ -59,7 +59,7 @@ def test_Copier(assets, source): assert (dstdir / "subdir" / "bar").is_file() -def test_Copier_config_file_dry_run(assets): +def test_fs_Copier_config_file_dry_run(assets): dstdir, cfgdict, _ = assets assert not (dstdir / "foo").exists() assert not (dstdir / "subdir" / "bar").exists() @@ -69,7 +69,7 @@ def test_Copier_config_file_dry_run(assets): iotaa.dryrun(False) -def test_Copier_no_targetdir_abspath_pass(assets): +def test_fs_Copier_no_targetdir_abspath_pass(assets): dstdir, cfgdict, _ = assets old = cfgdict["a"]["b"] cfgdict = {str(dstdir / "foo"): old["foo"], str(dstdir / "bar"): old["subdir/bar"]} @@ -77,23 +77,23 @@ def test_Copier_no_targetdir_abspath_pass(assets): assert all(asset.ready() for asset in assets) # type: ignore -def test_Copier_no_targetdir_relpath_fail(assets): +def test_fs_Copier_no_targetdir_relpath_fail(assets): _, cfgdict, _ = assets with raises(UWConfigError) as e: fs.Copier(config=cfgdict, keys=["a", "b"]).go() - errmsg = "Relative path '%s' requires the target directory to be specified" + errmsg = "Relative path '%s' requires target directory to be specified" assert errmsg % "foo" in str(e.value) @mark.parametrize("source", ("dict", "file")) -def test_FilerStager(assets, source): +def test_fs_FilerStager(assets, source): dstdir, cfgdict, cfgfile = assets config = cfgdict if source == "dict" else cfgfile assert fs.FileStager(target_dir=dstdir, config=config, keys=["a", "b"]) @mark.parametrize("source", ("dict", "file")) -def test_Linker(assets, source): +def test_fs_Linker(assets, source): dstdir, cfgdict, cfgfile = assets config = cfgdict if source == "dict" else cfgfile assert not (dstdir / "foo").exists() @@ -104,7 +104,7 @@ def test_Linker(assets, source): @mark.parametrize("source", ("dict", "file")) -def test_Stager__config_block_fail_bad_key_path(assets, source): +def test_fs_Stager__config_block_fail_bad_key_path(assets, source): dstdir, cfgdict, cfgfile = assets config = cfgdict if source == "dict" else cfgfile with raises(UWConfigError) as e: @@ -113,7 +113,7 @@ def test_Stager__config_block_fail_bad_key_path(assets, source): @mark.parametrize("val", [None, True, False, "str", 42, 3.14, [], tuple()]) -def test_Stager__config_block_fails_bad_type(assets, val): +def test_fs_Stager__config_block_fails_bad_type(assets, val): dstdir, cfgdict, _ = assets cfgdict["a"]["b"] = val with raises(UWConfigError) as e: From e02d346a4ce18faf94be5ab8da439531d1a815f3 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Thu, 5 Dec 2024 01:29:29 +0000 Subject: [PATCH 03/34] Work on tests --- src/uwtools/fs.py | 31 ++++++++----------- src/uwtools/tests/api/test_fs.py | 2 +- src/uwtools/tests/test_fs.py | 51 +++++++++++++++++++++++++++----- 3 files changed, 57 insertions(+), 27 deletions(-) diff --git a/src/uwtools/fs.py b/src/uwtools/fs.py index 3df81d09f..76f81f9cc 100644 --- a/src/uwtools/fs.py +++ b/src/uwtools/fs.py @@ -66,26 +66,21 @@ def _check_destination_paths(self) -> None: :raises: UWConfigError when a bad path is detected. """ - msg = [ - "Path '%s' invalid when target directory is specified", - "Non-filesystem destination path '%s' not currently supported", - "Relative path '%s' requires target directory to be specified", - ] - errors = [] - report = lambda dst, i: errors.append(msg[i] % dst) for dst in self._dst_paths: scheme = urlparse(dst).scheme - if self._target_dir: - if scheme: - report(dst, 0) - else: - if scheme and scheme != "file": - report(dst, 1) - else: - if not Path(dst).is_absolute(): - report(dst, 2) - if errors: - raise UWConfigError("\n".join(errors)) + absolute = Path(dst).is_absolute() + if scheme and scheme != "file": + msg = "Non-filesystem destination path '%s' not currently supported" + raise UWConfigError(msg % dst) + if self._target_dir and scheme: + msg = "Path '%s' invalid when target directory is specified" + raise UWConfigError(msg % dst) + if self._target_dir and absolute: + msg = "When target directory is specified, path '%s' must be relative" + raise UWConfigError(msg % dst) + if not self._target_dir and not absolute: + msg = "Relative path '%s' requires target directory to be specified" + raise UWConfigError(msg % dst) def _set_config_block(self) -> None: """ diff --git a/src/uwtools/tests/api/test_fs.py b/src/uwtools/tests/api/test_fs.py index 71d48b5cb..650dc05be 100644 --- a/src/uwtools/tests/api/test_fs.py +++ b/src/uwtools/tests/api/test_fs.py @@ -17,7 +17,7 @@ def kwargs(tmp_path): f.touch() config = {"a": {"b": {str(dstdir / "f1"): str(srcfile1), str(dstdir / "f2"): str(srcfile2)}}} return { - "target_dir": dstdir, + "target_dir": None, "config": config, "cycle": dt.datetime.now(), "leadtime": dt.timedelta(hours=6), diff --git a/src/uwtools/tests/test_fs.py b/src/uwtools/tests/test_fs.py index 4a62d95e6..4795977e3 100644 --- a/src/uwtools/tests/test_fs.py +++ b/src/uwtools/tests/test_fs.py @@ -1,7 +1,10 @@ # pylint: disable=missing-class-docstring # pylint: disable=missing-function-docstring +# pylint: disable=protected-access # pylint: disable=redefined-outer-name +from unittest.mock import Mock + import iotaa import yaml from pytest import fixture, mark, raises @@ -77,14 +80,6 @@ def test_fs_Copier_no_targetdir_abspath_pass(assets): assert all(asset.ready() for asset in assets) # type: ignore -def test_fs_Copier_no_targetdir_relpath_fail(assets): - _, cfgdict, _ = assets - with raises(UWConfigError) as e: - fs.Copier(config=cfgdict, keys=["a", "b"]).go() - errmsg = "Relative path '%s' requires target directory to be specified" - assert errmsg % "foo" in str(e.value) - - @mark.parametrize("source", ("dict", "file")) def test_fs_FilerStager(assets, source): dstdir, cfgdict, cfgfile = assets @@ -103,6 +98,46 @@ def test_fs_Linker(assets, source): assert (dstdir / "subdir" / "bar").is_symlink() +def test_fs_Stager__check_destination_paths_fail_absolute_with_target_dir(): + path = "/other/path" + obj = Mock(_dst_paths=[path], _target_dir="/some/path") + with raises(UWConfigError) as e: + fs.Stager._check_destination_paths(obj) + assert str(e.value) == f"When target directory is specified, path '{path}' must be relative" + + +def test_fs_Stager__check_destination_paths_fail_bad_scheme(): + path = "s3://bucket/a/b" + obj = Mock(_dst_paths=[path], _target_dir=None) + with raises(UWConfigError) as e: + fs.Stager._check_destination_paths(obj) + assert str(e.value) == f"Non-filesystem destination path '{path}' not currently supported" + + +def test_fs_Stager__check_destination_paths_fail_need_target_dir(): + path = "relpath" + obj = Mock(_dst_paths=[path], _target_dir=None) + with raises(UWConfigError) as e: + fs.Stager._check_destination_paths(obj) + assert str(e.value) == f"Relative path '{path}' requires target directory to be specified" + + +def test_fs_Stager__check_destination_paths_fail_scheme_with_target_dir(): + path = "file://foo.com/a/b" + obj = Mock(_dst_paths=[path], _target_dir="/some/path") + with raises(UWConfigError) as e: + fs.Stager._check_destination_paths(obj) + assert str(e.value) == f"Path '{path}' invalid when target directory is specified" + + +# def test_fs_Copier_no_targetdir_relpath_fail(assets): +# _, cfgdict, _ = assets +# with raises(UWConfigError) as e: +# fs.Copier(config=cfgdict, keys=["a", "b"]).go() +# errmsg = "Relative path '%s' requires target directory to be specified" +# assert errmsg % "foo" in str(e.value) + + @mark.parametrize("source", ("dict", "file")) def test_fs_Stager__config_block_fail_bad_key_path(assets, source): dstdir, cfgdict, cfgfile = assets From e74d2439f89e9e5beceb8b48aeca2d96a4c69eb7 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Thu, 5 Dec 2024 05:45:57 +0000 Subject: [PATCH 04/34] Remove unneeded test --- src/uwtools/tests/test_fs.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/uwtools/tests/test_fs.py b/src/uwtools/tests/test_fs.py index 4795977e3..c255ac627 100644 --- a/src/uwtools/tests/test_fs.py +++ b/src/uwtools/tests/test_fs.py @@ -130,14 +130,6 @@ def test_fs_Stager__check_destination_paths_fail_scheme_with_target_dir(): assert str(e.value) == f"Path '{path}' invalid when target directory is specified" -# def test_fs_Copier_no_targetdir_relpath_fail(assets): -# _, cfgdict, _ = assets -# with raises(UWConfigError) as e: -# fs.Copier(config=cfgdict, keys=["a", "b"]).go() -# errmsg = "Relative path '%s' requires target directory to be specified" -# assert errmsg % "foo" in str(e.value) - - @mark.parametrize("source", ("dict", "file")) def test_fs_Stager__config_block_fail_bad_key_path(assets, source): dstdir, cfgdict, cfgfile = assets From d447b5d5fc1e73bb7a4be9142796189d38f160be Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Thu, 5 Dec 2024 15:27:31 +0000 Subject: [PATCH 05/34] Improve error messages --- src/uwtools/fs.py | 9 +++++++-- src/uwtools/tests/test_fs.py | 6 ++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/uwtools/fs.py b/src/uwtools/fs.py index 76f81f9cc..6f534f830 100644 --- a/src/uwtools/fs.py +++ b/src/uwtools/fs.py @@ -73,15 +73,20 @@ def _check_destination_paths(self) -> None: msg = "Non-filesystem destination path '%s' not currently supported" raise UWConfigError(msg % dst) if self._target_dir and scheme: - msg = "Path '%s' invalid when target directory is specified" + msg = "Non-filesystem path '%s' invalid when target directory is specified" raise UWConfigError(msg % dst) if self._target_dir and absolute: - msg = "When target directory is specified, path '%s' must be relative" + msg = "Path '%s' must be relative when target directory is specified" raise UWConfigError(msg % dst) if not self._target_dir and not absolute: msg = "Relative path '%s' requires target directory to be specified" raise UWConfigError(msg % dst) + def _check_target_dir(self) -> None: + """ + PM WRITEME. + """ + def _set_config_block(self) -> None: """ Navigate keys to a config block. diff --git a/src/uwtools/tests/test_fs.py b/src/uwtools/tests/test_fs.py index c255ac627..f100f6c70 100644 --- a/src/uwtools/tests/test_fs.py +++ b/src/uwtools/tests/test_fs.py @@ -103,7 +103,7 @@ def test_fs_Stager__check_destination_paths_fail_absolute_with_target_dir(): obj = Mock(_dst_paths=[path], _target_dir="/some/path") with raises(UWConfigError) as e: fs.Stager._check_destination_paths(obj) - assert str(e.value) == f"When target directory is specified, path '{path}' must be relative" + assert str(e.value) == f"Path '{path}' must be relative when target directory is specified" def test_fs_Stager__check_destination_paths_fail_bad_scheme(): @@ -127,7 +127,9 @@ def test_fs_Stager__check_destination_paths_fail_scheme_with_target_dir(): obj = Mock(_dst_paths=[path], _target_dir="/some/path") with raises(UWConfigError) as e: fs.Stager._check_destination_paths(obj) - assert str(e.value) == f"Path '{path}' invalid when target directory is specified" + assert ( + str(e.value) == f"Non-filesystem path '{path}' invalid when target directory is specified" + ) @mark.parametrize("source", ("dict", "file")) From 4c09249d67ba63d2cd059b4240de4be2ba5e95c3 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Thu, 5 Dec 2024 15:42:50 +0000 Subject: [PATCH 06/34] Parametrize tests --- src/uwtools/fs.py | 8 ++++- src/uwtools/tests/test_fs.py | 58 +++++++++++++++++++----------------- 2 files changed, 37 insertions(+), 29 deletions(-) diff --git a/src/uwtools/fs.py b/src/uwtools/fs.py index 6f534f830..c061c068b 100644 --- a/src/uwtools/fs.py +++ b/src/uwtools/fs.py @@ -58,6 +58,7 @@ def __init__( self._config = yaml_config.data self._set_config_block() self._validate() + # self._check_target_dir() self._check_destination_paths() def _check_destination_paths(self) -> None: @@ -84,8 +85,13 @@ def _check_destination_paths(self) -> None: def _check_target_dir(self) -> None: """ - PM WRITEME. + Check that target directory is valid. + + :raises: UWConfigError when a bad path is detected. """ + if self._target_dir and (scheme := urlparse(self._target_dir).scheme) and scheme != "file": + msg = "Non-filesystem path '%s' invalid as target directory" + raise UWConfigError(msg % self._target_dir) def _set_config_block(self) -> None: """ diff --git a/src/uwtools/tests/test_fs.py b/src/uwtools/tests/test_fs.py index f100f6c70..b6f982d4d 100644 --- a/src/uwtools/tests/test_fs.py +++ b/src/uwtools/tests/test_fs.py @@ -98,38 +98,40 @@ def test_fs_Linker(assets, source): assert (dstdir / "subdir" / "bar").is_symlink() -def test_fs_Stager__check_destination_paths_fail_absolute_with_target_dir(): - path = "/other/path" - obj = Mock(_dst_paths=[path], _target_dir="/some/path") +@mark.parametrize( + "path,target_dir,msg", + [ + ( + "/other/path", + "/some/path", + "Path '%s' must be relative when target directory is specified", + ), + ( + "s3://bucket/a/b", + None, + "Non-filesystem destination path '%s' not currently supported", + ), + ( + "relpath", + None, + "Relative path '%s' requires target directory to be specified", + ), + ( + "file://foo.com/a/b", + "/some/path", + "Non-filesystem path '%s' invalid when target directory is specified", + ), + ], +) +def test_fs_Stager__check_destination_paths_fail(path, target_dir, msg): + obj = Mock(_dst_paths=[path], _target_dir=target_dir) with raises(UWConfigError) as e: fs.Stager._check_destination_paths(obj) - assert str(e.value) == f"Path '{path}' must be relative when target directory is specified" + assert str(e.value) == msg % path -def test_fs_Stager__check_destination_paths_fail_bad_scheme(): - path = "s3://bucket/a/b" - obj = Mock(_dst_paths=[path], _target_dir=None) - with raises(UWConfigError) as e: - fs.Stager._check_destination_paths(obj) - assert str(e.value) == f"Non-filesystem destination path '{path}' not currently supported" - - -def test_fs_Stager__check_destination_paths_fail_need_target_dir(): - path = "relpath" - obj = Mock(_dst_paths=[path], _target_dir=None) - with raises(UWConfigError) as e: - fs.Stager._check_destination_paths(obj) - assert str(e.value) == f"Relative path '{path}' requires target directory to be specified" - - -def test_fs_Stager__check_destination_paths_fail_scheme_with_target_dir(): - path = "file://foo.com/a/b" - obj = Mock(_dst_paths=[path], _target_dir="/some/path") - with raises(UWConfigError) as e: - fs.Stager._check_destination_paths(obj) - assert ( - str(e.value) == f"Non-filesystem path '{path}' invalid when target directory is specified" - ) +def test_fs_Stager__check_target_dir_fail_bad_scheme(): + pass @mark.parametrize("source", ("dict", "file")) From 6182577f3de8281e3668e2d3028f146ad8087c22 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Thu, 5 Dec 2024 15:53:51 +0000 Subject: [PATCH 07/34] More tests --- src/uwtools/fs.py | 8 ++++++-- src/uwtools/tests/test_fs.py | 39 +++++++++++++++++++++++++++++------- 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/src/uwtools/fs.py b/src/uwtools/fs.py index c061c068b..4988640f6 100644 --- a/src/uwtools/fs.py +++ b/src/uwtools/fs.py @@ -58,7 +58,7 @@ def __init__( self._config = yaml_config.data self._set_config_block() self._validate() - # self._check_target_dir() + self._check_target_dir() self._check_destination_paths() def _check_destination_paths(self) -> None: @@ -89,7 +89,11 @@ def _check_target_dir(self) -> None: :raises: UWConfigError when a bad path is detected. """ - if self._target_dir and (scheme := urlparse(self._target_dir).scheme) and scheme != "file": + if ( + self._target_dir + and (scheme := urlparse(str(self._target_dir)).scheme) + and scheme != "file" + ): msg = "Non-filesystem path '%s' invalid as target directory" raise UWConfigError(msg % self._target_dir) diff --git a/src/uwtools/tests/test_fs.py b/src/uwtools/tests/test_fs.py index b6f982d4d..d20d2058f 100644 --- a/src/uwtools/tests/test_fs.py +++ b/src/uwtools/tests/test_fs.py @@ -99,39 +99,64 @@ def test_fs_Linker(assets, source): @mark.parametrize( - "path,target_dir,msg", + "path,target_dir,msg,fail_expected", [ ( "/other/path", "/some/path", "Path '%s' must be relative when target directory is specified", + True, ), ( "s3://bucket/a/b", None, "Non-filesystem destination path '%s' not currently supported", + True, ), ( "relpath", None, "Relative path '%s' requires target directory to be specified", + True, ), ( "file://foo.com/a/b", "/some/path", "Non-filesystem path '%s' invalid when target directory is specified", + True, + ), + ( + "other/path", + "/some/path", + None, + False, + ), + ( + "other/path", + "file:///some/path", + None, + False, ), ], ) -def test_fs_Stager__check_destination_paths_fail(path, target_dir, msg): +def test_fs_Stager__check_destination_paths_fail(path, target_dir, msg, fail_expected): obj = Mock(_dst_paths=[path], _target_dir=target_dir) - with raises(UWConfigError) as e: - fs.Stager._check_destination_paths(obj) - assert str(e.value) == msg % path + if fail_expected: + with raises(UWConfigError) as e: + fs.Stager._check_destination_paths(obj) + assert str(e.value) == msg % path -def test_fs_Stager__check_target_dir_fail_bad_scheme(): - pass +@mark.parametrize( + "path,fail_expected", + [("s3://bucket/a/b", True), ("/some/path", False), ("file:///some/path", False)], +) +def test_fs_Stager__check_target_dir_fail_bad_scheme(path, fail_expected): + obj = Mock(_target_dir="s3://bucket/a/b") + if fail_expected: + with raises(UWConfigError) as e: + fs.Stager._check_target_dir(obj) + assert str(e.value) == "Non-filesystem path '%s' invalid as target directory" % path @mark.parametrize("source", ("dict", "file")) From a9f7daa3c95f0ad14c4ad01e45e0c247c5789db9 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Thu, 5 Dec 2024 15:55:53 +0000 Subject: [PATCH 08/34] Formatting --- src/uwtools/tests/test_fs.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/src/uwtools/tests/test_fs.py b/src/uwtools/tests/test_fs.py index d20d2058f..ddf8ffa9c 100644 --- a/src/uwtools/tests/test_fs.py +++ b/src/uwtools/tests/test_fs.py @@ -125,18 +125,8 @@ def test_fs_Linker(assets, source): "Non-filesystem path '%s' invalid when target directory is specified", True, ), - ( - "other/path", - "/some/path", - None, - False, - ), - ( - "other/path", - "file:///some/path", - None, - False, - ), + ("other/path", "/some/path", None, False), + ("other/path", "file:///some/path", None, False), ], ) def test_fs_Stager__check_destination_paths_fail(path, target_dir, msg, fail_expected): From 314c871ce51a5934f9d7ec114e24b38eb6f0af54 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Thu, 5 Dec 2024 17:05:46 +0000 Subject: [PATCH 09/34] Generalize Copier --- src/uwtools/fs.py | 21 +++++++++++++++++++-- src/uwtools/tests/test_fs.py | 15 ++++++++++++++- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/src/uwtools/fs.py b/src/uwtools/fs.py index 4988640f6..11650db6a 100644 --- a/src/uwtools/fs.py +++ b/src/uwtools/fs.py @@ -164,14 +164,31 @@ class Copier(FileStager): Stage files by copying. """ + # >>> urlparse("path/to/file") + # ParseResult(scheme='', netloc='', path='path/to/file', params='', query='', fragment='') + # >>> urlparse("/path/to/file") + # ParseResult(scheme='', netloc='', path='/path/to/file', params='', query='', fragment='') + # >>> urlparse("file:///path/to/file") + # ParseResult(scheme='file', netloc='', path='/path/to/file', params='', query='', fragment='') + # >>> urlparse("https://foo.com/path/to/file") + # ParseResult(scheme='https', netloc='foo.com', + # path='/path/to/file', params='', query='', fragment='') + @tasks def go(self): """ Copy files. """ - dst = lambda k: Path(self._target_dir / k if self._target_dir else k) yield "File copies" - yield [filecopy(src=Path(v), dst=dst(k)) for k, v in self._config.items()] + reqs = [] + for dst, src in self._config.items(): + dst = Path((self._target_dir or "")) / dst + info = {x: urlparse(str(x)) for x in (dst, src)} + dst, src = [ + Path(info[x].path) if info[x].scheme == "file" else Path(x) for x in (dst, src) + ] + reqs.append(filecopy(src=src, dst=dst)) + yield reqs class Linker(FileStager): diff --git a/src/uwtools/tests/test_fs.py b/src/uwtools/tests/test_fs.py index ddf8ffa9c..b92dfe38b 100644 --- a/src/uwtools/tests/test_fs.py +++ b/src/uwtools/tests/test_fs.py @@ -3,7 +3,8 @@ # pylint: disable=protected-access # pylint: disable=redefined-outer-name -from unittest.mock import Mock +from pathlib import Path +from unittest.mock import Mock, patch import iotaa import yaml @@ -62,6 +63,18 @@ def test_fs_Copier(assets, source): assert (dstdir / "subdir" / "bar").is_file() +@mark.parametrize( + "dst,target_dir", + [("/dst/file", None), ("file:///dst/file", None), ("file", "/dst"), ("file", "file:///dst")], +) +def test_fs_Copier_scheme_file(dst, target_dir): + src = "/src/file" + obj = Mock(_config={dst: src}, _target_dir=target_dir) + with patch.object(fs, "filecopy") as filecopy: + fs.Copier.go(obj) + filecopy.assert_called_once_with(src=Path(src), dst=Path("/dst/file")) + + def test_fs_Copier_config_file_dry_run(assets): dstdir, cfgdict, _ = assets assert not (dstdir / "foo").exists() From fa5a0c7db529db1c1cd3b56403b1ec94ebe105db Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Thu, 5 Dec 2024 17:23:33 +0000 Subject: [PATCH 10/34] Generalize Copier --- src/uwtools/fs.py | 9 +++++---- src/uwtools/tests/test_fs.py | 19 +++++++++++++++---- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/src/uwtools/fs.py b/src/uwtools/fs.py index 11650db6a..bf262793a 100644 --- a/src/uwtools/fs.py +++ b/src/uwtools/fs.py @@ -184,10 +184,11 @@ def go(self): for dst, src in self._config.items(): dst = Path((self._target_dir or "")) / dst info = {x: urlparse(str(x)) for x in (dst, src)} - dst, src = [ - Path(info[x].path) if info[x].scheme == "file" else Path(x) for x in (dst, src) - ] - reqs.append(filecopy(src=src, dst=dst)) + dst, src = [info[x].path if info[x].scheme == "file" else x for x in (dst, src)] + if (scheme := info[src].scheme) in ("", "file"): + reqs.append(filecopy(src=Path(src), dst=Path(dst))) + else: + raise UWConfigError(f"Support for scheme '{scheme}' not implemented") yield reqs diff --git a/src/uwtools/tests/test_fs.py b/src/uwtools/tests/test_fs.py index b92dfe38b..727631cb1 100644 --- a/src/uwtools/tests/test_fs.py +++ b/src/uwtools/tests/test_fs.py @@ -64,17 +64,28 @@ def test_fs_Copier(assets, source): @mark.parametrize( - "dst,target_dir", - [("/dst/file", None), ("file:///dst/file", None), ("file", "/dst"), ("file", "file:///dst")], + "dst,src,target_dir", + [ + ("/dst/file", "/src/file", None), + ("file:///dst/file", "/src/file", None), + ("file", "/src/file", "/dst"), + ("file", "/src/file", "file:///dst"), + ], ) -def test_fs_Copier_scheme_file(dst, target_dir): - src = "/src/file" +def test_fs_Copier_schemes(dst, src, target_dir): obj = Mock(_config={dst: src}, _target_dir=target_dir) with patch.object(fs, "filecopy") as filecopy: fs.Copier.go(obj) filecopy.assert_called_once_with(src=Path(src), dst=Path("/dst/file")) +def test_fs_Copier_schemes_fail(): + obj = Mock(_config={"/dst/file": "foo://x/y/z"}, _target_dir=None) + with raises(UWConfigError) as e: + fs.Copier.go(obj) + assert str(e.value) == "Support for scheme 'foo' not implemented" + + def test_fs_Copier_config_file_dry_run(assets): dstdir, cfgdict, _ = assets assert not (dstdir / "foo").exists() From 9d6654bdab806a136c460456b0bd917a2947358a Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Thu, 5 Dec 2024 17:26:57 +0000 Subject: [PATCH 11/34] Generalize Copier --- src/uwtools/fs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/uwtools/fs.py b/src/uwtools/fs.py index bf262793a..0f4ca4832 100644 --- a/src/uwtools/fs.py +++ b/src/uwtools/fs.py @@ -185,7 +185,8 @@ def go(self): dst = Path((self._target_dir or "")) / dst info = {x: urlparse(str(x)) for x in (dst, src)} dst, src = [info[x].path if info[x].scheme == "file" else x for x in (dst, src)] - if (scheme := info[src].scheme) in ("", "file"): + scheme = info[src].scheme + if scheme in ("", "file"): reqs.append(filecopy(src=Path(src), dst=Path(dst))) else: raise UWConfigError(f"Support for scheme '{scheme}' not implemented") From 4ccb95d0db97cbb4c534bd8b1bb8b28b104651b6 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Thu, 5 Dec 2024 18:58:53 +0000 Subject: [PATCH 12/34] Push scheme handling into filecopy() --- src/uwtools/fs.py | 16 +------------- src/uwtools/tests/test_fs.py | 32 ++++----------------------- src/uwtools/tests/utils/test_tasks.py | 30 +++++++++++++++++++++++++ src/uwtools/utils/tasks.py | 29 +++++++++++++++++++----- 4 files changed, 59 insertions(+), 48 deletions(-) diff --git a/src/uwtools/fs.py b/src/uwtools/fs.py index 0f4ca4832..c2e11f5b2 100644 --- a/src/uwtools/fs.py +++ b/src/uwtools/fs.py @@ -164,16 +164,6 @@ class Copier(FileStager): Stage files by copying. """ - # >>> urlparse("path/to/file") - # ParseResult(scheme='', netloc='', path='path/to/file', params='', query='', fragment='') - # >>> urlparse("/path/to/file") - # ParseResult(scheme='', netloc='', path='/path/to/file', params='', query='', fragment='') - # >>> urlparse("file:///path/to/file") - # ParseResult(scheme='file', netloc='', path='/path/to/file', params='', query='', fragment='') - # >>> urlparse("https://foo.com/path/to/file") - # ParseResult(scheme='https', netloc='foo.com', - # path='/path/to/file', params='', query='', fragment='') - @tasks def go(self): """ @@ -185,11 +175,7 @@ def go(self): dst = Path((self._target_dir or "")) / dst info = {x: urlparse(str(x)) for x in (dst, src)} dst, src = [info[x].path if info[x].scheme == "file" else x for x in (dst, src)] - scheme = info[src].scheme - if scheme in ("", "file"): - reqs.append(filecopy(src=Path(src), dst=Path(dst))) - else: - raise UWConfigError(f"Support for scheme '{scheme}' not implemented") + reqs.append(filecopy(src=src, dst=dst)) yield reqs diff --git a/src/uwtools/tests/test_fs.py b/src/uwtools/tests/test_fs.py index 727631cb1..4246fb463 100644 --- a/src/uwtools/tests/test_fs.py +++ b/src/uwtools/tests/test_fs.py @@ -3,8 +3,7 @@ # pylint: disable=protected-access # pylint: disable=redefined-outer-name -from pathlib import Path -from unittest.mock import Mock, patch +from unittest.mock import Mock import iotaa import yaml @@ -63,29 +62,6 @@ def test_fs_Copier(assets, source): assert (dstdir / "subdir" / "bar").is_file() -@mark.parametrize( - "dst,src,target_dir", - [ - ("/dst/file", "/src/file", None), - ("file:///dst/file", "/src/file", None), - ("file", "/src/file", "/dst"), - ("file", "/src/file", "file:///dst"), - ], -) -def test_fs_Copier_schemes(dst, src, target_dir): - obj = Mock(_config={dst: src}, _target_dir=target_dir) - with patch.object(fs, "filecopy") as filecopy: - fs.Copier.go(obj) - filecopy.assert_called_once_with(src=Path(src), dst=Path("/dst/file")) - - -def test_fs_Copier_schemes_fail(): - obj = Mock(_config={"/dst/file": "foo://x/y/z"}, _target_dir=None) - with raises(UWConfigError) as e: - fs.Copier.go(obj) - assert str(e.value) == "Support for scheme 'foo' not implemented" - - def test_fs_Copier_config_file_dry_run(assets): dstdir, cfgdict, _ = assets assert not (dstdir / "foo").exists() @@ -132,7 +108,7 @@ def test_fs_Linker(assets, source): True, ), ( - "s3://bucket/a/b", + "foo://bucket/a/b", None, "Non-filesystem destination path '%s' not currently supported", True, @@ -163,10 +139,10 @@ def test_fs_Stager__check_destination_paths_fail(path, target_dir, msg, fail_exp @mark.parametrize( "path,fail_expected", - [("s3://bucket/a/b", True), ("/some/path", False), ("file:///some/path", False)], + [("foo://bucket/a/b", True), ("/some/path", False), ("file:///some/path", False)], ) def test_fs_Stager__check_target_dir_fail_bad_scheme(path, fail_expected): - obj = Mock(_target_dir="s3://bucket/a/b") + obj = Mock(_target_dir="foo://bucket/a/b") if fail_expected: with raises(UWConfigError) as e: fs.Stager._check_target_dir(obj) diff --git a/src/uwtools/tests/utils/test_tasks.py b/src/uwtools/tests/utils/test_tasks.py index aa606ed09..7f908067e 100644 --- a/src/uwtools/tests/utils/test_tasks.py +++ b/src/uwtools/tests/utils/test_tasks.py @@ -2,13 +2,24 @@ import os import stat +from pathlib import Path from unittest.mock import patch +from iotaa import asset, external +from pytest import mark, raises + +from uwtools.exceptions import UWConfigError from uwtools.utils import tasks # Helpers +@external +def exists(x): + yield x + yield asset(x, lambda: True) + + def ready(taskval): return taskval.ready() @@ -89,6 +100,25 @@ def test_tasks_filecopy_directory_hierarchy(tmp_path): assert dst.is_file() +@mark.parametrize( + "src,ok", + [("/src/file", True), ("file:///src/file", True), ("foo://bucket/a/b", False)], +) +def test_tasks_filecopy_source_local(src, ok): + dst = "/dst/file" + if ok: + with patch.object(tasks, "file", exists): + with patch.object(tasks, "copy") as copy: + with patch.object(tasks.Path, "mkdir") as mkdir: + tasks.filecopy(src=src, dst=dst) + mkdir.assert_called_once_with(parents=True, exist_ok=True) + copy.assert_called_once_with(Path(src), Path(dst)) + else: + with raises(UWConfigError) as e: + tasks.filecopy(src=src, dst=dst) + assert str(e.value) == "Support for scheme 'foo' not implemented" + + def test_tasks_symlink_simple(tmp_path): target = tmp_path / "target" link = tmp_path / "link" diff --git a/src/uwtools/utils/tasks.py b/src/uwtools/utils/tasks.py index 7659fe2be..4eddae0fc 100644 --- a/src/uwtools/utils/tasks.py +++ b/src/uwtools/utils/tasks.py @@ -6,9 +6,12 @@ from pathlib import Path from shutil import copy, which from typing import Union +from urllib.parse import urlparse from iotaa import asset, external, task +from uwtools.exceptions import UWConfigError + @task def directory(path: Path): @@ -58,8 +61,19 @@ def file(path: Path, context: str = ""): yield asset(path, path.is_file) +# >>> urlparse("path/to/file") +# ParseResult(scheme='', netloc='', path='path/to/file', params='', query='', fragment='') +# >>> urlparse("/path/to/file") +# ParseResult(scheme='', netloc='', path='/path/to/file', params='', query='', fragment='') +# >>> urlparse("file:///path/to/file") +# ParseResult(scheme='file', netloc='', path='/path/to/file', params='', query='', fragment='') +# >>> urlparse("https://foo.com/path/to/file") +# ParseResult(scheme='https', netloc='foo.com', path='/path/to/file', params='', query='', +# fragment='') + + @task -def filecopy(src: Path, dst: Path): +def filecopy(src: Union[Path, str], dst: Union[Path, str]): """ A copy of an existing file. @@ -67,10 +81,15 @@ def filecopy(src: Path, dst: Path): :param dst: Path to the destination file to create. """ yield "Copy %s -> %s" % (src, dst) - yield asset(dst, dst.is_file) - yield file(src) - dst.parent.mkdir(parents=True, exist_ok=True) - copy(src, dst) + yield asset(Path(dst), Path(dst).is_file) + scheme = urlparse(str(src)).scheme + if scheme in ("", "file"): + src, dst = map(Path, [src, dst]) + yield file(src) + dst.parent.mkdir(parents=True, exist_ok=True) + copy(src, dst) + else: + raise UWConfigError(f"Support for scheme '{scheme}' not implemented") @task From 9b573fa2365d8fc14423ee1e483fcc866d6426af Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Thu, 5 Dec 2024 19:50:46 +0000 Subject: [PATCH 13/34] Work on existing() HTTP support --- src/uwtools/utils/tasks.py | 58 +++++++++++++++++++++++++------------- 1 file changed, 39 insertions(+), 19 deletions(-) diff --git a/src/uwtools/utils/tasks.py b/src/uwtools/utils/tasks.py index 4eddae0fc..9557973dc 100644 --- a/src/uwtools/utils/tasks.py +++ b/src/uwtools/utils/tasks.py @@ -5,13 +5,17 @@ import os from pathlib import Path from shutil import copy, which -from typing import Union +from types import SimpleNamespace as ns +from typing import NoReturn, Union from urllib.parse import urlparse +import requests from iotaa import asset, external, task from uwtools.exceptions import UWConfigError +SCHEMES = ns(http=("http", "https"), local=("", "file")) + @task def directory(path: Path): @@ -38,14 +42,24 @@ def executable(program: Union[Path, str]): @external -def existing(path: Path): +def existing(path: Union[Path, str]): """ - An existing filesystem item (file, directory, or symlink). + An existing file, directory, symlink, or remote object. :param path: Path to the item. + :raises: UWConfigError for unsupported URL schemes. """ - yield "Filesystem item %s" % path - yield asset(path, path.exists) + scheme = urlparse(str(path)).scheme + if scheme in SCHEMES.local: + path = Path(path) + yield "Filesystem item %s" % path + yield asset(path, path.exists) + elif scheme in SCHEMES.http: + path = str(path) + yield "Remote item %s" % path + yield asset(path, lambda: requests.head(path, timeout=5).status_code == 200) + else: + _bad_scheme(scheme) @external @@ -61,17 +75,6 @@ def file(path: Path, context: str = ""): yield asset(path, path.is_file) -# >>> urlparse("path/to/file") -# ParseResult(scheme='', netloc='', path='path/to/file', params='', query='', fragment='') -# >>> urlparse("/path/to/file") -# ParseResult(scheme='', netloc='', path='/path/to/file', params='', query='', fragment='') -# >>> urlparse("file:///path/to/file") -# ParseResult(scheme='file', netloc='', path='/path/to/file', params='', query='', fragment='') -# >>> urlparse("https://foo.com/path/to/file") -# ParseResult(scheme='https', netloc='foo.com', path='/path/to/file', params='', query='', -# fragment='') - - @task def filecopy(src: Union[Path, str], dst: Union[Path, str]): """ @@ -79,17 +82,21 @@ def filecopy(src: Union[Path, str], dst: Union[Path, str]): :param src: Path to the source file. :param dst: Path to the destination file to create. + :raises: UWConfigError for unsupported URL schemes. """ yield "Copy %s -> %s" % (src, dst) yield asset(Path(dst), Path(dst).is_file) + dst = Path(dst) # currently no support for remote destinations scheme = urlparse(str(src)).scheme - if scheme in ("", "file"): - src, dst = map(Path, [src, dst]) + if scheme in SCHEMES.local: + src = Path(src) yield file(src) dst.parent.mkdir(parents=True, exist_ok=True) copy(src, dst) + elif scheme in SCHEMES.http: + pass else: - raise UWConfigError(f"Support for scheme '{scheme}' not implemented") + _bad_scheme(scheme) @task @@ -108,3 +115,16 @@ def symlink(target: Path, linkname: Path): src=target if target.is_absolute() else os.path.relpath(target, linkname.parent), dst=linkname, ) + + +# Private helpers + + +def _bad_scheme(scheme: str) -> NoReturn: + """ + Fail on an unsupported URL scheme. + + :param scheme: The scheme. + :raises: UWConfigError. + """ + raise UWConfigError(f"Support for scheme '{scheme}' not implemented") From e0c211b12a22001a7452debb871f451c9a95de01 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Thu, 5 Dec 2024 19:55:54 +0000 Subject: [PATCH 14/34] Accept HTTP 301 --- src/uwtools/utils/tasks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/uwtools/utils/tasks.py b/src/uwtools/utils/tasks.py index 9557973dc..bdf29d91b 100644 --- a/src/uwtools/utils/tasks.py +++ b/src/uwtools/utils/tasks.py @@ -55,9 +55,10 @@ def existing(path: Union[Path, str]): yield "Filesystem item %s" % path yield asset(path, path.exists) elif scheme in SCHEMES.http: + okcodes = (200, 301) path = str(path) yield "Remote item %s" % path - yield asset(path, lambda: requests.head(path, timeout=5).status_code == 200) + yield asset(path, lambda: requests.head(path, timeout=3).status_code in okcodes) else: _bad_scheme(scheme) From dbf8f4f19234c7e2cad0196fa1497e96579c091f Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Thu, 5 Dec 2024 20:03:28 +0000 Subject: [PATCH 15/34] Work on existing() HTTP support --- src/uwtools/utils/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/uwtools/utils/tasks.py b/src/uwtools/utils/tasks.py index bdf29d91b..91262d452 100644 --- a/src/uwtools/utils/tasks.py +++ b/src/uwtools/utils/tasks.py @@ -55,10 +55,10 @@ def existing(path: Union[Path, str]): yield "Filesystem item %s" % path yield asset(path, path.exists) elif scheme in SCHEMES.http: - okcodes = (200, 301) path = str(path) + ready = lambda: requests.head(path, allow_redirects=True, timeout=3).status_code == 200 yield "Remote item %s" % path - yield asset(path, lambda: requests.head(path, timeout=3).status_code in okcodes) + yield asset(path, ready) else: _bad_scheme(scheme) From 2e7916f7981c748177972ec6853589775e81f6c3 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Thu, 5 Dec 2024 20:52:24 +0000 Subject: [PATCH 16/34] Improve tests --- src/uwtools/tests/utils/test_tasks.py | 28 ++++++++++++++++++--------- src/uwtools/utils/tasks.py | 5 +++-- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/src/uwtools/tests/utils/test_tasks.py b/src/uwtools/tests/utils/test_tasks.py index 7f908067e..3ace767ee 100644 --- a/src/uwtools/tests/utils/test_tasks.py +++ b/src/uwtools/tests/utils/test_tasks.py @@ -48,29 +48,39 @@ def test_tasks_executable(tmp_path): assert ready(tasks.executable(program=p)) -def test_tasks_existing_missing(tmp_path): - path = tmp_path / "x" +@mark.parametrize("prefix", ["", "file://"]) +def test_tasks_existing_local_missing(prefix, tmp_path): + base = tmp_path / "x" + path = prefix + str(base) if prefix else base assert not ready(tasks.existing(path=path)) -def test_tasks_existing_present_directory(tmp_path): +def test_tasks_existing_local_present_directory(tmp_path): path = tmp_path / "directory" path.mkdir() assert ready(tasks.existing(path=path)) -def test_tasks_existing_present_file(tmp_path): - path = tmp_path / "file" - path.touch() +@mark.parametrize("prefix", ["", "file://"]) +def test_tasks_existing_local_present_file(prefix, tmp_path): + base = tmp_path / "file" + base.touch() + path = prefix + str(base) if prefix else base assert ready(tasks.existing(path=path)) -def test_tasks_existing_present_symlink(tmp_path): - path = tmp_path / "symlink" - path.symlink_to(os.devnull) +@mark.parametrize("prefix", ["", "file://"]) +def test_tasks_existing_local_present_symlink(prefix, tmp_path): + base = tmp_path / "symlink" + base.symlink_to(os.devnull) + path = prefix + str(base) if prefix else base assert ready(tasks.existing(path=path)) +def test_tasks_existing_remote(): + pass # PM FIXME + + def test_tasks_file_missing(tmp_path): path = tmp_path / "file" assert not ready(tasks.file(path=path)) diff --git a/src/uwtools/utils/tasks.py b/src/uwtools/utils/tasks.py index 91262d452..e54c9f16c 100644 --- a/src/uwtools/utils/tasks.py +++ b/src/uwtools/utils/tasks.py @@ -49,9 +49,10 @@ def existing(path: Union[Path, str]): :param path: Path to the item. :raises: UWConfigError for unsupported URL schemes. """ - scheme = urlparse(str(path)).scheme + info = urlparse(str(path)) + scheme = info.scheme if scheme in SCHEMES.local: - path = Path(path) + path = Path(info.path if scheme == "file" else path) yield "Filesystem item %s" % path yield asset(path, path.exists) elif scheme in SCHEMES.http: From ac91f61c865e764347948924556024838ab57a4f Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Thu, 5 Dec 2024 21:14:05 +0000 Subject: [PATCH 17/34] Improve tests --- src/uwtools/tests/utils/test_tasks.py | 37 ++++++++++++++++++++------- src/uwtools/utils/tasks.py | 2 +- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/src/uwtools/tests/utils/test_tasks.py b/src/uwtools/tests/utils/test_tasks.py index 3ace767ee..3d57d39ee 100644 --- a/src/uwtools/tests/utils/test_tasks.py +++ b/src/uwtools/tests/utils/test_tasks.py @@ -1,14 +1,16 @@ # pylint: disable=missing-function-docstring +import logging import os import stat from pathlib import Path -from unittest.mock import patch +from unittest.mock import Mock, patch from iotaa import asset, external from pytest import mark, raises from uwtools.exceptions import UWConfigError +from uwtools.tests.support import logged from uwtools.utils import tasks # Helpers @@ -49,36 +51,53 @@ def test_tasks_executable(tmp_path): @mark.parametrize("prefix", ["", "file://"]) -def test_tasks_existing_local_missing(prefix, tmp_path): +def test_tasks_existing_local_missing(caplog, prefix, tmp_path): + logging.getLogger().setLevel(logging.INFO) base = tmp_path / "x" path = prefix + str(base) if prefix else base assert not ready(tasks.existing(path=path)) + assert logged(caplog, "Filesystem item %s: State: Not Ready (external asset)" % base) -def test_tasks_existing_local_present_directory(tmp_path): +def test_tasks_existing_local_present_directory(caplog, tmp_path): + logging.getLogger().setLevel(logging.INFO) path = tmp_path / "directory" path.mkdir() assert ready(tasks.existing(path=path)) + assert logged(caplog, "Filesystem item %s: State: Ready" % path) @mark.parametrize("prefix", ["", "file://"]) -def test_tasks_existing_local_present_file(prefix, tmp_path): +def test_tasks_existing_local_present_file(caplog, prefix, tmp_path): + logging.getLogger().setLevel(logging.INFO) base = tmp_path / "file" base.touch() path = prefix + str(base) if prefix else base assert ready(tasks.existing(path=path)) + assert logged(caplog, "Filesystem item %s: State: Ready" % base) @mark.parametrize("prefix", ["", "file://"]) -def test_tasks_existing_local_present_symlink(prefix, tmp_path): +def test_tasks_existing_local_present_symlink(caplog, prefix, tmp_path): + logging.getLogger().setLevel(logging.INFO) base = tmp_path / "symlink" base.symlink_to(os.devnull) path = prefix + str(base) if prefix else base assert ready(tasks.existing(path=path)) - - -def test_tasks_existing_remote(): - pass # PM FIXME + assert logged(caplog, "Filesystem item %s: State: Ready" % base) + + +@mark.parametrize("scheme", ["http", "https"]) +@mark.parametrize("code,expected", [(200, True), (404, False)]) +def test_tasks_existing_remote(caplog, code, expected, scheme): + logging.getLogger().setLevel(logging.INFO) + path = f"{scheme}://foo.com/obj" + with patch.object(tasks.requests, "head", return_value=Mock(status_code=code)) as head: + state = ready(tasks.existing(path=path)) + assert state is expected + head.assert_called_with(path, allow_redirects=True, timeout=3) + msg = "Remote object %s: State: %s" % (path, "Ready" if state else "Not Ready (external asset)") + assert logged(caplog, msg) def test_tasks_file_missing(tmp_path): diff --git a/src/uwtools/utils/tasks.py b/src/uwtools/utils/tasks.py index e54c9f16c..57dc92001 100644 --- a/src/uwtools/utils/tasks.py +++ b/src/uwtools/utils/tasks.py @@ -58,7 +58,7 @@ def existing(path: Union[Path, str]): elif scheme in SCHEMES.http: path = str(path) ready = lambda: requests.head(path, allow_redirects=True, timeout=3).status_code == 200 - yield "Remote item %s" % path + yield "Remote object %s" % path yield asset(path, ready) else: _bad_scheme(scheme) From adf74229346184ad8cbea3730ae923b5de46460c Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Thu, 5 Dec 2024 21:18:10 +0000 Subject: [PATCH 18/34] Add test --- src/uwtools/tests/utils/test_tasks.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/uwtools/tests/utils/test_tasks.py b/src/uwtools/tests/utils/test_tasks.py index 3d57d39ee..72bc721dd 100644 --- a/src/uwtools/tests/utils/test_tasks.py +++ b/src/uwtools/tests/utils/test_tasks.py @@ -100,6 +100,13 @@ def test_tasks_existing_remote(caplog, code, expected, scheme): assert logged(caplog, msg) +def test_tasks_existing_bad_scheme(): + path = "foo://bucket/a/b" + with raises(UWConfigError) as e: + tasks.existing(path=path) + assert str(e.value) == "Support for scheme 'foo' not implemented" + + def test_tasks_file_missing(tmp_path): path = tmp_path / "file" assert not ready(tasks.file(path=path)) From 7a152473598955e96dd08ac235a629c85f6eb1b8 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Thu, 5 Dec 2024 21:28:07 +0000 Subject: [PATCH 19/34] Work on HTTP GET --- src/uwtools/utils/tasks.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/uwtools/utils/tasks.py b/src/uwtools/utils/tasks.py index 57dc92001..340761786 100644 --- a/src/uwtools/utils/tasks.py +++ b/src/uwtools/utils/tasks.py @@ -13,6 +13,8 @@ from iotaa import asset, external, task from uwtools.exceptions import UWConfigError +from uwtools.logging import log +from uwtools.utils.file import writable SCHEMES = ns(http=("http", "https"), local=("", "file")) @@ -96,7 +98,14 @@ def filecopy(src: Union[Path, str], dst: Union[Path, str]): dst.parent.mkdir(parents=True, exist_ok=True) copy(src, dst) elif scheme in SCHEMES.http: - pass + src = str(src) + yield existing(src) + response = requests.get(src, allow_redirects=True, timeout=3) + if (code := response.status_code) == 200: + with writable(dst) as f: + f.write(response.content) + else: + log.error("Could not get '%s', HTTP status was: %s", src, code) else: _bad_scheme(scheme) From c02c2c9528c54f8cfbd229549251401c0815e224 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Thu, 5 Dec 2024 21:35:31 +0000 Subject: [PATCH 20/34] Work on HTTP GET --- src/uwtools/utils/tasks.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/uwtools/utils/tasks.py b/src/uwtools/utils/tasks.py index 340761786..c01735474 100644 --- a/src/uwtools/utils/tasks.py +++ b/src/uwtools/utils/tasks.py @@ -14,7 +14,6 @@ from uwtools.exceptions import UWConfigError from uwtools.logging import log -from uwtools.utils.file import writable SCHEMES = ns(http=("http", "https"), local=("", "file")) @@ -102,7 +101,7 @@ def filecopy(src: Union[Path, str], dst: Union[Path, str]): yield existing(src) response = requests.get(src, allow_redirects=True, timeout=3) if (code := response.status_code) == 200: - with writable(dst) as f: + with open(dst, "wb") as f: f.write(response.content) else: log.error("Could not get '%s', HTTP status was: %s", src, code) From 9877c3c14099a5e88d5afe51ea68524c43e497a5 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Thu, 5 Dec 2024 22:53:59 +0000 Subject: [PATCH 21/34] Work on HTTP GET tests --- src/uwtools/tests/utils/test_tasks.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/uwtools/tests/utils/test_tasks.py b/src/uwtools/tests/utils/test_tasks.py index 72bc721dd..f446c51ba 100644 --- a/src/uwtools/tests/utils/test_tasks.py +++ b/src/uwtools/tests/utils/test_tasks.py @@ -136,6 +136,20 @@ def test_tasks_filecopy_directory_hierarchy(tmp_path): assert dst.is_file() +def test_tasks_filecopy_source_http(tmp_path): + src = "http://foo.com/obj" + dst = tmp_path / "a-file" + assert not dst.is_file() + with patch.object(tasks, "existing", exists): + with patch.object(tasks, "requests") as requests: + response = requests.get() + response.status_code = 200 + response.content = "data".encode("utf-8") + tasks.filecopy(src=src, dst=dst) + requests.get.assert_called_with(src, allow_redirects=True, timeout=3) + assert dst.is_file() + + @mark.parametrize( "src,ok", [("/src/file", True), ("file:///src/file", True), ("foo://bucket/a/b", False)], From 4ba3d9927b3e0248f8c0744346308960cb116254 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 7 Dec 2024 00:24:13 +0000 Subject: [PATCH 22/34] Use uwtools log object in tests --- src/uwtools/tests/utils/test_tasks.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/uwtools/tests/utils/test_tasks.py b/src/uwtools/tests/utils/test_tasks.py index f446c51ba..0a4465a3e 100644 --- a/src/uwtools/tests/utils/test_tasks.py +++ b/src/uwtools/tests/utils/test_tasks.py @@ -10,6 +10,7 @@ from pytest import mark, raises from uwtools.exceptions import UWConfigError +from uwtools.logging import log from uwtools.tests.support import logged from uwtools.utils import tasks @@ -52,7 +53,7 @@ def test_tasks_executable(tmp_path): @mark.parametrize("prefix", ["", "file://"]) def test_tasks_existing_local_missing(caplog, prefix, tmp_path): - logging.getLogger().setLevel(logging.INFO) + log.setLevel(logging.INFO) base = tmp_path / "x" path = prefix + str(base) if prefix else base assert not ready(tasks.existing(path=path)) @@ -60,7 +61,7 @@ def test_tasks_existing_local_missing(caplog, prefix, tmp_path): def test_tasks_existing_local_present_directory(caplog, tmp_path): - logging.getLogger().setLevel(logging.INFO) + log.setLevel(logging.INFO) path = tmp_path / "directory" path.mkdir() assert ready(tasks.existing(path=path)) @@ -69,7 +70,7 @@ def test_tasks_existing_local_present_directory(caplog, tmp_path): @mark.parametrize("prefix", ["", "file://"]) def test_tasks_existing_local_present_file(caplog, prefix, tmp_path): - logging.getLogger().setLevel(logging.INFO) + log.setLevel(logging.INFO) base = tmp_path / "file" base.touch() path = prefix + str(base) if prefix else base @@ -79,7 +80,7 @@ def test_tasks_existing_local_present_file(caplog, prefix, tmp_path): @mark.parametrize("prefix", ["", "file://"]) def test_tasks_existing_local_present_symlink(caplog, prefix, tmp_path): - logging.getLogger().setLevel(logging.INFO) + log.setLevel(logging.INFO) base = tmp_path / "symlink" base.symlink_to(os.devnull) path = prefix + str(base) if prefix else base @@ -90,7 +91,7 @@ def test_tasks_existing_local_present_symlink(caplog, prefix, tmp_path): @mark.parametrize("scheme", ["http", "https"]) @mark.parametrize("code,expected", [(200, True), (404, False)]) def test_tasks_existing_remote(caplog, code, expected, scheme): - logging.getLogger().setLevel(logging.INFO) + log.setLevel(logging.INFO) path = f"{scheme}://foo.com/obj" with patch.object(tasks.requests, "head", return_value=Mock(status_code=code)) as head: state = ready(tasks.existing(path=path)) @@ -136,18 +137,20 @@ def test_tasks_filecopy_directory_hierarchy(tmp_path): assert dst.is_file() -def test_tasks_filecopy_source_http(tmp_path): - src = "http://foo.com/obj" +@mark.parametrize("code,expected", [(200, True), (404, False)]) +@mark.parametrize("src", ["http://foo.com/obj", "https://foo.com/obj"]) +def test_tasks_filecopy_source_http(code, expected, src, tmp_path): + log.setLevel(logging.INFO) dst = tmp_path / "a-file" assert not dst.is_file() with patch.object(tasks, "existing", exists): with patch.object(tasks, "requests") as requests: response = requests.get() - response.status_code = 200 + response.status_code = code response.content = "data".encode("utf-8") tasks.filecopy(src=src, dst=dst) requests.get.assert_called_with(src, allow_redirects=True, timeout=3) - assert dst.is_file() + assert dst.is_file() is expected @mark.parametrize( From 46c34ab55cc38ee0051c90104ecc75fe2342026e Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 7 Dec 2024 01:07:21 +0000 Subject: [PATCH 23/34] Always make directory for dst file --- src/uwtools/tests/utils/test_tasks.py | 20 ++++++++++---------- src/uwtools/utils/tasks.py | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/uwtools/tests/utils/test_tasks.py b/src/uwtools/tests/utils/test_tasks.py index 0a4465a3e..996de3cf9 100644 --- a/src/uwtools/tests/utils/test_tasks.py +++ b/src/uwtools/tests/utils/test_tasks.py @@ -159,17 +159,17 @@ def test_tasks_filecopy_source_http(code, expected, src, tmp_path): ) def test_tasks_filecopy_source_local(src, ok): dst = "/dst/file" - if ok: - with patch.object(tasks, "file", exists): - with patch.object(tasks, "copy") as copy: - with patch.object(tasks.Path, "mkdir") as mkdir: + with patch.object(tasks.Path, "mkdir") as mkdir: + if ok: + with patch.object(tasks, "file", exists): + with patch.object(tasks, "copy") as copy: tasks.filecopy(src=src, dst=dst) - mkdir.assert_called_once_with(parents=True, exist_ok=True) - copy.assert_called_once_with(Path(src), Path(dst)) - else: - with raises(UWConfigError) as e: - tasks.filecopy(src=src, dst=dst) - assert str(e.value) == "Support for scheme 'foo' not implemented" + copy.assert_called_once_with(Path(src), Path(dst)) + else: + with raises(UWConfigError) as e: + tasks.filecopy(src=src, dst=dst) + assert str(e.value) == "Support for scheme 'foo' not implemented" + mkdir.assert_called_once_with(parents=True, exist_ok=True) def test_tasks_symlink_simple(tmp_path): diff --git a/src/uwtools/utils/tasks.py b/src/uwtools/utils/tasks.py index c01735474..1bf21f514 100644 --- a/src/uwtools/utils/tasks.py +++ b/src/uwtools/utils/tasks.py @@ -90,11 +90,11 @@ def filecopy(src: Union[Path, str], dst: Union[Path, str]): yield "Copy %s -> %s" % (src, dst) yield asset(Path(dst), Path(dst).is_file) dst = Path(dst) # currently no support for remote destinations + dst.parent.mkdir(parents=True, exist_ok=True) scheme = urlparse(str(src)).scheme if scheme in SCHEMES.local: src = Path(src) yield file(src) - dst.parent.mkdir(parents=True, exist_ok=True) copy(src, dst) elif scheme in SCHEMES.http: src = str(src) From 89941b54479ae367de7933af04f544c879aaedd2 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 7 Dec 2024 01:07:26 +0000 Subject: [PATCH 24/34] Update docs --- docs/sections/user_guide/cli/tools/fs.rst | 6 ++- .../user_guide/cli/tools/fs/copy-config.yaml | 1 + .../tools/fs/copy-exec-no-target-dir-err.out | 6 +-- .../user_guide/cli/tools/fs/copy-exec.out | 39 +++++++++++-------- .../tools/fs/link-exec-no-target-dir-err.out | 6 +-- .../fs/makedirs-exec-no-target-dir-err.out | 6 +-- 6 files changed, 37 insertions(+), 27 deletions(-) diff --git a/docs/sections/user_guide/cli/tools/fs.rst b/docs/sections/user_guide/cli/tools/fs.rst index cb61dd7bf..edd06c5c6 100644 --- a/docs/sections/user_guide/cli/tools/fs.rst +++ b/docs/sections/user_guide/cli/tools/fs.rst @@ -15,6 +15,8 @@ The ``uw`` mode for handling filesystem items (files and directories). The ``copy`` action stages files in a target directory by copying files. Any ``KEY`` positional arguments are used to navigate, in the order given, from the top of the config to the :ref:`file block `. +Source paths prefixed with ``http://`` or ``https://`` will be copied from their upstream network locations to the local filesystem. + .. literalinclude:: fs/copy-help.cmd :emphasize-lines: 1 .. literalinclude:: fs/copy-help.out @@ -23,7 +25,7 @@ The ``copy`` action stages files in a target directory by copying files. Any ``K Examples ^^^^^^^^ -Given ``copy-config.yaml`` containing +Given ``copy-config.yaml`` containing a mapping from local-filesystem destination paths to source paths. .. literalinclude:: fs/copy-config.yaml :language: yaml @@ -32,7 +34,7 @@ Given ``copy-config.yaml`` containing .. literalinclude:: fs/copy-exec.out :language: text -Here, ``foo`` and ``bar`` are copies of their respective source files. +Here, ``foo`` and ``bar`` are copies of their respective local-filesystem source files, and ``gpl`` is a copy of the upstream network source of the GNU General Public License. The ``--cycle`` and ``--leadtime`` options can be used to make Python ``datetime`` and ``timedelta`` objects, respectively, available for use in Jinja2 expression in the config. For example: diff --git a/docs/sections/user_guide/cli/tools/fs/copy-config.yaml b/docs/sections/user_guide/cli/tools/fs/copy-config.yaml index 17c45a3e6..98d02f202 100644 --- a/docs/sections/user_guide/cli/tools/fs/copy-config.yaml +++ b/docs/sections/user_guide/cli/tools/fs/copy-config.yaml @@ -1,4 +1,5 @@ config: files: foo: src/foo + licenses/gpl: https://www.gnu.org/licenses/gpl-3.0.txt subdir/bar: src/bar diff --git a/docs/sections/user_guide/cli/tools/fs/copy-exec-no-target-dir-err.out b/docs/sections/user_guide/cli/tools/fs/copy-exec-no-target-dir-err.out index 71b4ca3a1..17a9f799f 100644 --- a/docs/sections/user_guide/cli/tools/fs/copy-exec-no-target-dir-err.out +++ b/docs/sections/user_guide/cli/tools/fs/copy-exec-no-target-dir-err.out @@ -1,3 +1,3 @@ -[2024-08-26T23:03:40] INFO Validating config against internal schema: files-to-stage -[2024-08-26T23:03:40] INFO 0 UW schema-validation errors found in fs config -[2024-08-26T23:03:40] ERROR Relative path 'foo' requires the target directory to be specified +[2024-12-07T01:01:51] INFO Validating config against internal schema: files-to-stage +[2024-12-07T01:01:53] INFO 0 UW schema-validation errors found in fs config +[2024-12-07T01:01:53] ERROR Relative path 'foo' requires target directory to be specified diff --git a/docs/sections/user_guide/cli/tools/fs/copy-exec.out b/docs/sections/user_guide/cli/tools/fs/copy-exec.out index 57221e756..46a06048e 100644 --- a/docs/sections/user_guide/cli/tools/fs/copy-exec.out +++ b/docs/sections/user_guide/cli/tools/fs/copy-exec.out @@ -1,22 +1,29 @@ -[2024-08-26T23:03:41] INFO Validating config against internal schema: files-to-stage -[2024-08-26T23:03:41] INFO 0 UW schema-validation errors found in fs config -[2024-08-26T23:03:41] INFO File copies: Initial state: Not Ready -[2024-08-26T23:03:41] INFO File copies: Checking requirements -[2024-08-26T23:03:41] INFO Copy src/foo -> copy-dst/foo: Initial state: Not Ready -[2024-08-26T23:03:41] INFO Copy src/foo -> copy-dst/foo: Checking requirements -[2024-08-26T23:03:41] INFO Copy src/foo -> copy-dst/foo: Requirement(s) ready -[2024-08-26T23:03:41] INFO Copy src/foo -> copy-dst/foo: Executing -[2024-08-26T23:03:41] INFO Copy src/foo -> copy-dst/foo: Final state: Ready -[2024-08-26T23:03:41] INFO Copy src/bar -> copy-dst/subdir/bar: Initial state: Not Ready -[2024-08-26T23:03:41] INFO Copy src/bar -> copy-dst/subdir/bar: Checking requirements -[2024-08-26T23:03:41] INFO Copy src/bar -> copy-dst/subdir/bar: Requirement(s) ready -[2024-08-26T23:03:41] INFO Copy src/bar -> copy-dst/subdir/bar: Executing -[2024-08-26T23:03:41] INFO Copy src/bar -> copy-dst/subdir/bar: Final state: Ready -[2024-08-26T23:03:41] INFO File copies: Final state: Ready +[2024-12-07T01:01:56] INFO Validating config against internal schema: files-to-stage +[2024-12-07T01:01:56] INFO 0 UW schema-validation errors found in fs config +[2024-12-07T01:01:56] INFO File copies: Initial state: Not Ready +[2024-12-07T01:01:56] INFO File copies: Checking requirements +[2024-12-07T01:01:56] INFO Copy src/foo -> copy-dst/foo: Initial state: Not Ready +[2024-12-07T01:01:56] INFO Copy src/foo -> copy-dst/foo: Checking requirements +[2024-12-07T01:01:56] INFO Copy src/foo -> copy-dst/foo: Requirement(s) ready +[2024-12-07T01:01:56] INFO Copy src/foo -> copy-dst/foo: Executing +[2024-12-07T01:01:56] INFO Copy src/foo -> copy-dst/foo: Final state: Ready +[2024-12-07T01:01:56] INFO Copy https://www.gnu.org/licenses/gpl-3.0.txt -> copy-dst/licenses/gpl: Initial state: Not Ready +[2024-12-07T01:01:56] INFO Copy https://www.gnu.org/licenses/gpl-3.0.txt -> copy-dst/licenses/gpl: Checking requirements +[2024-12-07T01:01:58] INFO Copy https://www.gnu.org/licenses/gpl-3.0.txt -> copy-dst/licenses/gpl: Requirement(s) ready +[2024-12-07T01:01:58] INFO Copy https://www.gnu.org/licenses/gpl-3.0.txt -> copy-dst/licenses/gpl: Executing +[2024-12-07T01:01:58] INFO Copy https://www.gnu.org/licenses/gpl-3.0.txt -> copy-dst/licenses/gpl: Final state: Ready +[2024-12-07T01:01:58] INFO Copy src/bar -> copy-dst/subdir/bar: Initial state: Not Ready +[2024-12-07T01:01:58] INFO Copy src/bar -> copy-dst/subdir/bar: Checking requirements +[2024-12-07T01:01:58] INFO Copy src/bar -> copy-dst/subdir/bar: Requirement(s) ready +[2024-12-07T01:01:58] INFO Copy src/bar -> copy-dst/subdir/bar: Executing +[2024-12-07T01:01:58] INFO Copy src/bar -> copy-dst/subdir/bar: Final state: Ready +[2024-12-07T01:01:58] INFO File copies: Final state: Ready copy-dst ├── foo +├── licenses +│   └── gpl └── subdir └── bar -2 directories, 2 files +3 directories, 3 files diff --git a/docs/sections/user_guide/cli/tools/fs/link-exec-no-target-dir-err.out b/docs/sections/user_guide/cli/tools/fs/link-exec-no-target-dir-err.out index dcb5593ed..7118df558 100644 --- a/docs/sections/user_guide/cli/tools/fs/link-exec-no-target-dir-err.out +++ b/docs/sections/user_guide/cli/tools/fs/link-exec-no-target-dir-err.out @@ -1,3 +1,3 @@ -[2024-08-26T23:03:41] INFO Validating config against internal schema: files-to-stage -[2024-08-26T23:03:41] INFO 0 UW schema-validation errors found in fs config -[2024-08-26T23:03:41] ERROR Relative path 'foo' requires the target directory to be specified +[2024-12-07T01:01:55] INFO Validating config against internal schema: files-to-stage +[2024-12-07T01:01:55] INFO 0 UW schema-validation errors found in fs config +[2024-12-07T01:01:55] ERROR Relative path 'foo' requires target directory to be specified diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-exec-no-target-dir-err.out b/docs/sections/user_guide/cli/tools/fs/makedirs-exec-no-target-dir-err.out index 84c7710bf..63c47798e 100644 --- a/docs/sections/user_guide/cli/tools/fs/makedirs-exec-no-target-dir-err.out +++ b/docs/sections/user_guide/cli/tools/fs/makedirs-exec-no-target-dir-err.out @@ -1,3 +1,3 @@ -[2024-08-26T23:03:44] INFO Validating config against internal schema: makedirs -[2024-08-26T23:03:45] INFO 0 UW schema-validation errors found in fs config -[2024-08-26T23:03:45] ERROR Relative path 'foo' requires the target directory to be specified +[2024-12-07T01:01:55] INFO Validating config against internal schema: makedirs +[2024-12-07T01:01:55] INFO 0 UW schema-validation errors found in fs config +[2024-12-07T01:01:55] ERROR Relative path 'foo' requires target directory to be specified From 7f05b41e3283f5a0cbfef566b2ecb27b74d4fc37 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 7 Dec 2024 01:18:37 +0000 Subject: [PATCH 25/34] Only make dst dir when src req is satisfied --- src/uwtools/tests/utils/test_tasks.py | 2 +- src/uwtools/utils/tasks.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/uwtools/tests/utils/test_tasks.py b/src/uwtools/tests/utils/test_tasks.py index 996de3cf9..f6bbd2162 100644 --- a/src/uwtools/tests/utils/test_tasks.py +++ b/src/uwtools/tests/utils/test_tasks.py @@ -164,12 +164,12 @@ def test_tasks_filecopy_source_local(src, ok): with patch.object(tasks, "file", exists): with patch.object(tasks, "copy") as copy: tasks.filecopy(src=src, dst=dst) + mkdir.assert_called_once_with(parents=True, exist_ok=True) copy.assert_called_once_with(Path(src), Path(dst)) else: with raises(UWConfigError) as e: tasks.filecopy(src=src, dst=dst) assert str(e.value) == "Support for scheme 'foo' not implemented" - mkdir.assert_called_once_with(parents=True, exist_ok=True) def test_tasks_symlink_simple(tmp_path): diff --git a/src/uwtools/utils/tasks.py b/src/uwtools/utils/tasks.py index 1bf21f514..73ebf77d6 100644 --- a/src/uwtools/utils/tasks.py +++ b/src/uwtools/utils/tasks.py @@ -90,15 +90,16 @@ def filecopy(src: Union[Path, str], dst: Union[Path, str]): yield "Copy %s -> %s" % (src, dst) yield asset(Path(dst), Path(dst).is_file) dst = Path(dst) # currently no support for remote destinations - dst.parent.mkdir(parents=True, exist_ok=True) scheme = urlparse(str(src)).scheme if scheme in SCHEMES.local: src = Path(src) yield file(src) + dst.parent.mkdir(parents=True, exist_ok=True) copy(src, dst) elif scheme in SCHEMES.http: src = str(src) yield existing(src) + dst.parent.mkdir(parents=True, exist_ok=True) response = requests.get(src, allow_redirects=True, timeout=3) if (code := response.status_code) == 200: with open(dst, "wb") as f: From e8738346292aedc2a66847f46035e4ed582840f4 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 7 Dec 2024 17:28:07 +0000 Subject: [PATCH 26/34] Doc fixes --- docs/sections/user_guide/cli/tools/fs.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/sections/user_guide/cli/tools/fs.rst b/docs/sections/user_guide/cli/tools/fs.rst index edd06c5c6..153404e98 100644 --- a/docs/sections/user_guide/cli/tools/fs.rst +++ b/docs/sections/user_guide/cli/tools/fs.rst @@ -25,7 +25,7 @@ Source paths prefixed with ``http://`` or ``https://`` will be copied from their Examples ^^^^^^^^ -Given ``copy-config.yaml`` containing a mapping from local-filesystem destination paths to source paths. +Given ``copy-config.yaml`` containing a mapping from local-filesystem destination paths to source paths .. literalinclude:: fs/copy-config.yaml :language: yaml @@ -34,7 +34,7 @@ Given ``copy-config.yaml`` containing a mapping from local-filesystem destinatio .. literalinclude:: fs/copy-exec.out :language: text -Here, ``foo`` and ``bar`` are copies of their respective local-filesystem source files, and ``gpl`` is a copy of the upstream network source of the GNU General Public License. +Here, ``foo`` and ``bar`` are copies of their respective local-filesystem source files, and ``gpl`` is a copy of the upstream network source. The ``--cycle`` and ``--leadtime`` options can be used to make Python ``datetime`` and ``timedelta`` objects, respectively, available for use in Jinja2 expression in the config. For example: From ada3aa733d9f06832f65375b8eb8b40c194fba1f Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 7 Dec 2024 17:44:38 +0000 Subject: [PATCH 27/34] Update --- src/uwtools/fs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/uwtools/fs.py b/src/uwtools/fs.py index c2e11f5b2..1e1bf58ad 100644 --- a/src/uwtools/fs.py +++ b/src/uwtools/fs.py @@ -69,7 +69,7 @@ def _check_destination_paths(self) -> None: """ for dst in self._dst_paths: scheme = urlparse(dst).scheme - absolute = Path(dst).is_absolute() + absolute = scheme or Path(dst).is_absolute() if scheme and scheme != "file": msg = "Non-filesystem destination path '%s' not currently supported" raise UWConfigError(msg % dst) From 7316d6097f5fcdb1804897f7a7c2c62c0c683ecb Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 7 Dec 2024 17:48:52 +0000 Subject: [PATCH 28/34] Use STR for "file" URL scheme string --- src/uwtools/fs.py | 8 +++++--- src/uwtools/strings.py | 1 + 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/uwtools/fs.py b/src/uwtools/fs.py index 1e1bf58ad..008b3cd03 100644 --- a/src/uwtools/fs.py +++ b/src/uwtools/fs.py @@ -70,7 +70,7 @@ def _check_destination_paths(self) -> None: for dst in self._dst_paths: scheme = urlparse(dst).scheme absolute = scheme or Path(dst).is_absolute() - if scheme and scheme != "file": + if scheme and scheme != STR.url_scheme_file: msg = "Non-filesystem destination path '%s' not currently supported" raise UWConfigError(msg % dst) if self._target_dir and scheme: @@ -92,7 +92,7 @@ def _check_target_dir(self) -> None: if ( self._target_dir and (scheme := urlparse(str(self._target_dir)).scheme) - and scheme != "file" + and scheme != STR.url_scheme_file ): msg = "Non-filesystem path '%s' invalid as target directory" raise UWConfigError(msg % self._target_dir) @@ -174,7 +174,9 @@ def go(self): for dst, src in self._config.items(): dst = Path((self._target_dir or "")) / dst info = {x: urlparse(str(x)) for x in (dst, src)} - dst, src = [info[x].path if info[x].scheme == "file" else x for x in (dst, src)] + dst, src = [ + info[x].path if info[x].scheme == STR.url_scheme_file else x for x in (dst, src) + ] reqs.append(filecopy(src=src, dst=dst)) yield reqs diff --git a/src/uwtools/strings.py b/src/uwtools/strings.py index 234ef728a..fa4a71d81 100644 --- a/src/uwtools/strings.py +++ b/src/uwtools/strings.py @@ -143,6 +143,7 @@ class STR: updatefmt: str = "update_format" updatevalues: str = "update_values" upp: str = "upp" + url_scheme_file: str = "file" validate: str = "validate" valsfile: str = "values_file" valsfmt: str = "values_format" From f9a0fb26b425ded84b7c125d62185aa415b821eb Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 7 Dec 2024 18:20:18 +0000 Subject: [PATCH 29/34] Simplify --- src/uwtools/fs.py | 22 +++++++++++++--------- src/uwtools/tests/test_fs.py | 14 +++++++++++--- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/src/uwtools/fs.py b/src/uwtools/fs.py index 008b3cd03..404204624 100644 --- a/src/uwtools/fs.py +++ b/src/uwtools/fs.py @@ -170,15 +170,19 @@ def go(self): Copy files. """ yield "File copies" - reqs = [] - for dst, src in self._config.items(): - dst = Path((self._target_dir or "")) / dst - info = {x: urlparse(str(x)) for x in (dst, src)} - dst, src = [ - info[x].path if info[x].scheme == STR.url_scheme_file else x for x in (dst, src) - ] - reqs.append(filecopy(src=src, dst=dst)) - yield reqs + yield [ + filecopy(src=self._local(src), dst=self._local(self._target_dir) / self._local(dst)) + for dst, src in self._config.items() + ] + + @staticmethod + def _local(path: Union[Path, str]) -> Path: + """ + Convert a path, potentially prefixed with scheme file://, into a simple path. + + :param path: The path to convert. + """ + return Path(urlparse(str(path)).path) class Linker(FileStager): diff --git a/src/uwtools/tests/test_fs.py b/src/uwtools/tests/test_fs.py index 4246fb463..c8654df35 100644 --- a/src/uwtools/tests/test_fs.py +++ b/src/uwtools/tests/test_fs.py @@ -3,6 +3,7 @@ # pylint: disable=protected-access # pylint: disable=redefined-outer-name +from pathlib import Path from unittest.mock import Mock import iotaa @@ -52,7 +53,7 @@ def _schema(self): @mark.parametrize("source", ("dict", "file")) -def test_fs_Copier(assets, source): +def test_fs_Copier_go(assets, source): dstdir, cfgdict, cfgfile = assets config = cfgdict if source == "dict" else cfgfile assert not (dstdir / "foo").exists() @@ -62,7 +63,7 @@ def test_fs_Copier(assets, source): assert (dstdir / "subdir" / "bar").is_file() -def test_fs_Copier_config_file_dry_run(assets): +def test_fs_Copier_go_config_file_dry_run(assets): dstdir, cfgdict, _ = assets assert not (dstdir / "foo").exists() assert not (dstdir / "subdir" / "bar").exists() @@ -72,7 +73,7 @@ def test_fs_Copier_config_file_dry_run(assets): iotaa.dryrun(False) -def test_fs_Copier_no_targetdir_abspath_pass(assets): +def test_fs_Copier_go_no_targetdir_abspath_pass(assets): dstdir, cfgdict, _ = assets old = cfgdict["a"]["b"] cfgdict = {str(dstdir / "foo"): old["foo"], str(dstdir / "bar"): old["subdir/bar"]} @@ -80,6 +81,13 @@ def test_fs_Copier_no_targetdir_abspath_pass(assets): assert all(asset.ready() for asset in assets) # type: ignore +def test_fs_Copier__local(): + assert fs.Copier._local("relative/path") == Path("relative/path") + assert fs.Copier._local("/absolute/path") == Path("/absolute/path") + assert fs.Copier._local("file:///absolute/path") == Path("/absolute/path") + assert fs.Copier._local("") == Path("") + + @mark.parametrize("source", ("dict", "file")) def test_fs_FilerStager(assets, source): dstdir, cfgdict, cfgfile = assets From ae394382d083771013dc275ec66a4268a937c202 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 7 Dec 2024 18:22:21 +0000 Subject: [PATCH 30/34] Rename --- src/uwtools/fs.py | 6 +++--- src/uwtools/tests/test_fs.py | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/uwtools/fs.py b/src/uwtools/fs.py index 404204624..a9dee3464 100644 --- a/src/uwtools/fs.py +++ b/src/uwtools/fs.py @@ -171,14 +171,14 @@ def go(self): """ yield "File copies" yield [ - filecopy(src=self._local(src), dst=self._local(self._target_dir) / self._local(dst)) + filecopy(src=self._simple(src), dst=self._simple(self._target_dir) / self._simple(dst)) for dst, src in self._config.items() ] @staticmethod - def _local(path: Union[Path, str]) -> Path: + def _simple(path: Union[Path, str]) -> Path: """ - Convert a path, potentially prefixed with scheme file://, into a simple path. + Convert a path, potentially prefixed with scheme file://, into a simple filesystem path. :param path: The path to convert. """ diff --git a/src/uwtools/tests/test_fs.py b/src/uwtools/tests/test_fs.py index c8654df35..30b5c77d5 100644 --- a/src/uwtools/tests/test_fs.py +++ b/src/uwtools/tests/test_fs.py @@ -81,11 +81,11 @@ def test_fs_Copier_go_no_targetdir_abspath_pass(assets): assert all(asset.ready() for asset in assets) # type: ignore -def test_fs_Copier__local(): - assert fs.Copier._local("relative/path") == Path("relative/path") - assert fs.Copier._local("/absolute/path") == Path("/absolute/path") - assert fs.Copier._local("file:///absolute/path") == Path("/absolute/path") - assert fs.Copier._local("") == Path("") +def test_fs_Copier__simple(): + assert fs.Copier._simple("relative/path") == Path("relative/path") + assert fs.Copier._simple("/absolute/path") == Path("/absolute/path") + assert fs.Copier._simple("file:///absolute/path") == Path("/absolute/path") + assert fs.Copier._simple("") == Path("") @mark.parametrize("source", ("dict", "file")) From ac7a90f5c7a7c5dd3768839ee8a0740b99a739bc Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Tue, 10 Dec 2024 01:24:42 +0000 Subject: [PATCH 31/34] Fix test --- src/uwtools/tests/test_fs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/uwtools/tests/test_fs.py b/src/uwtools/tests/test_fs.py index 352c38fd4..535d06155 100644 --- a/src/uwtools/tests/test_fs.py +++ b/src/uwtools/tests/test_fs.py @@ -85,7 +85,7 @@ def test_Copier_no_targetdir_relpath_fail(assets): _, cfgdict, _ = assets with raises(UWConfigError) as e: fs.Copier(config=cfgdict, key_path=["a", "b"]).go() - errmsg = "Relative path '%s' requires the target directory to be specified" + errmsg = "Relative path '%s' requires target directory to be specified" assert errmsg % "foo" in str(e.value) From 3a74abe91e358ec56c0b07c6466bebebf2f6517d Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Tue, 10 Dec 2024 04:34:18 +0000 Subject: [PATCH 32/34] More tests --- src/uwtools/tests/utils/test_tasks.py | 51 +++++++++++++++++++++------ src/uwtools/utils/tasks.py | 40 ++++++++++++++------- 2 files changed, 69 insertions(+), 22 deletions(-) diff --git a/src/uwtools/tests/utils/test_tasks.py b/src/uwtools/tests/utils/test_tasks.py index f6bbd2162..1fe044bb4 100644 --- a/src/uwtools/tests/utils/test_tasks.py +++ b/src/uwtools/tests/utils/test_tasks.py @@ -1,9 +1,10 @@ -# pylint: disable=missing-function-docstring +# pylint: disable=missing-function-docstring,protected-access import logging import os import stat from pathlib import Path +from typing import Union from unittest.mock import Mock, patch from iotaa import asset, external @@ -105,17 +106,21 @@ def test_tasks_existing_bad_scheme(): path = "foo://bucket/a/b" with raises(UWConfigError) as e: tasks.existing(path=path) - assert str(e.value) == "Support for scheme 'foo' not implemented" + assert str(e.value) == f"Scheme 'foo' in '{path}' not supported" -def test_tasks_file_missing(tmp_path): +@mark.parametrize("prefix", ["", "file://"]) +def test_tasks_file_missing(prefix, tmp_path): path = tmp_path / "file" + path = "%s%s" % (prefix, path) if prefix else path assert not ready(tasks.file(path=path)) -def test_tasks_file_present(tmp_path): +@mark.parametrize("prefix", ["", "file://"]) +def test_tasks_file_present(prefix, tmp_path): path = tmp_path / "file" path.touch() + path = "%s%s" % (prefix, path) if prefix else path assert ready(tasks.file(path=path)) @@ -165,26 +170,52 @@ def test_tasks_filecopy_source_local(src, ok): with patch.object(tasks, "copy") as copy: tasks.filecopy(src=src, dst=dst) mkdir.assert_called_once_with(parents=True, exist_ok=True) - copy.assert_called_once_with(Path(src), Path(dst)) + copy.assert_called_once_with(Path("/src/file"), Path(dst)) else: with raises(UWConfigError) as e: tasks.filecopy(src=src, dst=dst) - assert str(e.value) == "Support for scheme 'foo' not implemented" + assert str(e.value) == f"Scheme 'foo' in '{src}' not supported" -def test_tasks_symlink_simple(tmp_path): +@mark.parametrize("prefix", ["", "file://"]) +def test_tasks_symlink_simple(prefix, tmp_path): target = tmp_path / "target" link = tmp_path / "link" target.touch() assert not link.is_file() - tasks.symlink(target=target, linkname=link) + t2, l2 = ["%s%s" % (prefix, x) if prefix else x for x in (target, link)] + tasks.symlink(target=t2, linkname=l2) assert link.is_symlink() -def test_tasks_symlink_directory_hierarchy(tmp_path): +@mark.parametrize("prefix", ["", "file://"]) +def test_tasks_symlink_directory_hierarchy(prefix, tmp_path): target = tmp_path / "target" link = tmp_path / "foo" / "bar" / "link" target.touch() assert not link.is_file() - tasks.symlink(target=target, linkname=link) + t2, l2 = ["%s%s" % (prefix, x) if prefix else x for x in (target, link)] + tasks.symlink(target=t2, linkname=l2) assert link.is_symlink() + + +def test__bad_scheme(): + path = "foo://bucket/a/b" + with raises(UWConfigError) as e: + tasks.existing(path=path) + assert str(e.value) == f"Scheme 'foo' in '{path}' not supported" + + +def test__local_path_fail(): + path = "foo://bucket/a/b" + with patch.object(tasks, "_bad_scheme") as _bad_scheme: + tasks._local_path(path) + _bad_scheme.assert_called_once_with(path, "foo") + + +@mark.parametrize("prefix", ["", "file://"]) +@mark.parametrize("wrapper", [str, Path]) +def test__local_path_pass(prefix, wrapper): + path = "/some/file" + p2: Union[str, Path] = str(f"{prefix}{path}") if wrapper == str else Path(path) + assert tasks._local_path(p2) == Path(path) diff --git a/src/uwtools/utils/tasks.py b/src/uwtools/utils/tasks.py index 73ebf77d6..de66e3c9b 100644 --- a/src/uwtools/utils/tasks.py +++ b/src/uwtools/utils/tasks.py @@ -53,7 +53,7 @@ def existing(path: Union[Path, str]): info = urlparse(str(path)) scheme = info.scheme if scheme in SCHEMES.local: - path = Path(info.path if scheme == "file" else path) + path = _local_path(path) yield "Filesystem item %s" % path yield asset(path, path.exists) elif scheme in SCHEMES.http: @@ -62,17 +62,18 @@ def existing(path: Union[Path, str]): yield "Remote object %s" % path yield asset(path, ready) else: - _bad_scheme(scheme) + _bad_scheme(path, scheme) @external -def file(path: Path, context: str = ""): +def file(path: Union[Path, str], context: str = ""): """ An existing file or symlink to an existing file. :param path: Path to the file. :param context: Optional additional context for the file. """ + path = _local_path(path) suffix = f" ({context})" if context else "" yield "File %s%s" % (path, suffix) yield asset(path, path.is_file) @@ -89,14 +90,14 @@ def filecopy(src: Union[Path, str], dst: Union[Path, str]): """ yield "Copy %s -> %s" % (src, dst) yield asset(Path(dst), Path(dst).is_file) - dst = Path(dst) # currently no support for remote destinations - scheme = urlparse(str(src)).scheme - if scheme in SCHEMES.local: - src = Path(src) + dst = _local_path(dst) # currently no support for remote destinations + src_scheme = urlparse(str(src)).scheme + if src_scheme in SCHEMES.local: + src = _local_path(src) yield file(src) dst.parent.mkdir(parents=True, exist_ok=True) copy(src, dst) - elif scheme in SCHEMES.http: + elif src_scheme in SCHEMES.http: src = str(src) yield existing(src) dst.parent.mkdir(parents=True, exist_ok=True) @@ -107,17 +108,18 @@ def filecopy(src: Union[Path, str], dst: Union[Path, str]): else: log.error("Could not get '%s', HTTP status was: %s", src, code) else: - _bad_scheme(scheme) + _bad_scheme(src, src_scheme) @task -def symlink(target: Path, linkname: Path): +def symlink(target: Union[Path, str], linkname: Union[Path, str]): """ A symbolic link. :param target: The existing file or directory. :param linkname: The symlink to create. """ + target, linkname = map(_local_path, [target, linkname]) yield "Link %s -> %s" % (linkname, target) yield asset(linkname, linkname.exists) yield existing(target) @@ -131,11 +133,25 @@ def symlink(target: Path, linkname: Path): # Private helpers -def _bad_scheme(scheme: str) -> NoReturn: +def _bad_scheme(path: Union[Path, str], scheme: str) -> NoReturn: """ Fail on an unsupported URL scheme. + :param path: The path with a bad scheme. :param scheme: The scheme. :raises: UWConfigError. """ - raise UWConfigError(f"Support for scheme '{scheme}' not implemented") + raise UWConfigError(f"Scheme '{scheme}' in '{path}' not supported") + + +def _local_path(path: Union[Path, str]) -> Path: + """ + Ensure path is local and return simple version. + + :param path: The local path to check. + :raises: UWConfigError if a non-local scheme is specified. + """ + info = urlparse(str(path)) + if info.scheme and info.scheme not in SCHEMES.local: + _bad_scheme(path, info.scheme) + return Path(info.path) From d0ecc19ebe5b265269750a27648cb8f2bd010f49 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Tue, 10 Dec 2024 04:38:58 +0000 Subject: [PATCH 33/34] Fix --- src/uwtools/fs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/uwtools/fs.py b/src/uwtools/fs.py index aeb3ba8f0..f7d214a49 100644 --- a/src/uwtools/fs.py +++ b/src/uwtools/fs.py @@ -151,7 +151,7 @@ def go(self): """ yield "File copies" yield [ - filecopy(src=self._simple(src), dst=self._simple(self._target_dir) / self._simple(dst)) + filecopy(src=src, dst=self._simple(self._target_dir) / self._simple(dst)) for dst, src in self._config.items() ] From a05d302fb909874fd7b0c6cb51722ae813dfc9cb Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Tue, 10 Dec 2024 05:00:09 +0000 Subject: [PATCH 34/34] More tests --- src/uwtools/tests/test_fs.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/uwtools/tests/test_fs.py b/src/uwtools/tests/test_fs.py index 535d06155..227c52764 100644 --- a/src/uwtools/tests/test_fs.py +++ b/src/uwtools/tests/test_fs.py @@ -4,7 +4,7 @@ # pylint: disable=redefined-outer-name from pathlib import Path -from unittest.mock import Mock +from unittest.mock import Mock, patch import iotaa import yaml @@ -52,8 +52,19 @@ def _schema(self): # Tests +@mark.parametrize("src_fn", [str, Path]) +@mark.parametrize("dst_fn", [str, Path]) +@mark.parametrize("td_fn", [str, Path]) +def test_fs_Copier_go(src_fn, dst_fn, td_fn): + src, td, dst = src_fn("/src/file"), td_fn("/dst"), dst_fn("file") + obj = Mock(_config={dst: src}, _simple=fs.Copier._simple, _target_dir=td) + with patch.object(fs, "filecopy") as filecopy: + fs.Copier.go(obj) + filecopy.assert_called_once_with(src=src, dst=Path("/dst/file")) + + @mark.parametrize("source", ("dict", "file")) -def test_fs_Copier_go(assets, source): +def test_fs_Copier_go_live(assets, source): dstdir, cfgdict, cfgfile = assets config = cfgdict if source == "dict" else cfgfile assert not (dstdir / "foo").exists() @@ -63,7 +74,7 @@ def test_fs_Copier_go(assets, source): assert (dstdir / "subdir" / "bar").is_file() -def test_fs_Copier_go_config_file_dry_run(assets): +def test_fs_Copier_go_live_config_file_dry_run(assets): dstdir, cfgdict, _ = assets assert not (dstdir / "foo").exists() assert not (dstdir / "subdir" / "bar").exists() @@ -73,7 +84,7 @@ def test_fs_Copier_go_config_file_dry_run(assets): iotaa.dryrun(False) -def test_fs_Copier_go_no_targetdir_abspath_pass(assets): +def test_fs_Copier_go_live_no_targetdir_abspath_pass(assets): dstdir, cfgdict, _ = assets old = cfgdict["a"]["b"] cfgdict = {str(dstdir / "foo"): old["foo"], str(dstdir / "bar"): old["subdir/bar"]}