diff --git a/ush/python/pygw/src/pygw/file_utils.py b/ush/python/pygw/src/pygw/file_utils.py new file mode 100644 index 0000000000..062a707d05 --- /dev/null +++ b/ush/python/pygw/src/pygw/file_utils.py @@ -0,0 +1,73 @@ +from .fsutils import cp, mkdir + +__all__ = ['FileHandler'] + + +class FileHandler: + """Class to manipulate files in bulk for a given configuration + + Parameters + ---------- + config : dict + A dictionary containing the "action" and the "act" in the form of a list + + NOTE + ---- + "action" can be one of mkdir", "copy", etc. + Corresponding "act" would be ['dir1', 'dir2'], [['src1', 'dest1'], ['src2', 'dest2']] + + Attributes + ---------- + config : dict + Dictionary of files to manipulate + """ + + def __init__(self, config): + + self.config = config + + def sync(self): + """ + Method to execute bulk actions on files described in the configuration + """ + sync_factory = { + 'copy': self._copy_files, + 'mkdir': self._make_dirs, + } + # loop through the configuration keys + for action, files in self.config.items(): + sync_factory[action](files) + + @staticmethod + def _copy_files(filelist): + """Function to copy all files specified in the list + + `filelist` should be in the form: + - [src, dest] + + Parameters + ---------- + filelist : list + List of lists of [src, dest] + """ + for sublist in filelist: + if len(sublist) != 2: + raise Exception( + f"List must be of the form ['src', 'dest'], not {sublist}") + src = sublist[0] + dest = sublist[1] + cp(src, dest) + print(f'Copied {src} to {dest}') # TODO use logger + + @staticmethod + def _make_dirs(dirlist): + """Function to make all directories specified in the list + + Parameters + ---------- + dirlist : list + List of directories to create + """ + for dd in dirlist: + mkdir(dd) + print(f'Created {dd}') # TODO use logger diff --git a/ush/python/pygw/src/pygw/fsutils.py b/ush/python/pygw/src/pygw/fsutils.py index 66a62455f5..f0d41fb54f 100644 --- a/ush/python/pygw/src/pygw/fsutils.py +++ b/ush/python/pygw/src/pygw/fsutils.py @@ -3,7 +3,7 @@ import shutil import contextlib -__all__ = ['mkdir', 'mkdir_p', 'rmdir', 'chdir', 'rm_p'] +__all__ = ['mkdir', 'mkdir_p', 'rmdir', 'chdir', 'rm_p', 'cp'] def mkdir_p(path): @@ -45,3 +45,12 @@ def rm_p(path): pass else: raise OSError(f"unable to remove {path}") + + +def cp(src, dest): + try: + shutil.copyfile(src, dest) + except OSError as exc: + raise OSError(f"unable to copy {src} to {dest}") + except FileNotFoundError as exc: + raise FileNotFoundError(exc) diff --git a/ush/python/pygw/src/tests/test_file_utils.py b/ush/python/pygw/src/tests/test_file_utils.py new file mode 100644 index 0000000000..684c76b650 --- /dev/null +++ b/ush/python/pygw/src/tests/test_file_utils.py @@ -0,0 +1,66 @@ +import os +from pygw.file_utils import FileHandler + + +def test_mkdir(tmp_path): + """ + Test for creating directories: + Parameters + ---------- + tmp_path - pytest fixture + """ + + dir_path = tmp_path / 'my_test_dir' + d1 = f'{dir_path}1' + d2 = f'{dir_path}2' + d3 = f'{dir_path}3' + + # Create config object for FileHandler + config = {'mkdir': [d1, d2, d3]} + + # Create d1, d2, d3 + FileHandler(config).sync() + + # Check if d1, d2, d3 were indeed created + for dd in config['mkdir']: + assert os.path.exists(dd) + + +def test_copy(tmp_path): + """ + Test for copying files: + Parameters + ---------- + tmp_path - pytest fixture + """ + + input_dir_path = tmp_path / 'my_input_dir' + + # Create the input directory + config = {'mkdir': [input_dir_path]} + FileHandler(config).sync() + + # Put empty files in input_dir_path + src_files = [input_dir_path / 'a.txt', input_dir_path / 'b.txt'] + for ff in src_files: + ff.touch() + + # Create output_dir_path and expected file names + output_dir_path = tmp_path / 'my_output_dir' + config = {'mkdir': [output_dir_path]} + FileHandler(config).sync() + dest_files = [output_dir_path / 'a.txt', output_dir_path / 'bb.txt'] + + copy_list = [] + for src, dest in zip(src_files, dest_files): + copy_list.append([src, dest]) + + # Create config object for FileHandler + config = {'copy': copy_list} + + # Copy input files to output files + FileHandler(config).sync() + + # Check if files were indeed copied + for ff in dest_files: + assert os.path.isfile(ff) diff --git a/ush/python/pygw/src/tests/test_template.py b/ush/python/pygw/src/tests/test_template.py index 5d7bb378b9..f6d594b2d9 100644 --- a/ush/python/pygw/src/tests/test_template.py +++ b/ush/python/pygw/src/tests/test_template.py @@ -114,9 +114,9 @@ def test_substitute_with_dependencies(): 'a': 1, 'b': 2 }, - 'dd': { '2': 'a', '1': 'b' }, - 'ee': { '3': 'a', '1': 'b' }, - 'ff': { '4': 'a', '1': 'b $(greeting)' }, + 'dd': {'2': 'a', '1': 'b'}, + 'ee': {'3': 'a', '1': 'b'}, + 'ff': {'4': 'a', '1': 'b $(greeting)'}, 'host': { 'name': 'xenon', 'config': '$(root)/hosts', @@ -128,21 +128,20 @@ def test_substitute_with_dependencies(): } } output = {'complex': {'a': 1, 'b': 2}, - 'config': '/home/user/config/config.yaml', - 'config_file': 'config.yaml', - 'dd': {'1': 'b', '2': 'a'}, - 'dictionary': {'a': 1, 'b': 2}, - 'ee': {'1': 'b', '3': 'a'}, - 'ff': {'1': 'b hello world', '4': 'a'}, - 'greeting': 'hello world', - 'host': {'config': '/home/user/hosts', - 'config_file': '/home/user/config/config.yaml/xenon.config.yaml', - 'name': 'xenon', - 'proxy2': {'config': '/home/user/xenon.hello world.yaml', - 'list': [['/home/user/xenon', 'toto.xenon.hello world'], - 'config.yaml']}}, - 'root': '/home/user', - 'world': 'world'} - + 'config': '/home/user/config/config.yaml', + 'config_file': 'config.yaml', + 'dd': {'1': 'b', '2': 'a'}, + 'dictionary': {'a': 1, 'b': 2}, + 'ee': {'1': 'b', '3': 'a'}, + 'ff': {'1': 'b hello world', '4': 'a'}, + 'greeting': 'hello world', + 'host': {'config': '/home/user/hosts', + 'config_file': '/home/user/config/config.yaml/xenon.config.yaml', + 'name': 'xenon', + 'proxy2': {'config': '/home/user/xenon.hello world.yaml', + 'list': [['/home/user/xenon', 'toto.xenon.hello world'], + 'config.yaml']}}, + 'root': '/home/user', + 'world': 'world'} assert Template.substitute_with_dependencies(input, input, TemplateConstants.DOLLAR_PARENTHESES) == output