From 3cac1d1f95ec09dd3f952dd3fd5c599b5ba29124 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Mon, 30 Sep 2024 09:11:08 -0500 Subject: [PATCH 01/35] initial push --- smartsim/_core/generation/operations.py | 100 ++++++++++++++++++++ tests/test_operations.py | 117 ++++++++++++++++++++++++ 2 files changed, 217 insertions(+) create mode 100644 smartsim/_core/generation/operations.py create mode 100644 tests/test_operations.py diff --git a/smartsim/_core/generation/operations.py b/smartsim/_core/generation/operations.py new file mode 100644 index 0000000000..016465ff6e --- /dev/null +++ b/smartsim/_core/generation/operations.py @@ -0,0 +1,100 @@ +from ..commands import Command +import typing as t +import sys +import pathlib +from dataclasses import dataclass, field + +file_op_entry_point = "smartsim._core.entrypoints.file_operations" + +class GenerationContext(): + """Context for file system generation operations.""" + def __init__(self, job_root_path: pathlib.Path): + self.job_root_path = job_root_path + """The Job root path""" + + +class GenerationProtocol(t.Protocol): + """Protocol for Generation Operations.""" + def format(self) -> Command: + """Return a formatted Command that can be executed by a Launcher""" + +def create_final_dest(job_root_path: pathlib.Path, dest: pathlib.Path) -> str: + return str(job_root_path / dest) + +class CopyOperation(GenerationProtocol): + """Copy Operation""" + def __init__(self, src: pathlib.Path, dest: t.Union[pathlib.Path, None]) -> None: + self.src = src + self.dest = dest + + def format(self, context: GenerationContext) -> Command: + """Create Command to invoke copy fs entry point""" + final_dest = create_final_dest(context.job_root_path, self.dest) + return Command([sys.executable, "-m", file_op_entry_point, + "copy", self.src, final_dest]) + + +class SymlinkOperation(GenerationProtocol): + """Symlink Operation""" + def __init__(self, src: pathlib.Path, dest: t.Union[pathlib.Path, None]) -> None: + self.src = src + self.dest = dest + + def format(self, context: GenerationContext) -> Command: + """Create Command to invoke symlink fs entry point""" + final_dest = create_final_dest(context.job_root_path, self.dest) + return Command([sys.executable, "-m", file_op_entry_point, + "symlink", self.src, final_dest]) + + +class ConfigureOperation(GenerationProtocol): + """Configure Operation""" + def __init__(self, src: pathlib.Path, dest: t.Union[pathlib.Path, None], tag: t.Optional[str] = None) -> None: + self.src = src + self.dest = dest + self.tag = tag if tag else ";" + + # TODO discuss format as function name + def format(self, context: GenerationContext) -> Command: + """Create Command to invoke configure fs entry point""" + final_dest = create_final_dest(context.job_root_path, self.dest) + return Command([sys.executable, "-m", file_op_entry_point, + "configure", self.src, final_dest, self.tag, "encoded_dict"]) + +@dataclass +class FileSysOperationSet(): + """Dataclass to represent a set of FS Operation Objects""" + + # disallow modification - dunder function (post ticket to reevaluate API objects) + operations: t.List[GenerationContext] = field(default_factory=list) + """Set of FS Objects that match the GenerationProtocol""" + + def add_copy(self, src: pathlib.Path, dest: t.Optional[pathlib.Path] = None) -> None: + """Add a copy operation to the operations list""" + self.operations.append(CopyOperation(src, dest)) + + def add_symlink(self, src: pathlib.Path, dest: t.Optional[pathlib.Path] = None) -> None: + """Add a symlink operation to the operations list""" + self.operations.append(SymlinkOperation(src, dest)) + + def add_configuration(self, src: pathlib.Path, dest: t.Optional[pathlib.Path] = None, tag: t.Optional[str] = None) -> None: + """Add a configure operation to the operations list""" + self.operations.append(ConfigureOperation(src, dest, tag)) + + # entirely for introspection + # create generic filter operation that takes in a class type -> filter() -> public + # properties will call filter function - t.List of operation objects + @property + def copy_operations(self) -> t.List[CopyOperation]: + """Property to get the list of copy files.""" # return dict instead of operation list + return [x for x in self.operations if isinstance(x, CopyOperation)] + + @property + def symlink_operations(self) -> t.List[SymlinkOperation]: + """Property to get the list of symlink files.""" + return [x for x in self.operations if isinstance(x, SymlinkOperation)] + + @property + def configure_operations(self) -> t.List[ConfigureOperation]: + """Property to get the list of configure files.""" + return [x for x in self.operations if isinstance(x, ConfigureOperation)] \ No newline at end of file diff --git a/tests/test_operations.py b/tests/test_operations.py new file mode 100644 index 0000000000..fb79473b13 --- /dev/null +++ b/tests/test_operations.py @@ -0,0 +1,117 @@ +from smartsim._core.generation.operations import GenerationContext, CopyOperation, SymlinkOperation, ConfigureOperation, FileSysOperationSet +from smartsim._core.commands import Command +import pathlib +import pytest + +# TODO test python protocol? +# TODO add encoded dict into configure op +# TODO create a better way to append the paths together + +@pytest.fixture +def generation_context(test_dir: str): + return GenerationContext(pathlib.Path(test_dir)) + +@pytest.fixture +def mock_src(test_dir: str): + return pathlib.Path(test_dir) / pathlib.Path("mock_src") + +@pytest.fixture +def mock_dest(test_dir: str): + return pathlib.Path(test_dir) / pathlib.Path("mock_dest") + +@pytest.fixture +def copy_operation(mock_src: pathlib.Path, mock_dest: pathlib.Path): + return CopyOperation(src=mock_src, dest=mock_dest) + +def test_init_generation_context(test_dir: str, generation_context: GenerationContext): + """Validate GenerationContext init""" + assert isinstance(generation_context, GenerationContext) + assert generation_context.job_root_path == pathlib.Path(test_dir) + +def test_init_copy_operation(copy_operation: CopyOperation, mock_src: pathlib.Path, mock_dest: pathlib.Path): + assert copy_operation.src == mock_src + assert copy_operation.dest == mock_dest + +# def test_copy_operation_format(mock_src: str, mock_dest: str, generation_context: GenerationContext): +# copy_op = CopyOperation(src=mock_src, dest=mock_dest) +# exec = copy_op.format(generation_context) +# assert isinstance(exec, Command) +# # assert ( +# # str(mock_src) +# # and (mock_dest + str(generation_context.job_root_path)) in exec.command +# # ) + +# def test_init_symlink_operation(mock_src: str, mock_dest: str): +# symlink_op = SymlinkOperation(src=mock_src, dest=mock_dest) +# assert symlink_op.src == mock_src +# assert symlink_op.dest == mock_dest + +# def test_symlink_operation_format(mock_src: str, mock_dest: str, generation_context: GenerationContext): +# symlink_op = SymlinkOperation(src=mock_src, dest=mock_dest) +# exec = symlink_op.format(generation_context) +# assert isinstance(exec, Command) +# # assert ( +# # str(mock_src) +# # and (mock_dest + str(generation_context.job_root_path)) in exec.command +# # ) + +# def test_init_configure_operation(mock_src: str, mock_dest: str): +# config_op = ConfigureOperation(src=mock_src, dest=mock_dest) +# assert config_op.src == mock_src +# assert config_op.dest == mock_dest + +# def test_configure_operation_format(mock_src: str, mock_dest: str, generation_context: GenerationContext): +# config_op = ConfigureOperation(src=mock_src, dest=mock_dest) +# exec = config_op.format(generation_context) +# assert isinstance(exec, Command) +# # assert ( +# # str(mock_src) +# # and (mock_dest + str(generation_context.job_root_path)) in exec.command +# # ) + +# def test_init_file_sys_operation_set(): +# config_op = ConfigureOperation(src=mock_src, dest=mock_dest) +# file_sys_op_set = FileSysOperationSet([config_op]) +# assert len(file_sys_op_set.operations) == 1 + +# def test_add_copy_operation(): +# copy_op = CopyOperation(src=mock_src, dest=mock_dest) +# file_sys_op_set = FileSysOperationSet() +# file_sys_op_set.add_copy(copy_op) +# assert len(file_sys_op_set.operations) == 1 + +# def test_add_symlink_operation(): +# symlink_op = SymlinkOperation(src=mock_src, dest=mock_dest) +# file_sys_op_set = FileSysOperationSet() +# file_sys_op_set.add_symlink(symlink_op) +# assert len(file_sys_op_set.operations) == 1 + +# def test_add_configure_operation(): +# config_op = ConfigureOperation(src=mock_src, dest=mock_dest) +# file_sys_op_set = FileSysOperationSet() +# file_sys_op_set.add_symlink(config_op) +# assert len(file_sys_op_set.operations) == 1 + +# def test_copy_property(): +# config_op = ConfigureOperation(src=mock_src, dest=mock_dest) +# symlink_op = SymlinkOperation(src=mock_src, dest=mock_dest) +# copy_op = CopyOperation(src=mock_src, dest=mock_dest) +# file_sys_op_set = FileSysOperationSet([config_op,symlink_op,copy_op]) + +# assert file_sys_op_set.copy_operations == [copy_op] + +# def test_symlink_property(): +# config_op = ConfigureOperation(src=mock_src, dest=mock_dest) +# symlink_op = SymlinkOperation(src=mock_src, dest=mock_dest) +# copy_op = CopyOperation(src=mock_src, dest=mock_dest) +# file_sys_op_set = FileSysOperationSet([config_op,symlink_op,copy_op]) + +# assert file_sys_op_set.symlink_operations == [symlink_op] + +# def test_configure_property(): +# config_op = ConfigureOperation(src=mock_src, dest=mock_dest) +# symlink_op = SymlinkOperation(src=mock_src, dest=mock_dest) +# copy_op = CopyOperation(src=mock_src, dest=mock_dest) +# file_sys_op_set = FileSysOperationSet([config_op,symlink_op,copy_op]) + +# assert file_sys_op_set.configure_operations == [config_op] \ No newline at end of file From b6b5022bf9037d6866dd5ec70e9ada726a42b89b Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Mon, 30 Sep 2024 13:21:34 -0500 Subject: [PATCH 02/35] operations module close to finish --- smartsim/_core/commands/command.py | 4 + smartsim/_core/generation/operations.py | 126 +++++++++---- tests/test_operations.py | 239 +++++++++++++++--------- 3 files changed, 249 insertions(+), 120 deletions(-) diff --git a/smartsim/_core/commands/command.py b/smartsim/_core/commands/command.py index 3f41f32fe9..4b342652e2 100644 --- a/smartsim/_core/commands/command.py +++ b/smartsim/_core/commands/command.py @@ -35,6 +35,10 @@ class Command(MutableSequence[str]): """Basic container for command information""" def __init__(self, command: t.List[str]) -> None: + if not command: + raise ValueError("Command list cannot be empty") + if not all(isinstance(item, str) for item in command): + raise ValueError("All items in the command list must be strings") """Command constructor""" self._command = command diff --git a/smartsim/_core/generation/operations.py b/smartsim/_core/generation/operations.py index 016465ff6e..1cf3f549f6 100644 --- a/smartsim/_core/generation/operations.py +++ b/smartsim/_core/generation/operations.py @@ -1,13 +1,41 @@ -from ..commands import Command -import typing as t -import sys import pathlib +import sys +import typing as t from dataclasses import dataclass, field -file_op_entry_point = "smartsim._core.entrypoints.file_operations" +from ..commands import Command + +entry_point_path = "smartsim._core.entrypoints.file_operations" +"""Path to file operations module.""" -class GenerationContext(): +copy_cmd = "copy" +symlink_cmd = "symlink" +configure_cmd = "configure" + + +def create_final_dest(job_root_path: pathlib.Path, dest: t.Union[pathlib.Path, None]) -> str: + """Combine the job root path and destination path. Return as a string for + entry point consumption. + + :param job_root_path: Job root path + :param dest: Destination path + :return: Combined path + :raises ValueError: An error occurred during path combination + """ + if dest is None: + dest = pathlib.Path("") + if job_root_path is None or job_root_path == pathlib.Path("") or job_root_path.suffix: + raise ValueError(f"Job root path '{job_root_path}' is not a directory.") + try: + combined_path = job_root_path / dest + return str(combined_path) + except Exception as e: + raise ValueError(f"Error combining paths: {e}") + + +class GenerationContext: """Context for file system generation operations.""" + def __init__(self, job_root_path: pathlib.Path): self.job_root_path = job_root_path """The Job root path""" @@ -15,14 +43,14 @@ def __init__(self, job_root_path: pathlib.Path): class GenerationProtocol(t.Protocol): """Protocol for Generation Operations.""" - def format(self) -> Command: - """Return a formatted Command that can be executed by a Launcher""" -def create_final_dest(job_root_path: pathlib.Path, dest: pathlib.Path) -> str: - return str(job_root_path / dest) + def format(self, context: GenerationContext) -> Command: + """Return a formatted Command.""" + class CopyOperation(GenerationProtocol): """Copy Operation""" + def __init__(self, src: pathlib.Path, dest: t.Union[pathlib.Path, None]) -> None: self.src = src self.dest = dest @@ -30,12 +58,14 @@ def __init__(self, src: pathlib.Path, dest: t.Union[pathlib.Path, None]) -> None def format(self, context: GenerationContext) -> Command: """Create Command to invoke copy fs entry point""" final_dest = create_final_dest(context.job_root_path, self.dest) - return Command([sys.executable, "-m", file_op_entry_point, - "copy", self.src, final_dest]) + return Command( + [sys.executable, "-m", entry_point_path, copy_cmd, str(self.src), final_dest] + ) class SymlinkOperation(GenerationProtocol): """Symlink Operation""" + def __init__(self, src: pathlib.Path, dest: t.Union[pathlib.Path, None]) -> None: self.src = src self.dest = dest @@ -43,58 +73,86 @@ def __init__(self, src: pathlib.Path, dest: t.Union[pathlib.Path, None]) -> None def format(self, context: GenerationContext) -> Command: """Create Command to invoke symlink fs entry point""" final_dest = create_final_dest(context.job_root_path, self.dest) - return Command([sys.executable, "-m", file_op_entry_point, - "symlink", self.src, final_dest]) + return Command( + [sys.executable, "-m", entry_point_path, symlink_cmd, str(self.src), final_dest] + ) class ConfigureOperation(GenerationProtocol): """Configure Operation""" - def __init__(self, src: pathlib.Path, dest: t.Union[pathlib.Path, None], tag: t.Optional[str] = None) -> None: + + def __init__( + self, + src: pathlib.Path, + dest: t.Union[pathlib.Path, None], + tag: t.Optional[str] = None, + ) -> None: self.src = src self.dest = dest self.tag = tag if tag else ";" - + # TODO discuss format as function name def format(self, context: GenerationContext) -> Command: """Create Command to invoke configure fs entry point""" final_dest = create_final_dest(context.job_root_path, self.dest) - return Command([sys.executable, "-m", file_op_entry_point, - "configure", self.src, final_dest, self.tag, "encoded_dict"]) + return Command( + [ + sys.executable, + "-m", + entry_point_path, + configure_cmd, + str(self.src), + final_dest, + self.tag, + "encoded_dict", + ] + ) + +T = t.TypeVar('T', bound=GenerationProtocol) @dataclass -class FileSysOperationSet(): +class FileSysOperationSet: """Dataclass to represent a set of FS Operation Objects""" - + # disallow modification - dunder function (post ticket to reevaluate API objects) - operations: t.List[GenerationContext] = field(default_factory=list) + operations: t.List[GenerationProtocol] = field(default_factory=list) """Set of FS Objects that match the GenerationProtocol""" - - def add_copy(self, src: pathlib.Path, dest: t.Optional[pathlib.Path] = None) -> None: + + def add_copy( + self, src: pathlib.Path, dest: t.Optional[pathlib.Path] = None + ) -> None: """Add a copy operation to the operations list""" self.operations.append(CopyOperation(src, dest)) - def add_symlink(self, src: pathlib.Path, dest: t.Optional[pathlib.Path] = None) -> None: + def add_symlink( + self, src: pathlib.Path, dest: t.Optional[pathlib.Path] = None + ) -> None: """Add a symlink operation to the operations list""" self.operations.append(SymlinkOperation(src, dest)) - def add_configuration(self, src: pathlib.Path, dest: t.Optional[pathlib.Path] = None, tag: t.Optional[str] = None) -> None: + def add_configuration( + self, + src: pathlib.Path, + dest: t.Optional[pathlib.Path] = None, + tag: t.Optional[str] = None, + ) -> None: """Add a configure operation to the operations list""" self.operations.append(ConfigureOperation(src, dest, tag)) - - # entirely for introspection - # create generic filter operation that takes in a class type -> filter() -> public - # properties will call filter function - t.List of operation objects + @property def copy_operations(self) -> t.List[CopyOperation]: - """Property to get the list of copy files.""" # return dict instead of operation list - return [x for x in self.operations if isinstance(x, CopyOperation)] - + """Property to get the list of copy files.""" + return t.cast(t.List[CopyOperation], self._filter(CopyOperation)) + @property def symlink_operations(self) -> t.List[SymlinkOperation]: """Property to get the list of symlink files.""" - return [x for x in self.operations if isinstance(x, SymlinkOperation)] - + return t.cast(t.List[SymlinkOperation], self._filter(SymlinkOperation)) + @property def configure_operations(self) -> t.List[ConfigureOperation]: """Property to get the list of configure files.""" - return [x for x in self.operations if isinstance(x, ConfigureOperation)] \ No newline at end of file + return t.cast(t.List[ConfigureOperation], self._filter(ConfigureOperation)) + + def _filter(self, type: t.Type[T]) -> t.List[T]: + return [x for x in self.operations if isinstance(x, type)] \ No newline at end of file diff --git a/tests/test_operations.py b/tests/test_operations.py index fb79473b13..6e379b2c84 100644 --- a/tests/test_operations.py +++ b/tests/test_operations.py @@ -1,117 +1,184 @@ -from smartsim._core.generation.operations import GenerationContext, CopyOperation, SymlinkOperation, ConfigureOperation, FileSysOperationSet -from smartsim._core.commands import Command import pathlib + import pytest +from smartsim._core.commands import Command +from smartsim._core.generation.operations import ( + ConfigureOperation, + CopyOperation, + FileSysOperationSet, + GenerationContext, + SymlinkOperation, + create_final_dest, + copy_cmd, + symlink_cmd, + configure_cmd, +) + +# QUESTIONS # TODO test python protocol? # TODO add encoded dict into configure op # TODO create a better way to append the paths together +# TODO do I allow the paths to combine if src is empty? + @pytest.fixture def generation_context(test_dir: str): + """Fixture to create a GenerationContext object.""" return GenerationContext(pathlib.Path(test_dir)) + @pytest.fixture def mock_src(test_dir: str): + """Fixture to create a mock source path.""" return pathlib.Path(test_dir) / pathlib.Path("mock_src") + @pytest.fixture def mock_dest(test_dir: str): + """Fixture to create a mock destination path.""" return pathlib.Path(test_dir) / pathlib.Path("mock_dest") + @pytest.fixture def copy_operation(mock_src: pathlib.Path, mock_dest: pathlib.Path): + """Fixture to create a CopyOperation object.""" return CopyOperation(src=mock_src, dest=mock_dest) + +@pytest.fixture +def symlink_operation(mock_src: pathlib.Path, mock_dest: pathlib.Path): + """Fixture to create a CopyOperation object.""" + return SymlinkOperation(src=mock_src, dest=mock_dest) + + +@pytest.fixture +def configure_operation(mock_src: pathlib.Path, mock_dest: pathlib.Path): + """Fixture to create a CopyOperation object.""" + return ConfigureOperation(src=mock_src, dest=mock_dest, ) + + +@pytest.fixture +def file_system_operation_set(copy_operation: CopyOperation, symlink_operation: SymlinkOperation, configure_operation: ConfigureOperation): + """Fixture to create a FileSysOperationSet object.""" + return FileSysOperationSet([copy_operation,symlink_operation,configure_operation]) + + +@pytest.mark.parametrize( + "job_root_path, dest, expected", + ( + pytest.param( + pathlib.Path("/valid/root"), + pathlib.Path("valid/dest"), + "/valid/root/valid/dest", + id="Valid paths", + ), + # pytest.param( + # pathlib.Path("/valid/root/"), + # pathlib.Path("/valid/dest.txt"), + # "/valid/root/valid/dest.txt", + # id="Valid_file_path", + # ), + pytest.param( + pathlib.Path("/valid/root"), + pathlib.Path(""), + "/valid/root", + id="Empty destination path", + ), + pytest.param( + pathlib.Path("/valid/root"), + None, + "/valid/root", + id="Empty dest path", + ), + ), +) +def test_create_final_dest_valid(job_root_path, dest, expected): + """Test valid path inputs for operations.create_final_dest""" + assert create_final_dest(job_root_path, dest) == expected + + +@pytest.mark.parametrize( + "job_root_path, dest", + ( + pytest.param(None, pathlib.Path("valid/dest"), id="None as root path"), + pytest.param("", pathlib.Path("valid/dest"), id="Empty str as root path"), + pytest.param(pathlib.Path("/invalid/root.py"), pathlib.Path("valid/dest"), id="File as root path"), + ), +) +def test_create_final_dest_invalid(job_root_path, dest): + """Test invalid path inputs for operations.create_final_dest""" + with pytest.raises(ValueError): + create_final_dest(job_root_path, dest) + + def test_init_generation_context(test_dir: str, generation_context: GenerationContext): """Validate GenerationContext init""" assert isinstance(generation_context, GenerationContext) assert generation_context.job_root_path == pathlib.Path(test_dir) -def test_init_copy_operation(copy_operation: CopyOperation, mock_src: pathlib.Path, mock_dest: pathlib.Path): + +def test_init_copy_operation( + copy_operation: CopyOperation, mock_src: pathlib.Path, mock_dest: pathlib.Path +): + """Validate CopyOperation init""" + assert isinstance(copy_operation, CopyOperation) assert copy_operation.src == mock_src assert copy_operation.dest == mock_dest -# def test_copy_operation_format(mock_src: str, mock_dest: str, generation_context: GenerationContext): -# copy_op = CopyOperation(src=mock_src, dest=mock_dest) -# exec = copy_op.format(generation_context) -# assert isinstance(exec, Command) -# # assert ( -# # str(mock_src) -# # and (mock_dest + str(generation_context.job_root_path)) in exec.command -# # ) - -# def test_init_symlink_operation(mock_src: str, mock_dest: str): -# symlink_op = SymlinkOperation(src=mock_src, dest=mock_dest) -# assert symlink_op.src == mock_src -# assert symlink_op.dest == mock_dest - -# def test_symlink_operation_format(mock_src: str, mock_dest: str, generation_context: GenerationContext): -# symlink_op = SymlinkOperation(src=mock_src, dest=mock_dest) -# exec = symlink_op.format(generation_context) -# assert isinstance(exec, Command) -# # assert ( -# # str(mock_src) -# # and (mock_dest + str(generation_context.job_root_path)) in exec.command -# # ) - -# def test_init_configure_operation(mock_src: str, mock_dest: str): -# config_op = ConfigureOperation(src=mock_src, dest=mock_dest) -# assert config_op.src == mock_src -# assert config_op.dest == mock_dest - -# def test_configure_operation_format(mock_src: str, mock_dest: str, generation_context: GenerationContext): -# config_op = ConfigureOperation(src=mock_src, dest=mock_dest) -# exec = config_op.format(generation_context) -# assert isinstance(exec, Command) -# # assert ( -# # str(mock_src) -# # and (mock_dest + str(generation_context.job_root_path)) in exec.command -# # ) - -# def test_init_file_sys_operation_set(): -# config_op = ConfigureOperation(src=mock_src, dest=mock_dest) -# file_sys_op_set = FileSysOperationSet([config_op]) -# assert len(file_sys_op_set.operations) == 1 - -# def test_add_copy_operation(): -# copy_op = CopyOperation(src=mock_src, dest=mock_dest) -# file_sys_op_set = FileSysOperationSet() -# file_sys_op_set.add_copy(copy_op) -# assert len(file_sys_op_set.operations) == 1 - -# def test_add_symlink_operation(): -# symlink_op = SymlinkOperation(src=mock_src, dest=mock_dest) -# file_sys_op_set = FileSysOperationSet() -# file_sys_op_set.add_symlink(symlink_op) -# assert len(file_sys_op_set.operations) == 1 - -# def test_add_configure_operation(): -# config_op = ConfigureOperation(src=mock_src, dest=mock_dest) -# file_sys_op_set = FileSysOperationSet() -# file_sys_op_set.add_symlink(config_op) -# assert len(file_sys_op_set.operations) == 1 - -# def test_copy_property(): -# config_op = ConfigureOperation(src=mock_src, dest=mock_dest) -# symlink_op = SymlinkOperation(src=mock_src, dest=mock_dest) -# copy_op = CopyOperation(src=mock_src, dest=mock_dest) -# file_sys_op_set = FileSysOperationSet([config_op,symlink_op,copy_op]) - -# assert file_sys_op_set.copy_operations == [copy_op] - -# def test_symlink_property(): -# config_op = ConfigureOperation(src=mock_src, dest=mock_dest) -# symlink_op = SymlinkOperation(src=mock_src, dest=mock_dest) -# copy_op = CopyOperation(src=mock_src, dest=mock_dest) -# file_sys_op_set = FileSysOperationSet([config_op,symlink_op,copy_op]) - -# assert file_sys_op_set.symlink_operations == [symlink_op] - -# def test_configure_property(): -# config_op = ConfigureOperation(src=mock_src, dest=mock_dest) -# symlink_op = SymlinkOperation(src=mock_src, dest=mock_dest) -# copy_op = CopyOperation(src=mock_src, dest=mock_dest) -# file_sys_op_set = FileSysOperationSet([config_op,symlink_op,copy_op]) - -# assert file_sys_op_set.configure_operations == [config_op] \ No newline at end of file + +def test_copy_operation_format(copy_operation: CopyOperation, mock_src: str, mock_dest: str, generation_context: GenerationContext): + """Validate CopyOperation.format""" + exec = copy_operation.format(generation_context) + assert isinstance(exec, Command) + assert str(mock_src) in exec.command + assert copy_cmd in exec.command + assert create_final_dest(mock_src, mock_dest) in exec.command + +def test_init_symlink_operation(symlink_operation: SymlinkOperation, mock_src: str, mock_dest: str): + """Validate SymlinkOperation init""" + assert isinstance(symlink_operation, SymlinkOperation) + assert symlink_operation.src == mock_src + assert symlink_operation.dest == mock_dest + +def test_symlink_operation_format(symlink_operation: SymlinkOperation, mock_src: str, mock_dest: str, generation_context: GenerationContext): + """Validate SymlinkOperation.format""" + exec = symlink_operation.format(generation_context) + assert isinstance(exec, Command) + assert str(mock_src) in exec.command + assert symlink_cmd in exec.command + assert create_final_dest(mock_src, mock_dest) in exec.command + +def test_init_configure_operation(configure_operation: ConfigureOperation, mock_src: str, mock_dest: str): + """Validate ConfigureOperation init""" + assert isinstance(configure_operation, ConfigureOperation) + assert configure_operation.src == mock_src + assert configure_operation.dest == mock_dest + assert configure_operation.tag == ";" + +def test_configure_operation_format(configure_operation: ConfigureOperation, mock_src: str, mock_dest: str, generation_context: GenerationContext): + """Validate ConfigureOperation.format""" + exec = configure_operation.format(generation_context) + assert isinstance(exec, Command) + assert str(mock_src) in exec.command + assert configure_cmd in exec.command + assert create_final_dest(mock_src, mock_dest) in exec.command + +def test_init_file_sys_operation_set(file_system_operation_set: FileSysOperationSet): + assert isinstance(file_system_operation_set.operations, list) + assert len(file_system_operation_set.operations) == 3 + +def test_add_copy_operation(file_system_operation_set: FileSysOperationSet, copy_operation: CopyOperation): + assert len(file_system_operation_set.copy_operations) == 1 + file_system_operation_set.add_copy(copy_operation) + assert len(file_system_operation_set.copy_operations) == 2 + +def test_add_symlink_operation(file_system_operation_set: FileSysOperationSet, symlink_operation: SymlinkOperation): + assert len(file_system_operation_set.symlink_operations) == 1 + file_system_operation_set.add_symlink(symlink_operation) + assert len(file_system_operation_set.symlink_operations) == 2 + +def test_add_configure_operation(file_system_operation_set: FileSysOperationSet, configure_operation: ConfigureOperation): + assert len(file_system_operation_set.configure_operations) == 1 + file_system_operation_set.add_configuration(configure_operation) + assert len(file_system_operation_set.configure_operations) == 2 From 6e88a405b2aafaadbd317170a7f1365f94a3fe3c Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Mon, 30 Sep 2024 14:47:59 -0500 Subject: [PATCH 03/35] styling and mypy and partial test passing --- smartsim/_core/generation/operations.py | 41 +++++++++++--- tests/test_operations.py | 73 ++++++++++++++++++++----- 2 files changed, 91 insertions(+), 23 deletions(-) diff --git a/smartsim/_core/generation/operations.py b/smartsim/_core/generation/operations.py index 1cf3f549f6..f7319442ba 100644 --- a/smartsim/_core/generation/operations.py +++ b/smartsim/_core/generation/operations.py @@ -13,7 +13,9 @@ configure_cmd = "configure" -def create_final_dest(job_root_path: pathlib.Path, dest: t.Union[pathlib.Path, None]) -> str: +def create_final_dest( + job_root_path: pathlib.Path, dest: t.Union[pathlib.Path, None] +) -> str: """Combine the job root path and destination path. Return as a string for entry point consumption. @@ -24,7 +26,12 @@ def create_final_dest(job_root_path: pathlib.Path, dest: t.Union[pathlib.Path, N """ if dest is None: dest = pathlib.Path("") - if job_root_path is None or job_root_path == pathlib.Path("") or job_root_path.suffix: + if ( + job_root_path is None + or job_root_path == pathlib.Path("") + or "" + or job_root_path.suffix + ): raise ValueError(f"Job root path '{job_root_path}' is not a directory.") try: combined_path = job_root_path / dest @@ -59,7 +66,14 @@ def format(self, context: GenerationContext) -> Command: """Create Command to invoke copy fs entry point""" final_dest = create_final_dest(context.job_root_path, self.dest) return Command( - [sys.executable, "-m", entry_point_path, copy_cmd, str(self.src), final_dest] + [ + sys.executable, + "-m", + entry_point_path, + copy_cmd, + str(self.src), + final_dest, + ] ) @@ -74,7 +88,14 @@ def format(self, context: GenerationContext) -> Command: """Create Command to invoke symlink fs entry point""" final_dest = create_final_dest(context.job_root_path, self.dest) return Command( - [sys.executable, "-m", entry_point_path, symlink_cmd, str(self.src), final_dest] + [ + sys.executable, + "-m", + entry_point_path, + symlink_cmd, + str(self.src), + final_dest, + ] ) @@ -108,7 +129,9 @@ def format(self, context: GenerationContext) -> Command: ] ) -T = t.TypeVar('T', bound=GenerationProtocol) + +T = t.TypeVar("T", bound=GenerationProtocol) + @dataclass class FileSysOperationSet: @@ -142,17 +165,17 @@ def add_configuration( @property def copy_operations(self) -> t.List[CopyOperation]: """Property to get the list of copy files.""" - return t.cast(t.List[CopyOperation], self._filter(CopyOperation)) + return self._filter(CopyOperation) @property def symlink_operations(self) -> t.List[SymlinkOperation]: """Property to get the list of symlink files.""" - return t.cast(t.List[SymlinkOperation], self._filter(SymlinkOperation)) + return self._filter(SymlinkOperation) @property def configure_operations(self) -> t.List[ConfigureOperation]: """Property to get the list of configure files.""" - return t.cast(t.List[ConfigureOperation], self._filter(ConfigureOperation)) + return self._filter(ConfigureOperation) def _filter(self, type: t.Type[T]) -> t.List[T]: - return [x for x in self.operations if isinstance(x, type)] \ No newline at end of file + return [x for x in self.operations if isinstance(x, type)] diff --git a/tests/test_operations.py b/tests/test_operations.py index 6e379b2c84..ff2667e2d4 100644 --- a/tests/test_operations.py +++ b/tests/test_operations.py @@ -9,10 +9,10 @@ FileSysOperationSet, GenerationContext, SymlinkOperation, - create_final_dest, + configure_cmd, copy_cmd, + create_final_dest, symlink_cmd, - configure_cmd, ) # QUESTIONS @@ -55,13 +55,20 @@ def symlink_operation(mock_src: pathlib.Path, mock_dest: pathlib.Path): @pytest.fixture def configure_operation(mock_src: pathlib.Path, mock_dest: pathlib.Path): """Fixture to create a CopyOperation object.""" - return ConfigureOperation(src=mock_src, dest=mock_dest, ) + return ConfigureOperation( + src=mock_src, + dest=mock_dest, + ) @pytest.fixture -def file_system_operation_set(copy_operation: CopyOperation, symlink_operation: SymlinkOperation, configure_operation: ConfigureOperation): +def file_system_operation_set( + copy_operation: CopyOperation, + symlink_operation: SymlinkOperation, + configure_operation: ConfigureOperation, +): """Fixture to create a FileSysOperationSet object.""" - return FileSysOperationSet([copy_operation,symlink_operation,configure_operation]) + return FileSysOperationSet([copy_operation, symlink_operation, configure_operation]) @pytest.mark.parametrize( @@ -103,7 +110,11 @@ def test_create_final_dest_valid(job_root_path, dest, expected): ( pytest.param(None, pathlib.Path("valid/dest"), id="None as root path"), pytest.param("", pathlib.Path("valid/dest"), id="Empty str as root path"), - pytest.param(pathlib.Path("/invalid/root.py"), pathlib.Path("valid/dest"), id="File as root path"), + pytest.param( + pathlib.Path("/invalid/root.py"), + pathlib.Path("valid/dest"), + id="File as root path", + ), ), ) def test_create_final_dest_invalid(job_root_path, dest): @@ -127,7 +138,12 @@ def test_init_copy_operation( assert copy_operation.dest == mock_dest -def test_copy_operation_format(copy_operation: CopyOperation, mock_src: str, mock_dest: str, generation_context: GenerationContext): +def test_copy_operation_format( + copy_operation: CopyOperation, + mock_src: str, + mock_dest: str, + generation_context: GenerationContext, +): """Validate CopyOperation.format""" exec = copy_operation.format(generation_context) assert isinstance(exec, Command) @@ -135,13 +151,22 @@ def test_copy_operation_format(copy_operation: CopyOperation, mock_src: str, moc assert copy_cmd in exec.command assert create_final_dest(mock_src, mock_dest) in exec.command -def test_init_symlink_operation(symlink_operation: SymlinkOperation, mock_src: str, mock_dest: str): + +def test_init_symlink_operation( + symlink_operation: SymlinkOperation, mock_src: str, mock_dest: str +): """Validate SymlinkOperation init""" assert isinstance(symlink_operation, SymlinkOperation) assert symlink_operation.src == mock_src assert symlink_operation.dest == mock_dest -def test_symlink_operation_format(symlink_operation: SymlinkOperation, mock_src: str, mock_dest: str, generation_context: GenerationContext): + +def test_symlink_operation_format( + symlink_operation: SymlinkOperation, + mock_src: str, + mock_dest: str, + generation_context: GenerationContext, +): """Validate SymlinkOperation.format""" exec = symlink_operation.format(generation_context) assert isinstance(exec, Command) @@ -149,14 +174,23 @@ def test_symlink_operation_format(symlink_operation: SymlinkOperation, mock_src: assert symlink_cmd in exec.command assert create_final_dest(mock_src, mock_dest) in exec.command -def test_init_configure_operation(configure_operation: ConfigureOperation, mock_src: str, mock_dest: str): + +def test_init_configure_operation( + configure_operation: ConfigureOperation, mock_src: str, mock_dest: str +): """Validate ConfigureOperation init""" assert isinstance(configure_operation, ConfigureOperation) assert configure_operation.src == mock_src assert configure_operation.dest == mock_dest assert configure_operation.tag == ";" -def test_configure_operation_format(configure_operation: ConfigureOperation, mock_src: str, mock_dest: str, generation_context: GenerationContext): + +def test_configure_operation_format( + configure_operation: ConfigureOperation, + mock_src: str, + mock_dest: str, + generation_context: GenerationContext, +): """Validate ConfigureOperation.format""" exec = configure_operation.format(generation_context) assert isinstance(exec, Command) @@ -164,21 +198,32 @@ def test_configure_operation_format(configure_operation: ConfigureOperation, moc assert configure_cmd in exec.command assert create_final_dest(mock_src, mock_dest) in exec.command + def test_init_file_sys_operation_set(file_system_operation_set: FileSysOperationSet): assert isinstance(file_system_operation_set.operations, list) assert len(file_system_operation_set.operations) == 3 -def test_add_copy_operation(file_system_operation_set: FileSysOperationSet, copy_operation: CopyOperation): + +def test_add_copy_operation( + file_system_operation_set: FileSysOperationSet, copy_operation: CopyOperation +): assert len(file_system_operation_set.copy_operations) == 1 file_system_operation_set.add_copy(copy_operation) assert len(file_system_operation_set.copy_operations) == 2 -def test_add_symlink_operation(file_system_operation_set: FileSysOperationSet, symlink_operation: SymlinkOperation): + +def test_add_symlink_operation( + file_system_operation_set: FileSysOperationSet, symlink_operation: SymlinkOperation +): assert len(file_system_operation_set.symlink_operations) == 1 file_system_operation_set.add_symlink(symlink_operation) assert len(file_system_operation_set.symlink_operations) == 2 -def test_add_configure_operation(file_system_operation_set: FileSysOperationSet, configure_operation: ConfigureOperation): + +def test_add_configure_operation( + file_system_operation_set: FileSysOperationSet, + configure_operation: ConfigureOperation, +): assert len(file_system_operation_set.configure_operations) == 1 file_system_operation_set.add_configuration(configure_operation) assert len(file_system_operation_set.configure_operations) == 2 From c93c85d1205fa441c63b71652daf592845603a2f Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Mon, 30 Sep 2024 17:52:27 -0700 Subject: [PATCH 04/35] saving spot --- smartsim/_core/entrypoints/file_operations.py | 1 + smartsim/_core/generation/generator.py | 90 +++++++------ smartsim/_core/generation/operations.py | 9 +- smartsim/entity/application.py | 31 ++--- tests/test_generator.py | 125 +++++++++++++++++- 5 files changed, 195 insertions(+), 61 deletions(-) diff --git a/smartsim/_core/entrypoints/file_operations.py b/smartsim/_core/entrypoints/file_operations.py index a714eff6a4..9529ffdcd1 100644 --- a/smartsim/_core/entrypoints/file_operations.py +++ b/smartsim/_core/entrypoints/file_operations.py @@ -159,6 +159,7 @@ def copy(parsed_args: argparse.Namespace) -> None: not includedm and the destination file already exists, a FileExistsError will be raised """ + print(parsed_args.source) if os.path.isdir(parsed_args.source): shutil.copytree( parsed_args.source, diff --git a/smartsim/_core/generation/generator.py b/smartsim/_core/generation/generator.py index 6d31fe2ce8..4482a6d9b9 100644 --- a/smartsim/_core/generation/generator.py +++ b/smartsim/_core/generation/generator.py @@ -30,15 +30,16 @@ import pickle import subprocess import sys -import time import typing as t from collections import namedtuple from datetime import datetime from ...entity.files import EntityFiles +from .operations import FileSysOperationSet from ...launchable import Job from ...log import get_logger from ..commands import Command, CommandList +from .operations import GenerationContext, GenerationProtocol, CopyOperation logger = get_logger(__name__) logger.propagate = False @@ -48,7 +49,7 @@ class _GenerableProtocol(t.Protocol): """Ensures functions using job.entity continue if attrs file and params are supported.""" - files: t.Union[EntityFiles, None] + files: FileSysOperationSet file_parameters: t.Mapping[str, str] @@ -201,30 +202,32 @@ def _build_commands( :param job_path: The file path for the Job run folder :return: A CommandList containing the file operation commands """ + context = GenerationContext(job_path) cmd_list = CommandList() cmd_list.commands.append(cls._mkdir_file(job_path)) cmd_list.commands.append(cls._mkdir_file(log_path)) entity = job.entity if isinstance(entity, _GenerableProtocol): - helpers: t.List[ - t.Callable[ - [t.Union[EntityFiles, None], pathlib.Path], - t.Union[CommandList, None], - ] - ] = [ - cls._copy_files, - cls._symlink_files, - lambda files, path: cls._write_tagged_files( - files, entity.file_parameters, path - ), - ] - - for method in helpers: - return_cmd_list = method(entity.files, job_path) - if return_cmd_list: - cmd_list.commands.extend(return_cmd_list.commands) - + ret = cls._copy_files(entity.files.copy_operations, context, job_path) + cmd_list.commands.extend(ret.commands) return cmd_list + # helpers: t.List[ + # t.Callable[ + # [t.Union[EntityFiles, None], pathlib.Path], + # t.Union[CommandList, None], + # ] + # ] = [ + # cls._copy_files, + # cls._symlink_files, + # lambda files, path: cls._write_tagged_files( + # files, entity.file_parameters, path + # ), + # ] + + # for method in helpers: + # return_cmd_list = method(entity.files, job_path) + # if return_cmd_list: + # cmd_list.commands.extend(return_cmd_list.commands) @classmethod def _execute_commands(cls, cmd_list: CommandList) -> None: @@ -245,7 +248,7 @@ def _mkdir_file(file_path: pathlib.Path) -> Command: @staticmethod def _copy_files( - files: t.Union[EntityFiles, None], dest: pathlib.Path + files: t.List[CopyOperation], context: GenerationContext, run_path: pathlib.Path ) -> t.Optional[CommandList]: """Build command to copy files/directories from specified paths to a destination directory. @@ -257,29 +260,32 @@ def _copy_files( :param dest: The destination path to the Job's run directory. :return: A CommandList containing the copy commands, or None if no files are provided. """ - if files is None: - return None cmd_list = CommandList() - for src in files.copy: - cmd = Command( - [ - sys.executable, - "-m", - "smartsim._core.entrypoints.file_operations", - "copy", - src, - ] - ) - destination = str(dest) - if os.path.isdir(src): - base_source_name = os.path.basename(src) - destination = os.path.join(dest, base_source_name) - cmd.append(str(destination)) - cmd.append("--dirs_exist_ok") - else: - cmd.append(str(dest)) - cmd_list.commands.append(cmd) + for file in files: + cmd_list.append(file.format(context)) + print(cmd_list) return cmd_list + # cmd_list = CommandList() + # for src in files.copy: + # cmd = Command( + # [ + # sys.executable, + # "-m", + # "smartsim._core.entrypoints.file_operations", + # "copy", + # src, + # ] + # ) + # destination = str(dest) + # if os.path.isdir(src): + # base_source_name = os.path.basename(src) + # destination = os.path.join(dest, base_source_name) + # cmd.append(str(destination)) + # cmd.append("--dirs_exist_ok") + # else: + # cmd.append(str(dest)) + # cmd_list.commands.append(cmd) + # return cmd_list @staticmethod def _symlink_files( diff --git a/smartsim/_core/generation/operations.py b/smartsim/_core/generation/operations.py index f7319442ba..d0b97ff5c6 100644 --- a/smartsim/_core/generation/operations.py +++ b/smartsim/_core/generation/operations.py @@ -26,10 +26,11 @@ def create_final_dest( """ if dest is None: dest = pathlib.Path("") + # these need to be more descriptive if ( job_root_path is None or job_root_path == pathlib.Path("") - or "" + or isinstance(job_root_path, str) or job_root_path.suffix ): raise ValueError(f"Job root path '{job_root_path}' is not a directory.") @@ -73,6 +74,7 @@ def format(self, context: GenerationContext) -> Command: copy_cmd, str(self.src), final_dest, + "--dirs_exist_ok", ] ) @@ -86,6 +88,11 @@ def __init__(self, src: pathlib.Path, dest: t.Union[pathlib.Path, None]) -> None def format(self, context: GenerationContext) -> Command: """Create Command to invoke symlink fs entry point""" + + # USE CASES + + # if dest is None: src is copied into run + # if dest has val: append dest onto run and do things! final_dest = create_final_dest(context.job_root_path, self.dest) return Command( [ diff --git a/smartsim/entity/application.py b/smartsim/entity/application.py index fb3ed2a7ef..e4e29be7c2 100644 --- a/smartsim/entity/application.py +++ b/smartsim/entity/application.py @@ -36,6 +36,7 @@ from ..log import get_logger from .entity import SmartSimEntity from .files import EntityFiles +from .._core.generation.operations import FileSysOperationSet logger = get_logger(__name__) @@ -88,7 +89,7 @@ def __init__( """The executable to run""" self._exe_args = self._build_exe_args(exe_args) or [] """The executable arguments""" - self._files = copy.deepcopy(files) if files else None + self.files = FileSysOperationSet([]) """Files to be copied, symlinked, and/or configured prior to execution""" self._file_parameters = ( copy.deepcopy(file_parameters) if file_parameters else {} @@ -139,23 +140,23 @@ def add_exe_args(self, args: t.Union[str, t.List[str], None]) -> None: args = self._build_exe_args(args) self._exe_args.extend(args) - @property - def files(self) -> t.Union[EntityFiles, None]: - """Return attached EntityFiles object. + # @property + # def files(self) -> t.Union[EntityFiles, None]: + # """Return attached EntityFiles object. - :return: the EntityFiles object of files to be copied, symlinked, - and/or configured prior to execution - """ - return self._files + # :return: the EntityFiles object of files to be copied, symlinked, + # and/or configured prior to execution + # """ + # return self._files - @files.setter - def files(self, value: t.Optional[EntityFiles]) -> None: - """Set the EntityFiles object. + # @files.setter + # def files(self, value: t.Optional[EntityFiles]) -> None: + # """Set the EntityFiles object. - :param value: the EntityFiles object of files to be copied, symlinked, - and/or configured prior to execution - """ - self._files = copy.deepcopy(value) + # :param value: the EntityFiles object of files to be copied, symlinked, + # and/or configured prior to execution + # """ + # self._files = copy.deepcopy(value) @property def file_parameters(self) -> t.Mapping[str, str]: diff --git a/tests/test_generator.py b/tests/test_generator.py index 4c25ccd05f..ca2a43ea4f 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -43,6 +43,18 @@ from smartsim.launchable import Job from smartsim.settings import LaunchSettings +from smartsim._core.generation.operations import ( + ConfigureOperation, + CopyOperation, + FileSysOperationSet, + GenerationContext, + SymlinkOperation, + configure_cmd, + copy_cmd, + create_final_dest, + symlink_cmd, +) + # TODO Add JobGroup tests when JobGroup becomes a Launchable pytestmark = pytest.mark.group_a @@ -83,20 +95,50 @@ def generator_instance(test_dir: str) -> Generator: def get_gen_file(fileutils, filename: str): return fileutils.get_test_conf_path(osp.join("generator_files", filename)) +@pytest.fixture +def mock_src(test_dir: str): + """Fixture to create a mock source path.""" + return pathlib.Path(test_dir) / pathlib.Path("mock_src") + + +@pytest.fixture +def mock_dest(test_dir: str): + """Fixture to create a mock destination path.""" + return pathlib.Path(test_dir) / pathlib.Path("mock_dest") + +@pytest.fixture +def copy_operation(mock_src: pathlib.Path, mock_dest: pathlib.Path): + """Fixture to create a CopyOperation object.""" + return CopyOperation(src=mock_src, dest=mock_dest) + + +@pytest.fixture +def symlink_operation(mock_src: pathlib.Path, mock_dest: pathlib.Path): + """Fixture to create a CopyOperation object.""" + return SymlinkOperation(src=mock_src, dest=mock_dest) + + +@pytest.fixture +def configure_operation(mock_src: pathlib.Path, mock_dest: pathlib.Path): + """Fixture to create a CopyOperation object.""" + return ConfigureOperation( + src=mock_src, + dest=mock_dest, + ) class EchoHelloWorldEntity(entity.SmartSimEntity): """A simple smartsim entity that meets the `ExecutableProtocol` protocol""" def __init__(self): self.name = "entity_name" - self.files = None + self.files = FileSysOperationSet([]) self.file_parameters = None def as_executable_sequence(self): return ("echo", "Hello", "World!") - def files(): - return ["file_path"] + # def files(): + # return FileSysOperationSet([CopyOperation(src="mock_path", dest="mock_src")]) @pytest.fixture @@ -468,3 +510,80 @@ def _check_generated(param_0, param_1, dir): _check_generated(1, 3, jobs_dir / "ensemble-name-3-3" / Generator.run_directory) _check_generated(0, 2, jobs_dir / "ensemble-name-0-0" / Generator.run_directory) ids.clear() + + + + + +# REDO + +# COPY + +# test making file copies without a dest +def test_1_copy_files(test_dir, fileutils, generator_instance, mock_job: Job, copy_operation, symlink_operation, configure_operation): + correct_path = fileutils.get_test_conf_path( + osp.join("generator_files", "easy", "correct/") + ) + correct_files = sorted(glob(correct_path + "/*")) + for file in correct_files: + mock_job.entity.files.add_copy(src=pathlib.Path(file)) + ret = generator_instance._build_commands(mock_job,pathlib.Path(test_dir), "log_path") + +# test making directory +def test_2_copy_directory(test_dir, fileutils, generator_instance, mock_job: Job, copy_operation, symlink_operation, configure_operation): + correct_path = fileutils.get_test_conf_path( + osp.join("generator_files", "easy", "correct/") + ) + mock_job.entity.files.add_copy(src=pathlib.Path(correct_path)) + ret = generator_instance._build_commands(mock_job,pathlib.Path(test_dir), "log_path") + print(ret) + +# test making file copies without a dest +def test_3_copy_files(test_dir, fileutils, generator_instance, mock_job: Job, copy_operation, symlink_operation, configure_operation): + correct_path = fileutils.get_test_conf_path( + osp.join("generator_files", "easy", "correct/") + ) + correct_files = sorted(glob(correct_path + "/*")) + for file in correct_files: + mock_job.entity.files.add_copy(src=pathlib.Path(file), dest="mock") + ret = generator_instance._build_commands(mock_job,pathlib.Path(test_dir), "log_path") + print(ret) + +# test making file copies without a dest +def test_4_copy_directory(test_dir, fileutils, generator_instance, mock_job: Job, copy_operation, symlink_operation, configure_operation): + correct_path = fileutils.get_test_conf_path( + osp.join("generator_files", "easy", "correct/") + ) + mock_job.entity.files.add_copy(src=pathlib.Path(correct_path), dest="mock") + ret = generator_instance._build_commands(mock_job,pathlib.Path(test_dir), "log_path") + print(ret) + +def test_1_execute_the_copy(test_dir, fileutils, mock_job,generator_instance): + correct_path = fileutils.get_test_conf_path( + osp.join("generator_files", "easy", "correct/") + ) + correct_files = sorted(glob(correct_path + "/*")) + for file in correct_files: + mock_job.entity.files.add_copy(src=pathlib.Path(file)) + ret = generator_instance._build_commands(mock_job,pathlib.Path(test_dir), "log_path") + generator_instance._execute_commands(ret) + print(listdir(test_dir)) + assert pathlib.Path(test_dir).is_dir() + for file in correct_files: + assert pathlib.Path(file).name in listdir(test_dir) + +def test_2_execute_the_copy(test_dir, fileutils, mock_job,generator_instance): + correct_path = fileutils.get_test_conf_path( + osp.join("generator_files", "easy", "correct/") + ) + mock_job.entity.files.add_copy(src=pathlib.Path(correct_path), dest=pathlib.Path("mock")) + ret = generator_instance._build_commands(mock_job,pathlib.Path(test_dir), "log_path") + generator_instance._execute_commands(ret) + path = pathlib.Path(test_dir) / pathlib.Path("mock") + pathlib.Path(path).is_dir() + correct_files = sorted(glob(correct_path + "/*")) + for file in correct_files: + assert pathlib.Path(file).name in listdir(pathlib.Path(test_dir) / pathlib.Path("mock")) + + +# SYMLINK \ No newline at end of file From 3866bbda9f000f217a43765f7d8e9e6310d0bc08 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Mon, 30 Sep 2024 18:27:25 -0700 Subject: [PATCH 05/35] very ugly testing --- smartsim/_core/generation/generator.py | 27 +- smartsim/_core/generation/operations.py | 13 +- tests/test_generator.py | 455 +++++++++++++----------- 3 files changed, 254 insertions(+), 241 deletions(-) diff --git a/smartsim/_core/generation/generator.py b/smartsim/_core/generation/generator.py index 4482a6d9b9..fbc0f48b07 100644 --- a/smartsim/_core/generation/generator.py +++ b/smartsim/_core/generation/generator.py @@ -39,7 +39,7 @@ from ...launchable import Job from ...log import get_logger from ..commands import Command, CommandList -from .operations import GenerationContext, GenerationProtocol, CopyOperation +from .operations import GenerationContext, GenerationProtocol, CopyOperation, SymlinkOperation logger = get_logger(__name__) logger.propagate = False @@ -210,6 +210,8 @@ def _build_commands( if isinstance(entity, _GenerableProtocol): ret = cls._copy_files(entity.files.copy_operations, context, job_path) cmd_list.commands.extend(ret.commands) + ret = cls._symlink_files(entity.files.symlink_operations, context, job_path) + cmd_list.commands.extend(ret.commands) return cmd_list # helpers: t.List[ # t.Callable[ @@ -289,7 +291,7 @@ def _copy_files( @staticmethod def _symlink_files( - files: t.Union[EntityFiles, None], dest: pathlib.Path + files: t.List[CopyOperation], context: GenerationContext, run_path: pathlib.Path ) -> t.Optional[CommandList]: """Build command to symlink files/directories from specified paths to a destination directory. @@ -301,26 +303,9 @@ def _symlink_files( :param dest: The destination path to the Job's run directory. :return: A CommandList containing the symlink commands, or None if no files are provided. """ - if files is None: - return None cmd_list = CommandList() - for src in files.link: - # Normalize the path to remove trailing slashes - normalized_path = os.path.normpath(src) - # Get the parent directory (last folder) - parent_dir = os.path.basename(normalized_path) - new_dest = os.path.join(str(dest), parent_dir) - cmd = Command( - [ - sys.executable, - "-m", - "smartsim._core.entrypoints.file_operations", - "symlink", - src, - new_dest, - ] - ) - cmd_list.append(cmd) + for src in files: + cmd_list.append(src.format(context)) return cmd_list @staticmethod diff --git a/smartsim/_core/generation/operations.py b/smartsim/_core/generation/operations.py index d0b97ff5c6..3b3fb19f9c 100644 --- a/smartsim/_core/generation/operations.py +++ b/smartsim/_core/generation/operations.py @@ -1,6 +1,7 @@ import pathlib import sys import typing as t +import os from dataclasses import dataclass, field from ..commands import Command @@ -88,12 +89,12 @@ def __init__(self, src: pathlib.Path, dest: t.Union[pathlib.Path, None]) -> None def format(self, context: GenerationContext) -> Command: """Create Command to invoke symlink fs entry point""" - - # USE CASES - - # if dest is None: src is copied into run - # if dest has val: append dest onto run and do things! + normalized_path = os.path.normpath(self.src) + # # Get the parent directory (last folder) + parent_dir = os.path.basename(normalized_path) final_dest = create_final_dest(context.job_root_path, self.dest) + new_dest = os.path.join(final_dest, parent_dir) + print(f"here: {new_dest}") return Command( [ sys.executable, @@ -101,7 +102,7 @@ def format(self, context: GenerationContext) -> Command: entry_point_path, symlink_cmd, str(self.src), - final_dest, + new_dest, ] ) diff --git a/tests/test_generator.py b/tests/test_generator.py index ca2a43ea4f..6dca19e5b0 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -297,219 +297,219 @@ def test_mkdir_file(generator_instance: Generator, test_dir: str): assert cmd.command == ["mkdir", "-p", test_dir] -def test_copy_file(generator_instance: Generator, fileutils): - """Test Generator._copy_files helper function with file""" - script = fileutils.get_test_conf_path("sleep.py") - files = EntityFiles(copy=script) - cmd_list = generator_instance._copy_files(files, generator_instance.root) - assert isinstance(cmd_list, CommandList) - assert len(cmd_list) == 1 - assert str(generator_instance.root) and script in cmd_list.commands[0].command - - -def test_copy_directory(get_gen_copy_dir, generator_instance: Generator): - """Test Generator._copy_files helper function with directory""" - files = EntityFiles(copy=get_gen_copy_dir) - cmd_list = generator_instance._copy_files(files, generator_instance.root) - assert isinstance(cmd_list, CommandList) - assert len(cmd_list) == 1 - assert ( - str(generator_instance.root) - and get_gen_copy_dir in cmd_list.commands[0].command - ) - - -def test_symlink_file(get_gen_symlink_dir, generator_instance: Generator): - """Test Generator._symlink_files helper function with file list""" - symlink_files = sorted(glob(get_gen_symlink_dir + "/*")) - files = EntityFiles(symlink=symlink_files) - cmd_list = generator_instance._symlink_files(files, generator_instance.root) - assert isinstance(cmd_list, CommandList) - for file, cmd in zip(symlink_files, cmd_list): - assert file in cmd.command - - -def test_symlink_directory(generator_instance: Generator, get_gen_symlink_dir): - """Test Generator._symlink_files helper function with directory""" - files = EntityFiles(symlink=get_gen_symlink_dir) - cmd_list = generator_instance._symlink_files(files, generator_instance.root) - symlinked_folder = generator_instance.root / os.path.basename(get_gen_symlink_dir) - assert isinstance(cmd_list, CommandList) - assert str(symlinked_folder) in cmd_list.commands[0].command - - -def test_write_tagged_file(fileutils, generator_instance: Generator): - """Test Generator._write_tagged_files helper function with file list""" - conf_path = fileutils.get_test_conf_path( - osp.join("generator_files", "easy", "marked/") - ) - tagged_files = sorted(glob(conf_path + "/*")) - files = EntityFiles(tagged=tagged_files) - param_set = { - "5": 10, - "FIRST": "SECOND", - "17": 20, - "65": "70", - "placeholder": "group leftupper region", - "1200": "120", - "VALID": "valid", - } - cmd_list = generator_instance._write_tagged_files( - files=files, params=param_set, dest=generator_instance.root - ) - assert isinstance(cmd_list, CommandList) - for file, cmd in zip(tagged_files, cmd_list): - assert file in cmd.command - - -def test_write_tagged_directory(fileutils, generator_instance: Generator): - """Test Generator._write_tagged_files helper function with directory path""" - config = get_gen_file(fileutils, "tag_dir_template") - files = EntityFiles(tagged=[config]) - param_set = {"PARAM0": "param_value_1", "PARAM1": "param_value_2"} - cmd_list = generator_instance._write_tagged_files( - files=files, params=param_set, dest=generator_instance.root - ) - - assert isinstance(cmd_list, CommandList) - assert str(config) in cmd_list.commands[0].command - - -# INTEGRATED TESTS - - -def test_exp_private_generate_method( - mock_job: unittest.mock.MagicMock, test_dir: str, generator_instance: Generator -): - """Test that Experiment._generate returns expected tuple.""" - mock_index = 1 - exp = Experiment(name="experiment_name", exp_path=test_dir) - job_paths = exp._generate(generator_instance, mock_job, mock_index) - assert osp.isdir(job_paths.run_path) - assert job_paths.out_path.name == f"{mock_job.entity.name}.out" - assert job_paths.err_path.name == f"{mock_job.entity.name}.err" - - -def test_generate_ensemble_directory_start( - test_dir: str, wlmutils, monkeypatch: pytest.MonkeyPatch -): - """Test that Experiment._generate returns expected tuple.""" - monkeypatch.setattr( - "smartsim._core.dispatch._LauncherAdapter.start", - lambda launch, exe, job_execution_path, env, out, err: random_id(), - ) - ensemble = Ensemble("ensemble-name", "echo", replicas=2) - launch_settings = LaunchSettings(wlmutils.get_test_launcher()) - job_list = ensemble.build_jobs(launch_settings) - exp = Experiment(name="exp_name", exp_path=test_dir) - exp.start(*job_list) - run_dir = listdir(test_dir) - jobs_dir_path = pathlib.Path(test_dir) / run_dir[0] / "jobs" - list_of_job_dirs = jobs_dir_path.iterdir() - for job in list_of_job_dirs: - run_path = jobs_dir_path / job / Generator.run_directory - assert run_path.is_dir() - log_path = jobs_dir_path / job / Generator.log_directory - assert log_path.is_dir() - ids.clear() - - -def test_generate_ensemble_copy( - test_dir: str, wlmutils, monkeypatch: pytest.MonkeyPatch, get_gen_copy_dir -): - monkeypatch.setattr( - "smartsim._core.dispatch._LauncherAdapter.start", - lambda launch, exe, job_execution_path, env, out, err: random_id(), - ) - ensemble = Ensemble( - "ensemble-name", "echo", replicas=2, files=EntityFiles(copy=get_gen_copy_dir) - ) - launch_settings = LaunchSettings(wlmutils.get_test_launcher()) - job_list = ensemble.build_jobs(launch_settings) - exp = Experiment(name="exp_name", exp_path=test_dir) - exp.start(*job_list) - run_dir = listdir(test_dir) - jobs_dir = pathlib.Path(test_dir) / run_dir[0] / "jobs" - job_dir = jobs_dir.iterdir() - for ensemble_dir in job_dir: - copy_folder_path = ( - jobs_dir / ensemble_dir / Generator.run_directory / "to_copy_dir" - ) - assert copy_folder_path.is_dir() - ids.clear() - - -def test_generate_ensemble_symlink( - test_dir: str, wlmutils, monkeypatch: pytest.MonkeyPatch, get_gen_symlink_dir -): - monkeypatch.setattr( - "smartsim._core.dispatch._LauncherAdapter.start", - lambda launch, exe, job_execution_path, env, out, err: random_id(), - ) - ensemble = Ensemble( - "ensemble-name", - "echo", - replicas=2, - files=EntityFiles(symlink=get_gen_symlink_dir), - ) - launch_settings = LaunchSettings(wlmutils.get_test_launcher()) - job_list = ensemble.build_jobs(launch_settings) - exp = Experiment(name="exp_name", exp_path=test_dir) - _ = exp.start(*job_list) - run_dir = listdir(test_dir) - jobs_dir = pathlib.Path(test_dir) / run_dir[0] / "jobs" - job_dir = jobs_dir.iterdir() - for ensemble_dir in job_dir: - sym_file_path = pathlib.Path(jobs_dir) / ensemble_dir / "run" / "to_symlink_dir" - assert sym_file_path.is_dir() - assert sym_file_path.is_symlink() - assert os.fspath(sym_file_path.resolve()) == osp.realpath(get_gen_symlink_dir) - ids.clear() - - -def test_generate_ensemble_configure( - test_dir: str, wlmutils, monkeypatch: pytest.MonkeyPatch, get_gen_configure_dir -): - monkeypatch.setattr( - "smartsim._core.dispatch._LauncherAdapter.start", - lambda launch, exe, job_execution_path, env, out, err: random_id(), - ) - param_set = {"PARAM0": [0, 1], "PARAM1": [2, 3]} - tagged_files = sorted(glob(get_gen_configure_dir + "/*")) - ensemble = Ensemble( - "ensemble-name", - "echo", - replicas=1, - files=EntityFiles(tagged=tagged_files), - file_parameters=param_set, - ) - launch_settings = LaunchSettings(wlmutils.get_test_launcher()) - job_list = ensemble.build_jobs(launch_settings) - exp = Experiment(name="exp_name", exp_path=test_dir) - _ = exp.start(*job_list) - run_dir = listdir(test_dir) - jobs_dir = pathlib.Path(test_dir) / run_dir[0] / "jobs" - - def _check_generated(param_0, param_1, dir): - assert dir.is_dir() - tagged_0 = dir / "tagged_0.sh" - tagged_1 = dir / "tagged_1.sh" - assert tagged_0.is_file() - assert tagged_1.is_file() - - with open(tagged_0) as f: - line = f.readline() - assert line.strip() == f'echo "Hello with parameter 0 = {param_0}"' - - with open(tagged_1) as f: - line = f.readline() - assert line.strip() == f'echo "Hello with parameter 1 = {param_1}"' - - _check_generated(0, 3, jobs_dir / "ensemble-name-1-1" / Generator.run_directory) - _check_generated(1, 2, jobs_dir / "ensemble-name-2-2" / Generator.run_directory) - _check_generated(1, 3, jobs_dir / "ensemble-name-3-3" / Generator.run_directory) - _check_generated(0, 2, jobs_dir / "ensemble-name-0-0" / Generator.run_directory) - ids.clear() +# def test_copy_file(generator_instance: Generator, fileutils): +# """Test Generator._copy_files helper function with file""" +# script = fileutils.get_test_conf_path("sleep.py") +# files = EntityFiles(copy=script) +# cmd_list = generator_instance._copy_files(files, generator_instance.root) +# assert isinstance(cmd_list, CommandList) +# assert len(cmd_list) == 1 +# assert str(generator_instance.root) and script in cmd_list.commands[0].command + + +# def test_copy_directory(get_gen_copy_dir, generator_instance: Generator): +# """Test Generator._copy_files helper function with directory""" +# files = EntityFiles(copy=get_gen_copy_dir) +# cmd_list = generator_instance._copy_files(files, generator_instance.root) +# assert isinstance(cmd_list, CommandList) +# assert len(cmd_list) == 1 +# assert ( +# str(generator_instance.root) +# and get_gen_copy_dir in cmd_list.commands[0].command +# ) + + +# def test_symlink_file(get_gen_symlink_dir, generator_instance: Generator): +# """Test Generator._symlink_files helper function with file list""" +# symlink_files = sorted(glob(get_gen_symlink_dir + "/*")) +# files = EntityFiles(symlink=symlink_files) +# cmd_list = generator_instance._symlink_files(files, generator_instance.root) +# assert isinstance(cmd_list, CommandList) +# for file, cmd in zip(symlink_files, cmd_list): +# assert file in cmd.command + + +# def test_symlink_directory(generator_instance: Generator, get_gen_symlink_dir): +# """Test Generator._symlink_files helper function with directory""" +# files = EntityFiles(symlink=get_gen_symlink_dir) +# cmd_list = generator_instance._symlink_files(files, generator_instance.root) +# symlinked_folder = generator_instance.root / os.path.basename(get_gen_symlink_dir) +# assert isinstance(cmd_list, CommandList) +# assert str(symlinked_folder) in cmd_list.commands[0].command + + +# def test_write_tagged_file(fileutils, generator_instance: Generator): +# """Test Generator._write_tagged_files helper function with file list""" +# conf_path = fileutils.get_test_conf_path( +# osp.join("generator_files", "easy", "marked/") +# ) +# tagged_files = sorted(glob(conf_path + "/*")) +# files = EntityFiles(tagged=tagged_files) +# param_set = { +# "5": 10, +# "FIRST": "SECOND", +# "17": 20, +# "65": "70", +# "placeholder": "group leftupper region", +# "1200": "120", +# "VALID": "valid", +# } +# cmd_list = generator_instance._write_tagged_files( +# files=files, params=param_set, dest=generator_instance.root +# ) +# assert isinstance(cmd_list, CommandList) +# for file, cmd in zip(tagged_files, cmd_list): +# assert file in cmd.command + + +# def test_write_tagged_directory(fileutils, generator_instance: Generator): +# """Test Generator._write_tagged_files helper function with directory path""" +# config = get_gen_file(fileutils, "tag_dir_template") +# files = EntityFiles(tagged=[config]) +# param_set = {"PARAM0": "param_value_1", "PARAM1": "param_value_2"} +# cmd_list = generator_instance._write_tagged_files( +# files=files, params=param_set, dest=generator_instance.root +# ) + +# assert isinstance(cmd_list, CommandList) +# assert str(config) in cmd_list.commands[0].command + + +# # INTEGRATED TESTS + + +# def test_exp_private_generate_method( +# mock_job: unittest.mock.MagicMock, test_dir: str, generator_instance: Generator +# ): +# """Test that Experiment._generate returns expected tuple.""" +# mock_index = 1 +# exp = Experiment(name="experiment_name", exp_path=test_dir) +# job_paths = exp._generate(generator_instance, mock_job, mock_index) +# assert osp.isdir(job_paths.run_path) +# assert job_paths.out_path.name == f"{mock_job.entity.name}.out" +# assert job_paths.err_path.name == f"{mock_job.entity.name}.err" + + +# def test_generate_ensemble_directory_start( +# test_dir: str, wlmutils, monkeypatch: pytest.MonkeyPatch +# ): +# """Test that Experiment._generate returns expected tuple.""" +# monkeypatch.setattr( +# "smartsim._core.dispatch._LauncherAdapter.start", +# lambda launch, exe, job_execution_path, env, out, err: random_id(), +# ) +# ensemble = Ensemble("ensemble-name", "echo", replicas=2) +# launch_settings = LaunchSettings(wlmutils.get_test_launcher()) +# job_list = ensemble.build_jobs(launch_settings) +# exp = Experiment(name="exp_name", exp_path=test_dir) +# exp.start(*job_list) +# run_dir = listdir(test_dir) +# jobs_dir_path = pathlib.Path(test_dir) / run_dir[0] / "jobs" +# list_of_job_dirs = jobs_dir_path.iterdir() +# for job in list_of_job_dirs: +# run_path = jobs_dir_path / job / Generator.run_directory +# assert run_path.is_dir() +# log_path = jobs_dir_path / job / Generator.log_directory +# assert log_path.is_dir() +# ids.clear() + + +# def test_generate_ensemble_copy( +# test_dir: str, wlmutils, monkeypatch: pytest.MonkeyPatch, get_gen_copy_dir +# ): +# monkeypatch.setattr( +# "smartsim._core.dispatch._LauncherAdapter.start", +# lambda launch, exe, job_execution_path, env, out, err: random_id(), +# ) +# ensemble = Ensemble( +# "ensemble-name", "echo", replicas=2, files=EntityFiles(copy=get_gen_copy_dir) +# ) +# launch_settings = LaunchSettings(wlmutils.get_test_launcher()) +# job_list = ensemble.build_jobs(launch_settings) +# exp = Experiment(name="exp_name", exp_path=test_dir) +# exp.start(*job_list) +# run_dir = listdir(test_dir) +# jobs_dir = pathlib.Path(test_dir) / run_dir[0] / "jobs" +# job_dir = jobs_dir.iterdir() +# for ensemble_dir in job_dir: +# copy_folder_path = ( +# jobs_dir / ensemble_dir / Generator.run_directory / "to_copy_dir" +# ) +# assert copy_folder_path.is_dir() +# ids.clear() + + +# def test_generate_ensemble_symlink( +# test_dir: str, wlmutils, monkeypatch: pytest.MonkeyPatch, get_gen_symlink_dir +# ): +# monkeypatch.setattr( +# "smartsim._core.dispatch._LauncherAdapter.start", +# lambda launch, exe, job_execution_path, env, out, err: random_id(), +# ) +# ensemble = Ensemble( +# "ensemble-name", +# "echo", +# replicas=2, +# files=EntityFiles(symlink=get_gen_symlink_dir), +# ) +# launch_settings = LaunchSettings(wlmutils.get_test_launcher()) +# job_list = ensemble.build_jobs(launch_settings) +# exp = Experiment(name="exp_name", exp_path=test_dir) +# _ = exp.start(*job_list) +# run_dir = listdir(test_dir) +# jobs_dir = pathlib.Path(test_dir) / run_dir[0] / "jobs" +# job_dir = jobs_dir.iterdir() +# for ensemble_dir in job_dir: +# sym_file_path = pathlib.Path(jobs_dir) / ensemble_dir / "run" / "to_symlink_dir" +# assert sym_file_path.is_dir() +# assert sym_file_path.is_symlink() +# assert os.fspath(sym_file_path.resolve()) == osp.realpath(get_gen_symlink_dir) +# ids.clear() + + +# def test_generate_ensemble_configure( +# test_dir: str, wlmutils, monkeypatch: pytest.MonkeyPatch, get_gen_configure_dir +# ): +# monkeypatch.setattr( +# "smartsim._core.dispatch._LauncherAdapter.start", +# lambda launch, exe, job_execution_path, env, out, err: random_id(), +# ) +# param_set = {"PARAM0": [0, 1], "PARAM1": [2, 3]} +# tagged_files = sorted(glob(get_gen_configure_dir + "/*")) +# ensemble = Ensemble( +# "ensemble-name", +# "echo", +# replicas=1, +# files=EntityFiles(tagged=tagged_files), +# file_parameters=param_set, +# ) +# launch_settings = LaunchSettings(wlmutils.get_test_launcher()) +# job_list = ensemble.build_jobs(launch_settings) +# exp = Experiment(name="exp_name", exp_path=test_dir) +# _ = exp.start(*job_list) +# run_dir = listdir(test_dir) +# jobs_dir = pathlib.Path(test_dir) / run_dir[0] / "jobs" + +# def _check_generated(param_0, param_1, dir): +# assert dir.is_dir() +# tagged_0 = dir / "tagged_0.sh" +# tagged_1 = dir / "tagged_1.sh" +# assert tagged_0.is_file() +# assert tagged_1.is_file() + +# with open(tagged_0) as f: +# line = f.readline() +# assert line.strip() == f'echo "Hello with parameter 0 = {param_0}"' + +# with open(tagged_1) as f: +# line = f.readline() +# assert line.strip() == f'echo "Hello with parameter 1 = {param_1}"' + +# _check_generated(0, 3, jobs_dir / "ensemble-name-1-1" / Generator.run_directory) +# _check_generated(1, 2, jobs_dir / "ensemble-name-2-2" / Generator.run_directory) +# _check_generated(1, 3, jobs_dir / "ensemble-name-3-3" / Generator.run_directory) +# _check_generated(0, 2, jobs_dir / "ensemble-name-0-0" / Generator.run_directory) +# ids.clear() @@ -586,4 +586,31 @@ def test_2_execute_the_copy(test_dir, fileutils, mock_job,generator_instance): assert pathlib.Path(file).name in listdir(pathlib.Path(test_dir) / pathlib.Path("mock")) -# SYMLINK \ No newline at end of file +# SYMLINK +def test_1_symlink_files(test_dir, fileutils, generator_instance, mock_job: Job, copy_operation, symlink_operation, configure_operation): + symlink_dir = get_gen_file(fileutils, "to_symlink_dir") + correct_files = sorted(glob(symlink_dir + "/*")) + for file in correct_files: + mock_job.entity.files.add_symlink(src=pathlib.Path(file)) + ret = generator_instance._build_commands(mock_job,pathlib.Path(test_dir), "log_path") + +def test_2_symlink_directory(test_dir, fileutils, generator_instance, mock_job: Job, copy_operation, symlink_operation, configure_operation): + symlink_dir = get_gen_file(fileutils, "to_symlink_dir") + mock_job.entity.files.add_symlink(src=pathlib.Path(symlink_dir)) + ret = generator_instance._build_commands(mock_job,pathlib.Path(test_dir), "log_path") + print(ret) + +def test_3_symlink_files(test_dir, fileutils, generator_instance, mock_job: Job, copy_operation, symlink_operation, configure_operation): + symlink_dir = get_gen_file(fileutils, "to_symlink_dir") + correct_files = sorted(glob(symlink_dir + "/*")) + for file in correct_files: + mock_job.entity.files.add_symlink(src=pathlib.Path(file)) + ret = generator_instance._build_commands(mock_job,pathlib.Path(test_dir), "log_path") + print(ret) + generator_instance._execute_commands(ret) + +def test_4_symlink_directory(test_dir, fileutils, generator_instance, mock_job: Job, copy_operation, symlink_operation, configure_operation): + symlink_dir = get_gen_file(fileutils, "to_symlink_dir") + mock_job.entity.files.add_symlink(src=pathlib.Path(symlink_dir)) + ret = generator_instance._build_commands(mock_job,pathlib.Path(test_dir), "log_path") + generator_instance._execute_commands(ret) \ No newline at end of file From b2c54215f23cd3ea446e05d1cf1414fe8a0ef66a Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Thu, 3 Oct 2024 14:04:37 -0700 Subject: [PATCH 06/35] adding configuration tests --- smartsim/_core/generation/generator.py | 217 ++++---- smartsim/_core/generation/operations.py | 39 +- smartsim/entity/application.py | 94 +--- smartsim/entity/files.py | 1 + tests/test_generator.py | 675 +++++++++++------------- 5 files changed, 425 insertions(+), 601 deletions(-) diff --git a/smartsim/_core/generation/generator.py b/smartsim/_core/generation/generator.py index fbc0f48b07..6287b7b7e3 100644 --- a/smartsim/_core/generation/generator.py +++ b/smartsim/_core/generation/generator.py @@ -35,11 +35,18 @@ from datetime import datetime from ...entity.files import EntityFiles -from .operations import FileSysOperationSet +from ...entity import entity from ...launchable import Job from ...log import get_logger from ..commands import Command, CommandList -from .operations import GenerationContext, GenerationProtocol, CopyOperation, SymlinkOperation +from .operations import ( + CopyOperation, + FileSysOperationSet, + GenerationContext, + ConfigureOperation, + GenerationProtocol, + SymlinkOperation, +) logger = get_logger(__name__) logger.propagate = False @@ -47,41 +54,43 @@ @t.runtime_checkable class _GenerableProtocol(t.Protocol): - """Ensures functions using job.entity continue if attrs file and params are supported.""" + """Ensures functions using job.entity proceed if both attribute files and + parameters are supported.""" files: FileSysOperationSet + # TODO might need to review if file_params is taken off file_parameters: t.Mapping[str, str] Job_Path = namedtuple("Job_Path", ["run_path", "out_path", "err_path"]) -"""Paths related to the Job's execution.""" +"""Stores the Job's run path, output path, and error file path.""" class Generator: - """The primary responsibility of the Generator class is to create the directory structure - for a SmartSim Job and to build and execute file operation commands.""" + """ The Generator class creates the directory structure for a SmartSim Job by building + and executing file operation commands. + """ run_directory = "run" - """The name of the directory where run-related files are stored.""" + """The name of the directory storing run-related files.""" log_directory = "log" - """The name of the directory where log files are stored.""" + """The name of the directory storing log-related files.""" def __init__(self, root: pathlib.Path) -> None: """Initialize a Generator object - The Generator class constructs a Job's directory structure, including: + The Generator class is responsible for constructing a Job's directory, including + the following tasks: - - The run and log directories - - Output and error files - - The "smartsim_params.txt" settings file - - Additionally, it manages symlinking, copying, and configuring files associated - with a Job's entity. + - Creating the run and log directories + - Generating the output and error files + - Building the parameter settings file + - Managing symlinking, copying, and configuration of attached files :param root: Job base path """ self.root = root - """The root path under which to generate files""" + """The root directory under which all generated files and directories will be placed.""" def _build_job_base_path(self, job: Job, job_index: int) -> pathlib.Path: """Build and return a Job's base directory. The path is created by combining the @@ -175,7 +184,7 @@ def generate_job(self, job: Job, job_index: int) -> Job_Path: out_file = self._build_out_file_path(log_path, job.entity.name) err_file = self._build_err_file_path(log_path, job.entity.name) - cmd_list = self._build_commands(job, job_path, log_path) + cmd_list = self._build_commands(job.entity, job_path, log_path) self._execute_commands(cmd_list) @@ -189,7 +198,7 @@ def generate_job(self, job: Job, job_index: int) -> Job_Path: @classmethod def _build_commands( - cls, job: Job, job_path: pathlib.Path, log_path: pathlib.Path + cls, entity: entity.SmartSimEntity, job_path: pathlib.Path, log_path: pathlib.Path ) -> CommandList: """Build file operation commands for a Job's entity. @@ -200,36 +209,47 @@ def _build_commands( :param job: Job object :param job_path: The file path for the Job run folder + :param log_path: The file path for the Job log folder :return: A CommandList containing the file operation commands """ context = GenerationContext(job_path) cmd_list = CommandList() - cmd_list.commands.append(cls._mkdir_file(job_path)) - cmd_list.commands.append(cls._mkdir_file(log_path)) - entity = job.entity + + cls._append_mkdir_commands(cmd_list, job_path, log_path) + if isinstance(entity, _GenerableProtocol): - ret = cls._copy_files(entity.files.copy_operations, context, job_path) - cmd_list.commands.extend(ret.commands) - ret = cls._symlink_files(entity.files.symlink_operations, context, job_path) - cmd_list.commands.extend(ret.commands) + cls._append_file_operations(cmd_list, entity, context) + return cmd_list - # helpers: t.List[ - # t.Callable[ - # [t.Union[EntityFiles, None], pathlib.Path], - # t.Union[CommandList, None], - # ] - # ] = [ - # cls._copy_files, - # cls._symlink_files, - # lambda files, path: cls._write_tagged_files( - # files, entity.file_parameters, path - # ), - # ] - - # for method in helpers: - # return_cmd_list = method(entity.files, job_path) - # if return_cmd_list: - # cmd_list.commands.extend(return_cmd_list.commands) + + @classmethod + def _append_mkdir_commands(cls, cmd_list: CommandList, job_path: pathlib.Path, log_path: pathlib.Path) -> None: + """Append file operation Commands (mkdir) for a Job's run and log directory. + + :param cmd_list: A CommandList object containing the commands to be executed + :param job_path: The file path for the Job run folder + :param log_path: The file path for the Job log folder + """ + cmd_list.commands.append(cls._mkdir_file(job_path)) + cmd_list.commands.append(cls._mkdir_file(log_path)) + + @classmethod + def _append_file_operations(cls, cmd_list: CommandList, entity: _GenerableProtocol, context: GenerationContext) -> None: + """Append file operation Commands (copy, symlink, configure) for all + files attached to the entity. + + :param cmd_list: A CommandList object containing the commands to be executed + :param entity: The Job's attached entity + :param context: A GenerationContext object that holds the Job's run directory + """ + copy_ret = cls._copy_files(entity.files.copy_operations, context) + cmd_list.commands.extend(copy_ret.commands) + + symlink_ret = cls._symlink_files(entity.files.symlink_operations, context) + cmd_list.commands.extend(symlink_ret.commands) + + configure_ret = cls._configure_files(entity.files.configure_operations, context) + cmd_list.commands.extend(configure_ret.commands) @classmethod def _execute_commands(cls, cmd_list: CommandList) -> None: @@ -245,105 +265,60 @@ def _execute_commands(cls, cmd_list: CommandList) -> None: @staticmethod def _mkdir_file(file_path: pathlib.Path) -> Command: + """Build a Command to create a directory, including any + necessary parent directories. + + :param file_path: The directory path to be created + :return: A Command object to execute the directory creation + """ cmd = Command(["mkdir", "-p", str(file_path)]) return cmd @staticmethod def _copy_files( - files: t.List[CopyOperation], context: GenerationContext, run_path: pathlib.Path - ) -> t.Optional[CommandList]: - """Build command to copy files/directories from specified paths to a destination directory. - - This method creates commands to copy files/directories from the source paths provided in the - `files` parameter to the specified destination directory. If the source is a directory, - it copies the directory while allowing existing directories to remain intact. + files: t.List[CopyOperation], context: GenerationContext + ) -> CommandList: + """Build commands to copy files/directories from specified source paths + to an optional destination in the run directory. - :param files: An EntityFiles object containing the paths to copy, or None. - :param dest: The destination path to the Job's run directory. - :return: A CommandList containing the copy commands, or None if no files are provided. + :param files: A list of CopyOperation objects + :param context: A GenerationContext object that holds the Job's run directory + :return: A CommandList containing the copy commands """ cmd_list = CommandList() for file in files: cmd_list.append(file.format(context)) - print(cmd_list) return cmd_list - # cmd_list = CommandList() - # for src in files.copy: - # cmd = Command( - # [ - # sys.executable, - # "-m", - # "smartsim._core.entrypoints.file_operations", - # "copy", - # src, - # ] - # ) - # destination = str(dest) - # if os.path.isdir(src): - # base_source_name = os.path.basename(src) - # destination = os.path.join(dest, base_source_name) - # cmd.append(str(destination)) - # cmd.append("--dirs_exist_ok") - # else: - # cmd.append(str(dest)) - # cmd_list.commands.append(cmd) - # return cmd_list @staticmethod def _symlink_files( - files: t.List[CopyOperation], context: GenerationContext, run_path: pathlib.Path - ) -> t.Optional[CommandList]: - """Build command to symlink files/directories from specified paths to a destination directory. - - This method creates commands to symlink files/directories from the source paths provided in the - `files` parameter to the specified destination directory. If the source is a directory, - it copies the directory while allowing existing directories to remain intact. + files: t.List[SymlinkOperation], context: GenerationContext + ) -> CommandList: + """Build commands to symlink files/directories from specified source paths + to an optional destination in the run directory. - :param files: An EntityFiles object containing the paths to symlink, or None. - :param dest: The destination path to the Job's run directory. - :return: A CommandList containing the symlink commands, or None if no files are provided. + :param files: A list of SymlinkOperation objects + :param context: A GenerationContext object that holds the Job's run directory + :return: A CommandList containing the symlink commands """ cmd_list = CommandList() - for src in files: - cmd_list.append(src.format(context)) + for file in files: + cmd_list.append(file.format(context)) return cmd_list @staticmethod - def _write_tagged_files( - files: t.Union[EntityFiles, None], - params: t.Mapping[str, str], - dest: pathlib.Path, - ) -> t.Optional[CommandList]: - """Build command to configure files/directories from specified paths to a destination directory. - - This method processes tagged files by reading their configurations, - serializing the provided parameters, and generating commands to - write these configurations to the destination directory. - - :param files: An EntityFiles object containing the paths to configure, or None. - :param params: A dictionary of params - :param dest: The destination path to the Job's run directory. - :return: A CommandList containing the configuration commands, or None if no files are provided. + def _configure_files( + files: t.List[ConfigureOperation], + context: GenerationContext, + ) -> CommandList: + """Build commands to configure files/directories from specified source paths + to an optional destination in the run directory. + + :param files: A list of ConfigurationOperation objects + :param context: A GenerationContext object that holds the Job's run directory + :return: A CommandList containing the configuration commands """ - if files is None: - return None cmd_list = CommandList() - if files.tagged: - tag_delimiter = ";" - pickled_dict = pickle.dumps(params) - encoded_dict = base64.b64encode(pickled_dict).decode("ascii") - for path in files.tagged: - cmd = Command( - [ - sys.executable, - "-m", - "smartsim._core.entrypoints.file_operations", - "configure", - path, - str(dest), - tag_delimiter, - encoded_dict, - ] - ) - cmd_list.commands.append(cmd) + for file in files: + cmd_list.append(file.format(context)) return cmd_list diff --git a/smartsim/_core/generation/operations.py b/smartsim/_core/generation/operations.py index 3b3fb19f9c..d543823ed1 100644 --- a/smartsim/_core/generation/operations.py +++ b/smartsim/_core/generation/operations.py @@ -1,7 +1,9 @@ +import os import pathlib import sys import typing as t -import os +import pickle +import base64 from dataclasses import dataclass, field from ..commands import Command @@ -25,18 +27,22 @@ def create_final_dest( :return: Combined path :raises ValueError: An error occurred during path combination """ - if dest is None: - dest = pathlib.Path("") - # these need to be more descriptive + if dest is not None and not isinstance(dest, pathlib.Path): + raise ValueError(f"Must be absolute path") + if isinstance(dest, pathlib.Path) and not dest.is_absolute(): + raise ValueError("Invalid destination path") + if isinstance(dest, pathlib.Path) and " " in str(dest): + raise ValueError("Path contains spaces, which are not allowed") if ( job_root_path is None or job_root_path == pathlib.Path("") or isinstance(job_root_path, str) - or job_root_path.suffix ): raise ValueError(f"Job root path '{job_root_path}' is not a directory.") try: - combined_path = job_root_path / dest + combined_path = job_root_path + if dest: + combined_path = job_root_path / dest return str(combined_path) except Exception as e: raise ValueError(f"Error combining paths: {e}") @@ -60,7 +66,9 @@ def format(self, context: GenerationContext) -> Command: class CopyOperation(GenerationProtocol): """Copy Operation""" - def __init__(self, src: pathlib.Path, dest: t.Union[pathlib.Path, None]) -> None: + def __init__( + self, src: pathlib.Path, dest: t.Optional[pathlib.Path] = None + ) -> None: self.src = src self.dest = dest @@ -83,7 +91,7 @@ def format(self, context: GenerationContext) -> Command: class SymlinkOperation(GenerationProtocol): """Symlink Operation""" - def __init__(self, src: pathlib.Path, dest: t.Union[pathlib.Path, None]) -> None: + def __init__(self, src: pathlib.Path, dest: t.Optional[pathlib.Path] = None) -> None: self.src = src self.dest = dest @@ -94,7 +102,6 @@ def format(self, context: GenerationContext) -> Command: parent_dir = os.path.basename(normalized_path) final_dest = create_final_dest(context.job_root_path, self.dest) new_dest = os.path.join(final_dest, parent_dir) - print(f"here: {new_dest}") return Command( [ sys.executable, @@ -113,14 +120,17 @@ class ConfigureOperation(GenerationProtocol): def __init__( self, src: pathlib.Path, - dest: t.Union[pathlib.Path, None], + file_parameters: t.Mapping[str,str], + dest: t.Optional[pathlib.Path] = None, tag: t.Optional[str] = None, ) -> None: self.src = src self.dest = dest + pickled_dict = pickle.dumps(file_parameters) + encoded_dict = base64.b64encode(pickled_dict).decode("ascii") + self.file_parameters = encoded_dict self.tag = tag if tag else ";" - # TODO discuss format as function name def format(self, context: GenerationContext) -> Command: """Create Command to invoke configure fs entry point""" final_dest = create_final_dest(context.job_root_path, self.dest) @@ -133,7 +143,7 @@ def format(self, context: GenerationContext) -> Command: str(self.src), final_dest, self.tag, - "encoded_dict", + self.file_parameters, ] ) @@ -164,11 +174,12 @@ def add_symlink( def add_configuration( self, src: pathlib.Path, + file_parameters: t.Mapping[str,str], dest: t.Optional[pathlib.Path] = None, tag: t.Optional[str] = None, ) -> None: """Add a configure operation to the operations list""" - self.operations.append(ConfigureOperation(src, dest, tag)) + self.operations.append(ConfigureOperation(src, file_parameters, dest, tag)) @property def copy_operations(self) -> t.List[CopyOperation]: @@ -186,4 +197,4 @@ def configure_operations(self) -> t.List[ConfigureOperation]: return self._filter(ConfigureOperation) def _filter(self, type: t.Type[T]) -> t.List[T]: - return [x for x in self.operations if isinstance(x, type)] + return [x for x in self.operations if isinstance(x, type)] \ No newline at end of file diff --git a/smartsim/entity/application.py b/smartsim/entity/application.py index e4e29be7c2..11b1a36bab 100644 --- a/smartsim/entity/application.py +++ b/smartsim/entity/application.py @@ -32,11 +32,11 @@ import typing as t from os import path as osp +from .._core.generation.operations import FileSysOperationSet from .._core.utils.helpers import expand_exe_path from ..log import get_logger from .entity import SmartSimEntity from .files import EntityFiles -from .._core.generation.operations import FileSysOperationSet logger = get_logger(__name__) @@ -59,9 +59,9 @@ def __init__( self, name: str, exe: str, + file_parameters: t.Mapping[str,str], # TODO remove when Ensemble is addressed + files: t.Optional[EntityFiles] = None, # TODO remove when Ensemble is addressed exe_args: t.Optional[t.Union[str, t.Sequence[str]]] = None, - files: t.Optional[EntityFiles] = None, - file_parameters: t.Mapping[str, str] | None = None, ) -> None: """Initialize an ``Application`` @@ -78,10 +78,6 @@ def __init__( :param name: name of the application :param exe: executable to run :param exe_args: executable arguments - :param files: files to be copied, symlinked, and/or configured prior to - execution - :param file_parameters: parameters and values to be used when configuring - files """ super().__init__(name) """The name of the application""" @@ -91,10 +87,6 @@ def __init__( """The executable arguments""" self.files = FileSysOperationSet([]) """Files to be copied, symlinked, and/or configured prior to execution""" - self._file_parameters = ( - copy.deepcopy(file_parameters) if file_parameters else {} - ) - """Parameters and values to be used when configuring files""" self._incoming_entities: t.List[SmartSimEntity] = [] """Entities for which the prefix will have to be known by other entities""" self._key_prefixing_enabled = False @@ -140,24 +132,6 @@ def add_exe_args(self, args: t.Union[str, t.List[str], None]) -> None: args = self._build_exe_args(args) self._exe_args.extend(args) - # @property - # def files(self) -> t.Union[EntityFiles, None]: - # """Return attached EntityFiles object. - - # :return: the EntityFiles object of files to be copied, symlinked, - # and/or configured prior to execution - # """ - # return self._files - - # @files.setter - # def files(self, value: t.Optional[EntityFiles]) -> None: - # """Set the EntityFiles object. - - # :param value: the EntityFiles object of files to be copied, symlinked, - # and/or configured prior to execution - # """ - # self._files = copy.deepcopy(value) - @property def file_parameters(self) -> t.Mapping[str, str]: """Return file parameters. @@ -213,62 +187,6 @@ def as_executable_sequence(self) -> t.Sequence[str]: """ return [self.exe, *self.exe_args] - def attach_generator_files( - self, - to_copy: t.Optional[t.List[str]] = None, - to_symlink: t.Optional[t.List[str]] = None, - to_configure: t.Optional[t.List[str]] = None, - ) -> None: - """Attach files to an entity for generation - - Attach files needed for the entity that, upon generation, - will be located in the path of the entity. Invoking this method - after files have already been attached will overwrite - the previous list of entity files. - - During generation, files "to_copy" are copied into - the path of the entity, and files "to_symlink" are - symlinked into the path of the entity. - - Files "to_configure" are text based application input files where - parameters for the application are set. Note that only applications - support the "to_configure" field. These files must have - fields tagged that correspond to the values the user - would like to change. The tag is settable but defaults - to a semicolon e.g. THERMO = ;10; - - :param to_copy: files to copy - :param to_symlink: files to symlink - :param to_configure: input files with tagged parameters - :raises ValueError: if the generator file already exists - """ - to_copy = to_copy or [] - to_symlink = to_symlink or [] - to_configure = to_configure or [] - - # Check that no file collides with the parameter file written - # by Generator. We check the basename, even though it is more - # restrictive than what we need (but it avoids relative path issues) - for strategy in [to_copy, to_symlink, to_configure]: - if strategy is not None and any( - osp.basename(filename) == "smartsim_params.txt" for filename in strategy - ): - raise ValueError( - "`smartsim_params.txt` is a file automatically " - + "generated by SmartSim and cannot be ovewritten." - ) - self.files = EntityFiles(to_configure, to_copy, to_symlink) - - @property - def attached_files_table(self) -> str: - """Return a list of attached files as a plain text table - - :return: String version of table - """ - if not self.files: - return "No file attached to this application." - return str(self.files) - @staticmethod def _build_exe_args(exe_args: t.Union[str, t.Sequence[str], None]) -> t.List[str]: """Check and convert exe_args input to a desired collection format @@ -293,10 +211,6 @@ def _build_exe_args(exe_args: t.Union[str, t.Sequence[str], None]) -> t.List[str return list(exe_args) - def print_attached_files(self) -> None: - """Print a table of the attached files on std out""" - print(self.attached_files_table) - def __str__(self) -> str: # pragma: no cover exe_args_str = "\n".join(self.exe_args) entities_str = "\n".join(str(entity) for entity in self.incoming_entities) @@ -307,8 +221,6 @@ def __str__(self) -> str: # pragma: no cover {self.exe} Executable Arguments: {exe_args_str} - Entity Files: {self.files} - File Parameters: {self.file_parameters} Incoming Entities: {entities_str} Key Prefixing Enabled: {self.key_prefixing_enabled} diff --git a/smartsim/entity/files.py b/smartsim/entity/files.py index 08143fbfc2..42586f153e 100644 --- a/smartsim/entity/files.py +++ b/smartsim/entity/files.py @@ -29,6 +29,7 @@ from tabulate import tabulate +# TODO remove when Ensemble is addressed class EntityFiles: """EntityFiles are the files a user wishes to have available to applications and nodes within SmartSim. Each entity has a method diff --git a/tests/test_generator.py b/tests/test_generator.py index 6dca19e5b0..ba818f6f50 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -37,12 +37,6 @@ from smartsim import Experiment from smartsim._core.commands import Command, CommandList from smartsim._core.generation.generator import Generator -from smartsim.builders import Ensemble -from smartsim.entity import entity -from smartsim.entity.files import EntityFiles -from smartsim.launchable import Job -from smartsim.settings import LaunchSettings - from smartsim._core.generation.operations import ( ConfigureOperation, CopyOperation, @@ -54,6 +48,11 @@ create_final_dest, symlink_cmd, ) +from smartsim.builders import Ensemble +from smartsim.entity import SmartSimEntity +from smartsim.entity.files import EntityFiles +from smartsim.launchable import Job +from smartsim.settings import LaunchSettings # TODO Add JobGroup tests when JobGroup becomes a Launchable @@ -69,32 +68,13 @@ def random_id(): return next(_ID_GENERATOR) -@pytest.fixture -def get_gen_copy_dir(fileutils): - yield fileutils.get_test_conf_path(osp.join("generator_files", "to_copy_dir")) - - -@pytest.fixture -def get_gen_symlink_dir(fileutils): - yield fileutils.get_test_conf_path(osp.join("generator_files", "to_symlink_dir")) - - -@pytest.fixture -def get_gen_configure_dir(fileutils): - yield fileutils.get_test_conf_path(osp.join("generator_files", "tag_dir_template")) - - @pytest.fixture def generator_instance(test_dir: str) -> Generator: - """Fixture to create an instance of Generator.""" - root = pathlib.Path(test_dir, "temp_id") - os.mkdir(root) - yield Generator(root=root) + """Instance of Generator""" + # os.mkdir(root) + yield Generator(root=pathlib.Path(test_dir)) -def get_gen_file(fileutils, filename: str): - return fileutils.get_test_conf_path(osp.join("generator_files", filename)) - @pytest.fixture def mock_src(test_dir: str): """Fixture to create a mock source path.""" @@ -106,6 +86,11 @@ def mock_dest(test_dir: str): """Fixture to create a mock destination path.""" return pathlib.Path(test_dir) / pathlib.Path("mock_dest") +@pytest.fixture +def mock_index(): + """Fixture to create a mock destination path.""" + return 1 + @pytest.fixture def copy_operation(mock_src: pathlib.Path, mock_dest: pathlib.Path): """Fixture to create a CopyOperation object.""" @@ -126,7 +111,8 @@ def configure_operation(mock_src: pathlib.Path, mock_dest: pathlib.Path): dest=mock_dest, ) -class EchoHelloWorldEntity(entity.SmartSimEntity): + +class EchoHelloWorldEntity(SmartSimEntity): """A simple smartsim entity that meets the `ExecutableProtocol` protocol""" def __init__(self): @@ -137,9 +123,6 @@ def __init__(self): def as_executable_sequence(self): return ("echo", "Hello", "World!") - # def files(): - # return FileSysOperationSet([CopyOperation(src="mock_path", dest="mock_src")]) - @pytest.fixture def mock_job() -> unittest.mock.MagicMock: @@ -162,14 +145,13 @@ def mock_job() -> unittest.mock.MagicMock: def test_init_generator(generator_instance: Generator, test_dir: str): """Test Generator init""" - assert generator_instance.root == pathlib.Path(test_dir) / "temp_id" + assert generator_instance.root == pathlib.Path(test_dir) def test_build_job_base_path( - generator_instance: Generator, mock_job: unittest.mock.MagicMock + generator_instance: Generator, mock_job: unittest.mock.MagicMock, mock_index ): """Test Generator._build_job_base_path returns correct path""" - mock_index = 1 root_path = generator_instance._build_job_base_path(mock_job, mock_index) expected_path = ( generator_instance.root @@ -184,16 +166,16 @@ def test_build_job_run_path( mock_job: unittest.mock.MagicMock, generator_instance: Generator, monkeypatch: pytest.MonkeyPatch, + mock_index ): """Test Generator._build_job_run_path returns correct path""" - mock_index = 1 monkeypatch.setattr( Generator, "_build_job_base_path", lambda self, job, job_index: pathlib.Path(test_dir), ) run_path = generator_instance._build_job_run_path(mock_job, mock_index) - expected_run_path = pathlib.Path(test_dir) / "run" + expected_run_path = pathlib.Path(test_dir) / generator_instance.run_directory assert run_path == expected_run_path @@ -202,16 +184,16 @@ def test_build_job_log_path( mock_job: unittest.mock.MagicMock, generator_instance: Generator, monkeypatch: pytest.MonkeyPatch, + mock_index ): """Test Generator._build_job_log_path returns correct path""" - mock_index = 1 monkeypatch.setattr( Generator, "_build_job_base_path", lambda self, job, job_index: pathlib.Path(test_dir), ) log_path = generator_instance._build_job_log_path(mock_job, mock_index) - expected_log_path = pathlib.Path(test_dir) / "log" + expected_log_path = pathlib.Path(test_dir) / generator_instance.log_directory assert log_path == expected_log_path @@ -244,40 +226,15 @@ def test_build_err_file_path( def test_generate_job( mock_job: unittest.mock.MagicMock, generator_instance: Generator, + mock_index ): """Test Generator.generate_job returns correct paths""" - mock_index = 1 job_paths = generator_instance.generate_job(mock_job, mock_index) assert job_paths.run_path.name == Generator.run_directory assert job_paths.out_path.name == f"{mock_job.entity.name}.out" assert job_paths.err_path.name == f"{mock_job.entity.name}.err" -def test_build_commands( - mock_job: unittest.mock.MagicMock, generator_instance: Generator, test_dir: str -): - """Test Generator._build_commands calls correct helper functions""" - with ( - unittest.mock.patch( - "smartsim._core.generation.Generator._copy_files" - ) as mock_copy_files, - unittest.mock.patch( - "smartsim._core.generation.Generator._symlink_files" - ) as mock_symlink_files, - unittest.mock.patch( - "smartsim._core.generation.Generator._write_tagged_files" - ) as mock_write_tagged_files, - ): - generator_instance._build_commands( - mock_job, - pathlib.Path(test_dir) / generator_instance.run_directory, - pathlib.Path(test_dir) / generator_instance.log_directory, - ) - mock_copy_files.assert_called_once() - mock_symlink_files.assert_called_once() - mock_write_tagged_files.assert_called_once() - - def test_execute_commands(generator_instance: Generator): """Test Generator._execute_commands subprocess.run""" with ( @@ -296,321 +253,289 @@ def test_mkdir_file(generator_instance: Generator, test_dir: str): assert isinstance(cmd, Command) assert cmd.command == ["mkdir", "-p", test_dir] - -# def test_copy_file(generator_instance: Generator, fileutils): -# """Test Generator._copy_files helper function with file""" -# script = fileutils.get_test_conf_path("sleep.py") -# files = EntityFiles(copy=script) -# cmd_list = generator_instance._copy_files(files, generator_instance.root) -# assert isinstance(cmd_list, CommandList) -# assert len(cmd_list) == 1 -# assert str(generator_instance.root) and script in cmd_list.commands[0].command - - -# def test_copy_directory(get_gen_copy_dir, generator_instance: Generator): -# """Test Generator._copy_files helper function with directory""" -# files = EntityFiles(copy=get_gen_copy_dir) -# cmd_list = generator_instance._copy_files(files, generator_instance.root) -# assert isinstance(cmd_list, CommandList) -# assert len(cmd_list) == 1 -# assert ( -# str(generator_instance.root) -# and get_gen_copy_dir in cmd_list.commands[0].command -# ) - - -# def test_symlink_file(get_gen_symlink_dir, generator_instance: Generator): -# """Test Generator._symlink_files helper function with file list""" -# symlink_files = sorted(glob(get_gen_symlink_dir + "/*")) -# files = EntityFiles(symlink=symlink_files) -# cmd_list = generator_instance._symlink_files(files, generator_instance.root) -# assert isinstance(cmd_list, CommandList) -# for file, cmd in zip(symlink_files, cmd_list): -# assert file in cmd.command - - -# def test_symlink_directory(generator_instance: Generator, get_gen_symlink_dir): -# """Test Generator._symlink_files helper function with directory""" -# files = EntityFiles(symlink=get_gen_symlink_dir) -# cmd_list = generator_instance._symlink_files(files, generator_instance.root) -# symlinked_folder = generator_instance.root / os.path.basename(get_gen_symlink_dir) -# assert isinstance(cmd_list, CommandList) -# assert str(symlinked_folder) in cmd_list.commands[0].command - - -# def test_write_tagged_file(fileutils, generator_instance: Generator): -# """Test Generator._write_tagged_files helper function with file list""" -# conf_path = fileutils.get_test_conf_path( -# osp.join("generator_files", "easy", "marked/") -# ) -# tagged_files = sorted(glob(conf_path + "/*")) -# files = EntityFiles(tagged=tagged_files) -# param_set = { -# "5": 10, -# "FIRST": "SECOND", -# "17": 20, -# "65": "70", -# "placeholder": "group leftupper region", -# "1200": "120", -# "VALID": "valid", -# } -# cmd_list = generator_instance._write_tagged_files( -# files=files, params=param_set, dest=generator_instance.root -# ) -# assert isinstance(cmd_list, CommandList) -# for file, cmd in zip(tagged_files, cmd_list): -# assert file in cmd.command - - -# def test_write_tagged_directory(fileutils, generator_instance: Generator): -# """Test Generator._write_tagged_files helper function with directory path""" -# config = get_gen_file(fileutils, "tag_dir_template") -# files = EntityFiles(tagged=[config]) -# param_set = {"PARAM0": "param_value_1", "PARAM1": "param_value_2"} -# cmd_list = generator_instance._write_tagged_files( -# files=files, params=param_set, dest=generator_instance.root -# ) - -# assert isinstance(cmd_list, CommandList) -# assert str(config) in cmd_list.commands[0].command - - -# # INTEGRATED TESTS - - -# def test_exp_private_generate_method( -# mock_job: unittest.mock.MagicMock, test_dir: str, generator_instance: Generator -# ): -# """Test that Experiment._generate returns expected tuple.""" -# mock_index = 1 -# exp = Experiment(name="experiment_name", exp_path=test_dir) -# job_paths = exp._generate(generator_instance, mock_job, mock_index) -# assert osp.isdir(job_paths.run_path) -# assert job_paths.out_path.name == f"{mock_job.entity.name}.out" -# assert job_paths.err_path.name == f"{mock_job.entity.name}.err" - - -# def test_generate_ensemble_directory_start( -# test_dir: str, wlmutils, monkeypatch: pytest.MonkeyPatch -# ): -# """Test that Experiment._generate returns expected tuple.""" -# monkeypatch.setattr( -# "smartsim._core.dispatch._LauncherAdapter.start", -# lambda launch, exe, job_execution_path, env, out, err: random_id(), -# ) -# ensemble = Ensemble("ensemble-name", "echo", replicas=2) -# launch_settings = LaunchSettings(wlmutils.get_test_launcher()) -# job_list = ensemble.build_jobs(launch_settings) -# exp = Experiment(name="exp_name", exp_path=test_dir) -# exp.start(*job_list) -# run_dir = listdir(test_dir) -# jobs_dir_path = pathlib.Path(test_dir) / run_dir[0] / "jobs" -# list_of_job_dirs = jobs_dir_path.iterdir() -# for job in list_of_job_dirs: -# run_path = jobs_dir_path / job / Generator.run_directory -# assert run_path.is_dir() -# log_path = jobs_dir_path / job / Generator.log_directory -# assert log_path.is_dir() -# ids.clear() - - -# def test_generate_ensemble_copy( -# test_dir: str, wlmutils, monkeypatch: pytest.MonkeyPatch, get_gen_copy_dir -# ): -# monkeypatch.setattr( -# "smartsim._core.dispatch._LauncherAdapter.start", -# lambda launch, exe, job_execution_path, env, out, err: random_id(), -# ) -# ensemble = Ensemble( -# "ensemble-name", "echo", replicas=2, files=EntityFiles(copy=get_gen_copy_dir) -# ) -# launch_settings = LaunchSettings(wlmutils.get_test_launcher()) -# job_list = ensemble.build_jobs(launch_settings) -# exp = Experiment(name="exp_name", exp_path=test_dir) -# exp.start(*job_list) -# run_dir = listdir(test_dir) -# jobs_dir = pathlib.Path(test_dir) / run_dir[0] / "jobs" -# job_dir = jobs_dir.iterdir() -# for ensemble_dir in job_dir: -# copy_folder_path = ( -# jobs_dir / ensemble_dir / Generator.run_directory / "to_copy_dir" -# ) -# assert copy_folder_path.is_dir() -# ids.clear() - - -# def test_generate_ensemble_symlink( -# test_dir: str, wlmutils, monkeypatch: pytest.MonkeyPatch, get_gen_symlink_dir -# ): -# monkeypatch.setattr( -# "smartsim._core.dispatch._LauncherAdapter.start", -# lambda launch, exe, job_execution_path, env, out, err: random_id(), -# ) -# ensemble = Ensemble( -# "ensemble-name", -# "echo", -# replicas=2, -# files=EntityFiles(symlink=get_gen_symlink_dir), -# ) -# launch_settings = LaunchSettings(wlmutils.get_test_launcher()) -# job_list = ensemble.build_jobs(launch_settings) -# exp = Experiment(name="exp_name", exp_path=test_dir) -# _ = exp.start(*job_list) -# run_dir = listdir(test_dir) -# jobs_dir = pathlib.Path(test_dir) / run_dir[0] / "jobs" -# job_dir = jobs_dir.iterdir() -# for ensemble_dir in job_dir: -# sym_file_path = pathlib.Path(jobs_dir) / ensemble_dir / "run" / "to_symlink_dir" -# assert sym_file_path.is_dir() -# assert sym_file_path.is_symlink() -# assert os.fspath(sym_file_path.resolve()) == osp.realpath(get_gen_symlink_dir) -# ids.clear() - - -# def test_generate_ensemble_configure( -# test_dir: str, wlmutils, monkeypatch: pytest.MonkeyPatch, get_gen_configure_dir -# ): -# monkeypatch.setattr( -# "smartsim._core.dispatch._LauncherAdapter.start", -# lambda launch, exe, job_execution_path, env, out, err: random_id(), -# ) -# param_set = {"PARAM0": [0, 1], "PARAM1": [2, 3]} -# tagged_files = sorted(glob(get_gen_configure_dir + "/*")) -# ensemble = Ensemble( -# "ensemble-name", -# "echo", -# replicas=1, -# files=EntityFiles(tagged=tagged_files), -# file_parameters=param_set, -# ) -# launch_settings = LaunchSettings(wlmutils.get_test_launcher()) -# job_list = ensemble.build_jobs(launch_settings) -# exp = Experiment(name="exp_name", exp_path=test_dir) -# _ = exp.start(*job_list) -# run_dir = listdir(test_dir) -# jobs_dir = pathlib.Path(test_dir) / run_dir[0] / "jobs" - -# def _check_generated(param_0, param_1, dir): -# assert dir.is_dir() -# tagged_0 = dir / "tagged_0.sh" -# tagged_1 = dir / "tagged_1.sh" -# assert tagged_0.is_file() -# assert tagged_1.is_file() - -# with open(tagged_0) as f: -# line = f.readline() -# assert line.strip() == f'echo "Hello with parameter 0 = {param_0}"' - -# with open(tagged_1) as f: -# line = f.readline() -# assert line.strip() == f'echo "Hello with parameter 1 = {param_1}"' - -# _check_generated(0, 3, jobs_dir / "ensemble-name-1-1" / Generator.run_directory) -# _check_generated(1, 2, jobs_dir / "ensemble-name-2-2" / Generator.run_directory) -# _check_generated(1, 3, jobs_dir / "ensemble-name-3-3" / Generator.run_directory) -# _check_generated(0, 2, jobs_dir / "ensemble-name-0-0" / Generator.run_directory) -# ids.clear() - - - - - -# REDO - -# COPY - -# test making file copies without a dest -def test_1_copy_files(test_dir, fileutils, generator_instance, mock_job: Job, copy_operation, symlink_operation, configure_operation): - correct_path = fileutils.get_test_conf_path( - osp.join("generator_files", "easy", "correct/") - ) - correct_files = sorted(glob(correct_path + "/*")) - for file in correct_files: - mock_job.entity.files.add_copy(src=pathlib.Path(file)) - ret = generator_instance._build_commands(mock_job,pathlib.Path(test_dir), "log_path") - -# test making directory -def test_2_copy_directory(test_dir, fileutils, generator_instance, mock_job: Job, copy_operation, symlink_operation, configure_operation): - correct_path = fileutils.get_test_conf_path( +# might change this to files that can be configured +@pytest.fixture +def files(fileutils): + path_to_files = fileutils.get_test_conf_path( osp.join("generator_files", "easy", "correct/") ) - mock_job.entity.files.add_copy(src=pathlib.Path(correct_path)) - ret = generator_instance._build_commands(mock_job,pathlib.Path(test_dir), "log_path") - print(ret) + list_of_files_strs = sorted(glob(path_to_files + "/*")) + yield [pathlib.Path(str_path) for str_path in list_of_files_strs] -# test making file copies without a dest -def test_3_copy_files(test_dir, fileutils, generator_instance, mock_job: Job, copy_operation, symlink_operation, configure_operation): - correct_path = fileutils.get_test_conf_path( - osp.join("generator_files", "easy", "correct/") - ) - correct_files = sorted(glob(correct_path + "/*")) - for file in correct_files: - mock_job.entity.files.add_copy(src=pathlib.Path(file), dest="mock") - ret = generator_instance._build_commands(mock_job,pathlib.Path(test_dir), "log_path") - print(ret) - -# test making file copies without a dest -def test_4_copy_directory(test_dir, fileutils, generator_instance, mock_job: Job, copy_operation, symlink_operation, configure_operation): - correct_path = fileutils.get_test_conf_path( - osp.join("generator_files", "easy", "correct/") - ) - mock_job.entity.files.add_copy(src=pathlib.Path(correct_path), dest="mock") - ret = generator_instance._build_commands(mock_job,pathlib.Path(test_dir), "log_path") - print(ret) -def test_1_execute_the_copy(test_dir, fileutils, mock_job,generator_instance): - correct_path = fileutils.get_test_conf_path( +@pytest.fixture +def directory(fileutils): + directory = fileutils.get_test_conf_path( osp.join("generator_files", "easy", "correct/") ) - correct_files = sorted(glob(correct_path + "/*")) - for file in correct_files: - mock_job.entity.files.add_copy(src=pathlib.Path(file)) - ret = generator_instance._build_commands(mock_job,pathlib.Path(test_dir), "log_path") - generator_instance._execute_commands(ret) - print(listdir(test_dir)) - assert pathlib.Path(test_dir).is_dir() - for file in correct_files: - assert pathlib.Path(file).name in listdir(test_dir) - -def test_2_execute_the_copy(test_dir, fileutils, mock_job,generator_instance): - correct_path = fileutils.get_test_conf_path( + yield [pathlib.Path(directory)] + +@pytest.fixture +def source(request, files, directory): + if request.param == "files": + return files + elif request.param == "directory": + return directory + else: + raise ValueError(f"Unknown source fixture: {request.param}") + +@pytest.mark.parametrize( + "dest", + ( + pytest.param(123, id="dest as integer"), + pytest.param("", id="dest as empty str"), + pytest.param("/absolute/path", id="dest as absolute str"), + pytest.param(pathlib.Path("relative/path"), id="dest as relative Path"), + pytest.param(pathlib.Path("/path with spaces"), id="dest as Path with spaces"), + # pytest.param(pathlib.Path("/path/with/special!@#"), id="dest as Path with special char"), + ), +) +@pytest.mark.parametrize("source", ["files", "directory"], indirect=True) +def test_copy_files_invalid_dest(dest, source, generator_instance, test_dir): + to_copy = [SymlinkOperation(src=file, dest=dest) for file in source] + gen = GenerationContext(pathlib.Path(test_dir)) + with pytest.raises(ValueError): + generator_instance._symlink_files(files=to_copy, context=gen) + +@pytest.mark.parametrize( + "dest", + ( + pytest.param(None, id="dest as None"), + pytest.param( + pathlib.Path("/absolute/path"), + id="dest as valid path", + ), + ), +) +@pytest.mark.parametrize("source", ["files", "directory"], indirect=True) +def test_copy_files_valid_dest(dest, source, generator_instance, test_dir): + to_copy = [CopyOperation(src=file, dest=dest) for file in source] + gen = GenerationContext(pathlib.Path(test_dir)) + cmd_list = generator_instance._copy_files(files=to_copy, context=gen) + assert isinstance(cmd_list, CommandList) + # Extract file paths from commands + cmd_src_paths = set() + for cmd in cmd_list.commands: + # Assert destination path exists in cmd + assert create_final_dest(pathlib.Path(test_dir), dest) in cmd.command + src_index = cmd.command.index("copy") + 1 + cmd_src_paths.add(cmd.command[src_index]) + # Assert all file paths are in the command list + file_paths = {str(file) for file in source} + assert file_paths.issubset( + cmd_src_paths + ), "Not all file paths are in the command list" + +@pytest.mark.parametrize( + "dest", + ( + pytest.param(123, id="dest as integer"), + pytest.param("", id="dest as empty str"), + pytest.param("/absolute/path", id="dest as absolute str"), + pytest.param(pathlib.Path("relative/path"), id="dest as relative Path"), + pytest.param(pathlib.Path("/path with spaces"), id="dest as Path with spaces"), + # pytest.param(pathlib.Path("/path/with/special!@#"), id="dest as Path with special char"), + ), +) +@pytest.mark.parametrize("source", ["files", "directory"], indirect=True) +def test_symlink_files_invalid_dest(dest, source, generator_instance, test_dir): + to_copy = [SymlinkOperation(src=file, dest=dest) for file in source] + gen = GenerationContext(pathlib.Path(test_dir)) + with pytest.raises(ValueError): + generator_instance._symlink_files(files=to_copy, context=gen) + +@pytest.mark.parametrize( + "dest", + ( + pytest.param(None, id="dest as None"), + pytest.param( + pathlib.Path("/absolute/path"), + id="dest as valid path", + ), + ), +) +@pytest.mark.parametrize("source", ["files", "directory"], indirect=True) +def test_symlink_files_valid_dest(dest, source, generator_instance, test_dir): + to_copy = [CopyOperation(src=file, dest=dest) for file in source] + gen = GenerationContext(pathlib.Path(test_dir)) + cmd_list = generator_instance._copy_files(files=to_copy, context=gen) + assert isinstance(cmd_list, CommandList) + # Extract file paths from commands + cmd_src_paths = set() + for cmd in cmd_list.commands: + # Assert destination path exists in cmd + assert create_final_dest(pathlib.Path(test_dir), dest) in cmd.command + src_index = cmd.command.index("copy") + 1 + cmd_src_paths.add(cmd.command[src_index]) + # Assert all file paths are in the command list + file_paths = {str(file) for file in source} + assert file_paths.issubset( + cmd_src_paths + ), "Not all file paths are in the command list" + +@pytest.mark.parametrize( + "dest", + ( + pytest.param(123, id="dest as integer"), + pytest.param("", id="dest as empty str"), + pytest.param("/absolute/path", id="dest as absolute str"), + pytest.param(pathlib.Path("relative/path"), id="dest as relative Path"), + pytest.param(pathlib.Path("/path with spaces"), id="dest as Path with spaces"), + # pytest.param(pathlib.Path("/path/with/special!@#"), id="dest as Path with special char"), + ), +) +@pytest.mark.parametrize("source", ["files", "directory"], indirect=True) +def test_configure_files_invalid_dest(dest, source, generator_instance, test_dir): + to_configure = [ConfigureOperation(src=file, dest=dest, file_parameters={"FOO": "BAR"}) for file in source] + gen = GenerationContext(pathlib.Path(test_dir)) + with pytest.raises(ValueError): + generator_instance._configure_files(files=to_configure, context=gen) + +@pytest.mark.parametrize( + "dest", + ( + pytest.param(None, id="dest as None"), + pytest.param( + pathlib.Path("/absolute/path"), + id="dest as valid path", + ), + ), +) +@pytest.mark.parametrize("source", ["files", "directory"], indirect=True) +def test_configure_files_valid_dest(dest, source, generator_instance, test_dir): + file_param = { + "5": 10, + "FIRST": "SECOND", + "17": 20, + "65": "70", + "placeholder": "group leftupper region", + "1200": "120", + "VALID": "valid", + } + to_configure = [ConfigureOperation(src=file, dest=dest, file_parameters=file_param) for file in source] + gen = GenerationContext(pathlib.Path(test_dir)) + cmd_list = generator_instance._configure_files(files=to_configure, context=gen) + assert isinstance(cmd_list, CommandList) + # Extract file paths from commands + cmd_src_paths = set() + for cmd in cmd_list.commands: + # Assert destination path exists in cmd + assert create_final_dest(pathlib.Path(test_dir), dest) in cmd.command + src_index = cmd.command.index("configure") + 1 + cmd_src_paths.add(cmd.command[src_index]) + # Assert all file paths are in the command list + file_paths = {str(file) for file in source} + assert file_paths.issubset( + cmd_src_paths + ), "Not all file paths are in the command list" + +# TODO Add configure_file tests + +@pytest.fixture +def run_directory(test_dir, generator_instance): + return pathlib.Path(test_dir) / generator_instance.run_directory + +@pytest.fixture +def log_directory(test_dir, generator_instance): + return pathlib.Path(test_dir) / generator_instance.log_directory + +def test_build_commands( + generator_instance: Generator, run_directory: pathlib.Path, log_directory: pathlib.Path +): + """Test Generator._build_commands calls internal helper functions""" + with ( + unittest.mock.patch( + "smartsim._core.generation.Generator._append_mkdir_commands" + ) as mock_append_mkdir_commands, + unittest.mock.patch( + "smartsim._core.generation.Generator._append_file_operations" + ) as mock_append_file_operations + ): + generator_instance._build_commands( + EchoHelloWorldEntity(), + run_directory, + log_directory, + ) + mock_append_mkdir_commands.assert_called_once() + mock_append_file_operations.assert_called_once() + +def test_append_mkdir_commands(generator_instance: Generator, run_directory: pathlib.Path, log_directory: pathlib.Path): + with ( + unittest.mock.patch( + "smartsim._core.generation.Generator._mkdir_file" + ) as mock_mkdir_file, + ): + generator_instance._append_mkdir_commands( + CommandList(), + run_directory, + log_directory, + ) + assert mock_mkdir_file.call_count == 2 + +def test_append_file_operations(context, generator_instance): + with ( + unittest.mock.patch( + "smartsim._core.generation.Generator._copy_files" + ) as mock_copy_files, + unittest.mock.patch( + "smartsim._core.generation.Generator._symlink_files" + ) as mock_symlink_files, + unittest.mock.patch( + "smartsim._core.generation.Generator._configure_files" + ) as mock_configure_files, + ): + generator_instance._append_file_operations( + CommandList(), + EchoHelloWorldEntity(), + context, + ) + mock_copy_files.assert_called_once() + mock_symlink_files.assert_called_once() + mock_configure_files.assert_called_once() + +@pytest.fixture +def paths_to_copy(fileutils): + paths = fileutils.get_test_conf_path(osp.join("mock", "copy_mock")) + yield [pathlib.Path(path) for path in sorted(glob(paths + "/*"))] + +@pytest.fixture +def paths_to_symlink(fileutils): + paths = fileutils.get_test_conf_path(osp.join("mock", "symlink_mock")) + yield [pathlib.Path(path) for path in sorted(glob(paths + "/*"))] + +@pytest.fixture +def paths_to_configure(fileutils): + paths = fileutils.get_test_conf_path( osp.join("generator_files", "easy", "correct/") ) - mock_job.entity.files.add_copy(src=pathlib.Path(correct_path), dest=pathlib.Path("mock")) - ret = generator_instance._build_commands(mock_job,pathlib.Path(test_dir), "log_path") - generator_instance._execute_commands(ret) - path = pathlib.Path(test_dir) / pathlib.Path("mock") - pathlib.Path(path).is_dir() - correct_files = sorted(glob(correct_path + "/*")) - for file in correct_files: - assert pathlib.Path(file).name in listdir(pathlib.Path(test_dir) / pathlib.Path("mock")) - - -# SYMLINK -def test_1_symlink_files(test_dir, fileutils, generator_instance, mock_job: Job, copy_operation, symlink_operation, configure_operation): - symlink_dir = get_gen_file(fileutils, "to_symlink_dir") - correct_files = sorted(glob(symlink_dir + "/*")) - for file in correct_files: - mock_job.entity.files.add_symlink(src=pathlib.Path(file)) - ret = generator_instance._build_commands(mock_job,pathlib.Path(test_dir), "log_path") - -def test_2_symlink_directory(test_dir, fileutils, generator_instance, mock_job: Job, copy_operation, symlink_operation, configure_operation): - symlink_dir = get_gen_file(fileutils, "to_symlink_dir") - mock_job.entity.files.add_symlink(src=pathlib.Path(symlink_dir)) - ret = generator_instance._build_commands(mock_job,pathlib.Path(test_dir), "log_path") - print(ret) - -def test_3_symlink_files(test_dir, fileutils, generator_instance, mock_job: Job, copy_operation, symlink_operation, configure_operation): - symlink_dir = get_gen_file(fileutils, "to_symlink_dir") - correct_files = sorted(glob(symlink_dir + "/*")) - for file in correct_files: - mock_job.entity.files.add_symlink(src=pathlib.Path(file)) - ret = generator_instance._build_commands(mock_job,pathlib.Path(test_dir), "log_path") - print(ret) - generator_instance._execute_commands(ret) - -def test_4_symlink_directory(test_dir, fileutils, generator_instance, mock_job: Job, copy_operation, symlink_operation, configure_operation): - symlink_dir = get_gen_file(fileutils, "to_symlink_dir") - mock_job.entity.files.add_symlink(src=pathlib.Path(symlink_dir)) - ret = generator_instance._build_commands(mock_job,pathlib.Path(test_dir), "log_path") - generator_instance._execute_commands(ret) \ No newline at end of file + yield [pathlib.Path(path) for path in sorted(glob(paths + "/*"))] + +@pytest.fixture +def context(test_dir): + yield GenerationContext(pathlib.Path(test_dir)) +@pytest.fixture +def operations_list(paths_to_copy, paths_to_symlink, paths_to_configure): + merp = [] + for file in paths_to_copy: + merp.append(CopyOperation(src=file)) + for file in paths_to_symlink: + merp.append(SymlinkOperation(src=file)) + for file in paths_to_configure: + merp.append(SymlinkOperation(src=file)) + return merp + +@pytest.fixture +def formatted_command_list(operations_list, context): + new_list = CommandList() + for file in operations_list: + new_list.append(file.format(context)) + return new_list + +def test_execute_commands(operations_list, formatted_command_list, generator_instance, test_dir): + with ( + unittest.mock.patch( + "smartsim._core.generation.generator.subprocess.run" + ) as mock_run, + ): + generator_instance._execute_commands(formatted_command_list) + assert mock_run.call_count == len(operations_list) From e221b436d84d8ec908b6cb49f8e6a15bad6c0409 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Thu, 3 Oct 2024 15:14:11 -0700 Subject: [PATCH 07/35] pushing updates --- smartsim/_core/generation/generator.py | 26 ++-- smartsim/_core/generation/operations.py | 159 +++++++++++++++++++----- smartsim/entity/application.py | 11 +- tests/test_generator.py | 127 +++++++------------ tests/test_operations.py | 157 +++++++++++++++++++---- 5 files changed, 331 insertions(+), 149 deletions(-) diff --git a/smartsim/_core/generation/generator.py b/smartsim/_core/generation/generator.py index 6287b7b7e3..49a4c322f6 100644 --- a/smartsim/_core/generation/generator.py +++ b/smartsim/_core/generation/generator.py @@ -24,27 +24,21 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import base64 -import os import pathlib -import pickle import subprocess -import sys import typing as t from collections import namedtuple from datetime import datetime -from ...entity.files import EntityFiles from ...entity import entity from ...launchable import Job from ...log import get_logger from ..commands import Command, CommandList from .operations import ( + ConfigureOperation, CopyOperation, FileSysOperationSet, GenerationContext, - ConfigureOperation, - GenerationProtocol, SymlinkOperation, ) @@ -67,7 +61,7 @@ class _GenerableProtocol(t.Protocol): class Generator: - """ The Generator class creates the directory structure for a SmartSim Job by building + """The Generator class creates the directory structure for a SmartSim Job by building and executing file operation commands. """ @@ -198,7 +192,10 @@ def generate_job(self, job: Job, job_index: int) -> Job_Path: @classmethod def _build_commands( - cls, entity: entity.SmartSimEntity, job_path: pathlib.Path, log_path: pathlib.Path + cls, + entity: entity.SmartSimEntity, + job_path: pathlib.Path, + log_path: pathlib.Path, ) -> CommandList: """Build file operation commands for a Job's entity. @@ -223,7 +220,9 @@ def _build_commands( return cmd_list @classmethod - def _append_mkdir_commands(cls, cmd_list: CommandList, job_path: pathlib.Path, log_path: pathlib.Path) -> None: + def _append_mkdir_commands( + cls, cmd_list: CommandList, job_path: pathlib.Path, log_path: pathlib.Path + ) -> None: """Append file operation Commands (mkdir) for a Job's run and log directory. :param cmd_list: A CommandList object containing the commands to be executed @@ -234,7 +233,12 @@ def _append_mkdir_commands(cls, cmd_list: CommandList, job_path: pathlib.Path, l cmd_list.commands.append(cls._mkdir_file(log_path)) @classmethod - def _append_file_operations(cls, cmd_list: CommandList, entity: _GenerableProtocol, context: GenerationContext) -> None: + def _append_file_operations( + cls, + cmd_list: CommandList, + entity: _GenerableProtocol, + context: GenerationContext, + ) -> None: """Append file operation Commands (copy, symlink, configure) for all files attached to the entity. diff --git a/smartsim/_core/generation/operations.py b/smartsim/_core/generation/operations.py index d543823ed1..9b9149b16f 100644 --- a/smartsim/_core/generation/operations.py +++ b/smartsim/_core/generation/operations.py @@ -1,19 +1,22 @@ +import base64 import os import pathlib +import pickle import sys import typing as t -import pickle -import base64 from dataclasses import dataclass, field from ..commands import Command entry_point_path = "smartsim._core.entrypoints.file_operations" -"""Path to file operations module.""" +"""Path to file operations module""" copy_cmd = "copy" +"""Copy file operations command""" symlink_cmd = "symlink" +"""Symlink file operations command""" configure_cmd = "configure" +"""Configure file operations command""" def create_final_dest( @@ -27,18 +30,6 @@ def create_final_dest( :return: Combined path :raises ValueError: An error occurred during path combination """ - if dest is not None and not isinstance(dest, pathlib.Path): - raise ValueError(f"Must be absolute path") - if isinstance(dest, pathlib.Path) and not dest.is_absolute(): - raise ValueError("Invalid destination path") - if isinstance(dest, pathlib.Path) and " " in str(dest): - raise ValueError("Path contains spaces, which are not allowed") - if ( - job_root_path is None - or job_root_path == pathlib.Path("") - or isinstance(job_root_path, str) - ): - raise ValueError(f"Job root path '{job_root_path}' is not a directory.") try: combined_path = job_root_path if dest: @@ -48,12 +39,56 @@ def create_final_dest( raise ValueError(f"Error combining paths: {e}") +def check_src_and_dest_path( + src: pathlib.Path, dest: t.Union[pathlib.Path, None] +) -> None: + """Validate that the provided source and destination paths are + of type pathlib.Path + + :param src: The source path to be checked. + :param des: The destination path to be checked. + :raises TypeError: If either src or dest is not an instance of pathlib.Path + """ + if not isinstance(src, pathlib.Path): + raise TypeError(f"src must be of type pathlib.Path, not {type(src).__name__}") + if dest is not None and not isinstance(dest, pathlib.Path): + raise TypeError( + f"dest must be of type pathlib.Path or None, not {type(dest).__name__}" + ) + if isinstance(dest, pathlib.Path) and not dest.is_absolute(): + raise ValueError("Invalid destination path") + if isinstance(dest, pathlib.Path) and " " in str(dest): + raise ValueError("Path contains spaces, which are not allowed") + + # TODO I want to add the check below but I do not think this works for remote jobs + # full_path = path.abspath(file_path) + # if path.isfile(full_path): + # return full_path + # if path.isdir(full_path): + # return full_path + + +def check_run_path(run_path: pathlib.Path) -> None: + """Validate that the provided run path is of type pathlib.Path + + :param run_path: The run path to be checked + :raises TypeError: If either src or dest is not an instance of pathlib.Path + """ + if not isinstance(run_path, pathlib.Path): + raise TypeError( + f"run_path must be of type pathlib.Path, not {type(run_path).__name__}" + ) + if run_path == pathlib.Path(""): + raise ValueError(f"Job root path '{run_path}' is not a directory.") + + class GenerationContext: """Context for file system generation operations.""" def __init__(self, job_root_path: pathlib.Path): + check_run_path(job_root_path) self.job_root_path = job_root_path - """The Job root path""" + """The Job run path""" class GenerationProtocol(t.Protocol): @@ -69,11 +104,21 @@ class CopyOperation(GenerationProtocol): def __init__( self, src: pathlib.Path, dest: t.Optional[pathlib.Path] = None ) -> None: + """Initialize a CopyOperation object + + :param src: Path to source + :param dest: Path to destination + """ + check_src_and_dest_path(src, dest) self.src = src self.dest = dest def format(self, context: GenerationContext) -> Command: - """Create Command to invoke copy fs entry point""" + """Create Command to invoke copy fs entry point + + :param context: Context for copy operation + :return: Copy Command + """ final_dest = create_final_dest(context.job_root_path, self.dest) return Command( [ @@ -91,14 +136,25 @@ def format(self, context: GenerationContext) -> Command: class SymlinkOperation(GenerationProtocol): """Symlink Operation""" - def __init__(self, src: pathlib.Path, dest: t.Optional[pathlib.Path] = None) -> None: + def __init__( + self, src: pathlib.Path, dest: t.Optional[pathlib.Path] = None + ) -> None: + """Initialize a SymlinkOperation object + + :param src: Path to source + :param dest: Path to destination + """ + check_src_and_dest_path(src, dest) self.src = src self.dest = dest def format(self, context: GenerationContext) -> Command: - """Create Command to invoke symlink fs entry point""" + """Create Command to invoke symlink fs entry point + + :param context: Context for symlink operation + :return: Symlink Command + """ normalized_path = os.path.normpath(self.src) - # # Get the parent directory (last folder) parent_dir = os.path.basename(normalized_path) final_dest = create_final_dest(context.job_root_path, self.dest) new_dest = os.path.join(final_dest, parent_dir) @@ -120,10 +176,18 @@ class ConfigureOperation(GenerationProtocol): def __init__( self, src: pathlib.Path, - file_parameters: t.Mapping[str,str], + file_parameters: t.Mapping[str, str], dest: t.Optional[pathlib.Path] = None, tag: t.Optional[str] = None, ) -> None: + """Initialize a ConfigureOperation + + :param src: Path to source + :param file_parameters: File parameters to find and replace + :param dest: Path to destination + :param tag: Tag to use for find and replacement + """ + check_src_and_dest_path(src, dest) self.src = src self.dest = dest pickled_dict = pickle.dumps(file_parameters) @@ -132,7 +196,11 @@ def __init__( self.tag = tag if tag else ";" def format(self, context: GenerationContext) -> Command: - """Create Command to invoke configure fs entry point""" + """Create Command to invoke configure fs entry point + + :param context: Context for configure operation + :return: Configure Command + """ final_dest = create_final_dest(context.job_root_path, self.dest) return Command( [ @@ -155,46 +223,75 @@ def format(self, context: GenerationContext) -> Command: class FileSysOperationSet: """Dataclass to represent a set of FS Operation Objects""" - # disallow modification - dunder function (post ticket to reevaluate API objects) + # TODO disallow modification - dunder function (post ticket to reevaluate API objects) operations: t.List[GenerationProtocol] = field(default_factory=list) """Set of FS Objects that match the GenerationProtocol""" def add_copy( self, src: pathlib.Path, dest: t.Optional[pathlib.Path] = None ) -> None: - """Add a copy operation to the operations list""" + """Add a copy operation to the operations list + + :param src: Path to source + :param dest: Path to destination + """ self.operations.append(CopyOperation(src, dest)) def add_symlink( self, src: pathlib.Path, dest: t.Optional[pathlib.Path] = None ) -> None: - """Add a symlink operation to the operations list""" + """Add a symlink operation to the operations list + + :param src: Path to source + :param dest: Path to destination + """ self.operations.append(SymlinkOperation(src, dest)) def add_configuration( self, src: pathlib.Path, - file_parameters: t.Mapping[str,str], + file_parameters: t.Mapping[str, str], dest: t.Optional[pathlib.Path] = None, tag: t.Optional[str] = None, ) -> None: - """Add a configure operation to the operations list""" + """Add a configure operation to the operations list + + :param src: Path to source + :param file_parameters: File parameters to find and replace + :param dest: Path to destination + :param tag: Tag to use for find and replacement + """ self.operations.append(ConfigureOperation(src, file_parameters, dest, tag)) @property def copy_operations(self) -> t.List[CopyOperation]: - """Property to get the list of copy files.""" + """Property to get the list of copy files. + + :return: List of CopyOperation objects + """ return self._filter(CopyOperation) @property def symlink_operations(self) -> t.List[SymlinkOperation]: - """Property to get the list of symlink files.""" + """Property to get the list of symlink files. + + :return: List of SymlinkOperation objects + """ return self._filter(SymlinkOperation) @property def configure_operations(self) -> t.List[ConfigureOperation]: - """Property to get the list of configure files.""" + """Property to get the list of configure files. + + :return: List of ConfigureOperation objects + """ return self._filter(ConfigureOperation) def _filter(self, type: t.Type[T]) -> t.List[T]: - return [x for x in self.operations if isinstance(x, type)] \ No newline at end of file + """Filters the operations list to include only instances of the + specified type. + + :param type: The type of operations to filter + :return: A list of operations that are instances of the specified type + """ + return [x for x in self.operations if isinstance(x, type)] diff --git a/smartsim/entity/application.py b/smartsim/entity/application.py index 11b1a36bab..43082fe7a0 100644 --- a/smartsim/entity/application.py +++ b/smartsim/entity/application.py @@ -59,8 +59,10 @@ def __init__( self, name: str, exe: str, - file_parameters: t.Mapping[str,str], # TODO remove when Ensemble is addressed - files: t.Optional[EntityFiles] = None, # TODO remove when Ensemble is addressed + file_parameters: ( + t.Mapping[str, str] | None + ) = None, # TODO remove when Ensemble is addressed + files: t.Optional[EntityFiles] = None, # TODO remove when Ensemble is addressed exe_args: t.Optional[t.Union[str, t.Sequence[str]]] = None, ) -> None: """Initialize an ``Application`` @@ -86,6 +88,11 @@ def __init__( self._exe_args = self._build_exe_args(exe_args) or [] """The executable arguments""" self.files = FileSysOperationSet([]) + """Attach files""" + self._file_parameters = ( + copy.deepcopy(file_parameters) if file_parameters else {} + ) + """TODO MOCK until Ensemble is implemented""" """Files to be copied, symlinked, and/or configured prior to execution""" self._incoming_entities: t.List[SmartSimEntity] = [] """Entities for which the prefix will have to be known by other entities""" diff --git a/tests/test_generator.py b/tests/test_generator.py index ba818f6f50..82bce214f8 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -25,16 +25,13 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import itertools -import os import pathlib import unittest.mock from glob import glob -from os import listdir from os import path as osp import pytest -from smartsim import Experiment from smartsim._core.commands import Command, CommandList from smartsim._core.generation.generator import Generator from smartsim._core.generation.operations import ( @@ -43,18 +40,10 @@ FileSysOperationSet, GenerationContext, SymlinkOperation, - configure_cmd, - copy_cmd, create_final_dest, - symlink_cmd, ) -from smartsim.builders import Ensemble from smartsim.entity import SmartSimEntity -from smartsim.entity.files import EntityFiles from smartsim.launchable import Job -from smartsim.settings import LaunchSettings - -# TODO Add JobGroup tests when JobGroup becomes a Launchable pytestmark = pytest.mark.group_a @@ -86,11 +75,13 @@ def mock_dest(test_dir: str): """Fixture to create a mock destination path.""" return pathlib.Path(test_dir) / pathlib.Path("mock_dest") + @pytest.fixture def mock_index(): """Fixture to create a mock destination path.""" return 1 + @pytest.fixture def copy_operation(mock_src: pathlib.Path, mock_dest: pathlib.Path): """Fixture to create a CopyOperation object.""" @@ -166,7 +157,7 @@ def test_build_job_run_path( mock_job: unittest.mock.MagicMock, generator_instance: Generator, monkeypatch: pytest.MonkeyPatch, - mock_index + mock_index, ): """Test Generator._build_job_run_path returns correct path""" monkeypatch.setattr( @@ -184,7 +175,7 @@ def test_build_job_log_path( mock_job: unittest.mock.MagicMock, generator_instance: Generator, monkeypatch: pytest.MonkeyPatch, - mock_index + mock_index, ): """Test Generator._build_job_log_path returns correct path""" monkeypatch.setattr( @@ -224,9 +215,7 @@ def test_build_err_file_path( def test_generate_job( - mock_job: unittest.mock.MagicMock, - generator_instance: Generator, - mock_index + mock_job: unittest.mock.MagicMock, generator_instance: Generator, mock_index ): """Test Generator.generate_job returns correct paths""" job_paths = generator_instance.generate_job(mock_job, mock_index) @@ -253,6 +242,7 @@ def test_mkdir_file(generator_instance: Generator, test_dir: str): assert isinstance(cmd, Command) assert cmd.command == ["mkdir", "-p", test_dir] + # might change this to files that can be configured @pytest.fixture def files(fileutils): @@ -270,32 +260,14 @@ def directory(fileutils): ) yield [pathlib.Path(directory)] + @pytest.fixture def source(request, files, directory): if request.param == "files": return files elif request.param == "directory": return directory - else: - raise ValueError(f"Unknown source fixture: {request.param}") -@pytest.mark.parametrize( - "dest", - ( - pytest.param(123, id="dest as integer"), - pytest.param("", id="dest as empty str"), - pytest.param("/absolute/path", id="dest as absolute str"), - pytest.param(pathlib.Path("relative/path"), id="dest as relative Path"), - pytest.param(pathlib.Path("/path with spaces"), id="dest as Path with spaces"), - # pytest.param(pathlib.Path("/path/with/special!@#"), id="dest as Path with special char"), - ), -) -@pytest.mark.parametrize("source", ["files", "directory"], indirect=True) -def test_copy_files_invalid_dest(dest, source, generator_instance, test_dir): - to_copy = [SymlinkOperation(src=file, dest=dest) for file in source] - gen = GenerationContext(pathlib.Path(test_dir)) - with pytest.raises(ValueError): - generator_instance._symlink_files(files=to_copy, context=gen) @pytest.mark.parametrize( "dest", @@ -326,23 +298,6 @@ def test_copy_files_valid_dest(dest, source, generator_instance, test_dir): cmd_src_paths ), "Not all file paths are in the command list" -@pytest.mark.parametrize( - "dest", - ( - pytest.param(123, id="dest as integer"), - pytest.param("", id="dest as empty str"), - pytest.param("/absolute/path", id="dest as absolute str"), - pytest.param(pathlib.Path("relative/path"), id="dest as relative Path"), - pytest.param(pathlib.Path("/path with spaces"), id="dest as Path with spaces"), - # pytest.param(pathlib.Path("/path/with/special!@#"), id="dest as Path with special char"), - ), -) -@pytest.mark.parametrize("source", ["files", "directory"], indirect=True) -def test_symlink_files_invalid_dest(dest, source, generator_instance, test_dir): - to_copy = [SymlinkOperation(src=file, dest=dest) for file in source] - gen = GenerationContext(pathlib.Path(test_dir)) - with pytest.raises(ValueError): - generator_instance._symlink_files(files=to_copy, context=gen) @pytest.mark.parametrize( "dest", @@ -373,23 +328,6 @@ def test_symlink_files_valid_dest(dest, source, generator_instance, test_dir): cmd_src_paths ), "Not all file paths are in the command list" -@pytest.mark.parametrize( - "dest", - ( - pytest.param(123, id="dest as integer"), - pytest.param("", id="dest as empty str"), - pytest.param("/absolute/path", id="dest as absolute str"), - pytest.param(pathlib.Path("relative/path"), id="dest as relative Path"), - pytest.param(pathlib.Path("/path with spaces"), id="dest as Path with spaces"), - # pytest.param(pathlib.Path("/path/with/special!@#"), id="dest as Path with special char"), - ), -) -@pytest.mark.parametrize("source", ["files", "directory"], indirect=True) -def test_configure_files_invalid_dest(dest, source, generator_instance, test_dir): - to_configure = [ConfigureOperation(src=file, dest=dest, file_parameters={"FOO": "BAR"}) for file in source] - gen = GenerationContext(pathlib.Path(test_dir)) - with pytest.raises(ValueError): - generator_instance._configure_files(files=to_configure, context=gen) @pytest.mark.parametrize( "dest", @@ -404,15 +342,18 @@ def test_configure_files_invalid_dest(dest, source, generator_instance, test_dir @pytest.mark.parametrize("source", ["files", "directory"], indirect=True) def test_configure_files_valid_dest(dest, source, generator_instance, test_dir): file_param = { - "5": 10, - "FIRST": "SECOND", - "17": 20, - "65": "70", - "placeholder": "group leftupper region", - "1200": "120", - "VALID": "valid", - } - to_configure = [ConfigureOperation(src=file, dest=dest, file_parameters=file_param) for file in source] + "5": 10, + "FIRST": "SECOND", + "17": 20, + "65": "70", + "placeholder": "group leftupper region", + "1200": "120", + "VALID": "valid", + } + to_configure = [ + ConfigureOperation(src=file, dest=dest, file_parameters=file_param) + for file in source + ] gen = GenerationContext(pathlib.Path(test_dir)) cmd_list = generator_instance._configure_files(files=to_configure, context=gen) assert isinstance(cmd_list, CommandList) @@ -429,18 +370,24 @@ def test_configure_files_valid_dest(dest, source, generator_instance, test_dir): cmd_src_paths ), "Not all file paths are in the command list" + # TODO Add configure_file tests + @pytest.fixture def run_directory(test_dir, generator_instance): return pathlib.Path(test_dir) / generator_instance.run_directory + @pytest.fixture def log_directory(test_dir, generator_instance): return pathlib.Path(test_dir) / generator_instance.log_directory + def test_build_commands( - generator_instance: Generator, run_directory: pathlib.Path, log_directory: pathlib.Path + generator_instance: Generator, + run_directory: pathlib.Path, + log_directory: pathlib.Path, ): """Test Generator._build_commands calls internal helper functions""" with ( @@ -449,7 +396,7 @@ def test_build_commands( ) as mock_append_mkdir_commands, unittest.mock.patch( "smartsim._core.generation.Generator._append_file_operations" - ) as mock_append_file_operations + ) as mock_append_file_operations, ): generator_instance._build_commands( EchoHelloWorldEntity(), @@ -459,7 +406,12 @@ def test_build_commands( mock_append_mkdir_commands.assert_called_once() mock_append_file_operations.assert_called_once() -def test_append_mkdir_commands(generator_instance: Generator, run_directory: pathlib.Path, log_directory: pathlib.Path): + +def test_append_mkdir_commands( + generator_instance: Generator, + run_directory: pathlib.Path, + log_directory: pathlib.Path, +): with ( unittest.mock.patch( "smartsim._core.generation.Generator._mkdir_file" @@ -472,6 +424,7 @@ def test_append_mkdir_commands(generator_instance: Generator, run_directory: pat ) assert mock_mkdir_file.call_count == 2 + def test_append_file_operations(context, generator_instance): with ( unittest.mock.patch( @@ -493,16 +446,19 @@ def test_append_file_operations(context, generator_instance): mock_symlink_files.assert_called_once() mock_configure_files.assert_called_once() + @pytest.fixture def paths_to_copy(fileutils): paths = fileutils.get_test_conf_path(osp.join("mock", "copy_mock")) yield [pathlib.Path(path) for path in sorted(glob(paths + "/*"))] + @pytest.fixture def paths_to_symlink(fileutils): paths = fileutils.get_test_conf_path(osp.join("mock", "symlink_mock")) yield [pathlib.Path(path) for path in sorted(glob(paths + "/*"))] + @pytest.fixture def paths_to_configure(fileutils): paths = fileutils.get_test_conf_path( @@ -510,9 +466,12 @@ def paths_to_configure(fileutils): ) yield [pathlib.Path(path) for path in sorted(glob(paths + "/*"))] + @pytest.fixture def context(test_dir): yield GenerationContext(pathlib.Path(test_dir)) + + @pytest.fixture def operations_list(paths_to_copy, paths_to_symlink, paths_to_configure): merp = [] @@ -524,6 +483,7 @@ def operations_list(paths_to_copy, paths_to_symlink, paths_to_configure): merp.append(SymlinkOperation(src=file)) return merp + @pytest.fixture def formatted_command_list(operations_list, context): new_list = CommandList() @@ -531,7 +491,10 @@ def formatted_command_list(operations_list, context): new_list.append(file.format(context)) return new_list -def test_execute_commands(operations_list, formatted_command_list, generator_instance, test_dir): + +def test_execute_commands( + operations_list, formatted_command_list, generator_instance, test_dir +): with ( unittest.mock.patch( "smartsim._core.generation.generator.subprocess.run" diff --git a/tests/test_operations.py b/tests/test_operations.py index ff2667e2d4..41b465faa7 100644 --- a/tests/test_operations.py +++ b/tests/test_operations.py @@ -1,4 +1,9 @@ +import base64 +import os import pathlib +import pickle +from glob import glob +from os import path as osp import pytest @@ -17,8 +22,6 @@ # QUESTIONS # TODO test python protocol? -# TODO add encoded dict into configure op -# TODO create a better way to append the paths together # TODO do I allow the paths to combine if src is empty? @@ -48,16 +51,15 @@ def copy_operation(mock_src: pathlib.Path, mock_dest: pathlib.Path): @pytest.fixture def symlink_operation(mock_src: pathlib.Path, mock_dest: pathlib.Path): - """Fixture to create a CopyOperation object.""" + """Fixture to create a SymlinkOperation object.""" return SymlinkOperation(src=mock_src, dest=mock_dest) @pytest.fixture def configure_operation(mock_src: pathlib.Path, mock_dest: pathlib.Path): - """Fixture to create a CopyOperation object.""" + """Fixture to create a Configure object.""" return ConfigureOperation( - src=mock_src, - dest=mock_dest, + src=mock_src, dest=mock_dest, file_parameters={"FOO": "BAR"} ) @@ -80,12 +82,6 @@ def file_system_operation_set( "/valid/root/valid/dest", id="Valid paths", ), - # pytest.param( - # pathlib.Path("/valid/root/"), - # pathlib.Path("/valid/dest.txt"), - # "/valid/root/valid/dest.txt", - # id="Valid_file_path", - # ), pytest.param( pathlib.Path("/valid/root"), pathlib.Path(""), @@ -109,12 +105,8 @@ def test_create_final_dest_valid(job_root_path, dest, expected): "job_root_path, dest", ( pytest.param(None, pathlib.Path("valid/dest"), id="None as root path"), - pytest.param("", pathlib.Path("valid/dest"), id="Empty str as root path"), - pytest.param( - pathlib.Path("/invalid/root.py"), - pathlib.Path("valid/dest"), - id="File as root path", - ), + pytest.param(1234, pathlib.Path("valid/dest"), id="Number as root path"), + pytest.param(pathlib.Path("valid/dest"), 1234, id="Number as dest"), ), ) def test_create_final_dest_invalid(job_root_path, dest): @@ -123,12 +115,22 @@ def test_create_final_dest_invalid(job_root_path, dest): create_final_dest(job_root_path, dest) -def test_init_generation_context(test_dir: str, generation_context: GenerationContext): +def test_valid_init_generation_context( + test_dir: str, generation_context: GenerationContext +): """Validate GenerationContext init""" assert isinstance(generation_context, GenerationContext) assert generation_context.job_root_path == pathlib.Path(test_dir) +def test_invalid_init_generation_context(): + """Validate GenerationContext init""" + with pytest.raises(TypeError): + GenerationContext(1234) + with pytest.raises(TypeError): + GenerationContext("") + + def test_init_copy_operation( copy_operation: CopyOperation, mock_src: pathlib.Path, mock_dest: pathlib.Path ): @@ -172,7 +174,12 @@ def test_symlink_operation_format( assert isinstance(exec, Command) assert str(mock_src) in exec.command assert symlink_cmd in exec.command - assert create_final_dest(mock_src, mock_dest) in exec.command + + normalized_path = os.path.normpath(mock_src) + parent_dir = os.path.basename(normalized_path) + final_dest = create_final_dest(generation_context.job_root_path, mock_dest) + new_dest = os.path.join(final_dest, parent_dir) + assert new_dest in exec.command def test_init_configure_operation( @@ -183,6 +190,9 @@ def test_init_configure_operation( assert configure_operation.src == mock_src assert configure_operation.dest == mock_dest assert configure_operation.tag == ";" + decoded_dict = base64.b64decode(configure_operation.file_parameters.encode("ascii")) + unpickled_dict = pickle.loads(decoded_dict) + assert unpickled_dict == {"FOO": "BAR"} def test_configure_operation_format( @@ -200,6 +210,7 @@ def test_configure_operation_format( def test_init_file_sys_operation_set(file_system_operation_set: FileSysOperationSet): + """Test initialize FileSystemOperationSet""" assert isinstance(file_system_operation_set.operations, list) assert len(file_system_operation_set.operations) == 3 @@ -207,16 +218,18 @@ def test_init_file_sys_operation_set(file_system_operation_set: FileSysOperation def test_add_copy_operation( file_system_operation_set: FileSysOperationSet, copy_operation: CopyOperation ): + """Test FileSystemOperationSet.add_copy""" assert len(file_system_operation_set.copy_operations) == 1 - file_system_operation_set.add_copy(copy_operation) + file_system_operation_set.add_copy(src=pathlib.Path("src")) assert len(file_system_operation_set.copy_operations) == 2 def test_add_symlink_operation( file_system_operation_set: FileSysOperationSet, symlink_operation: SymlinkOperation ): + """Test FileSystemOperationSet.add_symlink""" assert len(file_system_operation_set.symlink_operations) == 1 - file_system_operation_set.add_symlink(symlink_operation) + file_system_operation_set.add_symlink(src=pathlib.Path("src")) assert len(file_system_operation_set.symlink_operations) == 2 @@ -224,6 +237,104 @@ def test_add_configure_operation( file_system_operation_set: FileSysOperationSet, configure_operation: ConfigureOperation, ): + """Test FileSystemOperationSet.add_configuration""" assert len(file_system_operation_set.configure_operations) == 1 - file_system_operation_set.add_configuration(configure_operation) + file_system_operation_set.add_configuration( + src=pathlib.Path("src"), file_parameters={"FOO": "BAR"} + ) assert len(file_system_operation_set.configure_operations) == 2 + + +# might change this to files that can be configured +@pytest.fixture +def files(fileutils): + path_to_files = fileutils.get_test_conf_path( + osp.join("generator_files", "easy", "correct/") + ) + list_of_files_strs = sorted(glob(path_to_files + "/*")) + yield [pathlib.Path(str_path) for str_path in list_of_files_strs] + + +@pytest.fixture +def directory(fileutils): + directory = fileutils.get_test_conf_path( + osp.join("generator_files", "easy", "correct/") + ) + yield [pathlib.Path(directory)] + + +@pytest.fixture +def source(request, files, directory): + if request.param == "files": + return files + elif request.param == "directory": + return directory + + +@pytest.mark.parametrize( + "dest,error", + ( + pytest.param(123, TypeError, id="dest as integer"), + pytest.param("", TypeError, id="dest as empty str"), + pytest.param("/absolute/path", TypeError, id="dest as absolute str"), + pytest.param( + pathlib.Path("relative/path"), ValueError, id="dest as relative Path" + ), + pytest.param( + pathlib.Path("/path with spaces"), ValueError, id="dest as Path with spaces" + ), + # TODO pytest.param(pathlib.Path("/path/with/special!@#"), id="dest as Path with special char"), + ), +) +@pytest.mark.parametrize("source", ["files", "directory"], indirect=True) +def test_copy_files_invalid_dest(dest, error, source): + """Test invalid copy destination""" + with pytest.raises(error): + _ = [CopyOperation(src=file, dest=dest) for file in source] + + +@pytest.mark.parametrize( + "dest,error", + ( + pytest.param(123, TypeError, id="dest as integer"), + pytest.param("", TypeError, id="dest as empty str"), + pytest.param("/absolute/path", TypeError, id="dest as absolute str"), + pytest.param( + pathlib.Path("relative/path"), ValueError, id="dest as relative Path" + ), + pytest.param( + pathlib.Path("/path with spaces"), ValueError, id="dest as Path with spaces" + ), + # TODO pytest.param(pathlib.Path("/path/with/special!@#"), id="dest as Path with special char"), + ), +) +@pytest.mark.parametrize("source", ["files", "directory"], indirect=True) +def test_symlink_files_invalid_dest(dest, error, source): + """Test invalid symlink destination""" + with pytest.raises(error): + _ = [SymlinkOperation(src=file, dest=dest) for file in source] + + +@pytest.mark.parametrize( + "dest,error", + ( + pytest.param(123, TypeError, id="dest as integer"), + pytest.param("", TypeError, id="dest as empty str"), + pytest.param("/absolute/path", TypeError, id="dest as absolute str"), + pytest.param( + pathlib.Path("relative/path"), ValueError, id="dest as relative Path" + ), + pytest.param( + pathlib.Path("/path with spaces"), ValueError, id="dest as Path with spaces" + ), + # TODO pytest.param(pathlib.Path("/path/with/special!@#"), id="dest as Path with special char"), + ), +) +@pytest.mark.parametrize("source", ["files", "directory"], indirect=True) +def test_configure_files_invalid_dest(dest, error, source): + """Test invalid configure destination""" + with pytest.raises(error): + _ = [ + ConfigureOperation(src=file, dest=dest, file_parameters={"FOO": "BAR"}) + for file in source + ] From 69f786688b4690506669cd0165ac842c94d8d113 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Thu, 3 Oct 2024 15:15:30 -0700 Subject: [PATCH 08/35] remove unused print statements --- smartsim/_core/entrypoints/file_operations.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/smartsim/_core/entrypoints/file_operations.py b/smartsim/_core/entrypoints/file_operations.py index 9529ffdcd1..9db576fd42 100644 --- a/smartsim/_core/entrypoints/file_operations.py +++ b/smartsim/_core/entrypoints/file_operations.py @@ -159,7 +159,6 @@ def copy(parsed_args: argparse.Namespace) -> None: not includedm and the destination file already exists, a FileExistsError will be raised """ - print(parsed_args.source) if os.path.isdir(parsed_args.source): shutil.copytree( parsed_args.source, @@ -227,7 +226,6 @@ def configure(parsed_args: argparse.Namespace) -> None: for file_name in filenames: src_file = os.path.join(dirpath, file_name) dst_file = os.path.join(new_dir_dest, file_name) - print(type(substitutions)) _process_file(substitutions, src_file, dst_file) else: dst_file = parsed_args.dest / os.path.basename(parsed_args.source) From 143ff4b2536400aa80758673b239448ddee31ed6 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Thu, 3 Oct 2024 15:15:53 -0700 Subject: [PATCH 09/35] fix typos --- smartsim/_core/entrypoints/file_operations.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/smartsim/_core/entrypoints/file_operations.py b/smartsim/_core/entrypoints/file_operations.py index 9db576fd42..69d7f7565e 100644 --- a/smartsim/_core/entrypoints/file_operations.py +++ b/smartsim/_core/entrypoints/file_operations.py @@ -154,9 +154,9 @@ def copy(parsed_args: argparse.Namespace) -> None: /absolute/file/dest/path: Path to destination directory or path to destination file --dirs_exist_ok: if the flag is included, the copying operation will - continue if the destination directory and files alrady exist, + continue if the destination directory and files already exist, and will be overwritten by corresponding files. If the flag is - not includedm and the destination file already exists, a + not included and the destination file already exists, a FileExistsError will be raised """ if os.path.isdir(parsed_args.source): From cdd82a4d330b6fd55ff76267a2a08ee8df253ff3 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Mon, 7 Oct 2024 11:57:08 -0700 Subject: [PATCH 10/35] updates to PR --- smartsim/_core/generation/generator.py | 27 ++++++------ smartsim/_core/generation/operations.py | 15 +++---- .../test_commands/test_commandList.py | 9 ++-- .../to_copy_dir/{mock.txt => mock_1.txt} | 0 .../generator_files/to_copy_dir/mock_2.txt | 0 .../generator_files/to_copy_dir/mock_3.txt | 0 .../generator_files/to_symlink_dir/mock_1.txt | 0 .../to_symlink_dir/{mock2.txt => mock_2.txt} | 0 .../generator_files/to_symlink_dir/mock_3.txt | 0 tests/test_generator.py | 42 ++++++++++++------- tests/test_operations.py | 2 + 11 files changed, 53 insertions(+), 42 deletions(-) rename tests/test_configs/generator_files/to_copy_dir/{mock.txt => mock_1.txt} (100%) create mode 100644 tests/test_configs/generator_files/to_copy_dir/mock_2.txt create mode 100644 tests/test_configs/generator_files/to_copy_dir/mock_3.txt create mode 100644 tests/test_configs/generator_files/to_symlink_dir/mock_1.txt rename tests/test_configs/generator_files/to_symlink_dir/{mock2.txt => mock_2.txt} (100%) create mode 100644 tests/test_configs/generator_files/to_symlink_dir/mock_3.txt diff --git a/smartsim/_core/generation/generator.py b/smartsim/_core/generation/generator.py index 49a4c322f6..8f74dc6335 100644 --- a/smartsim/_core/generation/generator.py +++ b/smartsim/_core/generation/generator.py @@ -48,16 +48,17 @@ @t.runtime_checkable class _GenerableProtocol(t.Protocol): - """Ensures functions using job.entity proceed if both attribute files and - parameters are supported.""" + """Protocol to ensure that an entity supports both file operations + and parameters.""" files: FileSysOperationSet - # TODO might need to review if file_params is taken off + # TODO change when file_parameters taken off Application during Ensemble refactor ticket file_parameters: t.Mapping[str, str] Job_Path = namedtuple("Job_Path", ["run_path", "out_path", "err_path"]) -"""Stores the Job's run path, output path, and error file path.""" +"""Namedtuple that stores a Job's run directory, output file path, and +error file path.""" class Generator: @@ -73,7 +74,7 @@ class Generator: def __init__(self, root: pathlib.Path) -> None: """Initialize a Generator object - The Generator class is responsible for constructing a Job's directory, including + The Generator class is responsible for constructing a Job's directory, performing the following tasks: - Creating the run and log directories @@ -81,7 +82,7 @@ def __init__(self, root: pathlib.Path) -> None: - Building the parameter settings file - Managing symlinking, copying, and configuration of attached files - :param root: Job base path + :param root: The base path for job-related files and directories """ self.root = root """The root directory under which all generated files and directories will be placed.""" @@ -102,8 +103,8 @@ def _build_job_base_path(self, job: Job, job_index: int) -> pathlib.Path: def _build_job_run_path(self, job: Job, job_index: int) -> pathlib.Path: """Build and return a Job's run directory. The path is formed by combining - the base directory with the `run` class-level variable, where run specifies - the name of the job's run folder. + the base directory with the `run_directory` class-level constant, which specifies + the name of the Job's run folder. :param job: Job object :param job_index: Job index @@ -114,8 +115,8 @@ def _build_job_run_path(self, job: Job, job_index: int) -> pathlib.Path: def _build_job_log_path(self, job: Job, job_index: int) -> pathlib.Path: """Build and return a Job's log directory. The path is formed by combining - the base directory with the `log` class-level variable, where log specifies - the name of the job's log folder. + the base directory with the `log_directory` class-level constant, which specifies + the name of the Job's log folder. :param job: Job object :param job_index: Job index @@ -126,7 +127,7 @@ def _build_job_log_path(self, job: Job, job_index: int) -> pathlib.Path: @staticmethod def _build_log_file_path(log_path: pathlib.Path) -> pathlib.Path: - """Build and return an entities file summarizing the parameters + """Build and return a parameters file summarizing the parameters used for the generation of the entity. :param log_path: Path to log directory @@ -159,7 +160,7 @@ def _build_err_file_path(log_path: pathlib.Path, job_name: str) -> pathlib.Path: return err_file_path def generate_job(self, job: Job, job_index: int) -> Job_Path: - """Build and return the Job's run directory, error file and out file. + """Build and return the Job's run directory, output file, and error file. This method creates the Job's run and log directories, generates the `smartsim_params.txt` file to log parameters used for the Job, and sets @@ -269,7 +270,7 @@ def _execute_commands(cls, cmd_list: CommandList) -> None: @staticmethod def _mkdir_file(file_path: pathlib.Path) -> Command: - """Build a Command to create a directory, including any + """Build a Command to create the directory along with any necessary parent directories. :param file_path: The directory path to be created diff --git a/smartsim/_core/generation/operations.py b/smartsim/_core/generation/operations.py index 9b9149b16f..4e6e1001b2 100644 --- a/smartsim/_core/generation/operations.py +++ b/smartsim/_core/generation/operations.py @@ -46,7 +46,7 @@ def check_src_and_dest_path( of type pathlib.Path :param src: The source path to be checked. - :param des: The destination path to be checked. + :param dest: The destination path to be checked. :raises TypeError: If either src or dest is not an instance of pathlib.Path """ if not isinstance(src, pathlib.Path): @@ -72,7 +72,8 @@ def check_run_path(run_path: pathlib.Path) -> None: """Validate that the provided run path is of type pathlib.Path :param run_path: The run path to be checked - :raises TypeError: If either src or dest is not an instance of pathlib.Path + :raises TypeError: If either run path is not an instance of pathlib.Path + :raises ValueError: If the run path is not a directory """ if not isinstance(run_path, pathlib.Path): raise TypeError( @@ -114,7 +115,7 @@ def __init__( self.dest = dest def format(self, context: GenerationContext) -> Command: - """Create Command to invoke copy fs entry point + """Create Command to invoke copy file system entry point :param context: Context for copy operation :return: Copy Command @@ -149,7 +150,7 @@ def __init__( self.dest = dest def format(self, context: GenerationContext) -> Command: - """Create Command to invoke symlink fs entry point + """Create Command to invoke symlink file system entry point :param context: Context for symlink operation :return: Symlink Command @@ -196,7 +197,7 @@ def __init__( self.tag = tag if tag else ";" def format(self, context: GenerationContext) -> Command: - """Create Command to invoke configure fs entry point + """Create Command to invoke configure file system entry point :param context: Context for configure operation :return: Configure Command @@ -221,11 +222,11 @@ def format(self, context: GenerationContext) -> Command: @dataclass class FileSysOperationSet: - """Dataclass to represent a set of FS Operation Objects""" + """Dataclass to represent a set of file system operation objects""" # TODO disallow modification - dunder function (post ticket to reevaluate API objects) operations: t.List[GenerationProtocol] = field(default_factory=list) - """Set of FS Objects that match the GenerationProtocol""" + """Set of file system objects that match the GenerationProtocol""" def add_copy( self, src: pathlib.Path, dest: t.Optional[pathlib.Path] = None diff --git a/tests/temp_tests/test_core/test_commands/test_commandList.py b/tests/temp_tests/test_core/test_commands/test_commandList.py index c6bc8d8347..8d91336e88 100644 --- a/tests/temp_tests/test_core/test_commands/test_commandList.py +++ b/tests/temp_tests/test_core/test_commands/test_commandList.py @@ -76,13 +76,10 @@ def test_command_setitem_fail(): cmd_list[0:1] = "fail" with pytest.raises(ValueError): cmd_list[0:1] = "fail" - cmd_1 = Command(command=["salloc", "-N", 1]) - cmd_2 = Command(command=["salloc", "-N", "1"]) - cmd_3 = Command(command=1) with pytest.raises(ValueError): - cmd_list[0:1] = [cmd_1, cmd_2] - with pytest.raises(ValueError): - cmd_list[0:1] = [cmd_3, cmd_2] + _ = Command(command=["salloc", "-N", 1]) + with pytest.raises(TypeError): + cmd_list[0:1] = [Command(command=["salloc", "-N", "1"]), Command(command=1)] def test_command_delitem(): diff --git a/tests/test_configs/generator_files/to_copy_dir/mock.txt b/tests/test_configs/generator_files/to_copy_dir/mock_1.txt similarity index 100% rename from tests/test_configs/generator_files/to_copy_dir/mock.txt rename to tests/test_configs/generator_files/to_copy_dir/mock_1.txt diff --git a/tests/test_configs/generator_files/to_copy_dir/mock_2.txt b/tests/test_configs/generator_files/to_copy_dir/mock_2.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/test_configs/generator_files/to_copy_dir/mock_3.txt b/tests/test_configs/generator_files/to_copy_dir/mock_3.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/test_configs/generator_files/to_symlink_dir/mock_1.txt b/tests/test_configs/generator_files/to_symlink_dir/mock_1.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/test_configs/generator_files/to_symlink_dir/mock2.txt b/tests/test_configs/generator_files/to_symlink_dir/mock_2.txt similarity index 100% rename from tests/test_configs/generator_files/to_symlink_dir/mock2.txt rename to tests/test_configs/generator_files/to_symlink_dir/mock_2.txt diff --git a/tests/test_configs/generator_files/to_symlink_dir/mock_3.txt b/tests/test_configs/generator_files/to_symlink_dir/mock_3.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/test_generator.py b/tests/test_generator.py index 82bce214f8..e955dba7f5 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -215,7 +215,7 @@ def test_build_err_file_path( def test_generate_job( - mock_job: unittest.mock.MagicMock, generator_instance: Generator, mock_index + mock_job: unittest.mock.MagicMock, generator_instance: Generator, mock_index: int ): """Test Generator.generate_job returns correct paths""" job_paths = generator_instance.generate_job(mock_job, mock_index) @@ -243,7 +243,6 @@ def test_mkdir_file(generator_instance: Generator, test_dir: str): assert cmd.command == ["mkdir", "-p", test_dir] -# might change this to files that can be configured @pytest.fixture def files(fileutils): path_to_files = fileutils.get_test_conf_path( @@ -280,7 +279,9 @@ def source(request, files, directory): ), ) @pytest.mark.parametrize("source", ["files", "directory"], indirect=True) -def test_copy_files_valid_dest(dest, source, generator_instance, test_dir): +def test_copy_files_valid_dest( + dest, source, generator_instance: Generator, test_dir: str +): to_copy = [CopyOperation(src=file, dest=dest) for file in source] gen = GenerationContext(pathlib.Path(test_dir)) cmd_list = generator_instance._copy_files(files=to_copy, context=gen) @@ -310,7 +311,9 @@ def test_copy_files_valid_dest(dest, source, generator_instance, test_dir): ), ) @pytest.mark.parametrize("source", ["files", "directory"], indirect=True) -def test_symlink_files_valid_dest(dest, source, generator_instance, test_dir): +def test_symlink_files_valid_dest( + dest, source, generator_instance: Generator, test_dir: str +): to_copy = [CopyOperation(src=file, dest=dest) for file in source] gen = GenerationContext(pathlib.Path(test_dir)) cmd_list = generator_instance._copy_files(files=to_copy, context=gen) @@ -340,7 +343,9 @@ def test_symlink_files_valid_dest(dest, source, generator_instance, test_dir): ), ) @pytest.mark.parametrize("source", ["files", "directory"], indirect=True) -def test_configure_files_valid_dest(dest, source, generator_instance, test_dir): +def test_configure_files_valid_dest( + dest, source, generator_instance: Generator, test_dir: str +): file_param = { "5": 10, "FIRST": "SECOND", @@ -412,6 +417,7 @@ def test_append_mkdir_commands( run_directory: pathlib.Path, log_directory: pathlib.Path, ): + """Test Generator._append_mkdir_commands calls Generator._mkdir_file twice""" with ( unittest.mock.patch( "smartsim._core.generation.Generator._mkdir_file" @@ -425,7 +431,10 @@ def test_append_mkdir_commands( assert mock_mkdir_file.call_count == 2 -def test_append_file_operations(context, generator_instance): +def test_append_file_operations( + context: GenerationContext, generator_instance: Generator +): + """Test Generator._append_file_operations calls all file operations""" with ( unittest.mock.patch( "smartsim._core.generation.Generator._copy_files" @@ -449,13 +458,13 @@ def test_append_file_operations(context, generator_instance): @pytest.fixture def paths_to_copy(fileutils): - paths = fileutils.get_test_conf_path(osp.join("mock", "copy_mock")) + paths = fileutils.get_test_conf_path(osp.join("generator_files", "to_copy_dir")) yield [pathlib.Path(path) for path in sorted(glob(paths + "/*"))] @pytest.fixture def paths_to_symlink(fileutils): - paths = fileutils.get_test_conf_path(osp.join("mock", "symlink_mock")) + paths = fileutils.get_test_conf_path(osp.join("generator_files", "to_symlink_dir")) yield [pathlib.Path(path) for path in sorted(glob(paths + "/*"))] @@ -468,24 +477,24 @@ def paths_to_configure(fileutils): @pytest.fixture -def context(test_dir): +def context(test_dir: str): yield GenerationContext(pathlib.Path(test_dir)) @pytest.fixture def operations_list(paths_to_copy, paths_to_symlink, paths_to_configure): - merp = [] + op_list = [] for file in paths_to_copy: - merp.append(CopyOperation(src=file)) + op_list.append(CopyOperation(src=file)) for file in paths_to_symlink: - merp.append(SymlinkOperation(src=file)) + op_list.append(SymlinkOperation(src=file)) for file in paths_to_configure: - merp.append(SymlinkOperation(src=file)) - return merp + op_list.append(SymlinkOperation(src=file)) + return op_list @pytest.fixture -def formatted_command_list(operations_list, context): +def formatted_command_list(operations_list: list, context: GenerationContext): new_list = CommandList() for file in operations_list: new_list.append(file.format(context)) @@ -493,8 +502,9 @@ def formatted_command_list(operations_list, context): def test_execute_commands( - operations_list, formatted_command_list, generator_instance, test_dir + operations_list: list, formatted_command_list, generator_instance: Generator ): + """Test Generator._execute_commands calls with appropriate type and num times""" with ( unittest.mock.patch( "smartsim._core.generation.generator.subprocess.run" diff --git a/tests/test_operations.py b/tests/test_operations.py index 41b465faa7..6281eb437d 100644 --- a/tests/test_operations.py +++ b/tests/test_operations.py @@ -24,6 +24,8 @@ # TODO test python protocol? # TODO do I allow the paths to combine if src is empty? +pytestmark = pytest.mark.group_a + @pytest.fixture def generation_context(test_dir: str): From 56eb31c8462428a076353551a35921323e06779d Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Tue, 8 Oct 2024 09:41:47 -0700 Subject: [PATCH 11/35] initial push --- .../_core/generation/builder_operations.py | 95 ++++++ smartsim/builders/ensemble.py | 134 ++++---- smartsim/builders/utils/strategies.py | 19 ++ smartsim/entity/application.py | 10 - tests/test_builder_operations.py | 297 ++++++++++++++++++ 5 files changed, 489 insertions(+), 66 deletions(-) create mode 100644 smartsim/_core/generation/builder_operations.py create mode 100644 tests/test_builder_operations.py diff --git a/smartsim/_core/generation/builder_operations.py b/smartsim/_core/generation/builder_operations.py new file mode 100644 index 0000000000..2d4a7704d8 --- /dev/null +++ b/smartsim/_core/generation/builder_operations.py @@ -0,0 +1,95 @@ +import pathlib +import typing as t +from dataclasses import dataclass, field + + + +class EnsembleGenerationProtocol(t.Protocol): + """Protocol for Generation Operations Ensemble.""" + src: pathlib.Path + dest: t.Optional[pathlib.Path] + + +class EnsembleCopyOperation(EnsembleGenerationProtocol): + """Copy Operation""" + + def __init__( + self, src: pathlib.Path, dest: t.Optional[pathlib.Path] = None + ) -> None: + self.src = src + self.dest = dest + + +class EnsembleSymlinkOperation(EnsembleGenerationProtocol): + """Symlink Operation""" + + def __init__(self, src: pathlib.Path, dest: t.Optional[pathlib.Path] = None) -> None: + self.src = src + self.dest = dest + + +class EnsembleConfigureOperation(EnsembleGenerationProtocol): + """Configure Operation""" + + def __init__( + self, + src: pathlib.Path, + file_parameters:t.Mapping[str,t.Sequence[str]], + dest: t.Optional[pathlib.Path] = None, + tag: t.Optional[str] = None, + ) -> None: + self.src = src + self.dest = dest + self.file_parameters = file_parameters + self.tag = tag if tag else ";" + + +U = t.TypeVar("U", bound=EnsembleGenerationProtocol) + + +@dataclass +class EnsembleFileSysOperationSet: + """Dataclass to represent a set of FS Operation Objects""" + + operations: t.List[EnsembleGenerationProtocol] = field(default_factory=list) + """Set of FS Objects that match the GenerationProtocol""" + + def add_copy( + self, src: pathlib.Path, dest: t.Optional[pathlib.Path] = None + ) -> None: + """Add a copy operation to the operations list""" + self.operations.append(EnsembleCopyOperation(src, dest)) + + def add_symlink( + self, src: pathlib.Path, dest: t.Optional[pathlib.Path] = None + ) -> None: + """Add a symlink operation to the operations list""" + self.operations.append(EnsembleSymlinkOperation(src, dest)) + + def add_configuration( + self, + src: pathlib.Path, + file_parameters: t.Mapping[str,t.Sequence[str]], + dest: t.Optional[pathlib.Path] = None, + tag: t.Optional[str] = None, + ) -> None: + """Add a configure operation to the operations list""" + self.operations.append(EnsembleConfigureOperation(src, file_parameters, dest, tag)) + + @property + def copy_operations(self) -> t.List[EnsembleCopyOperation]: + """Property to get the list of copy files.""" + return self._filter(EnsembleCopyOperation) + + @property + def symlink_operations(self) -> t.List[EnsembleSymlinkOperation]: + """Property to get the list of symlink files.""" + return self._filter(EnsembleSymlinkOperation) + + @property + def configure_operations(self) -> t.List[EnsembleConfigureOperation]: + """Property to get the list of configure files.""" + return self._filter(EnsembleConfigureOperation) + + def _filter(self, type: t.Type[U]) -> t.List[U]: + return [x for x in self.operations if isinstance(x, type)] \ No newline at end of file diff --git a/smartsim/builders/ensemble.py b/smartsim/builders/ensemble.py index c4a57175f5..f322d66f16 100644 --- a/smartsim/builders/ensemble.py +++ b/smartsim/builders/ensemble.py @@ -36,12 +36,38 @@ from smartsim.builders.utils.strategies import ParamSet from smartsim.entity import entity from smartsim.entity.application import Application -from smartsim.entity.files import EntityFiles from smartsim.launchable.job import Job +from smartsim._core.generation.builder_operations import EnsembleFileSysOperationSet, EnsembleConfigureOperation +from dataclasses import dataclass, field + if t.TYPE_CHECKING: from smartsim.settings.launch_settings import LaunchSettings +@dataclass(frozen=True) +class FileSet: + """ + Represents a set of file parameters and execution arguments as parameters. + """ + + file: EnsembleConfigureOperation + combinations: ParamSet + +@dataclass(frozen=True) +class Combo: + """ + Represents a set of file parameters and execution arguments as parameters. + """ + + file: EnsembleConfigureOperation + combination: ParamSet + +@dataclass(frozen=True) +class ComboSet: + """ + Represents a set of file parameters and execution arguments as parameters. + """ + combos: t.List[Combo] class Ensemble(entity.CompoundEntity): """An Ensemble is a builder class that parameterizes the creation of multiple @@ -54,8 +80,6 @@ def __init__( exe: str | os.PathLike[str], exe_args: t.Sequence[str] | None = None, exe_arg_parameters: t.Mapping[str, t.Sequence[t.Sequence[str]]] | None = None, - files: EntityFiles | None = None, - file_parameters: t.Mapping[str, t.Sequence[str]] | None = None, permutation_strategy: str | strategies.PermutationStrategyType = "all_perm", max_permutations: int = -1, replicas: int = 1, @@ -137,12 +161,8 @@ def __init__( copy.deepcopy(exe_arg_parameters) if exe_arg_parameters else {} ) """The parameters and values to be used when configuring entities""" - self._files = copy.deepcopy(files) if files else None + self.files = EnsembleFileSysOperationSet([]) """The files to be copied, symlinked, and/or configured prior to execution""" - self._file_parameters = ( - copy.deepcopy(file_parameters) if file_parameters else {} - ) - """The parameters and values to be used when configuring files""" self._permutation_strategy = permutation_strategy """The strategy to control how the param values are applied to the Ensemble""" self._max_permutations = max_permutations @@ -200,40 +220,6 @@ def exe_arg_parameters( """ self._exe_arg_parameters = copy.deepcopy(value) - @property - def files(self) -> t.Union[EntityFiles, None]: - """Return attached EntityFiles object. - - :return: the EntityFiles object of files to be copied, symlinked, - and/or configured prior to execution - """ - return self._files - - @files.setter - def files(self, value: t.Optional[EntityFiles]) -> None: - """Set the EntityFiles object. - - :param value: the EntityFiles object of files to be copied, symlinked, - and/or configured prior to execution - """ - self._files = copy.deepcopy(value) - - @property - def file_parameters(self) -> t.Mapping[str, t.Sequence[str]]: - """Return the attached file parameters. - - :return: the file parameters - """ - return self._file_parameters - - @file_parameters.setter - def file_parameters(self, value: t.Mapping[str, t.Sequence[str]]) -> None: - """Set the file parameters. - - :param value: the file parameters - """ - self._file_parameters = dict(value) - @property def permutation_strategy(self) -> str | strategies.PermutationStrategyType: """Return the permutation strategy @@ -293,25 +279,50 @@ def _create_applications(self) -> tuple[Application, ...]: :return: A tuple of Application instances """ + ls = [] permutation_strategy = strategies.resolve(self.permutation_strategy) - - combinations = permutation_strategy( - self.file_parameters, self.exe_arg_parameters, self.max_permutations - ) - combinations = combinations if combinations else [ParamSet({}, {})] - permutations_ = itertools.chain.from_iterable( - itertools.repeat(permutation, self.replicas) for permutation in combinations - ) - return tuple( - Application( + for file in self.files.configure_operations: + new_list = [] + combinations = permutation_strategy( + file.file_parameters, self.exe_arg_parameters, self.max_permutations + ) + combinations = combinations if combinations else [ParamSet({}, {})] + # permutations_ = itertools.chain.from_iterable( + # itertools.repeat(permutation, self.replicas) for permutation in combinations + # ) + for combo in combinations: + new_list.append(FileSet(file, combo)) + ls.append(new_list) + combo = self._cartesian_values(ls) + print(combo) + print(type(combo)) + print(len(combo)) + for item in combo: + print(type(item)) + print(item) + + # return tuple( + # self.create_app(i, item) + + # for i, item in combo + # ) + apps = [] + i = 0 + for item in combo: + i+=1 + apps.append(self.create_app(i, item)) + return tuple(apps) + + def create_app(self, i, item): + app = Application( name=f"{self.name}-{i}", exe=self.exe, exe_args=self.exe_args, - files=self.files, - file_parameters=permutation.params, ) - for i, permutation in enumerate(permutations_) - ) + for merp in item: + app.files.add_configuration(src=merp.file.src, file_parameters=merp.combinations.params) + return app + def build_jobs(self, settings: LaunchSettings) -> tuple[Job, ...]: """Expand an Ensemble into a list of deployable Jobs and apply @@ -347,3 +358,14 @@ def build_jobs(self, settings: LaunchSettings) -> tuple[Job, ...]: if not apps: raise ValueError("There are no members as part of this ensemble") return tuple(Job(app, settings, app.name) for app in apps) + + def _step_values(self, ls): + return list(zip(*ls)) + + def _cartesian_values(self, ls): # needs to return a list[tuples] + combo = itertools.product(ls) + yup: t.Iterable = ( + val for val in zip(combo) + ) + print(yup) + return list(yup) diff --git a/smartsim/builders/utils/strategies.py b/smartsim/builders/utils/strategies.py index e3a2527a52..0e4ae34325 100644 --- a/smartsim/builders/utils/strategies.py +++ b/smartsim/builders/utils/strategies.py @@ -260,3 +260,22 @@ def random_permutations( if 0 <= n_permutations < len(permutations): permutations = random.sample(permutations, n_permutations) return permutations + + +# def create_combos(file_sets: List[FileSet]) -> List[List[Combo]]: +# # Extract the combinations from each FileSet +# all_combinations = [file_set.combinations for file_set in file_sets] + +# # Generate the Cartesian product of all combinations +# product_combinations = itertools.product(*all_combinations) + +# # Create Combo instances for each combination in the product +# combo_lists = [] +# for combination_tuple in product_combinations: +# combo_list = [ +# Combo(file=file_set.file, combination=combination) +# for file_set, combination in zip(file_sets, combination_tuple) +# ] +# combo_lists.append(combo_list) + +# return combo_lists \ No newline at end of file diff --git a/smartsim/entity/application.py b/smartsim/entity/application.py index 43082fe7a0..9d25cb9132 100644 --- a/smartsim/entity/application.py +++ b/smartsim/entity/application.py @@ -59,10 +59,6 @@ def __init__( self, name: str, exe: str, - file_parameters: ( - t.Mapping[str, str] | None - ) = None, # TODO remove when Ensemble is addressed - files: t.Optional[EntityFiles] = None, # TODO remove when Ensemble is addressed exe_args: t.Optional[t.Union[str, t.Sequence[str]]] = None, ) -> None: """Initialize an ``Application`` @@ -88,12 +84,6 @@ def __init__( self._exe_args = self._build_exe_args(exe_args) or [] """The executable arguments""" self.files = FileSysOperationSet([]) - """Attach files""" - self._file_parameters = ( - copy.deepcopy(file_parameters) if file_parameters else {} - ) - """TODO MOCK until Ensemble is implemented""" - """Files to be copied, symlinked, and/or configured prior to execution""" self._incoming_entities: t.List[SmartSimEntity] = [] """Entities for which the prefix will have to be known by other entities""" self._key_prefixing_enabled = False diff --git a/tests/test_builder_operations.py b/tests/test_builder_operations.py new file mode 100644 index 0000000000..0d46e9bebe --- /dev/null +++ b/tests/test_builder_operations.py @@ -0,0 +1,297 @@ +import base64 +import os +import pathlib +import pickle +from glob import glob +from os import path as osp +import pickle + +import pytest + +from smartsim._core.commands import Command +from smartsim._core.generation.builder_operations import ( + EnsembleCopyOperation, + EnsembleSymlinkOperation, + EnsembleConfigureOperation, + EnsembleFileSysOperationSet +) +from smartsim.builders import Ensemble + +# QUESTIONS +# TODO test python protocol? +# TODO do I allow the paths to combine if src is empty? + +pytestmark = pytest.mark.group_a + + +@pytest.fixture +def mock_src(test_dir: str): + """Fixture to create a mock source path.""" + return pathlib.Path(test_dir) / pathlib.Path("mock_src") + + +@pytest.fixture +def mock_dest(test_dir: str): + """Fixture to create a mock destination path.""" + return pathlib.Path(test_dir) / pathlib.Path("mock_dest") + + +@pytest.fixture +def copy_operation(mock_src: pathlib.Path, mock_dest: pathlib.Path): + """Fixture to create a CopyOperation object.""" + return EnsembleCopyOperation(src=mock_src, dest=mock_dest) + + +@pytest.fixture +def symlink_operation(mock_src: pathlib.Path, mock_dest: pathlib.Path): + """Fixture to create a SymlinkOperation object.""" + return EnsembleSymlinkOperation(src=mock_src, dest=mock_dest) + + +@pytest.fixture +def configure_operation(mock_src: pathlib.Path, mock_dest: pathlib.Path): + """Fixture to create a Configure object.""" + return EnsembleConfigureOperation( + src=mock_src, dest=mock_dest, file_parameters={"FOO": ["BAR", "TOE"]} + ) + + +@pytest.fixture +def file_system_operation_set( + copy_operation: EnsembleCopyOperation, + symlink_operation: EnsembleSymlinkOperation, + configure_operation: EnsembleConfigureOperation, +): + """Fixture to create a FileSysOperationSet object.""" + return EnsembleFileSysOperationSet([copy_operation, symlink_operation, configure_operation]) + + +# @pytest.mark.parametrize( +# "job_root_path, dest, expected", +# ( +# pytest.param( +# pathlib.Path("/valid/root"), +# pathlib.Path("valid/dest"), +# "/valid/root/valid/dest", +# id="Valid paths", +# ), +# pytest.param( +# pathlib.Path("/valid/root"), +# pathlib.Path(""), +# "/valid/root", +# id="Empty destination path", +# ), +# pytest.param( +# pathlib.Path("/valid/root"), +# None, +# "/valid/root", +# id="Empty dest path", +# ), +# ), +# ) +# def test_create_final_dest_valid(job_root_path, dest, expected): +# """Test valid path inputs for operations.create_final_dest""" +# assert create_final_dest(job_root_path, dest) == expected + + +# @pytest.mark.parametrize( +# "job_root_path, dest", +# ( +# pytest.param(None, pathlib.Path("valid/dest"), id="None as root path"), +# pytest.param(1234, pathlib.Path("valid/dest"), id="Number as root path"), +# pytest.param(pathlib.Path("valid/dest"), 1234, id="Number as dest"), +# ), +# ) +# def test_create_final_dest_invalid(job_root_path, dest): +# """Test invalid path inputs for operations.create_final_dest""" +# with pytest.raises(ValueError): +# create_final_dest(job_root_path, dest) + + +# def test_valid_init_generation_context( +# test_dir: str, generation_context: GenerationContext +# ): +# """Validate GenerationContext init""" +# assert isinstance(generation_context, GenerationContext) +# assert generation_context.job_root_path == pathlib.Path(test_dir) + + +# def test_invalid_init_generation_context(): +# """Validate GenerationContext init""" +# with pytest.raises(TypeError): +# GenerationContext(1234) +# with pytest.raises(TypeError): +# GenerationContext("") + + +def test_init_copy_operation( + copy_operation: EnsembleCopyOperation, mock_src: pathlib.Path, mock_dest: pathlib.Path +): + """Validate CopyOperation init""" + assert isinstance(copy_operation, EnsembleCopyOperation) + assert copy_operation.src == mock_src + assert copy_operation.dest == mock_dest + + +def test_init_symlink_operation( + symlink_operation: EnsembleSymlinkOperation, mock_src: str, mock_dest: str +): + """Validate SymlinkOperation init""" + assert isinstance(symlink_operation, EnsembleSymlinkOperation) + assert symlink_operation.src == mock_src + assert symlink_operation.dest == mock_dest + + +def test_init_configure_operation( + configure_operation: EnsembleConfigureOperation, mock_src: str, mock_dest: str +): + """Validate ConfigureOperation init""" + assert isinstance(configure_operation, EnsembleConfigureOperation) + assert configure_operation.src == mock_src + assert configure_operation.dest == mock_dest + assert configure_operation.tag == ";" + assert configure_operation.file_parameters == {"FOO": ["BAR", "TOE"]} + + +def test_init_file_sys_operation_set(file_system_operation_set: EnsembleFileSysOperationSet): + """Test initialize FileSystemOperationSet""" + assert isinstance(file_system_operation_set.operations, list) + assert len(file_system_operation_set.operations) == 3 + + +def test_add_copy_operation( + file_system_operation_set: EnsembleFileSysOperationSet +): + """Test FileSystemOperationSet.add_copy""" + assert len(file_system_operation_set.copy_operations) == 1 + file_system_operation_set.add_copy(src=pathlib.Path("src")) + assert len(file_system_operation_set.copy_operations) == 2 + + +def test_add_symlink_operation( + file_system_operation_set: EnsembleFileSysOperationSet +): + """Test FileSystemOperationSet.add_symlink""" + assert len(file_system_operation_set.symlink_operations) == 1 + file_system_operation_set.add_symlink(src=pathlib.Path("src")) + assert len(file_system_operation_set.symlink_operations) == 2 + + +def test_add_configure_operation( + file_system_operation_set: EnsembleFileSysOperationSet, +): + """Test FileSystemOperationSet.add_configuration""" + assert len(file_system_operation_set.configure_operations) == 1 + file_system_operation_set.add_configuration( + src=pathlib.Path("src"), file_parameters={"FOO": "BAR"} + ) + assert len(file_system_operation_set.configure_operations) == 2 + + +# @pytest.fixture +# def files(fileutils): +# path_to_files = fileutils.get_test_conf_path( +# osp.join("generator_files", "easy", "correct/") +# ) +# list_of_files_strs = sorted(glob(path_to_files + "/*")) +# yield [pathlib.Path(str_path) for str_path in list_of_files_strs] + + +# @pytest.fixture +# def directory(fileutils): +# directory = fileutils.get_test_conf_path( +# osp.join("generator_files", "easy", "correct/") +# ) +# yield [pathlib.Path(directory)] + + +# @pytest.fixture +# def source(request, files, directory): +# if request.param == "files": +# return files +# elif request.param == "directory": +# return directory + + +# @pytest.mark.parametrize( +# "dest,error", +# ( +# pytest.param(123, TypeError, id="dest as integer"), +# pytest.param("", TypeError, id="dest as empty str"), +# pytest.param("/absolute/path", TypeError, id="dest as absolute str"), +# pytest.param( +# pathlib.Path("relative/path"), ValueError, id="dest as relative Path" +# ), +# pytest.param( +# pathlib.Path("/path with spaces"), ValueError, id="dest as Path with spaces" +# ), +# # TODO pytest.param(pathlib.Path("/path/with/special!@#"), id="dest as Path with special char"), +# ), +# ) +# @pytest.mark.parametrize("source", ["files", "directory"], indirect=True) +# def test_copy_files_invalid_dest(dest, error, source): +# """Test invalid copy destination""" +# with pytest.raises(error): +# _ = [CopyOperation(src=file, dest=dest) for file in source] + + +# @pytest.mark.parametrize( +# "dest,error", +# ( +# pytest.param(123, TypeError, id="dest as integer"), +# pytest.param("", TypeError, id="dest as empty str"), +# pytest.param("/absolute/path", TypeError, id="dest as absolute str"), +# pytest.param( +# pathlib.Path("relative/path"), ValueError, id="dest as relative Path" +# ), +# pytest.param( +# pathlib.Path("/path with spaces"), ValueError, id="dest as Path with spaces" +# ), +# # TODO pytest.param(pathlib.Path("/path/with/special!@#"), id="dest as Path with special char"), +# ), +# ) +# @pytest.mark.parametrize("source", ["files", "directory"], indirect=True) +# def test_symlink_files_invalid_dest(dest, error, source): +# """Test invalid symlink destination""" +# with pytest.raises(error): +# _ = [SymlinkOperation(src=file, dest=dest) for file in source] + + +# @pytest.mark.parametrize( +# "dest,error", +# ( +# pytest.param(123, TypeError, id="dest as integer"), +# pytest.param("", TypeError, id="dest as empty str"), +# pytest.param("/absolute/path", TypeError, id="dest as absolute str"), +# pytest.param( +# pathlib.Path("relative/path"), ValueError, id="dest as relative Path" +# ), +# pytest.param( +# pathlib.Path("/path with spaces"), ValueError, id="dest as Path with spaces" +# ), +# # TODO pytest.param(pathlib.Path("/path/with/special!@#"), id="dest as Path with special char"), +# ), +# ) +# @pytest.mark.parametrize("source", ["files", "directory"], indirect=True) +# def test_configure_files_invalid_dest(dest, error, source): +# """Test invalid configure destination""" +# with pytest.raises(error): +# _ = [ +# ConfigureOperation(src=file, dest=dest, file_parameters={"FOO": "BAR"}) +# for file in source +# ] + +# bug found that it does not properly permutate if exe_arg_params is not specified +# ISSUE with step, adds one file per application +def test_step_mock(): + ensemble = Ensemble("name", "echo", exe_arg_parameters = {"-N": ["1", "2"]}, permutation_strategy="step") + ensemble.files.add_configuration(pathlib.Path("src_1"), file_parameters={"FOO":["BAR", "TOE"]}) + ensemble.files.add_configuration(pathlib.Path("src_2"), file_parameters={"CAN":["TOM", "STO"]}) + apps = ensemble._create_applications() + print(apps) + for app in apps: + for config in app.files.configure_operations: + decoded_dict = base64.b64decode(config.file_parameters) + print(config.src) + deserialized_dict = pickle.loads(decoded_dict) + print(deserialized_dict) From 83e015a6c6bf0e3a3ba11328f6a75559dc5aa1fd Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Tue, 8 Oct 2024 12:19:21 -0700 Subject: [PATCH 12/35] partial address matts comments --- smartsim/_core/commands/command.py | 4 +- smartsim/_core/generation/generator.py | 25 ++----- smartsim/_core/generation/operations.py | 63 +++++++--------- smartsim/entity/application.py | 2 +- .../test_core/test_commands/test_command.py | 10 ++- .../test_commands/test_commandList.py | 2 +- tests/test_generator.py | 49 ++++-------- tests/test_operations.py | 75 ++++++++----------- 8 files changed, 92 insertions(+), 138 deletions(-) diff --git a/smartsim/_core/commands/command.py b/smartsim/_core/commands/command.py index 4b342652e2..b39f720dab 100644 --- a/smartsim/_core/commands/command.py +++ b/smartsim/_core/commands/command.py @@ -36,9 +36,9 @@ class Command(MutableSequence[str]): def __init__(self, command: t.List[str]) -> None: if not command: - raise ValueError("Command list cannot be empty") + raise TypeError("Command list cannot be empty") if not all(isinstance(item, str) for item in command): - raise ValueError("All items in the command list must be strings") + raise TypeError("All items in the command list must be strings") """Command constructor""" self._command = command diff --git a/smartsim/_core/generation/generator.py b/smartsim/_core/generation/generator.py index 8f74dc6335..27973b1b86 100644 --- a/smartsim/_core/generation/generator.py +++ b/smartsim/_core/generation/generator.py @@ -230,8 +230,8 @@ def _append_mkdir_commands( :param job_path: The file path for the Job run folder :param log_path: The file path for the Job log folder """ - cmd_list.commands.append(cls._mkdir_file(job_path)) - cmd_list.commands.append(cls._mkdir_file(log_path)) + cmd_list.append(cls._mkdir_file(job_path)) + cmd_list.append(cls._mkdir_file(log_path)) @classmethod def _append_file_operations( @@ -248,13 +248,13 @@ def _append_file_operations( :param context: A GenerationContext object that holds the Job's run directory """ copy_ret = cls._copy_files(entity.files.copy_operations, context) - cmd_list.commands.extend(copy_ret.commands) + cmd_list.extend(copy_ret) symlink_ret = cls._symlink_files(entity.files.symlink_operations, context) - cmd_list.commands.extend(symlink_ret.commands) + cmd_list.extend(symlink_ret) configure_ret = cls._configure_files(entity.files.configure_operations, context) - cmd_list.commands.extend(configure_ret.commands) + cmd_list.extend(configure_ret) @classmethod def _execute_commands(cls, cmd_list: CommandList) -> None: @@ -290,10 +290,7 @@ def _copy_files( :param context: A GenerationContext object that holds the Job's run directory :return: A CommandList containing the copy commands """ - cmd_list = CommandList() - for file in files: - cmd_list.append(file.format(context)) - return cmd_list + return CommandList([file.format(context) for file in files]) @staticmethod def _symlink_files( @@ -306,10 +303,7 @@ def _symlink_files( :param context: A GenerationContext object that holds the Job's run directory :return: A CommandList containing the symlink commands """ - cmd_list = CommandList() - for file in files: - cmd_list.append(file.format(context)) - return cmd_list + return CommandList([file.format(context) for file in files]) @staticmethod def _configure_files( @@ -323,7 +317,4 @@ def _configure_files( :param context: A GenerationContext object that holds the Job's run directory :return: A CommandList containing the configuration commands """ - cmd_list = CommandList() - for file in files: - cmd_list.append(file.format(context)) - return cmd_list + return CommandList([file.format(context) for file in files]) diff --git a/smartsim/_core/generation/operations.py b/smartsim/_core/generation/operations.py index 4e6e1001b2..5ad7de6884 100644 --- a/smartsim/_core/generation/operations.py +++ b/smartsim/_core/generation/operations.py @@ -8,6 +8,7 @@ from ..commands import Command +# pylint: disable=invalid-name entry_point_path = "smartsim._core.entrypoints.file_operations" """Path to file operations module""" @@ -19,7 +20,7 @@ """Configure file operations command""" -def create_final_dest( +def _create_final_dest( job_root_path: pathlib.Path, dest: t.Union[pathlib.Path, None] ) -> str: """Combine the job root path and destination path. Return as a string for @@ -30,16 +31,13 @@ def create_final_dest( :return: Combined path :raises ValueError: An error occurred during path combination """ - try: - combined_path = job_root_path - if dest: - combined_path = job_root_path / dest - return str(combined_path) - except Exception as e: - raise ValueError(f"Error combining paths: {e}") + combined_path = job_root_path + if dest: + combined_path = job_root_path / dest + return str(combined_path) -def check_src_and_dest_path( +def _check_src_and_dest_path( src: pathlib.Path, dest: t.Union[pathlib.Path, None] ) -> None: """Validate that the provided source and destination paths are @@ -55,20 +53,11 @@ def check_src_and_dest_path( raise TypeError( f"dest must be of type pathlib.Path or None, not {type(dest).__name__}" ) - if isinstance(dest, pathlib.Path) and not dest.is_absolute(): + if isinstance(dest, pathlib.Path) and dest.is_absolute(): raise ValueError("Invalid destination path") - if isinstance(dest, pathlib.Path) and " " in str(dest): - raise ValueError("Path contains spaces, which are not allowed") - # TODO I want to add the check below but I do not think this works for remote jobs - # full_path = path.abspath(file_path) - # if path.isfile(full_path): - # return full_path - # if path.isdir(full_path): - # return full_path - -def check_run_path(run_path: pathlib.Path) -> None: +def _check_run_path(run_path: pathlib.Path) -> None: """Validate that the provided run path is of type pathlib.Path :param run_path: The run path to be checked @@ -87,7 +76,7 @@ class GenerationContext: """Context for file system generation operations.""" def __init__(self, job_root_path: pathlib.Path): - check_run_path(job_root_path) + _check_run_path(job_root_path) self.job_root_path = job_root_path """The Job run path""" @@ -110,9 +99,9 @@ def __init__( :param src: Path to source :param dest: Path to destination """ - check_src_and_dest_path(src, dest) + _check_src_and_dest_path(src, dest) self.src = src - self.dest = dest + self.dest = dest or src.name def format(self, context: GenerationContext) -> Command: """Create Command to invoke copy file system entry point @@ -120,7 +109,7 @@ def format(self, context: GenerationContext) -> Command: :param context: Context for copy operation :return: Copy Command """ - final_dest = create_final_dest(context.job_root_path, self.dest) + final_dest = _create_final_dest(context.job_root_path, self.dest) return Command( [ sys.executable, @@ -145,9 +134,9 @@ def __init__( :param src: Path to source :param dest: Path to destination """ - check_src_and_dest_path(src, dest) + _check_src_and_dest_path(src, dest) self.src = src - self.dest = dest + self.dest = dest or src.name def format(self, context: GenerationContext) -> Command: """Create Command to invoke symlink file system entry point @@ -156,8 +145,8 @@ def format(self, context: GenerationContext) -> Command: :return: Symlink Command """ normalized_path = os.path.normpath(self.src) - parent_dir = os.path.basename(normalized_path) - final_dest = create_final_dest(context.job_root_path, self.dest) + parent_dir = os.path.dirname(normalized_path) + final_dest = _create_final_dest(context.job_root_path, self.dest) new_dest = os.path.join(final_dest, parent_dir) return Command( [ @@ -188,9 +177,9 @@ def __init__( :param dest: Path to destination :param tag: Tag to use for find and replacement """ - check_src_and_dest_path(src, dest) + _check_src_and_dest_path(src, dest) self.src = src - self.dest = dest + self.dest = dest or src.name pickled_dict = pickle.dumps(file_parameters) encoded_dict = base64.b64encode(pickled_dict).decode("ascii") self.file_parameters = encoded_dict @@ -202,7 +191,7 @@ def format(self, context: GenerationContext) -> Command: :param context: Context for configure operation :return: Configure Command """ - final_dest = create_final_dest(context.job_root_path, self.dest) + final_dest = _create_final_dest(context.job_root_path, self.dest) return Command( [ sys.executable, @@ -217,7 +206,7 @@ def format(self, context: GenerationContext) -> Command: ) -T = t.TypeVar("T", bound=GenerationProtocol) +GenerationProtocolT = t.TypeVar("GenerationProtocolT", bound=GenerationProtocol) @dataclass @@ -265,7 +254,7 @@ def add_configuration( self.operations.append(ConfigureOperation(src, file_parameters, dest, tag)) @property - def copy_operations(self) -> t.List[CopyOperation]: + def copy_operations(self) -> list[CopyOperation]: """Property to get the list of copy files. :return: List of CopyOperation objects @@ -273,7 +262,7 @@ def copy_operations(self) -> t.List[CopyOperation]: return self._filter(CopyOperation) @property - def symlink_operations(self) -> t.List[SymlinkOperation]: + def symlink_operations(self) -> list[SymlinkOperation]: """Property to get the list of symlink files. :return: List of SymlinkOperation objects @@ -281,18 +270,18 @@ def symlink_operations(self) -> t.List[SymlinkOperation]: return self._filter(SymlinkOperation) @property - def configure_operations(self) -> t.List[ConfigureOperation]: + def configure_operations(self) -> list[ConfigureOperation]: """Property to get the list of configure files. :return: List of ConfigureOperation objects """ return self._filter(ConfigureOperation) - def _filter(self, type: t.Type[T]) -> t.List[T]: + def _filter(self, type_: type[GenerationProtocolT]) -> list[GenerationProtocolT]: """Filters the operations list to include only instances of the specified type. :param type: The type of operations to filter :return: A list of operations that are instances of the specified type """ - return [x for x in self.operations if isinstance(x, type)] + return [x for x in self.operations if isinstance(x, type_)] diff --git a/smartsim/entity/application.py b/smartsim/entity/application.py index 43082fe7a0..d736617c18 100644 --- a/smartsim/entity/application.py +++ b/smartsim/entity/application.py @@ -59,11 +59,11 @@ def __init__( self, name: str, exe: str, + exe_args: t.Optional[t.Union[str, t.Sequence[str]]] = None, file_parameters: ( t.Mapping[str, str] | None ) = None, # TODO remove when Ensemble is addressed files: t.Optional[EntityFiles] = None, # TODO remove when Ensemble is addressed - exe_args: t.Optional[t.Union[str, t.Sequence[str]]] = None, ) -> None: """Initialize an ``Application`` diff --git a/tests/temp_tests/test_core/test_commands/test_command.py b/tests/temp_tests/test_core/test_commands/test_command.py index 2d1ddfbe84..2900b3c886 100644 --- a/tests/temp_tests/test_core/test_commands/test_command.py +++ b/tests/temp_tests/test_core/test_commands/test_command.py @@ -35,11 +35,15 @@ def test_command_init(): cmd = Command(command=["salloc", "-N", "1"]) assert cmd.command == ["salloc", "-N", "1"] +def test_command_invalid_init(): + cmd = Command(command=["salloc", "-N", "1"]) + assert cmd.command == ["salloc", "-N", "1"] def test_command_getitem_int(): - cmd = Command(command=["salloc", "-N", "1"]) - get_value = cmd[0] - assert get_value == "salloc" + with pytest.raises(TypeError): + _ = Command(command=[1]) + with pytest.raises(TypeError): + _ = Command(command=[]) def test_command_getitem_slice(): diff --git a/tests/temp_tests/test_core/test_commands/test_commandList.py b/tests/temp_tests/test_core/test_commands/test_commandList.py index 8d91336e88..28f9b07676 100644 --- a/tests/temp_tests/test_core/test_commands/test_commandList.py +++ b/tests/temp_tests/test_core/test_commands/test_commandList.py @@ -76,7 +76,7 @@ def test_command_setitem_fail(): cmd_list[0:1] = "fail" with pytest.raises(ValueError): cmd_list[0:1] = "fail" - with pytest.raises(ValueError): + with pytest.raises(TypeError): _ = Command(command=["salloc", "-N", 1]) with pytest.raises(TypeError): cmd_list[0:1] = [Command(command=["salloc", "-N", "1"]), Command(command=1)] diff --git a/tests/test_generator.py b/tests/test_generator.py index e955dba7f5..03fd6b3c1a 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -40,7 +40,7 @@ FileSysOperationSet, GenerationContext, SymlinkOperation, - create_final_dest, + _create_final_dest, ) from smartsim.entity import SmartSimEntity from smartsim.launchable import Job @@ -248,7 +248,7 @@ def files(fileutils): path_to_files = fileutils.get_test_conf_path( osp.join("generator_files", "easy", "correct/") ) - list_of_files_strs = sorted(glob(path_to_files + "/*")) + list_of_files_strs = glob(path_to_files + "/*") yield [pathlib.Path(str_path) for str_path in list_of_files_strs] @@ -260,12 +260,9 @@ def directory(fileutils): yield [pathlib.Path(directory)] -@pytest.fixture -def source(request, files, directory): - if request.param == "files": - return files - elif request.param == "directory": - return directory +@pytest.fixture(params=["files", "directory"]) +def source(request): + yield request.getfixturevalue(request.param) @pytest.mark.parametrize( @@ -273,12 +270,11 @@ def source(request, files, directory): ( pytest.param(None, id="dest as None"), pytest.param( - pathlib.Path("/absolute/path"), + pathlib.Path("absolute/path"), id="dest as valid path", ), ), ) -@pytest.mark.parametrize("source", ["files", "directory"], indirect=True) def test_copy_files_valid_dest( dest, source, generator_instance: Generator, test_dir: str ): @@ -289,15 +285,11 @@ def test_copy_files_valid_dest( # Extract file paths from commands cmd_src_paths = set() for cmd in cmd_list.commands: - # Assert destination path exists in cmd - assert create_final_dest(pathlib.Path(test_dir), dest) in cmd.command src_index = cmd.command.index("copy") + 1 cmd_src_paths.add(cmd.command[src_index]) # Assert all file paths are in the command list file_paths = {str(file) for file in source} - assert file_paths.issubset( - cmd_src_paths - ), "Not all file paths are in the command list" + assert file_paths == cmd_src_paths, "Not all file paths are in the command list" @pytest.mark.parametrize( @@ -305,31 +297,27 @@ def test_copy_files_valid_dest( ( pytest.param(None, id="dest as None"), pytest.param( - pathlib.Path("/absolute/path"), + pathlib.Path("absolute/path"), id="dest as valid path", ), ), ) -@pytest.mark.parametrize("source", ["files", "directory"], indirect=True) def test_symlink_files_valid_dest( dest, source, generator_instance: Generator, test_dir: str ): - to_copy = [CopyOperation(src=file, dest=dest) for file in source] + to_symlink = [SymlinkOperation(src=file, dest=dest) for file in source] gen = GenerationContext(pathlib.Path(test_dir)) - cmd_list = generator_instance._copy_files(files=to_copy, context=gen) + cmd_list = generator_instance._symlink_files(files=to_symlink, context=gen) assert isinstance(cmd_list, CommandList) # Extract file paths from commands cmd_src_paths = set() for cmd in cmd_list.commands: - # Assert destination path exists in cmd - assert create_final_dest(pathlib.Path(test_dir), dest) in cmd.command - src_index = cmd.command.index("copy") + 1 + print(cmd) + src_index = cmd.command.index("symlink") + 1 cmd_src_paths.add(cmd.command[src_index]) # Assert all file paths are in the command list file_paths = {str(file) for file in source} - assert file_paths.issubset( - cmd_src_paths - ), "Not all file paths are in the command list" + assert file_paths == cmd_src_paths, "Not all file paths are in the command list" @pytest.mark.parametrize( @@ -337,12 +325,11 @@ def test_symlink_files_valid_dest( ( pytest.param(None, id="dest as None"), pytest.param( - pathlib.Path("/absolute/path"), + pathlib.Path("absolute/path"), id="dest as valid path", ), ), ) -@pytest.mark.parametrize("source", ["files", "directory"], indirect=True) def test_configure_files_valid_dest( dest, source, generator_instance: Generator, test_dir: str ): @@ -365,15 +352,11 @@ def test_configure_files_valid_dest( # Extract file paths from commands cmd_src_paths = set() for cmd in cmd_list.commands: - # Assert destination path exists in cmd - assert create_final_dest(pathlib.Path(test_dir), dest) in cmd.command src_index = cmd.command.index("configure") + 1 cmd_src_paths.add(cmd.command[src_index]) # Assert all file paths are in the command list file_paths = {str(file) for file in source} - assert file_paths.issubset( - cmd_src_paths - ), "Not all file paths are in the command list" + assert file_paths == cmd_src_paths, "Not all file paths are in the command list" # TODO Add configure_file tests @@ -511,4 +494,4 @@ def test_execute_commands( ) as mock_run, ): generator_instance._execute_commands(formatted_command_list) - assert mock_run.call_count == len(operations_list) + assert mock_run.call_count == len(formatted_command_list) diff --git a/tests/test_operations.py b/tests/test_operations.py index 6281eb437d..98c3927bd1 100644 --- a/tests/test_operations.py +++ b/tests/test_operations.py @@ -16,13 +16,11 @@ SymlinkOperation, configure_cmd, copy_cmd, - create_final_dest, + _create_final_dest, symlink_cmd, ) -# QUESTIONS -# TODO test python protocol? -# TODO do I allow the paths to combine if src is empty? +# TODO ADD CHECK TO ENFORCE SRC AS RELATIVE pytestmark = pytest.mark.group_a @@ -40,9 +38,9 @@ def mock_src(test_dir: str): @pytest.fixture -def mock_dest(test_dir: str): +def mock_dest(): """Fixture to create a mock destination path.""" - return pathlib.Path(test_dir) / pathlib.Path("mock_dest") + return pathlib.Path("mock_dest") @pytest.fixture @@ -99,8 +97,8 @@ def file_system_operation_set( ), ) def test_create_final_dest_valid(job_root_path, dest, expected): - """Test valid path inputs for operations.create_final_dest""" - assert create_final_dest(job_root_path, dest) == expected + """Test valid path inputs for operations._create_final_dest""" + assert _create_final_dest(job_root_path, dest) == expected @pytest.mark.parametrize( @@ -112,15 +110,16 @@ def test_create_final_dest_valid(job_root_path, dest, expected): ), ) def test_create_final_dest_invalid(job_root_path, dest): - """Test invalid path inputs for operations.create_final_dest""" - with pytest.raises(ValueError): - create_final_dest(job_root_path, dest) + """Test invalid path inputs for operations._create_final_dest""" + with pytest.raises(TypeError): + _create_final_dest(job_root_path, dest) def test_valid_init_generation_context( - test_dir: str, generation_context: GenerationContext + test_dir: str ): """Validate GenerationContext init""" + generation_context = GenerationContext(pathlib.Path(test_dir)) assert isinstance(generation_context, GenerationContext) assert generation_context.job_root_path == pathlib.Path(test_dir) @@ -134,9 +133,10 @@ def test_invalid_init_generation_context(): def test_init_copy_operation( - copy_operation: CopyOperation, mock_src: pathlib.Path, mock_dest: pathlib.Path + mock_src: pathlib.Path, mock_dest: pathlib.Path ): """Validate CopyOperation init""" + copy_operation = CopyOperation(mock_src, mock_dest) assert isinstance(copy_operation, CopyOperation) assert copy_operation.src == mock_src assert copy_operation.dest == mock_dest @@ -144,22 +144,24 @@ def test_init_copy_operation( def test_copy_operation_format( copy_operation: CopyOperation, - mock_src: str, mock_dest: str, + mock_src: str, generation_context: GenerationContext, + test_dir: str ): """Validate CopyOperation.format""" exec = copy_operation.format(generation_context) assert isinstance(exec, Command) assert str(mock_src) in exec.command assert copy_cmd in exec.command - assert create_final_dest(mock_src, mock_dest) in exec.command + assert _create_final_dest(test_dir, mock_dest) in exec.command def test_init_symlink_operation( - symlink_operation: SymlinkOperation, mock_src: str, mock_dest: str + mock_src: str, mock_dest: str ): """Validate SymlinkOperation init""" + symlink_operation = SymlinkOperation(mock_src, mock_dest) assert isinstance(symlink_operation, SymlinkOperation) assert symlink_operation.src == mock_src assert symlink_operation.dest == mock_dest @@ -178,16 +180,17 @@ def test_symlink_operation_format( assert symlink_cmd in exec.command normalized_path = os.path.normpath(mock_src) - parent_dir = os.path.basename(normalized_path) - final_dest = create_final_dest(generation_context.job_root_path, mock_dest) + parent_dir = os.path.dirname(normalized_path) + final_dest = _create_final_dest(generation_context.job_root_path, mock_dest) new_dest = os.path.join(final_dest, parent_dir) assert new_dest in exec.command def test_init_configure_operation( - configure_operation: ConfigureOperation, mock_src: str, mock_dest: str + mock_src: str, mock_dest: str ): """Validate ConfigureOperation init""" + configure_operation = ConfigureOperation(src=mock_src, dest=mock_dest,file_parameters={"FOO": "BAR"}) assert isinstance(configure_operation, ConfigureOperation) assert configure_operation.src == mock_src assert configure_operation.dest == mock_dest @@ -199,8 +202,9 @@ def test_init_configure_operation( def test_configure_operation_format( configure_operation: ConfigureOperation, - mock_src: str, + test_dir: str, mock_dest: str, + mock_src: str, generation_context: GenerationContext, ): """Validate ConfigureOperation.format""" @@ -208,11 +212,15 @@ def test_configure_operation_format( assert isinstance(exec, Command) assert str(mock_src) in exec.command assert configure_cmd in exec.command - assert create_final_dest(mock_src, mock_dest) in exec.command + assert _create_final_dest(test_dir, mock_dest) in exec.command -def test_init_file_sys_operation_set(file_system_operation_set: FileSysOperationSet): +def test_init_file_sys_operation_set( + copy_operation: CopyOperation, + symlink_operation: SymlinkOperation, + configure_operation: ConfigureOperation): """Test initialize FileSystemOperationSet""" + file_system_operation_set = FileSysOperationSet([copy_operation,symlink_operation,configure_operation]) assert isinstance(file_system_operation_set.operations, list) assert len(file_system_operation_set.operations) == 3 @@ -222,7 +230,7 @@ def test_add_copy_operation( ): """Test FileSystemOperationSet.add_copy""" assert len(file_system_operation_set.copy_operations) == 1 - file_system_operation_set.add_copy(src=pathlib.Path("src")) + file_system_operation_set.add_copy(src=pathlib.Path("/src")) assert len(file_system_operation_set.copy_operations) == 2 @@ -279,13 +287,6 @@ def source(request, files, directory): pytest.param(123, TypeError, id="dest as integer"), pytest.param("", TypeError, id="dest as empty str"), pytest.param("/absolute/path", TypeError, id="dest as absolute str"), - pytest.param( - pathlib.Path("relative/path"), ValueError, id="dest as relative Path" - ), - pytest.param( - pathlib.Path("/path with spaces"), ValueError, id="dest as Path with spaces" - ), - # TODO pytest.param(pathlib.Path("/path/with/special!@#"), id="dest as Path with special char"), ), ) @pytest.mark.parametrize("source", ["files", "directory"], indirect=True) @@ -301,13 +302,6 @@ def test_copy_files_invalid_dest(dest, error, source): pytest.param(123, TypeError, id="dest as integer"), pytest.param("", TypeError, id="dest as empty str"), pytest.param("/absolute/path", TypeError, id="dest as absolute str"), - pytest.param( - pathlib.Path("relative/path"), ValueError, id="dest as relative Path" - ), - pytest.param( - pathlib.Path("/path with spaces"), ValueError, id="dest as Path with spaces" - ), - # TODO pytest.param(pathlib.Path("/path/with/special!@#"), id="dest as Path with special char"), ), ) @pytest.mark.parametrize("source", ["files", "directory"], indirect=True) @@ -323,13 +317,6 @@ def test_symlink_files_invalid_dest(dest, error, source): pytest.param(123, TypeError, id="dest as integer"), pytest.param("", TypeError, id="dest as empty str"), pytest.param("/absolute/path", TypeError, id="dest as absolute str"), - pytest.param( - pathlib.Path("relative/path"), ValueError, id="dest as relative Path" - ), - pytest.param( - pathlib.Path("/path with spaces"), ValueError, id="dest as Path with spaces" - ), - # TODO pytest.param(pathlib.Path("/path/with/special!@#"), id="dest as Path with special char"), ), ) @pytest.mark.parametrize("source", ["files", "directory"], indirect=True) From 70dd2a0bcafa03526000943164c573f9f4f3e4df Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Wed, 9 Oct 2024 09:28:20 -0700 Subject: [PATCH 13/35] addressed Matts comments, exclude Qs --- conftest.py | 56 ++++++++ smartsim/_core/generation/operations.py | 16 +-- smartsim/builders/ensemble.py | 1 - smartsim/entity/application.py | 1 - .../test_core/test_commands/test_command.py | 2 + tests/test_generator.py | 56 -------- tests/test_operations.py | 131 +++++++++--------- 7 files changed, 129 insertions(+), 134 deletions(-) diff --git a/conftest.py b/conftest.py index c407681d76..fe94ee5409 100644 --- a/conftest.py +++ b/conftest.py @@ -40,6 +40,8 @@ import typing as t import uuid import warnings +from glob import glob +from os import path as osp from collections import defaultdict from dataclasses import dataclass from subprocess import run @@ -53,6 +55,8 @@ from smartsim._core.config.config import Config from smartsim._core.launcher.dragon.dragon_connector import DragonConnector from smartsim._core.launcher.dragon.dragon_launcher import DragonLauncher +from smartsim._core.generation.operations import ConfigureOperation, CopyOperation, SymlinkOperation +from smartsim._core.generation.generator import Generator from smartsim._core.utils.telemetry.telemetry import JobEntity from smartsim.database import FeatureStore from smartsim.entity import Application @@ -469,6 +473,58 @@ def check_output_dir() -> None: def fsutils() -> t.Type[FSUtils]: return FSUtils +@pytest.fixture +def files(fileutils): + path_to_files = fileutils.get_test_conf_path( + osp.join("generator_files", "easy", "correct/") + ) + list_of_files_strs = glob(path_to_files + "/*") + yield [pathlib.Path(str_path) for str_path in list_of_files_strs] + + +@pytest.fixture +def directory(fileutils): + directory = fileutils.get_test_conf_path( + osp.join("generator_files", "easy", "correct/") + ) + yield [pathlib.Path(directory)] + + +@pytest.fixture(params=["files", "directory"]) +def source(request): + yield request.getfixturevalue(request.param) + + +@pytest.fixture +def mock_src(test_dir: str): + """Fixture to create a mock source path.""" + return pathlib.Path(test_dir) / pathlib.Path("mock_src") + + +@pytest.fixture +def mock_dest(): + """Fixture to create a mock destination path.""" + return pathlib.Path("mock_dest") + + +@pytest.fixture +def copy_operation(mock_src: pathlib.Path, mock_dest: pathlib.Path): + """Fixture to create a CopyOperation object.""" + return CopyOperation(src=mock_src, dest=mock_dest) + + +@pytest.fixture +def symlink_operation(mock_src: pathlib.Path, mock_dest: pathlib.Path): + """Fixture to create a CopyOperation object.""" + return SymlinkOperation(src=mock_src, dest=mock_dest) + + +@pytest.fixture +def configure_operation(mock_src: pathlib.Path, mock_dest: pathlib.Path): + """Fixture to create a Configure object.""" + return ConfigureOperation( + src=mock_src, dest=mock_dest, file_parameters={"FOO": "BAR"} + ) class FSUtils: @staticmethod diff --git a/smartsim/_core/generation/operations.py b/smartsim/_core/generation/operations.py index 5ad7de6884..ec9da381d7 100644 --- a/smartsim/_core/generation/operations.py +++ b/smartsim/_core/generation/operations.py @@ -20,9 +20,7 @@ """Configure file operations command""" -def _create_final_dest( - job_root_path: pathlib.Path, dest: t.Union[pathlib.Path, None] -) -> str: +def _create_final_dest(job_root_path: pathlib.Path, dest: pathlib.Path) -> str: """Combine the job root path and destination path. Return as a string for entry point consumption. @@ -31,9 +29,7 @@ def _create_final_dest( :return: Combined path :raises ValueError: An error occurred during path combination """ - combined_path = job_root_path - if dest: - combined_path = job_root_path / dest + combined_path = job_root_path / dest return str(combined_path) @@ -47,7 +43,7 @@ def _check_src_and_dest_path( :param dest: The destination path to be checked. :raises TypeError: If either src or dest is not an instance of pathlib.Path """ - if not isinstance(src, pathlib.Path): + if not isinstance(src, pathlib.Path) or not src.is_absolute(): raise TypeError(f"src must be of type pathlib.Path, not {type(src).__name__}") if dest is not None and not isinstance(dest, pathlib.Path): raise TypeError( @@ -101,7 +97,7 @@ def __init__( """ _check_src_and_dest_path(src, dest) self.src = src - self.dest = dest or src.name + self.dest = dest or pathlib.Path(src.name) def format(self, context: GenerationContext) -> Command: """Create Command to invoke copy file system entry point @@ -136,7 +132,7 @@ def __init__( """ _check_src_and_dest_path(src, dest) self.src = src - self.dest = dest or src.name + self.dest = dest or pathlib.Path(src.name) def format(self, context: GenerationContext) -> Command: """Create Command to invoke symlink file system entry point @@ -179,7 +175,7 @@ def __init__( """ _check_src_and_dest_path(src, dest) self.src = src - self.dest = dest or src.name + self.dest = dest or pathlib.Path(src.name) pickled_dict = pickle.dumps(file_parameters) encoded_dict = base64.b64encode(pickled_dict).decode("ascii") self.file_parameters = encoded_dict diff --git a/smartsim/builders/ensemble.py b/smartsim/builders/ensemble.py index c4a57175f5..317ff5c024 100644 --- a/smartsim/builders/ensemble.py +++ b/smartsim/builders/ensemble.py @@ -307,7 +307,6 @@ def _create_applications(self) -> tuple[Application, ...]: name=f"{self.name}-{i}", exe=self.exe, exe_args=self.exe_args, - files=self.files, file_parameters=permutation.params, ) for i, permutation in enumerate(permutations_) diff --git a/smartsim/entity/application.py b/smartsim/entity/application.py index d736617c18..a896c4272f 100644 --- a/smartsim/entity/application.py +++ b/smartsim/entity/application.py @@ -63,7 +63,6 @@ def __init__( file_parameters: ( t.Mapping[str, str] | None ) = None, # TODO remove when Ensemble is addressed - files: t.Optional[EntityFiles] = None, # TODO remove when Ensemble is addressed ) -> None: """Initialize an ``Application`` diff --git a/tests/temp_tests/test_core/test_commands/test_command.py b/tests/temp_tests/test_core/test_commands/test_command.py index 2900b3c886..faee568cfb 100644 --- a/tests/temp_tests/test_core/test_commands/test_command.py +++ b/tests/temp_tests/test_core/test_commands/test_command.py @@ -35,10 +35,12 @@ def test_command_init(): cmd = Command(command=["salloc", "-N", "1"]) assert cmd.command == ["salloc", "-N", "1"] + def test_command_invalid_init(): cmd = Command(command=["salloc", "-N", "1"]) assert cmd.command == ["salloc", "-N", "1"] + def test_command_getitem_int(): with pytest.raises(TypeError): _ = Command(command=[1]) diff --git a/tests/test_generator.py b/tests/test_generator.py index 03fd6b3c1a..d5617817e9 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -40,7 +40,6 @@ FileSysOperationSet, GenerationContext, SymlinkOperation, - _create_final_dest, ) from smartsim.entity import SmartSimEntity from smartsim.launchable import Job @@ -64,45 +63,12 @@ def generator_instance(test_dir: str) -> Generator: yield Generator(root=pathlib.Path(test_dir)) -@pytest.fixture -def mock_src(test_dir: str): - """Fixture to create a mock source path.""" - return pathlib.Path(test_dir) / pathlib.Path("mock_src") - - -@pytest.fixture -def mock_dest(test_dir: str): - """Fixture to create a mock destination path.""" - return pathlib.Path(test_dir) / pathlib.Path("mock_dest") - - @pytest.fixture def mock_index(): """Fixture to create a mock destination path.""" return 1 -@pytest.fixture -def copy_operation(mock_src: pathlib.Path, mock_dest: pathlib.Path): - """Fixture to create a CopyOperation object.""" - return CopyOperation(src=mock_src, dest=mock_dest) - - -@pytest.fixture -def symlink_operation(mock_src: pathlib.Path, mock_dest: pathlib.Path): - """Fixture to create a CopyOperation object.""" - return SymlinkOperation(src=mock_src, dest=mock_dest) - - -@pytest.fixture -def configure_operation(mock_src: pathlib.Path, mock_dest: pathlib.Path): - """Fixture to create a CopyOperation object.""" - return ConfigureOperation( - src=mock_src, - dest=mock_dest, - ) - - class EchoHelloWorldEntity(SmartSimEntity): """A simple smartsim entity that meets the `ExecutableProtocol` protocol""" @@ -243,28 +209,6 @@ def test_mkdir_file(generator_instance: Generator, test_dir: str): assert cmd.command == ["mkdir", "-p", test_dir] -@pytest.fixture -def files(fileutils): - path_to_files = fileutils.get_test_conf_path( - osp.join("generator_files", "easy", "correct/") - ) - list_of_files_strs = glob(path_to_files + "/*") - yield [pathlib.Path(str_path) for str_path in list_of_files_strs] - - -@pytest.fixture -def directory(fileutils): - directory = fileutils.get_test_conf_path( - osp.join("generator_files", "easy", "correct/") - ) - yield [pathlib.Path(directory)] - - -@pytest.fixture(params=["files", "directory"]) -def source(request): - yield request.getfixturevalue(request.param) - - @pytest.mark.parametrize( "dest", ( diff --git a/tests/test_operations.py b/tests/test_operations.py index 98c3927bd1..9b2b7e7a8b 100644 --- a/tests/test_operations.py +++ b/tests/test_operations.py @@ -2,8 +2,6 @@ import os import pathlib import pickle -from glob import glob -from os import path as osp import pytest @@ -14,9 +12,9 @@ FileSysOperationSet, GenerationContext, SymlinkOperation, + _create_final_dest, configure_cmd, copy_cmd, - _create_final_dest, symlink_cmd, ) @@ -88,12 +86,6 @@ def file_system_operation_set( "/valid/root", id="Empty destination path", ), - pytest.param( - pathlib.Path("/valid/root"), - None, - "/valid/root", - id="Empty dest path", - ), ), ) def test_create_final_dest_valid(job_root_path, dest, expected): @@ -115,9 +107,7 @@ def test_create_final_dest_invalid(job_root_path, dest): _create_final_dest(job_root_path, dest) -def test_valid_init_generation_context( - test_dir: str -): +def test_valid_init_generation_context(test_dir: str): """Validate GenerationContext init""" generation_context = GenerationContext(pathlib.Path(test_dir)) assert isinstance(generation_context, GenerationContext) @@ -132,9 +122,7 @@ def test_invalid_init_generation_context(): GenerationContext("") -def test_init_copy_operation( - mock_src: pathlib.Path, mock_dest: pathlib.Path -): +def test_init_copy_operation(mock_src: pathlib.Path, mock_dest: pathlib.Path): """Validate CopyOperation init""" copy_operation = CopyOperation(mock_src, mock_dest) assert isinstance(copy_operation, CopyOperation) @@ -147,7 +135,7 @@ def test_copy_operation_format( mock_dest: str, mock_src: str, generation_context: GenerationContext, - test_dir: str + test_dir: str, ): """Validate CopyOperation.format""" exec = copy_operation.format(generation_context) @@ -157,9 +145,7 @@ def test_copy_operation_format( assert _create_final_dest(test_dir, mock_dest) in exec.command -def test_init_symlink_operation( - mock_src: str, mock_dest: str -): +def test_init_symlink_operation(mock_src: str, mock_dest: str): """Validate SymlinkOperation init""" symlink_operation = SymlinkOperation(mock_src, mock_dest) assert isinstance(symlink_operation, SymlinkOperation) @@ -186,11 +172,11 @@ def test_symlink_operation_format( assert new_dest in exec.command -def test_init_configure_operation( - mock_src: str, mock_dest: str -): +def test_init_configure_operation(mock_src: str, mock_dest: str): """Validate ConfigureOperation init""" - configure_operation = ConfigureOperation(src=mock_src, dest=mock_dest,file_parameters={"FOO": "BAR"}) + configure_operation = ConfigureOperation( + src=mock_src, dest=mock_dest, file_parameters={"FOO": "BAR"} + ) assert isinstance(configure_operation, ConfigureOperation) assert configure_operation.src == mock_src assert configure_operation.dest == mock_dest @@ -218,9 +204,12 @@ def test_configure_operation_format( def test_init_file_sys_operation_set( copy_operation: CopyOperation, symlink_operation: SymlinkOperation, - configure_operation: ConfigureOperation): + configure_operation: ConfigureOperation, +): """Test initialize FileSystemOperationSet""" - file_system_operation_set = FileSysOperationSet([copy_operation,symlink_operation,configure_operation]) + file_system_operation_set = FileSysOperationSet( + [copy_operation, symlink_operation, configure_operation] + ) assert isinstance(file_system_operation_set.operations, list) assert len(file_system_operation_set.operations) == 3 @@ -229,56 +218,27 @@ def test_add_copy_operation( file_system_operation_set: FileSysOperationSet, copy_operation: CopyOperation ): """Test FileSystemOperationSet.add_copy""" - assert len(file_system_operation_set.copy_operations) == 1 + orig_num_ops = len(file_system_operation_set.copy_operations) file_system_operation_set.add_copy(src=pathlib.Path("/src")) - assert len(file_system_operation_set.copy_operations) == 2 + assert len(file_system_operation_set.copy_operations) == orig_num_ops + 1 -def test_add_symlink_operation( - file_system_operation_set: FileSysOperationSet, symlink_operation: SymlinkOperation -): +def test_add_symlink_operation(file_system_operation_set: FileSysOperationSet): """Test FileSystemOperationSet.add_symlink""" - assert len(file_system_operation_set.symlink_operations) == 1 - file_system_operation_set.add_symlink(src=pathlib.Path("src")) - assert len(file_system_operation_set.symlink_operations) == 2 + orig_num_ops = len(file_system_operation_set.symlink_operations) + file_system_operation_set.add_symlink(src=pathlib.Path("/src")) + assert len(file_system_operation_set.symlink_operations) == orig_num_ops + 1 def test_add_configure_operation( file_system_operation_set: FileSysOperationSet, - configure_operation: ConfigureOperation, ): """Test FileSystemOperationSet.add_configuration""" - assert len(file_system_operation_set.configure_operations) == 1 + orig_num_ops = len(file_system_operation_set.configure_operations) file_system_operation_set.add_configuration( - src=pathlib.Path("src"), file_parameters={"FOO": "BAR"} - ) - assert len(file_system_operation_set.configure_operations) == 2 - - -# might change this to files that can be configured -@pytest.fixture -def files(fileutils): - path_to_files = fileutils.get_test_conf_path( - osp.join("generator_files", "easy", "correct/") - ) - list_of_files_strs = sorted(glob(path_to_files + "/*")) - yield [pathlib.Path(str_path) for str_path in list_of_files_strs] - - -@pytest.fixture -def directory(fileutils): - directory = fileutils.get_test_conf_path( - osp.join("generator_files", "easy", "correct/") + src=pathlib.Path("/src"), file_parameters={"FOO": "BAR"} ) - yield [pathlib.Path(directory)] - - -@pytest.fixture -def source(request, files, directory): - if request.param == "files": - return files - elif request.param == "directory": - return directory + assert len(file_system_operation_set.configure_operations) == orig_num_ops + 1 @pytest.mark.parametrize( @@ -289,13 +249,26 @@ def source(request, files, directory): pytest.param("/absolute/path", TypeError, id="dest as absolute str"), ), ) -@pytest.mark.parametrize("source", ["files", "directory"], indirect=True) def test_copy_files_invalid_dest(dest, error, source): """Test invalid copy destination""" with pytest.raises(error): _ = [CopyOperation(src=file, dest=dest) for file in source] +@pytest.mark.parametrize( + "src,error", + ( + pytest.param(123, TypeError, id="src as integer"), + pytest.param("", TypeError, id="src as empty str"), + pytest.param("relative/path", TypeError, id="src as relative str"), + ), +) +def test_copy_files_invalid_src(src, error): + """Test invalid copy source""" + with pytest.raises(error): + _ = CopyOperation(src=src) + + @pytest.mark.parametrize( "dest,error", ( @@ -304,13 +277,26 @@ def test_copy_files_invalid_dest(dest, error, source): pytest.param("/absolute/path", TypeError, id="dest as absolute str"), ), ) -@pytest.mark.parametrize("source", ["files", "directory"], indirect=True) def test_symlink_files_invalid_dest(dest, error, source): """Test invalid symlink destination""" with pytest.raises(error): _ = [SymlinkOperation(src=file, dest=dest) for file in source] +@pytest.mark.parametrize( + "src,error", + ( + pytest.param(123, TypeError, id="src as integer"), + pytest.param("", TypeError, id="src as empty str"), + pytest.param("relative/path", TypeError, id="src as relative str"), + ), +) +def test_symlink_files_invalid_src(src, error): + """Test invalid symlink source""" + with pytest.raises(error): + _ = SymlinkOperation(src=src) + + @pytest.mark.parametrize( "dest,error", ( @@ -319,7 +305,6 @@ def test_symlink_files_invalid_dest(dest, error, source): pytest.param("/absolute/path", TypeError, id="dest as absolute str"), ), ) -@pytest.mark.parametrize("source", ["files", "directory"], indirect=True) def test_configure_files_invalid_dest(dest, error, source): """Test invalid configure destination""" with pytest.raises(error): @@ -327,3 +312,17 @@ def test_configure_files_invalid_dest(dest, error, source): ConfigureOperation(src=file, dest=dest, file_parameters={"FOO": "BAR"}) for file in source ] + + +@pytest.mark.parametrize( + "src,error", + ( + pytest.param(123, TypeError, id="src as integer"), + pytest.param("", TypeError, id="src as empty str"), + pytest.param("relative/path", TypeError, id="src as relative str"), + ), +) +def test_configure_files_invalid_src(src, error): + """Test invalid configure source""" + with pytest.raises(error): + _ = ConfigureOperation(src=src) From b9c01378f5b66e0ab5755f82879513d1737bc80c Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Wed, 9 Oct 2024 10:35:49 -0700 Subject: [PATCH 14/35] update to ensemble --- smartsim/builders/ensemble.py | 18 ++++++++++++------ tests/test_builder_operations.py | 12 ++++++++++++ 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/smartsim/builders/ensemble.py b/smartsim/builders/ensemble.py index f322d66f16..553a367462 100644 --- a/smartsim/builders/ensemble.py +++ b/smartsim/builders/ensemble.py @@ -279,10 +279,15 @@ def _create_applications(self) -> tuple[Application, ...]: :return: A tuple of Application instances """ + # Create a list ls = [] + # Grabbing the associated register function permutation_strategy = strategies.resolve(self.permutation_strategy) + # Open a loop to for configured files for file in self.files.configure_operations: + # list new_list = [] + # return a list of ParamSets combinations = permutation_strategy( file.file_parameters, self.exe_arg_parameters, self.max_permutations ) @@ -290,8 +295,10 @@ def _create_applications(self) -> tuple[Application, ...]: # permutations_ = itertools.chain.from_iterable( # itertools.repeat(permutation, self.replicas) for permutation in combinations # ) + # Attach each paramset with the associated file via dataset and append to the list for combo in combinations: new_list.append(FileSet(file, combo)) + # Add the list of (file, paramset) to a new list ls.append(new_list) combo = self._cartesian_values(ls) print(combo) @@ -360,12 +367,11 @@ def build_jobs(self, settings: LaunchSettings) -> tuple[Job, ...]: return tuple(Job(app, settings, app.name) for app in apps) def _step_values(self, ls): + #facilitate parallel iteration over multiple sequences return list(zip(*ls)) def _cartesian_values(self, ls): # needs to return a list[tuples] - combo = itertools.product(ls) - yup: t.Iterable = ( - val for val in zip(combo) - ) - print(yup) - return list(yup) + return list(itertools.product(*ls)) + + def _random_values(self, ls): + val = self._cartesian_values() diff --git a/tests/test_builder_operations.py b/tests/test_builder_operations.py index 0d46e9bebe..951d3826c5 100644 --- a/tests/test_builder_operations.py +++ b/tests/test_builder_operations.py @@ -295,3 +295,15 @@ def test_step_mock(): print(config.src) deserialized_dict = pickle.loads(decoded_dict) print(deserialized_dict) + +def test_all_perm_mock(): + ensemble = Ensemble("name", "echo", exe_arg_parameters = {"-N": ["1", "2"]}, permutation_strategy="step") + ensemble.files.add_configuration(pathlib.Path("src_1"), file_parameters={"FOO":["BAR", "TOE"]}) + ensemble.files.add_configuration(pathlib.Path("src_2"), file_parameters={"CAN":["TOM", "STO"]}) + apps = ensemble._create_applications() + for app in apps: + for config in app.files.configure_operations: + decoded_dict = base64.b64decode(config.file_parameters) + print(config.src) + deserialized_dict = pickle.loads(decoded_dict) + print(deserialized_dict) \ No newline at end of file From 3a2012e7fa60dbec87e0916029e25f6a52980ad2 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Mon, 14 Oct 2024 13:24:20 -0700 Subject: [PATCH 15/35] updates --- .../_core/generation/builder_operations.py | 84 +++++++++++++++---- smartsim/_core/generation/operations.py | 7 -- smartsim/builders/ensemble.py | 82 +++++++----------- tests/test_builder_operations.py | 29 +++++-- 4 files changed, 120 insertions(+), 82 deletions(-) diff --git a/smartsim/_core/generation/builder_operations.py b/smartsim/_core/generation/builder_operations.py index 2d4a7704d8..97e02184a6 100644 --- a/smartsim/_core/generation/builder_operations.py +++ b/smartsim/_core/generation/builder_operations.py @@ -5,31 +5,47 @@ class EnsembleGenerationProtocol(t.Protocol): - """Protocol for Generation Operations Ensemble.""" + """Protocol for Ensemble Generation Operations.""" src: pathlib.Path + """Path to source""" dest: t.Optional[pathlib.Path] + """Path to destination""" class EnsembleCopyOperation(EnsembleGenerationProtocol): - """Copy Operation""" + """Ensemble Copy Operation""" def __init__( self, src: pathlib.Path, dest: t.Optional[pathlib.Path] = None ) -> None: + """Initialize a EnsembleCopyOperation object + + :param src: Path to source + :param dest: Path to destination + """ self.src = src + """Path to source""" self.dest = dest + """Path to destination""" class EnsembleSymlinkOperation(EnsembleGenerationProtocol): - """Symlink Operation""" + """Ensemble Symlink Operation""" def __init__(self, src: pathlib.Path, dest: t.Optional[pathlib.Path] = None) -> None: + """Initialize a EnsembleSymlinkOperation object + + :param src: Path to source + :param dest: Path to destination + """ self.src = src + """Path to source""" self.dest = dest + """Path to destination""" class EnsembleConfigureOperation(EnsembleGenerationProtocol): - """Configure Operation""" + """Ensemble Configure Operation""" def __init__( self, @@ -38,32 +54,51 @@ def __init__( dest: t.Optional[pathlib.Path] = None, tag: t.Optional[str] = None, ) -> None: + """Initialize a EnsembleConfigureOperation + + :param src: Path to source + :param file_parameters: File parameters to find and replace + :param dest: Path to destination + :param tag: Tag to use for find and replacement + """ self.src = src + """Path to source""" self.dest = dest + """Path to destination""" self.file_parameters = file_parameters + """File parameters to find and replace""" self.tag = tag if tag else ";" + """Tag to use for the file""" -U = t.TypeVar("U", bound=EnsembleGenerationProtocol) +EnsembleGenerationProtocolT = t.TypeVar("EnsembleGenerationProtocolT", bound=EnsembleGenerationProtocol) @dataclass class EnsembleFileSysOperationSet: - """Dataclass to represent a set of FS Operation Objects""" + """Dataclass to represent a set of Ensemble File System Operation Objects""" operations: t.List[EnsembleGenerationProtocol] = field(default_factory=list) - """Set of FS Objects that match the GenerationProtocol""" + """Set of Ensemble File System Objects that match the EnsembleGenerationProtocol""" def add_copy( self, src: pathlib.Path, dest: t.Optional[pathlib.Path] = None ) -> None: - """Add a copy operation to the operations list""" + """Add a copy operation to the operations list + + :param src: Path to source + :param dest: Path to destination + """ self.operations.append(EnsembleCopyOperation(src, dest)) def add_symlink( self, src: pathlib.Path, dest: t.Optional[pathlib.Path] = None ) -> None: - """Add a symlink operation to the operations list""" + """Add a symlink operation to the operations list + + :param src: Path to source + :param dest: Path to destination + """ self.operations.append(EnsembleSymlinkOperation(src, dest)) def add_configuration( @@ -73,23 +108,44 @@ def add_configuration( dest: t.Optional[pathlib.Path] = None, tag: t.Optional[str] = None, ) -> None: - """Add a configure operation to the operations list""" + """Add a configure operation to the operations list + + :param src: Path to source + :param file_parameters: File parameters to find and replace + :param dest: Path to destination + :param tag: Tag to use for find and replacement + """ self.operations.append(EnsembleConfigureOperation(src, file_parameters, dest, tag)) @property def copy_operations(self) -> t.List[EnsembleCopyOperation]: - """Property to get the list of copy files.""" + """Property to get the list of copy files. + + :return: List of EnsembleCopyOperation objects + """ return self._filter(EnsembleCopyOperation) @property def symlink_operations(self) -> t.List[EnsembleSymlinkOperation]: - """Property to get the list of symlink files.""" + """Property to get the list of symlink files. + + :return: List of EnsembleSymlinkOperation objects + """ return self._filter(EnsembleSymlinkOperation) @property def configure_operations(self) -> t.List[EnsembleConfigureOperation]: - """Property to get the list of configure files.""" + """Property to get the list of configure files. + + :return: List of EnsembleConfigureOperation objects + """ return self._filter(EnsembleConfigureOperation) - def _filter(self, type: t.Type[U]) -> t.List[U]: + def _filter(self, type: t.Type[EnsembleGenerationProtocolT]) -> t.List[EnsembleGenerationProtocolT]: + """Filters the operations list to include only instances of the + specified type. + + :param type: The type of operations to filter + :return: A list of operations that are instances of the specified type + """ return [x for x in self.operations if isinstance(x, type)] \ No newline at end of file diff --git a/smartsim/_core/generation/operations.py b/smartsim/_core/generation/operations.py index 4e6e1001b2..eb841a22e8 100644 --- a/smartsim/_core/generation/operations.py +++ b/smartsim/_core/generation/operations.py @@ -60,13 +60,6 @@ def check_src_and_dest_path( if isinstance(dest, pathlib.Path) and " " in str(dest): raise ValueError("Path contains spaces, which are not allowed") - # TODO I want to add the check below but I do not think this works for remote jobs - # full_path = path.abspath(file_path) - # if path.isfile(full_path): - # return full_path - # if path.isdir(full_path): - # return full_path - def check_run_path(run_path: pathlib.Path) -> None: """Validate that the provided run path is of type pathlib.Path diff --git a/smartsim/builders/ensemble.py b/smartsim/builders/ensemble.py index 553a367462..d5b40d1ebd 100644 --- a/smartsim/builders/ensemble.py +++ b/smartsim/builders/ensemble.py @@ -31,6 +31,7 @@ import os import os.path import typing as t +import random from smartsim.builders.utils import strategies from smartsim.builders.utils.strategies import ParamSet @@ -279,56 +280,41 @@ def _create_applications(self) -> tuple[Application, ...]: :return: A tuple of Application instances """ - # Create a list - ls = [] - # Grabbing the associated register function + # resolve the permutation strategy permutation_strategy = strategies.resolve(self.permutation_strategy) - # Open a loop to for configured files - for file in self.files.configure_operations: - # list - new_list = [] - # return a list of ParamSets - combinations = permutation_strategy( - file.file_parameters, self.exe_arg_parameters, self.max_permutations - ) - combinations = combinations if combinations else [ParamSet({}, {})] - # permutations_ = itertools.chain.from_iterable( - # itertools.repeat(permutation, self.replicas) for permutation in combinations - # ) - # Attach each paramset with the associated file via dataset and append to the list - for combo in combinations: - new_list.append(FileSet(file, combo)) - # Add the list of (file, paramset) to a new list - ls.append(new_list) - combo = self._cartesian_values(ls) - print(combo) - print(type(combo)) - print(len(combo)) - for item in combo: - print(type(item)) - print(item) - - # return tuple( - # self.create_app(i, item) - - # for i, item in combo - # ) - apps = [] + # apply the permutation strategy to each attached config file + perm_list: t.List[t.List[FileSet]] = [self.perm_config_file(config_file, permutation_strategy) for config_file in self.files.configure_operations] + # group the files together + val: t.List[tuple[FileSet]] = self._cartesian_values(perm_list) + # duplicate if replicas + permutations_ = itertools.chain.from_iterable( + itertools.repeat(permutation, self.replicas) for permutation in val + ) + all_apps = [] i = 0 - for item in combo: + for item in permutations_: i+=1 - apps.append(self.create_app(i, item)) - return tuple(apps) - - def create_app(self, i, item): - app = Application( + app = Application( name=f"{self.name}-{i}", exe=self.exe, exe_args=self.exe_args, ) - for merp in item: - app.files.add_configuration(src=merp.file.src, file_parameters=merp.combinations.params) - return app + # apply the config files in the tuple + for file_set in item: + app.files.add_configuration(src=file_set.file.src, file_parameters=file_set.combinations.params) + all_apps.append(app) + return tuple(all_apps) + + + def perm_config_file(self, file, permutation_strategy): + combinations = permutation_strategy( + file.file_parameters, self.exe_arg_parameters, self.max_permutations + ) or [ParamSet({}, {})] + return [FileSet(file, combo) for combo in combinations] + + + def _cartesian_values(self, ls): + return list(itertools.product(*ls)) def build_jobs(self, settings: LaunchSettings) -> tuple[Job, ...]: @@ -365,13 +351,3 @@ def build_jobs(self, settings: LaunchSettings) -> tuple[Job, ...]: if not apps: raise ValueError("There are no members as part of this ensemble") return tuple(Job(app, settings, app.name) for app in apps) - - def _step_values(self, ls): - #facilitate parallel iteration over multiple sequences - return list(zip(*ls)) - - def _cartesian_values(self, ls): # needs to return a list[tuples] - return list(itertools.product(*ls)) - - def _random_values(self, ls): - val = self._cartesian_values() diff --git a/tests/test_builder_operations.py b/tests/test_builder_operations.py index 951d3826c5..0942090bda 100644 --- a/tests/test_builder_operations.py +++ b/tests/test_builder_operations.py @@ -16,6 +16,7 @@ EnsembleFileSysOperationSet ) from smartsim.builders import Ensemble +from smartsim.builders.utils import strategies # QUESTIONS # TODO test python protocol? @@ -30,24 +31,28 @@ def mock_src(test_dir: str): return pathlib.Path(test_dir) / pathlib.Path("mock_src") +# TODO remove when PR 732 is merged @pytest.fixture def mock_dest(test_dir: str): """Fixture to create a mock destination path.""" return pathlib.Path(test_dir) / pathlib.Path("mock_dest") +# TODO remove when PR 732 is merged @pytest.fixture def copy_operation(mock_src: pathlib.Path, mock_dest: pathlib.Path): """Fixture to create a CopyOperation object.""" return EnsembleCopyOperation(src=mock_src, dest=mock_dest) +# TODO remove when PR 732 is merged @pytest.fixture def symlink_operation(mock_src: pathlib.Path, mock_dest: pathlib.Path): """Fixture to create a SymlinkOperation object.""" return EnsembleSymlinkOperation(src=mock_src, dest=mock_dest) +# TODO remove when PR 732 is merged @pytest.fixture def configure_operation(mock_src: pathlib.Path, mock_dest: pathlib.Path): """Fixture to create a Configure object.""" @@ -57,7 +62,7 @@ def configure_operation(mock_src: pathlib.Path, mock_dest: pathlib.Path): @pytest.fixture -def file_system_operation_set( +def ensemble_file_system_operation_set( copy_operation: EnsembleCopyOperation, symlink_operation: EnsembleSymlinkOperation, configure_operation: EnsembleConfigureOperation, @@ -297,13 +302,21 @@ def test_step_mock(): print(deserialized_dict) def test_all_perm_mock(): - ensemble = Ensemble("name", "echo", exe_arg_parameters = {"-N": ["1", "2"]}, permutation_strategy="step") + ensemble = Ensemble("name", "echo", exe_arg_parameters = {"-N": ["1", "2"]}, permutation_strategy="step", replicas=2) ensemble.files.add_configuration(pathlib.Path("src_1"), file_parameters={"FOO":["BAR", "TOE"]}) ensemble.files.add_configuration(pathlib.Path("src_2"), file_parameters={"CAN":["TOM", "STO"]}) apps = ensemble._create_applications() - for app in apps: - for config in app.files.configure_operations: - decoded_dict = base64.b64decode(config.file_parameters) - print(config.src) - deserialized_dict = pickle.loads(decoded_dict) - print(deserialized_dict) \ No newline at end of file + print(len(apps)) + # for app in apps: + # for config in app.files.configure_operations: + # decoded_dict = base64.b64decode(config.file_parameters) + # print(config.src) + # deserialized_dict = pickle.loads(decoded_dict) + # print(deserialized_dict) + +def test_mock(): + ensemble = Ensemble("name", "echo", exe_arg_parameters = {"-N": ["1", "2"]}, permutation_strategy="step") + file = EnsembleConfigureOperation(src="src", file_parameters={"FOO":["BAR", "TOE"]}) + permutation_strategy = strategies.resolve("all_perm") + val = ensemble.perm_config_file(file, permutation_strategy) + print(val) \ No newline at end of file From 6c2d9087b4919e10c61c83c8018d3c57e48ecfe1 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Mon, 14 Oct 2024 16:33:54 -0700 Subject: [PATCH 16/35] updates --- conftest.py | 2 +- smartsim/_core/generation/generator.py | 2 +- .../generation/{ => operations}/operations.py | 71 ++++++------ smartsim/entity/application.py | 2 +- tests/test_application.py | 36 ------ tests/test_generator.py | 2 +- tests/test_operations.py | 105 +++++++++++++----- 7 files changed, 115 insertions(+), 105 deletions(-) rename smartsim/_core/generation/{ => operations}/operations.py (79%) diff --git a/conftest.py b/conftest.py index fe94ee5409..895fcc9adb 100644 --- a/conftest.py +++ b/conftest.py @@ -55,7 +55,7 @@ from smartsim._core.config.config import Config from smartsim._core.launcher.dragon.dragon_connector import DragonConnector from smartsim._core.launcher.dragon.dragon_launcher import DragonLauncher -from smartsim._core.generation.operations import ConfigureOperation, CopyOperation, SymlinkOperation +from smartsim._core.generation.operations.operations import ConfigureOperation, CopyOperation, SymlinkOperation from smartsim._core.generation.generator import Generator from smartsim._core.utils.telemetry.telemetry import JobEntity from smartsim.database import FeatureStore diff --git a/smartsim/_core/generation/generator.py b/smartsim/_core/generation/generator.py index 27973b1b86..08a6653b6b 100644 --- a/smartsim/_core/generation/generator.py +++ b/smartsim/_core/generation/generator.py @@ -34,7 +34,7 @@ from ...launchable import Job from ...log import get_logger from ..commands import Command, CommandList -from .operations import ( +from .operations.operations import ( ConfigureOperation, CopyOperation, FileSysOperationSet, diff --git a/smartsim/_core/generation/operations.py b/smartsim/_core/generation/operations/operations.py similarity index 79% rename from smartsim/_core/generation/operations.py rename to smartsim/_core/generation/operations/operations.py index aa2fa2bf88..3925dbcf05 100644 --- a/smartsim/_core/generation/operations.py +++ b/smartsim/_core/generation/operations/operations.py @@ -6,51 +6,30 @@ import typing as t from dataclasses import dataclass, field -from ..commands import Command +from ...commands import Command +from .utils.helpers import check_src_and_dest_path # pylint: disable=invalid-name entry_point_path = "smartsim._core.entrypoints.file_operations" """Path to file operations module""" copy_cmd = "copy" -"""Copy file operations command""" +"""Copy file operation command""" symlink_cmd = "symlink" -"""Symlink file operations command""" +"""Symlink file operation command""" configure_cmd = "configure" -"""Configure file operations command""" +"""Configure file operation command""" -def _create_final_dest(job_root_path: pathlib.Path, dest: pathlib.Path) -> str: - """Combine the job root path and destination path. Return as a string for +def _create_dest_path(job_run_path: pathlib.Path, dest: pathlib.Path) -> str: + """Combine the job run path and destination path. Return as a string for entry point consumption. - :param job_root_path: Job root path + :param job_run_path: Job run path :param dest: Destination path :return: Combined path - :raises ValueError: An error occurred during path combination """ - combined_path = job_root_path / dest - return str(combined_path) - - -def _check_src_and_dest_path( - src: pathlib.Path, dest: t.Union[pathlib.Path, None] -) -> None: - """Validate that the provided source and destination paths are - of type pathlib.Path - - :param src: The source path to be checked. - :param dest: The destination path to be checked. - :raises TypeError: If either src or dest is not an instance of pathlib.Path - """ - if not isinstance(src, pathlib.Path) or not src.is_absolute(): - raise TypeError(f"src must be of type pathlib.Path, not {type(src).__name__}") - if dest is not None and not isinstance(dest, pathlib.Path): - raise TypeError( - f"dest must be of type pathlib.Path or None, not {type(dest).__name__}" - ) - if isinstance(dest, pathlib.Path) and dest.is_absolute(): - raise ValueError("Invalid destination path") + return str(job_run_path / dest) def _check_run_path(run_path: pathlib.Path) -> None: @@ -62,16 +41,28 @@ def _check_run_path(run_path: pathlib.Path) -> None: """ if not isinstance(run_path, pathlib.Path): raise TypeError( - f"run_path must be of type pathlib.Path, not {type(run_path).__name__}" + f"The Job's run path must be of type pathlib.Path, not {type(run_path).__name__}" + ) + if not run_path.is_absolute(): + raise ValueError( + f"The Job's run path must be absolute." ) + # if not run_path.is_dir(): + # raise ValueError( + # "The Job's run path must be a directory." + # ) class GenerationContext: """Context for file system generation operations.""" - def __init__(self, job_root_path: pathlib.Path): - _check_run_path(job_root_path) - self.job_root_path = job_root_path + def __init__(self, job_run_path: pathlib.Path): + """Initialize a GenerationContext object + + :param job_run_path: Job's run path + """ + _check_run_path(job_run_path) + self.job_run_path = job_run_path """The Job run path""" @@ -93,7 +84,7 @@ def __init__( :param src: Path to source :param dest: Path to destination """ - _check_src_and_dest_path(src, dest) + check_src_and_dest_path(src, dest) self.src = src self.dest = dest or pathlib.Path(src.name) @@ -103,7 +94,7 @@ def format(self, context: GenerationContext) -> Command: :param context: Context for copy operation :return: Copy Command """ - final_dest = _create_final_dest(context.job_root_path, self.dest) + final_dest = _create_dest_path(context.job_run_path, self.dest) return Command( [ sys.executable, @@ -128,7 +119,7 @@ def __init__( :param src: Path to source :param dest: Path to destination """ - _check_src_and_dest_path(src, dest) + check_src_and_dest_path(src, dest) self.src = src self.dest = dest or pathlib.Path(src.name) @@ -140,7 +131,7 @@ def format(self, context: GenerationContext) -> Command: """ normalized_path = os.path.normpath(self.src) parent_dir = os.path.dirname(normalized_path) - final_dest = _create_final_dest(context.job_root_path, self.dest) + final_dest = _create_dest_path(context.job_run_path, self.dest) new_dest = os.path.join(final_dest, parent_dir) return Command( [ @@ -171,7 +162,7 @@ def __init__( :param dest: Path to destination :param tag: Tag to use for find and replacement """ - _check_src_and_dest_path(src, dest) + check_src_and_dest_path(src, dest) self.src = src self.dest = dest or pathlib.Path(src.name) pickled_dict = pickle.dumps(file_parameters) @@ -185,7 +176,7 @@ def format(self, context: GenerationContext) -> Command: :param context: Context for configure operation :return: Configure Command """ - final_dest = _create_final_dest(context.job_root_path, self.dest) + final_dest = _create_dest_path(context.job_run_path, self.dest) return Command( [ sys.executable, diff --git a/smartsim/entity/application.py b/smartsim/entity/application.py index 58353bd877..a9afa5f7cf 100644 --- a/smartsim/entity/application.py +++ b/smartsim/entity/application.py @@ -32,7 +32,7 @@ import typing as t from os import path as osp -from .._core.generation.operations import FileSysOperationSet +from .._core.generation.operations.operations import FileSysOperationSet from .._core.utils.helpers import expand_exe_path from ..log import get_logger from .entity import SmartSimEntity diff --git a/tests/test_application.py b/tests/test_application.py index d329321504..cd5cc5b3da 100644 --- a/tests/test_application.py +++ b/tests/test_application.py @@ -62,14 +62,6 @@ def test_application_exe_args_property(): assert exe_args is a.exe_args -def test_application_files_property(get_gen_configure_dir): - tagged_files = sorted(glob(get_gen_configure_dir + "/*")) - files = EntityFiles(tagged=tagged_files) - a = Application("test_name", exe="echo", exe_args=["spam", "eggs"], files=files) - files = a.files - assert files is a.files - - def test_application_file_parameters_property(): file_parameters = {"h": [5, 6, 7, 8]} a = Application( @@ -120,24 +112,6 @@ def test_type_exe_args(): application.exe_args = [1, 2, 3] -def test_type_files_property(): - application = Application( - "test_name", - exe="echo", - ) - with pytest.raises(TypeError): - application.files = "/path/to/file" - - -def test_type_file_parameters_property(): - application = Application( - "test_name", - exe="echo", - ) - with pytest.raises(TypeError): - application.file_parameters = {1: 2} - - def test_type_incoming_entities(): application = Application( "test_name", @@ -171,16 +145,6 @@ def test_application_type_exe_args(): application.exe_args = [1, 2, 3] -def test_application_type_files(): - application = Application( - "test_name", - exe="echo", - exe_args=["spam", "eggs"], - ) - with pytest.raises(TypeError, match="files argument was not of type EntityFiles"): - application.files = 2 - - @pytest.mark.parametrize( "file_params", ( diff --git a/tests/test_generator.py b/tests/test_generator.py index 91c150a08a..3915526a8b 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -34,7 +34,7 @@ from smartsim._core.commands import Command, CommandList from smartsim._core.generation.generator import Generator -from smartsim._core.generation.operations import ( +from smartsim._core.generation.operations.operations import ( ConfigureOperation, CopyOperation, FileSysOperationSet, diff --git a/tests/test_operations.py b/tests/test_operations.py index 9b2b7e7a8b..c565f6937b 100644 --- a/tests/test_operations.py +++ b/tests/test_operations.py @@ -6,20 +6,20 @@ import pytest from smartsim._core.commands import Command -from smartsim._core.generation.operations import ( +from smartsim._core.generation.operations.utils.helpers import check_src_and_dest_path +from smartsim._core.generation.operations.operations import ( ConfigureOperation, CopyOperation, FileSysOperationSet, GenerationContext, SymlinkOperation, - _create_final_dest, + _create_dest_path, + _check_run_path, configure_cmd, copy_cmd, symlink_cmd, ) -# TODO ADD CHECK TO ENFORCE SRC AS RELATIVE - pytestmark = pytest.mark.group_a @@ -70,48 +70,103 @@ def file_system_operation_set( """Fixture to create a FileSysOperationSet object.""" return FileSysOperationSet([copy_operation, symlink_operation, configure_operation]) +# TODO is this test even necessary +@pytest.mark.parametrize( + "job_run_path, dest", + ( + pytest.param( + pathlib.Path("/absolute/src"), + pathlib.Path("relative/dest"), + id="Valid paths", + ), + pytest.param( + pathlib.Path("/absolute/src"), + pathlib.Path(""), + id="Empty destination path", + ), + ), +) +def test_check_src_and_dest_path_valid(job_run_path, dest): + """Test valid path inputs for helpers.check_src_and_dest_path""" + check_src_and_dest_path(job_run_path, dest) + + +@pytest.mark.parametrize( + "job_run_path, dest, error", + ( + pytest.param( + pathlib.Path("relative/src"), + pathlib.Path("relative/dest"), + ValueError, + id="Relative src Path", + ), + pytest.param( + pathlib.Path("/absolute/src"), + pathlib.Path("/absolute/src"), + ValueError, + id="Absolute dest Path", + ), + pytest.param( + 123, + pathlib.Path("relative/dest"), + TypeError, + id="non Path src", + ), + pytest.param( + pathlib.Path("/absolute/src"), + 123, + TypeError, + id="non Path dest", + ), + ), +) +def test_check_src_and_dest_path_invalid(job_run_path, dest, error): + """Test invalid path inputs for helpers.check_src_and_dest_path""" + with pytest.raises(error): + check_src_and_dest_path(job_run_path, dest) + @pytest.mark.parametrize( - "job_root_path, dest, expected", + "job_run_path, dest, expected", ( pytest.param( - pathlib.Path("/valid/root"), - pathlib.Path("valid/dest"), - "/valid/root/valid/dest", + pathlib.Path("/absolute/root"), + pathlib.Path("relative/dest"), + "/absolute/root/relative/dest", id="Valid paths", ), pytest.param( - pathlib.Path("/valid/root"), + pathlib.Path("/absolute/root"), pathlib.Path(""), - "/valid/root", + "/absolute/root", id="Empty destination path", ), ), ) -def test_create_final_dest_valid(job_root_path, dest, expected): - """Test valid path inputs for operations._create_final_dest""" - assert _create_final_dest(job_root_path, dest) == expected +def test_create_dest_path_valid(job_run_path, dest, expected): + """Test valid path inputs for operations._create_dest_path""" + assert _create_dest_path(job_run_path, dest) == expected @pytest.mark.parametrize( - "job_root_path, dest", + "job_run_path, error", ( - pytest.param(None, pathlib.Path("valid/dest"), id="None as root path"), - pytest.param(1234, pathlib.Path("valid/dest"), id="Number as root path"), - pytest.param(pathlib.Path("valid/dest"), 1234, id="Number as dest"), + pytest.param(pathlib.Path("/valid/dest.py"), ValueError, id="Run path is not a directory"), + pytest.param(pathlib.Path("relative/path"), ValueError, id="Run path is not absolute"), + pytest.param(1234, TypeError, id="Run path is not pathlib.path"), ), ) -def test_create_final_dest_invalid(job_root_path, dest): - """Test invalid path inputs for operations._create_final_dest""" - with pytest.raises(TypeError): - _create_final_dest(job_root_path, dest) +def test_check_run_path_invalid(job_run_path, error): + """Test invalid path inputs for operations._check_run_path""" + with pytest.raises(error): + _check_run_path(job_run_path) def test_valid_init_generation_context(test_dir: str): """Validate GenerationContext init""" generation_context = GenerationContext(pathlib.Path(test_dir)) assert isinstance(generation_context, GenerationContext) - assert generation_context.job_root_path == pathlib.Path(test_dir) + assert generation_context.job_run_path == pathlib.Path(test_dir) def test_invalid_init_generation_context(): @@ -142,7 +197,7 @@ def test_copy_operation_format( assert isinstance(exec, Command) assert str(mock_src) in exec.command assert copy_cmd in exec.command - assert _create_final_dest(test_dir, mock_dest) in exec.command + assert _create_dest_path(test_dir, mock_dest) in exec.command def test_init_symlink_operation(mock_src: str, mock_dest: str): @@ -167,7 +222,7 @@ def test_symlink_operation_format( normalized_path = os.path.normpath(mock_src) parent_dir = os.path.dirname(normalized_path) - final_dest = _create_final_dest(generation_context.job_root_path, mock_dest) + final_dest = _create_dest_path(generation_context.job_run_path, mock_dest) new_dest = os.path.join(final_dest, parent_dir) assert new_dest in exec.command @@ -198,7 +253,7 @@ def test_configure_operation_format( assert isinstance(exec, Command) assert str(mock_src) in exec.command assert configure_cmd in exec.command - assert _create_final_dest(test_dir, mock_dest) in exec.command + assert _create_dest_path(test_dir, mock_dest) in exec.command def test_init_file_sys_operation_set( From f87d7cd1effbb158a9a7b7a942520abb06b69030 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Mon, 14 Oct 2024 18:25:14 -0700 Subject: [PATCH 17/35] styling up to date --- .../_core/generation/operations/operations.py | 5 ++-- .../generation/operations/utils/helpers.py | 27 +++++++++++++++++++ tests/test_operations.py | 14 +++++----- 3 files changed, 36 insertions(+), 10 deletions(-) create mode 100644 smartsim/_core/generation/operations/utils/helpers.py diff --git a/smartsim/_core/generation/operations/operations.py b/smartsim/_core/generation/operations/operations.py index 3925dbcf05..29b9db466a 100644 --- a/smartsim/_core/generation/operations/operations.py +++ b/smartsim/_core/generation/operations/operations.py @@ -44,9 +44,8 @@ def _check_run_path(run_path: pathlib.Path) -> None: f"The Job's run path must be of type pathlib.Path, not {type(run_path).__name__}" ) if not run_path.is_absolute(): - raise ValueError( - f"The Job's run path must be absolute." - ) + raise ValueError(f"The Job's run path must be absolute.") + # TODO # if not run_path.is_dir(): # raise ValueError( # "The Job's run path must be a directory." diff --git a/smartsim/_core/generation/operations/utils/helpers.py b/smartsim/_core/generation/operations/utils/helpers.py new file mode 100644 index 0000000000..9d99b0e8bf --- /dev/null +++ b/smartsim/_core/generation/operations/utils/helpers.py @@ -0,0 +1,27 @@ +import pathlib +import typing as t + + +def check_src_and_dest_path( + src: pathlib.Path, dest: t.Union[pathlib.Path, None] +) -> None: + """Validate that the provided source and destination paths are + of type pathlib.Path. Additionally, validate that destination is a + relative Path and source is a absolute Path. + + :param src: The source path to check + :param dest: The destination path to check + :raises TypeError: If either src or dest is not of type pathlib.Path + :raises ValueError: If source is not an absolute Path or if destination is not + a relative Path + """ + if not isinstance(src, pathlib.Path): + raise TypeError(f"src must be of type pathlib.Path, not {type(src).__name__}") + if dest is not None and not isinstance(dest, pathlib.Path): + raise TypeError( + f"dest must be of type pathlib.Path or None, not {type(dest).__name__}" + ) + if dest is not None and dest.is_absolute(): + raise ValueError(f"dest must be a relative Path") + if not src.is_absolute(): + raise ValueError(f"src must be an absolute Path") diff --git a/tests/test_operations.py b/tests/test_operations.py index c565f6937b..926e4b08e9 100644 --- a/tests/test_operations.py +++ b/tests/test_operations.py @@ -6,19 +6,19 @@ import pytest from smartsim._core.commands import Command -from smartsim._core.generation.operations.utils.helpers import check_src_and_dest_path from smartsim._core.generation.operations.operations import ( ConfigureOperation, CopyOperation, FileSysOperationSet, GenerationContext, SymlinkOperation, - _create_dest_path, _check_run_path, + _create_dest_path, configure_cmd, copy_cmd, symlink_cmd, ) +from smartsim._core.generation.operations.utils.helpers import check_src_and_dest_path pytestmark = pytest.mark.group_a @@ -70,6 +70,7 @@ def file_system_operation_set( """Fixture to create a FileSysOperationSet object.""" return FileSysOperationSet([copy_operation, symlink_operation, configure_operation]) + # TODO is this test even necessary @pytest.mark.parametrize( "job_run_path, dest", @@ -151,8 +152,9 @@ def test_create_dest_path_valid(job_run_path, dest, expected): @pytest.mark.parametrize( "job_run_path, error", ( - pytest.param(pathlib.Path("/valid/dest.py"), ValueError, id="Run path is not a directory"), - pytest.param(pathlib.Path("relative/path"), ValueError, id="Run path is not absolute"), + pytest.param( + pathlib.Path("relative/path"), ValueError, id="Run path is not absolute" + ), pytest.param(1234, TypeError, id="Run path is not pathlib.path"), ), ) @@ -269,9 +271,7 @@ def test_init_file_sys_operation_set( assert len(file_system_operation_set.operations) == 3 -def test_add_copy_operation( - file_system_operation_set: FileSysOperationSet, copy_operation: CopyOperation -): +def test_add_copy_operation(file_system_operation_set: FileSysOperationSet): """Test FileSystemOperationSet.add_copy""" orig_num_ops = len(file_system_operation_set.copy_operations) file_system_operation_set.add_copy(src=pathlib.Path("/src")) From a7960f1f568d001677e3f7e5c0a32dd81b2d3d9b Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Mon, 14 Oct 2024 18:28:04 -0700 Subject: [PATCH 18/35] file rename --- .../ensemble_operations.py} | 0 smartsim/builders/ensemble.py | 2 +- tests/test_builder_operations.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename smartsim/_core/generation/{builder_operations.py => operations/ensemble_operations.py} (100%) diff --git a/smartsim/_core/generation/builder_operations.py b/smartsim/_core/generation/operations/ensemble_operations.py similarity index 100% rename from smartsim/_core/generation/builder_operations.py rename to smartsim/_core/generation/operations/ensemble_operations.py diff --git a/smartsim/builders/ensemble.py b/smartsim/builders/ensemble.py index d7ec2c0869..b56da20c95 100644 --- a/smartsim/builders/ensemble.py +++ b/smartsim/builders/ensemble.py @@ -39,7 +39,7 @@ from smartsim.entity import entity from smartsim.entity.application import Application from smartsim.launchable.job import Job -from smartsim._core.generation.builder_operations import EnsembleFileSysOperationSet, EnsembleConfigureOperation +from smartsim._core.generation.operations.ensemble_operations import EnsembleFileSysOperationSet, EnsembleConfigureOperation from dataclasses import dataclass, field diff --git a/tests/test_builder_operations.py b/tests/test_builder_operations.py index 0942090bda..02c892aca7 100644 --- a/tests/test_builder_operations.py +++ b/tests/test_builder_operations.py @@ -9,7 +9,7 @@ import pytest from smartsim._core.commands import Command -from smartsim._core.generation.builder_operations import ( +from smartsim._core.generation.operations.ensemble_operations import ( EnsembleCopyOperation, EnsembleSymlinkOperation, EnsembleConfigureOperation, From a6391cf52d504bca7fda0cc9d87e213f2e95b0ef Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Mon, 14 Oct 2024 19:28:08 -0700 Subject: [PATCH 19/35] remove unused fixtures from operations --- tests/test_operations.py | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/tests/test_operations.py b/tests/test_operations.py index 926e4b08e9..5852b2575d 100644 --- a/tests/test_operations.py +++ b/tests/test_operations.py @@ -29,38 +29,6 @@ def generation_context(test_dir: str): return GenerationContext(pathlib.Path(test_dir)) -@pytest.fixture -def mock_src(test_dir: str): - """Fixture to create a mock source path.""" - return pathlib.Path(test_dir) / pathlib.Path("mock_src") - - -@pytest.fixture -def mock_dest(): - """Fixture to create a mock destination path.""" - return pathlib.Path("mock_dest") - - -@pytest.fixture -def copy_operation(mock_src: pathlib.Path, mock_dest: pathlib.Path): - """Fixture to create a CopyOperation object.""" - return CopyOperation(src=mock_src, dest=mock_dest) - - -@pytest.fixture -def symlink_operation(mock_src: pathlib.Path, mock_dest: pathlib.Path): - """Fixture to create a SymlinkOperation object.""" - return SymlinkOperation(src=mock_src, dest=mock_dest) - - -@pytest.fixture -def configure_operation(mock_src: pathlib.Path, mock_dest: pathlib.Path): - """Fixture to create a Configure object.""" - return ConfigureOperation( - src=mock_src, dest=mock_dest, file_parameters={"FOO": "BAR"} - ) - - @pytest.fixture def file_system_operation_set( copy_operation: CopyOperation, From e937f0dc5cf29b89b95768171c02126f8d3d0b9f Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Mon, 14 Oct 2024 19:35:06 -0700 Subject: [PATCH 20/35] small updates but pushing bc issue in gen branch --- .../operations/ensemble_operations.py | 12 ++++++++---- smartsim/builders/ensemble.py | 17 ----------------- tests/test_builder_operations.py | 3 --- 3 files changed, 8 insertions(+), 24 deletions(-) diff --git a/smartsim/_core/generation/operations/ensemble_operations.py b/smartsim/_core/generation/operations/ensemble_operations.py index 97e02184a6..4480dc63d4 100644 --- a/smartsim/_core/generation/operations/ensemble_operations.py +++ b/smartsim/_core/generation/operations/ensemble_operations.py @@ -1,9 +1,10 @@ import pathlib import typing as t from dataclasses import dataclass, field +from .utils.helpers import check_src_and_dest_path - - +# TODO do we need to add check for tags? +# TODO do I need to add checks for file_params? class EnsembleGenerationProtocol(t.Protocol): """Protocol for Ensemble Generation Operations.""" src: pathlib.Path @@ -23,6 +24,7 @@ def __init__( :param src: Path to source :param dest: Path to destination """ + check_src_and_dest_path(src, dest) self.src = src """Path to source""" self.dest = dest @@ -38,6 +40,7 @@ def __init__(self, src: pathlib.Path, dest: t.Optional[pathlib.Path] = None) -> :param src: Path to source :param dest: Path to destination """ + check_src_and_dest_path(src, dest) self.src = src """Path to source""" self.dest = dest @@ -61,6 +64,7 @@ def __init__( :param dest: Path to destination :param tag: Tag to use for find and replacement """ + check_src_and_dest_path(src, dest) self.src = src """Path to source""" self.dest = dest @@ -76,10 +80,10 @@ def __init__( @dataclass class EnsembleFileSysOperationSet: - """Dataclass to represent a set of Ensemble File System Operation Objects""" + """Dataclass to represent a set of Ensemble file system operation objects""" operations: t.List[EnsembleGenerationProtocol] = field(default_factory=list) - """Set of Ensemble File System Objects that match the EnsembleGenerationProtocol""" + """Set of Ensemble file system objects that match the EnsembleGenerationProtocol""" def add_copy( self, src: pathlib.Path, dest: t.Optional[pathlib.Path] = None diff --git a/smartsim/builders/ensemble.py b/smartsim/builders/ensemble.py index b56da20c95..67e9faf30a 100644 --- a/smartsim/builders/ensemble.py +++ b/smartsim/builders/ensemble.py @@ -54,23 +54,6 @@ class FileSet: file: EnsembleConfigureOperation combinations: ParamSet -@dataclass(frozen=True) -class Combo: - """ - Represents a set of file parameters and execution arguments as parameters. - """ - - file: EnsembleConfigureOperation - combination: ParamSet - -@dataclass(frozen=True) -class ComboSet: - """ - Represents a set of file parameters and execution arguments as parameters. - """ - - combos: t.List[Combo] - class Ensemble(entity.CompoundEntity): """An Ensemble is a builder class that parameterizes the creation of multiple Applications. diff --git a/tests/test_builder_operations.py b/tests/test_builder_operations.py index 02c892aca7..d62ec6e848 100644 --- a/tests/test_builder_operations.py +++ b/tests/test_builder_operations.py @@ -18,9 +18,6 @@ from smartsim.builders import Ensemble from smartsim.builders.utils import strategies -# QUESTIONS -# TODO test python protocol? -# TODO do I allow the paths to combine if src is empty? pytestmark = pytest.mark.group_a From 36e88019d8a9e9d1d17ff6157f958b4ce4c11170 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Mon, 14 Oct 2024 19:38:41 -0700 Subject: [PATCH 21/35] adding docstrings --- smartsim/_core/generation/operations/operations.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/smartsim/_core/generation/operations/operations.py b/smartsim/_core/generation/operations/operations.py index 29b9db466a..c0bb8c3033 100644 --- a/smartsim/_core/generation/operations/operations.py +++ b/smartsim/_core/generation/operations/operations.py @@ -85,7 +85,9 @@ def __init__( """ check_src_and_dest_path(src, dest) self.src = src + """Path to source""" self.dest = dest or pathlib.Path(src.name) + """Path to destination""" def format(self, context: GenerationContext) -> Command: """Create Command to invoke copy file system entry point @@ -120,7 +122,9 @@ def __init__( """ check_src_and_dest_path(src, dest) self.src = src + """Path to source""" self.dest = dest or pathlib.Path(src.name) + """Path to destination""" def format(self, context: GenerationContext) -> Command: """Create Command to invoke symlink file system entry point @@ -163,11 +167,15 @@ def __init__( """ check_src_and_dest_path(src, dest) self.src = src + """Path to source""" self.dest = dest or pathlib.Path(src.name) + """Path to destination""" pickled_dict = pickle.dumps(file_parameters) encoded_dict = base64.b64encode(pickled_dict).decode("ascii") self.file_parameters = encoded_dict + """File parameters to find and replace""" self.tag = tag if tag else ";" + """Tag to use for find and replacement""" def format(self, context: GenerationContext) -> Command: """Create Command to invoke configure file system entry point From 1fe771971333b6aa8df1ec516a164d0495e881a2 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Mon, 14 Oct 2024 19:46:46 -0700 Subject: [PATCH 22/35] pushing for an idea in gen --- ...rations.py => test_ensemble_operations.py} | 41 ++++++++++--------- 1 file changed, 22 insertions(+), 19 deletions(-) rename tests/{test_builder_operations.py => test_ensemble_operations.py} (89%) diff --git a/tests/test_builder_operations.py b/tests/test_ensemble_operations.py similarity index 89% rename from tests/test_builder_operations.py rename to tests/test_ensemble_operations.py index d62ec6e848..f26f2255b5 100644 --- a/tests/test_builder_operations.py +++ b/tests/test_ensemble_operations.py @@ -126,33 +126,36 @@ def ensemble_file_system_operation_set( # GenerationContext("") -def test_init_copy_operation( - copy_operation: EnsembleCopyOperation, mock_src: pathlib.Path, mock_dest: pathlib.Path +def test_init_ensemble_copy_operation( + mock_src: pathlib.Path, mock_dest: pathlib.Path ): - """Validate CopyOperation init""" - assert isinstance(copy_operation, EnsembleCopyOperation) - assert copy_operation.src == mock_src - assert copy_operation.dest == mock_dest + """Validate EnsembleCopyOperation init""" + ensemble_copy_operation = EnsembleCopyOperation(mock_src, mock_dest) + assert isinstance(ensemble_copy_operation, EnsembleCopyOperation) + assert ensemble_copy_operation.src == mock_src + assert ensemble_copy_operation.dest == mock_dest -def test_init_symlink_operation( - symlink_operation: EnsembleSymlinkOperation, mock_src: str, mock_dest: str +def test_init_ensemble_symlink_operation( + mock_src: str, mock_dest: str ): - """Validate SymlinkOperation init""" - assert isinstance(symlink_operation, EnsembleSymlinkOperation) - assert symlink_operation.src == mock_src - assert symlink_operation.dest == mock_dest + """Validate EnsembleSymlinkOperation init""" + ensemble_symlink_operation = EnsembleSymlinkOperation(mock_src, mock_dest) + assert isinstance(ensemble_symlink_operation, EnsembleSymlinkOperation) + assert ensemble_symlink_operation.src == mock_src + assert ensemble_symlink_operation.dest == mock_dest def test_init_configure_operation( - configure_operation: EnsembleConfigureOperation, mock_src: str, mock_dest: str + mock_src: str, mock_dest: str ): - """Validate ConfigureOperation init""" - assert isinstance(configure_operation, EnsembleConfigureOperation) - assert configure_operation.src == mock_src - assert configure_operation.dest == mock_dest - assert configure_operation.tag == ";" - assert configure_operation.file_parameters == {"FOO": ["BAR", "TOE"]} + """Validate EnsembleConfigureOperation init""" + ensemble_configure_operation = EnsembleConfigureOperation(mock_src, mock_dest) + assert isinstance(ensemble_configure_operation, EnsembleConfigureOperation) + assert ensemble_configure_operation.src == mock_src + assert ensemble_configure_operation.dest == mock_dest + assert ensemble_configure_operation.tag == ";" + assert ensemble_configure_operation.file_parameters == {"FOO": ["BAR", "TOE"]} def test_init_file_sys_operation_set(file_system_operation_set: EnsembleFileSysOperationSet): From b1dbbb37e78904734a9dc85fd58812c06f48b95a Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Mon, 14 Oct 2024 19:49:13 -0700 Subject: [PATCH 23/35] adding default tag --- smartsim/_core/generation/operations/operations.py | 5 ++++- tests/test_operations.py | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/smartsim/_core/generation/operations/operations.py b/smartsim/_core/generation/operations/operations.py index c0bb8c3033..54cb18a0b3 100644 --- a/smartsim/_core/generation/operations/operations.py +++ b/smartsim/_core/generation/operations/operations.py @@ -20,6 +20,9 @@ configure_cmd = "configure" """Configure file operation command""" +default_tag = ";" +"""Default configure tag""" + def _create_dest_path(job_run_path: pathlib.Path, dest: pathlib.Path) -> str: """Combine the job run path and destination path. Return as a string for @@ -174,7 +177,7 @@ def __init__( encoded_dict = base64.b64encode(pickled_dict).decode("ascii") self.file_parameters = encoded_dict """File parameters to find and replace""" - self.tag = tag if tag else ";" + self.tag = tag if tag else default_tag """Tag to use for find and replacement""" def format(self, context: GenerationContext) -> Command: diff --git a/tests/test_operations.py b/tests/test_operations.py index 5852b2575d..dab96e654d 100644 --- a/tests/test_operations.py +++ b/tests/test_operations.py @@ -17,6 +17,7 @@ configure_cmd, copy_cmd, symlink_cmd, + default_tag ) from smartsim._core.generation.operations.utils.helpers import check_src_and_dest_path @@ -205,7 +206,7 @@ def test_init_configure_operation(mock_src: str, mock_dest: str): assert isinstance(configure_operation, ConfigureOperation) assert configure_operation.src == mock_src assert configure_operation.dest == mock_dest - assert configure_operation.tag == ";" + assert configure_operation.tag == default_tag decoded_dict = base64.b64decode(configure_operation.file_parameters.encode("ascii")) unpickled_dict = pickle.loads(decoded_dict) assert unpickled_dict == {"FOO": "BAR"} From 8c7a7674cb7a3393e7797c0486176fd5338923da Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Mon, 14 Oct 2024 20:22:29 -0700 Subject: [PATCH 24/35] testing operations complete --- .../operations/ensemble_operations.py | 3 +- tests/test_ensemble_operations.py | 340 +++++++----------- 2 files changed, 134 insertions(+), 209 deletions(-) diff --git a/smartsim/_core/generation/operations/ensemble_operations.py b/smartsim/_core/generation/operations/ensemble_operations.py index 4480dc63d4..485dfca271 100644 --- a/smartsim/_core/generation/operations/ensemble_operations.py +++ b/smartsim/_core/generation/operations/ensemble_operations.py @@ -2,6 +2,7 @@ import typing as t from dataclasses import dataclass, field from .utils.helpers import check_src_and_dest_path +from .operations import default_tag # TODO do we need to add check for tags? # TODO do I need to add checks for file_params? @@ -71,7 +72,7 @@ def __init__( """Path to destination""" self.file_parameters = file_parameters """File parameters to find and replace""" - self.tag = tag if tag else ";" + self.tag = tag if tag else default_tag """Tag to use for the file""" diff --git a/tests/test_ensemble_operations.py b/tests/test_ensemble_operations.py index f26f2255b5..14c6d4a922 100644 --- a/tests/test_ensemble_operations.py +++ b/tests/test_ensemble_operations.py @@ -15,44 +15,30 @@ EnsembleConfigureOperation, EnsembleFileSysOperationSet ) +from smartsim._core.generation.operations.operations import default_tag from smartsim.builders import Ensemble from smartsim.builders.utils import strategies pytestmark = pytest.mark.group_a +# TODO missing test for _filter @pytest.fixture -def mock_src(test_dir: str): - """Fixture to create a mock source path.""" - return pathlib.Path(test_dir) / pathlib.Path("mock_src") - - -# TODO remove when PR 732 is merged -@pytest.fixture -def mock_dest(test_dir: str): - """Fixture to create a mock destination path.""" - return pathlib.Path(test_dir) / pathlib.Path("mock_dest") - - -# TODO remove when PR 732 is merged -@pytest.fixture -def copy_operation(mock_src: pathlib.Path, mock_dest: pathlib.Path): - """Fixture to create a CopyOperation object.""" +def ensemble_copy_operation(mock_src: pathlib.Path, mock_dest: pathlib.Path): + """Fixture to create a EnsembleCopyOperation object.""" return EnsembleCopyOperation(src=mock_src, dest=mock_dest) -# TODO remove when PR 732 is merged @pytest.fixture -def symlink_operation(mock_src: pathlib.Path, mock_dest: pathlib.Path): - """Fixture to create a SymlinkOperation object.""" +def ensemble_symlink_operation(mock_src: pathlib.Path, mock_dest: pathlib.Path): + """Fixture to create a EnsembleSymlinkOperation object.""" return EnsembleSymlinkOperation(src=mock_src, dest=mock_dest) -# TODO remove when PR 732 is merged @pytest.fixture -def configure_operation(mock_src: pathlib.Path, mock_dest: pathlib.Path): - """Fixture to create a Configure object.""" +def ensemble_configure_operation(mock_src: pathlib.Path, mock_dest: pathlib.Path): + """Fixture to create a EnsembleConfigureOperation object.""" return EnsembleConfigureOperation( src=mock_src, dest=mock_dest, file_parameters={"FOO": ["BAR", "TOE"]} ) @@ -60,70 +46,12 @@ def configure_operation(mock_src: pathlib.Path, mock_dest: pathlib.Path): @pytest.fixture def ensemble_file_system_operation_set( - copy_operation: EnsembleCopyOperation, - symlink_operation: EnsembleSymlinkOperation, - configure_operation: EnsembleConfigureOperation, + ensemble_copy_operation: EnsembleCopyOperation, + ensemble_symlink_operation: EnsembleSymlinkOperation, + ensemble_configure_operation: EnsembleConfigureOperation, ): """Fixture to create a FileSysOperationSet object.""" - return EnsembleFileSysOperationSet([copy_operation, symlink_operation, configure_operation]) - - -# @pytest.mark.parametrize( -# "job_root_path, dest, expected", -# ( -# pytest.param( -# pathlib.Path("/valid/root"), -# pathlib.Path("valid/dest"), -# "/valid/root/valid/dest", -# id="Valid paths", -# ), -# pytest.param( -# pathlib.Path("/valid/root"), -# pathlib.Path(""), -# "/valid/root", -# id="Empty destination path", -# ), -# pytest.param( -# pathlib.Path("/valid/root"), -# None, -# "/valid/root", -# id="Empty dest path", -# ), -# ), -# ) -# def test_create_final_dest_valid(job_root_path, dest, expected): -# """Test valid path inputs for operations.create_final_dest""" -# assert create_final_dest(job_root_path, dest) == expected - - -# @pytest.mark.parametrize( -# "job_root_path, dest", -# ( -# pytest.param(None, pathlib.Path("valid/dest"), id="None as root path"), -# pytest.param(1234, pathlib.Path("valid/dest"), id="Number as root path"), -# pytest.param(pathlib.Path("valid/dest"), 1234, id="Number as dest"), -# ), -# ) -# def test_create_final_dest_invalid(job_root_path, dest): -# """Test invalid path inputs for operations.create_final_dest""" -# with pytest.raises(ValueError): -# create_final_dest(job_root_path, dest) - - -# def test_valid_init_generation_context( -# test_dir: str, generation_context: GenerationContext -# ): -# """Validate GenerationContext init""" -# assert isinstance(generation_context, GenerationContext) -# assert generation_context.job_root_path == pathlib.Path(test_dir) - - -# def test_invalid_init_generation_context(): -# """Validate GenerationContext init""" -# with pytest.raises(TypeError): -# GenerationContext(1234) -# with pytest.raises(TypeError): -# GenerationContext("") + return EnsembleFileSysOperationSet([ensemble_copy_operation, ensemble_symlink_operation, ensemble_configure_operation]) def test_init_ensemble_copy_operation( @@ -146,145 +74,141 @@ def test_init_ensemble_symlink_operation( assert ensemble_symlink_operation.dest == mock_dest -def test_init_configure_operation( - mock_src: str, mock_dest: str +def test_init_ensemble_configure_operation( + mock_src: str ): """Validate EnsembleConfigureOperation init""" - ensemble_configure_operation = EnsembleConfigureOperation(mock_src, mock_dest) + ensemble_configure_operation = EnsembleConfigureOperation(mock_src, file_parameters={"FOO": ["BAR", "TOE"]}) assert isinstance(ensemble_configure_operation, EnsembleConfigureOperation) assert ensemble_configure_operation.src == mock_src - assert ensemble_configure_operation.dest == mock_dest - assert ensemble_configure_operation.tag == ";" + assert ensemble_configure_operation.dest == None + assert ensemble_configure_operation.tag == default_tag assert ensemble_configure_operation.file_parameters == {"FOO": ["BAR", "TOE"]} -def test_init_file_sys_operation_set(file_system_operation_set: EnsembleFileSysOperationSet): - """Test initialize FileSystemOperationSet""" - assert isinstance(file_system_operation_set.operations, list) - assert len(file_system_operation_set.operations) == 3 +def test_init_ensemble_file_sys_operation_set( + copy_operation: EnsembleCopyOperation, + symlink_operation: EnsembleSymlinkOperation, + configure_operation: EnsembleConfigureOperation): + """Test initialize EnsembleFileSysOperationSet""" + ensemble_fs_op_set = EnsembleFileSysOperationSet([copy_operation, symlink_operation, configure_operation]) + assert isinstance(ensemble_fs_op_set.operations, list) + assert len(ensemble_fs_op_set.operations) == 3 -def test_add_copy_operation( - file_system_operation_set: EnsembleFileSysOperationSet -): - """Test FileSystemOperationSet.add_copy""" - assert len(file_system_operation_set.copy_operations) == 1 - file_system_operation_set.add_copy(src=pathlib.Path("src")) - assert len(file_system_operation_set.copy_operations) == 2 +def test_add_ensemble_copy_operation(ensemble_file_system_operation_set: EnsembleFileSysOperationSet): + """Test EnsembleFileSysOperationSet.add_copy""" + orig_num_ops = len(ensemble_file_system_operation_set.copy_operations) + ensemble_file_system_operation_set.add_copy(src=pathlib.Path("/src")) + assert len(ensemble_file_system_operation_set.copy_operations) == orig_num_ops + 1 -def test_add_symlink_operation( - file_system_operation_set: EnsembleFileSysOperationSet -): - """Test FileSystemOperationSet.add_symlink""" - assert len(file_system_operation_set.symlink_operations) == 1 - file_system_operation_set.add_symlink(src=pathlib.Path("src")) - assert len(file_system_operation_set.symlink_operations) == 2 +def test_add_ensemble_symlink_operation(ensemble_file_system_operation_set: EnsembleFileSysOperationSet): + """Test EnsembleFileSysOperationSet.add_symlink""" + orig_num_ops = len(ensemble_file_system_operation_set.symlink_operations) + ensemble_file_system_operation_set.add_symlink(src=pathlib.Path("/src")) + assert len(ensemble_file_system_operation_set.symlink_operations) == orig_num_ops + 1 -def test_add_configure_operation( - file_system_operation_set: EnsembleFileSysOperationSet, +def test_add_ensemble_configure_operation( + ensemble_file_system_operation_set: EnsembleFileSysOperationSet, ): """Test FileSystemOperationSet.add_configuration""" - assert len(file_system_operation_set.configure_operations) == 1 - file_system_operation_set.add_configuration( - src=pathlib.Path("src"), file_parameters={"FOO": "BAR"} - ) - assert len(file_system_operation_set.configure_operations) == 2 - - -# @pytest.fixture -# def files(fileutils): -# path_to_files = fileutils.get_test_conf_path( -# osp.join("generator_files", "easy", "correct/") -# ) -# list_of_files_strs = sorted(glob(path_to_files + "/*")) -# yield [pathlib.Path(str_path) for str_path in list_of_files_strs] - - -# @pytest.fixture -# def directory(fileutils): -# directory = fileutils.get_test_conf_path( -# osp.join("generator_files", "easy", "correct/") -# ) -# yield [pathlib.Path(directory)] - - -# @pytest.fixture -# def source(request, files, directory): -# if request.param == "files": -# return files -# elif request.param == "directory": -# return directory - - -# @pytest.mark.parametrize( -# "dest,error", -# ( -# pytest.param(123, TypeError, id="dest as integer"), -# pytest.param("", TypeError, id="dest as empty str"), -# pytest.param("/absolute/path", TypeError, id="dest as absolute str"), -# pytest.param( -# pathlib.Path("relative/path"), ValueError, id="dest as relative Path" -# ), -# pytest.param( -# pathlib.Path("/path with spaces"), ValueError, id="dest as Path with spaces" -# ), -# # TODO pytest.param(pathlib.Path("/path/with/special!@#"), id="dest as Path with special char"), -# ), -# ) -# @pytest.mark.parametrize("source", ["files", "directory"], indirect=True) -# def test_copy_files_invalid_dest(dest, error, source): -# """Test invalid copy destination""" -# with pytest.raises(error): -# _ = [CopyOperation(src=file, dest=dest) for file in source] - - -# @pytest.mark.parametrize( -# "dest,error", -# ( -# pytest.param(123, TypeError, id="dest as integer"), -# pytest.param("", TypeError, id="dest as empty str"), -# pytest.param("/absolute/path", TypeError, id="dest as absolute str"), -# pytest.param( -# pathlib.Path("relative/path"), ValueError, id="dest as relative Path" -# ), -# pytest.param( -# pathlib.Path("/path with spaces"), ValueError, id="dest as Path with spaces" -# ), -# # TODO pytest.param(pathlib.Path("/path/with/special!@#"), id="dest as Path with special char"), -# ), -# ) -# @pytest.mark.parametrize("source", ["files", "directory"], indirect=True) -# def test_symlink_files_invalid_dest(dest, error, source): -# """Test invalid symlink destination""" -# with pytest.raises(error): -# _ = [SymlinkOperation(src=file, dest=dest) for file in source] - - -# @pytest.mark.parametrize( -# "dest,error", -# ( -# pytest.param(123, TypeError, id="dest as integer"), -# pytest.param("", TypeError, id="dest as empty str"), -# pytest.param("/absolute/path", TypeError, id="dest as absolute str"), -# pytest.param( -# pathlib.Path("relative/path"), ValueError, id="dest as relative Path" -# ), -# pytest.param( -# pathlib.Path("/path with spaces"), ValueError, id="dest as Path with spaces" -# ), -# # TODO pytest.param(pathlib.Path("/path/with/special!@#"), id="dest as Path with special char"), -# ), -# ) -# @pytest.mark.parametrize("source", ["files", "directory"], indirect=True) -# def test_configure_files_invalid_dest(dest, error, source): -# """Test invalid configure destination""" -# with pytest.raises(error): -# _ = [ -# ConfigureOperation(src=file, dest=dest, file_parameters={"FOO": "BAR"}) -# for file in source -# ] + orig_num_ops = len(ensemble_file_system_operation_set.configure_operations) + ensemble_file_system_operation_set.add_configuration(src=pathlib.Path("/src"), file_parameters={"FOO": "BAR"}) + assert len(ensemble_file_system_operation_set.configure_operations) == orig_num_ops + 1 + + +@pytest.mark.parametrize( + "dest,error", + ( + pytest.param(123, TypeError, id="dest as integer"), + pytest.param("", TypeError, id="dest as empty str"), + pytest.param(pathlib.Path("/absolute/path"), ValueError, id="dest as absolute str"), + ), +) +def test_ensemble_copy_files_invalid_dest(dest, error, source): + """Test invalid copy destination""" + with pytest.raises(error): + _ = [EnsembleCopyOperation(src=file, dest=dest) for file in source] + + +@pytest.mark.parametrize( + "src,error", + ( + pytest.param(123, TypeError, id="src as integer"), + pytest.param("", TypeError, id="src as empty str"), + pytest.param(pathlib.Path("relative/path"), ValueError, id="src as relative str"), + ), +) +def test_ensemble_copy_files_invalid_src(src, error): + """Test invalid copy source""" + with pytest.raises(error): + _ = EnsembleCopyOperation(src=src) + + +@pytest.mark.parametrize( + "dest,error", + ( + pytest.param(123, TypeError, id="dest as integer"), + pytest.param("", TypeError, id="dest as empty str"), + pytest.param(pathlib.Path("/absolute/path"), ValueError, id="dest as absolute str"), + ), +) +def test_ensemble_symlink_files_invalid_dest(dest, error, source): + """Test invalid symlink destination""" + with pytest.raises(error): + _ = [EnsembleSymlinkOperation(src=file, dest=dest) for file in source] + + +@pytest.mark.parametrize( + "src,error", + ( + pytest.param(123, TypeError, id="src as integer"), + pytest.param("", TypeError, id="src as empty str"), + pytest.param(pathlib.Path("relative/path"), ValueError, id="src as relative str"), + ), +) +def test_ensemble_symlink_files_invalid_src(src, error): + """Test invalid symlink source""" + with pytest.raises(error): + _ = EnsembleSymlinkOperation(src=src) + + +@pytest.mark.parametrize( + "dest,error", + ( + pytest.param(123, TypeError, id="dest as integer"), + pytest.param("", TypeError, id="dest as empty str"), + pytest.param(pathlib.Path("/absolute/path"), ValueError, id="dest as absolute str"), + ), +) +def test_ensemble_configure_files_invalid_dest(dest, error, source): + """Test invalid configure destination""" + with pytest.raises(error): + _ = [ + EnsembleConfigureOperation(src=file, dest=dest, file_parameters={"FOO": "BAR"}) + for file in source + ] + + +@pytest.mark.parametrize( + "src,error", + ( + pytest.param(123, TypeError, id="src as integer"), + pytest.param("", TypeError, id="src as empty str"), + pytest.param(pathlib.Path("relative/path"), ValueError, id="src as relative str"), + ), +) +def test_ensemble_configure_files_invalid_src(src, error): + """Test invalid configure source""" + with pytest.raises(error): + _ = EnsembleConfigureOperation(src=src, file_parameters={"FOO":["BAR", "TOE"]}) + + + + + # bug found that it does not properly permutate if exe_arg_params is not specified # ISSUE with step, adds one file per application From 06f530dd743812f15c53e95a63808a09f26417e2 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Mon, 14 Oct 2024 20:32:22 -0700 Subject: [PATCH 25/35] ensemble op test complete --- smartsim/builders/ensemble.py | 6 +---- tests/test_ensemble_operations.py | 40 ------------------------------- 2 files changed, 1 insertion(+), 45 deletions(-) diff --git a/smartsim/builders/ensemble.py b/smartsim/builders/ensemble.py index 67e9faf30a..4781230b14 100644 --- a/smartsim/builders/ensemble.py +++ b/smartsim/builders/ensemble.py @@ -55,7 +55,7 @@ class FileSet: combinations: ParamSet class Ensemble(entity.CompoundEntity): - """An Ensemble is a builder class that parameterizes the creation of multiple + """An Ensemble is a builder class to parameterize the creation of multiple Applications. """ @@ -128,10 +128,6 @@ def __init__( :param exe: executable to run :param exe_args: executable arguments :param exe_arg_parameters: parameters and values to be used when configuring entities - :param files: files to be copied, symlinked, and/or configured prior to - execution - :param file_parameters: parameters and values to be used when configuring - files :param permutation_strategy: strategy to control how the param values are applied to the Ensemble :param max_permutations: max parameter permutations to set for the ensemble :param replicas: number of identical entities to create within an Ensemble diff --git a/tests/test_ensemble_operations.py b/tests/test_ensemble_operations.py index 14c6d4a922..4c1f306235 100644 --- a/tests/test_ensemble_operations.py +++ b/tests/test_ensemble_operations.py @@ -204,43 +204,3 @@ def test_ensemble_configure_files_invalid_src(src, error): """Test invalid configure source""" with pytest.raises(error): _ = EnsembleConfigureOperation(src=src, file_parameters={"FOO":["BAR", "TOE"]}) - - - - - - -# bug found that it does not properly permutate if exe_arg_params is not specified -# ISSUE with step, adds one file per application -def test_step_mock(): - ensemble = Ensemble("name", "echo", exe_arg_parameters = {"-N": ["1", "2"]}, permutation_strategy="step") - ensemble.files.add_configuration(pathlib.Path("src_1"), file_parameters={"FOO":["BAR", "TOE"]}) - ensemble.files.add_configuration(pathlib.Path("src_2"), file_parameters={"CAN":["TOM", "STO"]}) - apps = ensemble._create_applications() - print(apps) - for app in apps: - for config in app.files.configure_operations: - decoded_dict = base64.b64decode(config.file_parameters) - print(config.src) - deserialized_dict = pickle.loads(decoded_dict) - print(deserialized_dict) - -def test_all_perm_mock(): - ensemble = Ensemble("name", "echo", exe_arg_parameters = {"-N": ["1", "2"]}, permutation_strategy="step", replicas=2) - ensemble.files.add_configuration(pathlib.Path("src_1"), file_parameters={"FOO":["BAR", "TOE"]}) - ensemble.files.add_configuration(pathlib.Path("src_2"), file_parameters={"CAN":["TOM", "STO"]}) - apps = ensemble._create_applications() - print(len(apps)) - # for app in apps: - # for config in app.files.configure_operations: - # decoded_dict = base64.b64decode(config.file_parameters) - # print(config.src) - # deserialized_dict = pickle.loads(decoded_dict) - # print(deserialized_dict) - -def test_mock(): - ensemble = Ensemble("name", "echo", exe_arg_parameters = {"-N": ["1", "2"]}, permutation_strategy="step") - file = EnsembleConfigureOperation(src="src", file_parameters={"FOO":["BAR", "TOE"]}) - permutation_strategy = strategies.resolve("all_perm") - val = ensemble.perm_config_file(file, permutation_strategy) - print(val) \ No newline at end of file From 8a9a577d7ddb469fdd973b5b9dc2016f60f78da0 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Mon, 14 Oct 2024 20:33:12 -0700 Subject: [PATCH 26/35] pushing to switch branches --- tests/test_ensemble.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tests/test_ensemble.py b/tests/test_ensemble.py index 1bfbd0b67a..0bdf0705d8 100644 --- a/tests/test_ensemble.py +++ b/tests/test_ensemble.py @@ -471,3 +471,39 @@ def test_random_strategy( replicas=replicas, ).build_jobs(mock_launcher_settings) assert len(jobs) == expected_num_jobs + + +# # bug found that it does not properly permutate if exe_arg_params is not specified +# # ISSUE with step, adds one file per application +# def test_step_mock(): +# ensemble = Ensemble("name", "echo", exe_arg_parameters = {"-N": ["1", "2"]}, permutation_strategy="step") +# ensemble.files.add_configuration(pathlib.Path("src_1"), file_parameters={"FOO":["BAR", "TOE"]}) +# ensemble.files.add_configuration(pathlib.Path("src_2"), file_parameters={"CAN":["TOM", "STO"]}) +# apps = ensemble._create_applications() +# print(apps) +# for app in apps: +# for config in app.files.configure_operations: +# decoded_dict = base64.b64decode(config.file_parameters) +# print(config.src) +# deserialized_dict = pickle.loads(decoded_dict) +# print(deserialized_dict) + +# def test_all_perm_mock(): +# ensemble = Ensemble("name", "echo", exe_arg_parameters = {"-N": ["1", "2"]}, permutation_strategy="step", replicas=2) +# ensemble.files.add_configuration(pathlib.Path("src_1"), file_parameters={"FOO":["BAR", "TOE"]}) +# ensemble.files.add_configuration(pathlib.Path("src_2"), file_parameters={"CAN":["TOM", "STO"]}) +# apps = ensemble._create_applications() +# print(len(apps)) +# # for app in apps: +# # for config in app.files.configure_operations: +# # decoded_dict = base64.b64decode(config.file_parameters) +# # print(config.src) +# # deserialized_dict = pickle.loads(decoded_dict) +# # print(deserialized_dict) + +# def test_mock(): +# ensemble = Ensemble("name", "echo", exe_arg_parameters = {"-N": ["1", "2"]}, permutation_strategy="step") +# file = EnsembleConfigureOperation(src="src", file_parameters={"FOO":["BAR", "TOE"]}) +# permutation_strategy = strategies.resolve("all_perm") +# val = ensemble.perm_config_file(file, permutation_strategy) +# print(val) \ No newline at end of file From e88f226ca26963b2e5aee201705a25d8a854721e Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Mon, 14 Oct 2024 20:36:32 -0700 Subject: [PATCH 27/35] pushing test updates --- tests/test_operations.py | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/tests/test_operations.py b/tests/test_operations.py index dab96e654d..abfc141d89 100644 --- a/tests/test_operations.py +++ b/tests/test_operations.py @@ -16,8 +16,8 @@ _create_dest_path, configure_cmd, copy_cmd, + default_tag, symlink_cmd, - default_tag ) from smartsim._core.generation.operations.utils.helpers import check_src_and_dest_path @@ -270,7 +270,9 @@ def test_add_configure_operation( ( pytest.param(123, TypeError, id="dest as integer"), pytest.param("", TypeError, id="dest as empty str"), - pytest.param("/absolute/path", TypeError, id="dest as absolute str"), + pytest.param( + pathlib.Path("/absolute/path"), ValueError, id="dest as absolute str" + ), ), ) def test_copy_files_invalid_dest(dest, error, source): @@ -284,7 +286,9 @@ def test_copy_files_invalid_dest(dest, error, source): ( pytest.param(123, TypeError, id="src as integer"), pytest.param("", TypeError, id="src as empty str"), - pytest.param("relative/path", TypeError, id="src as relative str"), + pytest.param( + pathlib.Path("relative/path"), ValueError, id="src as relative str" + ), ), ) def test_copy_files_invalid_src(src, error): @@ -298,7 +302,9 @@ def test_copy_files_invalid_src(src, error): ( pytest.param(123, TypeError, id="dest as integer"), pytest.param("", TypeError, id="dest as empty str"), - pytest.param("/absolute/path", TypeError, id="dest as absolute str"), + pytest.param( + pathlib.Path("/absolute/path"), ValueError, id="dest as absolute str" + ), ), ) def test_symlink_files_invalid_dest(dest, error, source): @@ -312,7 +318,9 @@ def test_symlink_files_invalid_dest(dest, error, source): ( pytest.param(123, TypeError, id="src as integer"), pytest.param("", TypeError, id="src as empty str"), - pytest.param("relative/path", TypeError, id="src as relative str"), + pytest.param( + pathlib.Path("relative/path"), ValueError, id="src as relative str" + ), ), ) def test_symlink_files_invalid_src(src, error): @@ -326,7 +334,9 @@ def test_symlink_files_invalid_src(src, error): ( pytest.param(123, TypeError, id="dest as integer"), pytest.param("", TypeError, id="dest as empty str"), - pytest.param("/absolute/path", TypeError, id="dest as absolute str"), + pytest.param( + pathlib.Path("/absolute/path"), ValueError, id="dest as absolute str" + ), ), ) def test_configure_files_invalid_dest(dest, error, source): @@ -343,10 +353,12 @@ def test_configure_files_invalid_dest(dest, error, source): ( pytest.param(123, TypeError, id="src as integer"), pytest.param("", TypeError, id="src as empty str"), - pytest.param("relative/path", TypeError, id="src as relative str"), + pytest.param( + pathlib.Path("relative/path"), ValueError, id="src as relative str" + ), ), ) def test_configure_files_invalid_src(src, error): """Test invalid configure source""" with pytest.raises(error): - _ = ConfigureOperation(src=src) + _ = ConfigureOperation(src=src, file_parameters={"FOO": "BAR"}) From d6325886f28604268499d9e073249be730e52ec1 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Tue, 15 Oct 2024 13:25:34 -0700 Subject: [PATCH 28/35] pushing to PR --- .../operations/ensemble_operations.py | 27 +- smartsim/builders/ensemble.py | 90 ++- smartsim/builders/utils/strategies.py | 8 +- tests/test_application.py | 12 - tests/test_ensemble.py | 556 ++++++++++-------- tests/test_ensemble_operations.py | 88 ++- 6 files changed, 454 insertions(+), 327 deletions(-) diff --git a/smartsim/_core/generation/operations/ensemble_operations.py b/smartsim/_core/generation/operations/ensemble_operations.py index 485dfca271..8bf72cbc19 100644 --- a/smartsim/_core/generation/operations/ensemble_operations.py +++ b/smartsim/_core/generation/operations/ensemble_operations.py @@ -1,13 +1,16 @@ import pathlib import typing as t from dataclasses import dataclass, field -from .utils.helpers import check_src_and_dest_path + from .operations import default_tag +from .utils.helpers import check_src_and_dest_path + # TODO do we need to add check for tags? # TODO do I need to add checks for file_params? class EnsembleGenerationProtocol(t.Protocol): """Protocol for Ensemble Generation Operations.""" + src: pathlib.Path """Path to source""" dest: t.Optional[pathlib.Path] @@ -35,7 +38,9 @@ def __init__( class EnsembleSymlinkOperation(EnsembleGenerationProtocol): """Ensemble Symlink Operation""" - def __init__(self, src: pathlib.Path, dest: t.Optional[pathlib.Path] = None) -> None: + def __init__( + self, src: pathlib.Path, dest: t.Optional[pathlib.Path] = None + ) -> None: """Initialize a EnsembleSymlinkOperation object :param src: Path to source @@ -54,7 +59,7 @@ class EnsembleConfigureOperation(EnsembleGenerationProtocol): def __init__( self, src: pathlib.Path, - file_parameters:t.Mapping[str,t.Sequence[str]], + file_parameters: t.Mapping[str, t.Sequence[str]], dest: t.Optional[pathlib.Path] = None, tag: t.Optional[str] = None, ) -> None: @@ -76,7 +81,9 @@ def __init__( """Tag to use for the file""" -EnsembleGenerationProtocolT = t.TypeVar("EnsembleGenerationProtocolT", bound=EnsembleGenerationProtocol) +EnsembleGenerationProtocolT = t.TypeVar( + "EnsembleGenerationProtocolT", bound=EnsembleGenerationProtocol +) @dataclass @@ -109,7 +116,7 @@ def add_symlink( def add_configuration( self, src: pathlib.Path, - file_parameters: t.Mapping[str,t.Sequence[str]], + file_parameters: t.Mapping[str, t.Sequence[str]], dest: t.Optional[pathlib.Path] = None, tag: t.Optional[str] = None, ) -> None: @@ -120,7 +127,9 @@ def add_configuration( :param dest: Path to destination :param tag: Tag to use for find and replacement """ - self.operations.append(EnsembleConfigureOperation(src, file_parameters, dest, tag)) + self.operations.append( + EnsembleConfigureOperation(src, file_parameters, dest, tag) + ) @property def copy_operations(self) -> t.List[EnsembleCopyOperation]: @@ -146,11 +155,13 @@ def configure_operations(self) -> t.List[EnsembleConfigureOperation]: """ return self._filter(EnsembleConfigureOperation) - def _filter(self, type: t.Type[EnsembleGenerationProtocolT]) -> t.List[EnsembleGenerationProtocolT]: + def _filter( + self, type: t.Type[EnsembleGenerationProtocolT] + ) -> t.List[EnsembleGenerationProtocolT]: """Filters the operations list to include only instances of the specified type. :param type: The type of operations to filter :return: A list of operations that are instances of the specified type """ - return [x for x in self.operations if isinstance(x, type)] \ No newline at end of file + return [x for x in self.operations if isinstance(x, type)] diff --git a/smartsim/builders/ensemble.py b/smartsim/builders/ensemble.py index 4781230b14..2ad9a171b4 100644 --- a/smartsim/builders/ensemble.py +++ b/smartsim/builders/ensemble.py @@ -31,20 +31,24 @@ import itertools import os import os.path -import typing as t import random +import typing as t +from dataclasses import dataclass, field +from smartsim._core.generation.operations.ensemble_operations import ( + EnsembleConfigureOperation, + EnsembleFileSysOperationSet, +) from smartsim.builders.utils import strategies from smartsim.builders.utils.strategies import ParamSet from smartsim.entity import entity from smartsim.entity.application import Application from smartsim.launchable.job import Job -from smartsim._core.generation.operations.ensemble_operations import EnsembleFileSysOperationSet, EnsembleConfigureOperation -from dataclasses import dataclass, field - if t.TYPE_CHECKING: from smartsim.settings.launch_settings import LaunchSettings + + @dataclass(frozen=True) class FileSet: """ @@ -54,6 +58,7 @@ class FileSet: file: EnsembleConfigureOperation combinations: ParamSet + class Ensemble(entity.CompoundEntity): """An Ensemble is a builder class to parameterize the creation of multiple Applications. @@ -306,6 +311,47 @@ def replicas(self, value: int) -> None: self._replicas = value + def _permutate_config_file( + self, + file: EnsembleConfigureOperation, + permutation_strategy: t.Callable[ + [ + t.Mapping[str, t.Sequence[str]], + t.Mapping[str, t.Sequence[t.Sequence[str]]], + int, + ], + list[ParamSet], + ], + ) -> t.List[FileSet]: + """Generate all possible permutations of file parameters using the given strategy, + and create corresponding FileSet objects. + + This method applies the provided permutation strategy to the file's parameters, + along with execution argument parameters and a maximum permutation limit. + It returns a list of FileSet objects, each containing one of the generated + ParamSets and an instance of the ConfigurationObject. + + :param file: The configuration file to be permuted + :param permutation_strategy: A function that generates permutations + of the file parameters + :returns: list[FileSet] + """ + combinations = permutation_strategy( + file.file_parameters, self.exe_arg_parameters, self.max_permutations + ) or [ParamSet({}, {})] + return [FileSet(file, combo) for combo in combinations] + + def _cartesian_values(self, ls: list[list[FileSet]]) -> list[tuple[FileSet, ...]]: + """Generate the Cartesian product of a list of lists of FileSets. + + This method takes a list of lists of FileSets and returns a list of tuples, + where each tuple contains one FileSet from each inner list. + + :param ls: A list of lists of FileSets + :returns: A list of tuples, each containing one FileSet from each inner list + """ + return list(itertools.product(*ls)) + def _create_applications(self) -> tuple[Application, ...]: """Generate a collection of Application instances based on the Ensembles attributes. @@ -315,43 +361,29 @@ def _create_applications(self) -> tuple[Application, ...]: :return: A tuple of Application instances """ - # resolve the permutation strategy permutation_strategy = strategies.resolve(self.permutation_strategy) - # apply the permutation strategy to each attached config file - perm_list: t.List[t.List[FileSet]] = [self.perm_config_file(config_file, permutation_strategy) for config_file in self.files.configure_operations] - # group the files together - val: t.List[tuple[FileSet]] = self._cartesian_values(perm_list) - # duplicate if replicas + perm_list: list[list[FileSet]] = [ + self._permutate_config_file(config_file, permutation_strategy) + for config_file in self.files.configure_operations + ] + val: list[tuple[FileSet, ...]] = self._cartesian_values(perm_list) permutations_ = itertools.chain.from_iterable( itertools.repeat(permutation, self.replicas) for permutation in val ) all_apps = [] - i = 0 - for item in permutations_: - i+=1 + for i, item in enumerate(permutations_, start=1): app = Application( name=f"{self.name}-{i}", exe=self.exe, exe_args=self.exe_args, ) - # apply the config files in the tuple for file_set in item: - app.files.add_configuration(src=file_set.file.src, file_parameters=file_set.combinations.params) + app.files.add_configuration( + src=file_set.file.src, file_parameters=file_set.combinations.params + ) all_apps.append(app) return tuple(all_apps) - - def perm_config_file(self, file, permutation_strategy): - combinations = permutation_strategy( - file.file_parameters, self.exe_arg_parameters, self.max_permutations - ) or [ParamSet({}, {})] - return [FileSet(file, combo) for combo in combinations] - - - def _cartesian_values(self, ls): - return list(itertools.product(*ls)) - - def build_jobs(self, settings: LaunchSettings) -> tuple[Job, ...]: """Expand an Ensemble into a list of deployable Jobs and apply identical LaunchSettings to each Job. @@ -384,8 +416,8 @@ def build_jobs(self, settings: LaunchSettings) -> tuple[Job, ...]: :raises TypeError: if the ids argument is not type LaunchSettings :raises ValueError: if the LaunchSettings provided are empty """ - if not isinstance(settings, LaunchSettings): - raise TypeError("ids argument was not of type LaunchSettings") + # if not isinstance(settings, LaunchSettings): + # raise TypeError("ids argument was not of type LaunchSettings") apps = self._create_applications() if not apps: raise ValueError("There are no members as part of this ensemble") diff --git a/smartsim/builders/utils/strategies.py b/smartsim/builders/utils/strategies.py index 0e4ae34325..9c3b2b55c5 100644 --- a/smartsim/builders/utils/strategies.py +++ b/smartsim/builders/utils/strategies.py @@ -265,10 +265,10 @@ def random_permutations( # def create_combos(file_sets: List[FileSet]) -> List[List[Combo]]: # # Extract the combinations from each FileSet # all_combinations = [file_set.combinations for file_set in file_sets] - + # # Generate the Cartesian product of all combinations # product_combinations = itertools.product(*all_combinations) - + # # Create Combo instances for each combination in the product # combo_lists = [] # for combination_tuple in product_combinations: @@ -277,5 +277,5 @@ def random_permutations( # for file_set, combination in zip(file_sets, combination_tuple) # ] # combo_lists.append(combo_list) - -# return combo_lists \ No newline at end of file + +# return combo_lists diff --git a/tests/test_application.py b/tests/test_application.py index cd5cc5b3da..5ec68ec7ba 100644 --- a/tests/test_application.py +++ b/tests/test_application.py @@ -62,18 +62,6 @@ def test_application_exe_args_property(): assert exe_args is a.exe_args -def test_application_file_parameters_property(): - file_parameters = {"h": [5, 6, 7, 8]} - a = Application( - "test_name", - exe="echo", - file_parameters=file_parameters, - ) - file_parameters = a.file_parameters - - assert file_parameters is a.file_parameters - - def test_application_key_prefixing_property(): key_prefixing_enabled = True a = Application("test_name", exe="echo", exe_args=["spam", "eggs"]) diff --git a/tests/test_ensemble.py b/tests/test_ensemble.py index 0bdf0705d8..32099b8d18 100644 --- a/tests/test_ensemble.py +++ b/tests/test_ensemble.py @@ -24,15 +24,22 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import os +import pathlib import typing as t from glob import glob from os import path as osp import pytest -from smartsim.builders.ensemble import Ensemble +from smartsim._core.generation.operations.ensemble_operations import ( + EnsembleConfigureOperation, + EnsembleCopyOperation, + EnsembleSymlinkOperation, +) +from smartsim.builders.ensemble import Ensemble, FileSet +from smartsim.builders.utils import strategies from smartsim.builders.utils.strategies import ParamSet -from smartsim.entity.files import EntityFiles from smartsim.settings.launch_settings import LaunchSettings pytestmark = pytest.mark.group_a @@ -54,99 +61,107 @@ def user_created_function( return [ParamSet({}, {})] +@pytest.fixture +def ensemble(): + return Ensemble( + name="ensemble_name", + exe="python", + exe_args="sleepy.py", + exe_arg_parameters={"-N": 2}, + permutation_strategy="all_perm", + max_permutations=2, + ) + + @pytest.fixture def mock_launcher_settings(wlmutils): return LaunchSettings(wlmutils.get_test_launcher(), {}, {}) -def test_exe_property(): - e = Ensemble(name="test", exe="path/to/example_simulation_program") - exe = e.exe - assert exe == e.exe +def test_ensemble_init(): + """Validate Ensemble init""" + ensemble = Ensemble(name="ensemble_name", exe="python") + assert isinstance(ensemble, Ensemble) + assert ensemble.name == "ensemble_name" + assert ensemble.exe == os.fspath("python") -def test_exe_args_property(): - e = Ensemble("test", exe="path/to/example_simulation_program", exe_args="sleepy.py") - exe_args = e.exe_args - assert exe_args == e.exe_args +def test_ensemble_init_empty_params(test_dir: str) -> None: + """Ensemble created without required args""" + with pytest.raises(TypeError): + Ensemble() -def test_exe_arg_parameters_property(): - exe_arg_parameters = {"-N": 2} - e = Ensemble( - "test", - exe="path/to/example_simulation_program", - exe_arg_parameters=exe_arg_parameters, - ) - exe_arg_parameters = e.exe_arg_parameters - assert exe_arg_parameters == e.exe_arg_parameters +def test_exe_property(ensemble): + """Validate Ensemble property""" + exe = ensemble.exe + assert exe == ensemble.exe -def test_files_property(get_gen_configure_dir): - tagged_files = sorted(glob(get_gen_configure_dir + "/*")) - files = EntityFiles(tagged=tagged_files) - e = Ensemble("test", exe="path/to/example_simulation_program", files=files) - files = e.files - assert files == e.files +@pytest.mark.parametrize( + "exe,error", + ( + pytest.param(123, TypeError, id="exe as integer"), + pytest.param(None, TypeError, id="exe as None"), + ), +) +def test_exe_set_invalid(ensemble, exe, error): + """Validate Ensemble exe setter throws""" + with pytest.raises(error): + ensemble.exe = exe -def test_file_parameters_property(): - file_parameters = {"h": [5, 6, 7, 8]} - e = Ensemble( - "test", - exe="path/to/example_simulation_program", - file_parameters=file_parameters, - ) - file_parameters = e.file_parameters - assert file_parameters == e.file_parameters +@pytest.mark.parametrize( + "exe", + ( + pytest.param(pathlib.Path("this/is/path"), id="exe as pathlib"), + pytest.param("this/is/path", id="exe as str"), + ), +) +def test_exe_set_valid(ensemble, exe): + """Validate Ensemble exe setter sets""" + ensemble.exe = exe + assert ensemble.exe == str(exe) -def test_ensemble_init_empty_params(test_dir: str) -> None: - """Ensemble created without required args""" - with pytest.raises(TypeError): - Ensemble() +def test_exe_args_property(ensemble): + exe_args = ensemble.exe_args + assert exe_args == ensemble.exe_args @pytest.mark.parametrize( - "bad_settings", - [pytest.param(None, id="Nullish"), pytest.param("invalid", id="String")], + "exe_args,error", + ( + pytest.param(123, TypeError, id="exe_args as integer"), + pytest.param(None, TypeError, id="exe_args as None"), + pytest.param(["script.py", 123], TypeError, id="exe_args as None"), + ), ) -def test_ensemble_incorrect_launch_settings_type(bad_settings): - """test starting an ensemble with invalid launch settings""" - ensemble = Ensemble("ensemble-name", "echo", replicas=2) - with pytest.raises(TypeError): - ensemble.build_jobs(bad_settings) - - -def test_ensemble_type_exe(): - ensemble = Ensemble( - "ensemble-name", - exe="valid", - exe_args=["spam", "eggs"], - ) - with pytest.raises( - TypeError, match="exe argument was not of type str or PathLike str" - ): - ensemble.exe = 2 +def test_exe_args_set_invalid(ensemble, exe_args, error): + """Validate Ensemble exe_arg setter throws""" + with pytest.raises(error): + ensemble.exe_args = exe_args @pytest.mark.parametrize( - "bad_settings", - [ - pytest.param([1, 2, 3], id="sequence of ints"), - pytest.param(0, id="null"), - pytest.param({"foo": "bar"}, id="dict"), - ], + "exe_args", + ( + pytest.param(["script.py", "another.py"], id="exe_args as pathlib"), + pytest.param([], id="exe_args as str"), + ), ) -def test_ensemble_type_exe_args(bad_settings): - ensemble = Ensemble( - "ensemble-name", - exe="echo", - ) - with pytest.raises( - TypeError, match="exe_args argument was not of type sequence of str" - ): - ensemble.exe_args = bad_settings +def test_exe_args_set_valid(ensemble, exe_args): + """Validate Ensemble exe_args setter sets""" + ensemble.exe_args = exe_args + assert ensemble.exe_args == exe_args + + +def test_exe_arg_parameters_property(ensemble): + exe_arg_parameters = ensemble.exe_arg_parameters + assert exe_arg_parameters == ensemble.exe_arg_parameters + + +# TODO need a valid test for exe args as params @pytest.mark.parametrize( @@ -167,7 +182,7 @@ def test_ensemble_type_exe_args(bad_settings): pytest.param({1: 2}, id="Values not mapping of str and str"), ), ) -def test_ensemble_type_exe_arg_parameters(exe_arg_params): +def test_exe_arg_parameters_set_invalid(exe_arg_params): ensemble = Ensemble( "ensemble-name", exe="echo", @@ -181,43 +196,12 @@ def test_ensemble_type_exe_arg_parameters(exe_arg_params): ensemble.exe_arg_parameters = exe_arg_params -def test_ensemble_type_files(): - ensemble = Ensemble( - "ensemble-name", - exe="echo", - exe_args=["spam", "eggs"], - ) - with pytest.raises(TypeError, match="files argument was not of type EntityFiles"): - ensemble.files = 2 +def test_permutation_strategy_property(ensemble): + permutation_strategy = ensemble.permutation_strategy + assert permutation_strategy == ensemble.permutation_strategy -@pytest.mark.parametrize( - "file_params", - ( - pytest.param(["invalid"], id="Not a mapping"), - pytest.param({"key": [1, 2, 3]}, id="Key is not sequence of sequences"), - ), -) -def test_ensemble_type_file_parameters(file_params): - ensemble = Ensemble( - "ensemble-name", - exe="echo", - exe_args=["spam", "eggs"], - ) - with pytest.raises( - TypeError, - match="file_parameters argument was not of type " - "mapping of str and sequence of str", - ): - ensemble.file_parameters = file_params - - -def test_ensemble_type_permutation_strategy(): - ensemble = Ensemble( - "ensemble-name", - exe="echo", - exe_args=["spam", "eggs"], - ) +def test_permutation_strategy_set_invalid(ensemble): with pytest.raises( TypeError, match="permutation_strategy argument was not of " @@ -226,49 +210,83 @@ def test_ensemble_type_permutation_strategy(): ensemble.permutation_strategy = 2 -def test_ensemble_type_max_permutations(): - ensemble = Ensemble( - "ensemble-name", - exe="echo", - exe_args=["spam", "eggs"], - ) - with pytest.raises( - TypeError, - match="max_permutations argument was not of type int", - ): - ensemble.max_permutations = "invalid" +# TODO add user created strategy +@pytest.mark.parametrize( + "strategy", + ( + pytest.param("all_perm", id="strategy as all_perm"), + pytest.param("step", id="strategy as step"), + pytest.param("random", id="strategy as random"), + ), +) +def test_permutation_strategy_set_valid(ensemble, strategy): + ensemble.permutation_strategy = strategy + assert ensemble.permutation_strategy == strategy -def test_ensemble_type_replicas(): - ensemble = Ensemble( - "ensemble-name", - exe="echo", - exe_args=["spam", "eggs"], - ) - with pytest.raises( - TypeError, - match="replicas argument was not of type int", - ): - ensemble.replicas = "invalid" +def test_max_permutations_property(ensemble): + max_permutations = ensemble.max_permutations + assert max_permutations == ensemble.max_permutations -def test_ensemble_type_replicas_negative(): - ensemble = Ensemble( - "ensemble-name", - exe="echo", - exe_args=["spam", "eggs"], - ) - with pytest.raises( - ValueError, - match="Number of replicas must be a positive integer", - ): - ensemble.replicas = -20 +@pytest.mark.parametrize( + "max_permutations", + ( + pytest.param(123, id="max_permutations as str"), + pytest.param(-1, id="max_permutations as float"), + ), +) +def test_max_permutations_set_valid(ensemble, max_permutations): + """Validate Ensemble max_permutations setter sets""" + ensemble.max_permutations = max_permutations + assert ensemble.max_permutations == max_permutations -def test_ensemble_type_build_jobs(): - ensemble = Ensemble("ensemble-name", "echo", replicas=2) - with pytest.raises(TypeError): - ensemble.build_jobs("invalid") +@pytest.mark.parametrize( + "max_permutations,error", + ( + pytest.param("str", TypeError, id="max_permutations as str"), + pytest.param(None, TypeError, id="max_permutations as None"), + pytest.param(0.1, TypeError, id="max_permutations as float"), + ), +) +def test_max_permutations_set_invalid(ensemble, max_permutations, error): + """Validate Ensemble exe_arg setter throws""" + with pytest.raises(error): + ensemble.max_permutations = max_permutations + + +def test_replicas_property(ensemble): + replicas = ensemble.replicas + assert replicas == ensemble.replicas + + +@pytest.mark.parametrize( + "replicas", + (pytest.param(123, id="replicas as str"),), +) +def test_replicas_set_valid(ensemble, replicas): + """Validate Ensemble replicas setter sets""" + ensemble.replicas = replicas + assert ensemble.replicas == replicas + + +@pytest.mark.parametrize( + "replicas,error", + ( + pytest.param("str", TypeError, id="replicas as str"), + pytest.param(None, TypeError, id="replicas as None"), + pytest.param(0.1, TypeError, id="replicas as float"), + pytest.param(-1, ValueError, id="replicas as negative int"), + ), +) +def test_replicas_set_invalid(ensemble, replicas, error): + """Validate Ensemble replicas setter throws""" + with pytest.raises(error): + ensemble.replicas = replicas + + +# END OF PROPERTY TESTS def test_ensemble_user_created_strategy(mock_launcher_settings, test_dir): @@ -289,7 +307,6 @@ def test_ensemble_without_any_members_raises_when_cast_to_jobs( "test_ensemble", "echo", ("hello", "world"), - file_parameters=_2x2_PARAMS, permutation_strategy="random", max_permutations=30, replicas=0, @@ -306,38 +323,38 @@ def test_strategy_error_raised_if_a_strategy_that_dne_is_requested(test_dir): )._create_applications() -@pytest.mark.parametrize( - "file_parameters", - ( - pytest.param({"SPAM": ["eggs"]}, id="Non-Empty Params"), - pytest.param({}, id="Empty Params"), - pytest.param(None, id="Nullish Params"), - ), -) -def test_replicated_applications_have_eq_deep_copies_of_parameters( - file_parameters, test_dir -): - apps = list( - Ensemble( - "test_ensemble", - "echo", - ("hello",), - replicas=4, - file_parameters=file_parameters, - )._create_applications() - ) - assert len(apps) >= 2 # Sanitiy check to make sure the test is valid - assert all( - app_1.file_parameters == app_2.file_parameters - for app_1 in apps - for app_2 in apps - ) - assert all( - app_1.file_parameters is not app_2.file_parameters - for app_1 in apps - for app_2 in apps - if app_1 is not app_2 - ) +# @pytest.mark.parametrize( +# "file_parameters", +# ( +# pytest.param({"SPAM": ["eggs"]}, id="Non-Empty Params"), +# pytest.param({}, id="Empty Params"), +# pytest.param(None, id="Nullish Params"), +# ), +# ) +# def test_replicated_applications_have_eq_deep_copies_of_parameters( +# file_parameters +# ): +# apps = list( +# Ensemble( +# "test_ensemble", +# "echo", +# ("hello",), +# replicas=4, +# file_parameters=file_parameters, +# )._create_applications() +# ) +# assert len(apps) >= 2 # Sanitiy check to make sure the test is valid +# assert all( +# app_1.file_parameters == app_2.file_parameters +# for app_1 in apps +# for app_2 in apps +# ) +# assert all( +# app_1.file_parameters is not app_2.file_parameters +# for app_1 in apps +# for app_2 in apps +# if app_1 is not app_2 +# ) # fmt: off @@ -346,9 +363,6 @@ def test_replicated_applications_have_eq_deep_copies_of_parameters( (pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, 30, 1, 16 , id="Set max permutation high"), pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, -1, 1, 16 , id="Set max permutation negative"), pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, 0, 1, 1 , id="Set max permutation zero"), - pytest.param(_2x2_PARAMS, None, 4, 1, 4 , id="No exe arg params or Replicas"), - pytest.param( None, _2x2_EXE_ARG, 4, 1, 4 , id="No Parameters or Replicas"), - pytest.param( None, None, 4, 1, 1 , id="No Parameters, Exe_Arg_Param or Replicas"), pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, 1, 1, 1 , id="Set max permutation to lowest"), pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, 6, 2, 12 , id="Set max permutation, set replicas"), pytest.param( {}, _2x2_EXE_ARG, 6, 2, 8 , id="Set params as dict, set max permutations and replicas"), @@ -367,42 +381,26 @@ def test_all_perm_strategy( mock_launcher_settings, test_dir, ): - jobs = Ensemble( + e = Ensemble( "test_ensemble", "echo", ("hello", "world"), - file_parameters=params, exe_arg_parameters=exe_arg_params, permutation_strategy="all_perm", max_permutations=max_perms, replicas=replicas, - ).build_jobs(mock_launcher_settings) + ) + e.files.add_configuration(src=pathlib.Path("/src_1"), file_parameters=params) + jobs = e.build_jobs(mock_launcher_settings) assert len(jobs) == expected_num_jobs -def test_all_perm_strategy_contents(mock_launcher_settings): - jobs = Ensemble( - "test_ensemble", - "echo", - ("hello", "world"), - file_parameters=_2x2_PARAMS, - exe_arg_parameters=_2x2_EXE_ARG, - permutation_strategy="all_perm", - max_permutations=16, - replicas=1, - ).build_jobs(mock_launcher_settings) - assert len(jobs) == 16 - - # fmt: off @pytest.mark.parametrize( " params, exe_arg_params, max_perms, replicas, expected_num_jobs", (pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, 30, 1, 2 , id="Set max permutation high"), pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, -1, 1, 2 , id="Set max permutation negtive"), pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, 0, 1, 1 , id="Set max permutation zero"), - pytest.param(_2x2_PARAMS, None, 4, 1, 1 , id="No exe arg params or Replicas"), - pytest.param( None, _2x2_EXE_ARG, 4, 1, 1 , id="No Parameters or Replicas"), - pytest.param( None, None, 4, 1, 1 , id="No Parameters, Exe_Arg_Param or Replicas"), pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, 1, 1, 1 , id="Set max permutation to lowest"), pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, 6, 2, 4 , id="Set max permutation, set replicas"), pytest.param( {}, _2x2_EXE_ARG, 6, 2, 2 , id="Set params as dict, set max permutations and replicas"), @@ -421,16 +419,17 @@ def test_step_strategy( mock_launcher_settings, test_dir, ): - jobs = Ensemble( + e = Ensemble( "test_ensemble", "echo", ("hello", "world"), - file_parameters=params, exe_arg_parameters=exe_arg_params, permutation_strategy="step", max_permutations=max_perms, replicas=replicas, - ).build_jobs(mock_launcher_settings) + ) + e.files.add_configuration(src=pathlib.Path("/src_1"), file_parameters=params) + jobs = e.build_jobs(mock_launcher_settings) assert len(jobs) == expected_num_jobs @@ -440,9 +439,6 @@ def test_step_strategy( (pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, 30, 1, 16 , id="Set max permutation high"), pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, -1, 1, 16 , id="Set max permutation negative"), pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, 0, 1, 1 , id="Set max permutation zero"), - pytest.param(_2x2_PARAMS, None, 4, 1, 4 , id="No exe arg params or Replicas"), - pytest.param( None, _2x2_EXE_ARG, 4, 1, 4 , id="No Parameters or Replicas"), - pytest.param( None, None, 4, 1, 1 , id="No Parameters, Exe_Arg_Param or Replicas"), pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, 1, 1, 1 , id="Set max permutation to lowest"), pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, 6, 2, 12 , id="Set max permutation, set replicas"), pytest.param( {}, _2x2_EXE_ARG, 6, 2, 8 , id="Set params as dict, set max permutations and replicas"), @@ -460,50 +456,122 @@ def test_random_strategy( # Other fixtures mock_launcher_settings, ): - jobs = Ensemble( + e = Ensemble( "test_ensemble", "echo", ("hello", "world"), - file_parameters=params, exe_arg_parameters=exe_arg_params, permutation_strategy="random", max_permutations=max_perms, replicas=replicas, - ).build_jobs(mock_launcher_settings) + ) + e.files.add_configuration(src=pathlib.Path("/src_1"), file_parameters=params) + jobs = e.build_jobs(mock_launcher_settings) assert len(jobs) == expected_num_jobs -# # bug found that it does not properly permutate if exe_arg_params is not specified -# # ISSUE with step, adds one file per application -# def test_step_mock(): -# ensemble = Ensemble("name", "echo", exe_arg_parameters = {"-N": ["1", "2"]}, permutation_strategy="step") -# ensemble.files.add_configuration(pathlib.Path("src_1"), file_parameters={"FOO":["BAR", "TOE"]}) -# ensemble.files.add_configuration(pathlib.Path("src_2"), file_parameters={"CAN":["TOM", "STO"]}) -# apps = ensemble._create_applications() -# print(apps) -# for app in apps: -# for config in app.files.configure_operations: -# decoded_dict = base64.b64decode(config.file_parameters) -# print(config.src) -# deserialized_dict = pickle.loads(decoded_dict) -# print(deserialized_dict) - -# def test_all_perm_mock(): -# ensemble = Ensemble("name", "echo", exe_arg_parameters = {"-N": ["1", "2"]}, permutation_strategy="step", replicas=2) -# ensemble.files.add_configuration(pathlib.Path("src_1"), file_parameters={"FOO":["BAR", "TOE"]}) -# ensemble.files.add_configuration(pathlib.Path("src_2"), file_parameters={"CAN":["TOM", "STO"]}) -# apps = ensemble._create_applications() -# print(len(apps)) -# # for app in apps: -# # for config in app.files.configure_operations: -# # decoded_dict = base64.b64decode(config.file_parameters) -# # print(config.src) -# # deserialized_dict = pickle.loads(decoded_dict) -# # print(deserialized_dict) - -# def test_mock(): -# ensemble = Ensemble("name", "echo", exe_arg_parameters = {"-N": ["1", "2"]}, permutation_strategy="step") -# file = EnsembleConfigureOperation(src="src", file_parameters={"FOO":["BAR", "TOE"]}) -# permutation_strategy = strategies.resolve("all_perm") -# val = ensemble.perm_config_file(file, permutation_strategy) -# print(val) \ No newline at end of file +@pytest.mark.parametrize( + " params, exe_arg_params, max_perms, strategy, expected_combinations", + ( + pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, 8, "all_perm", 8, id="1"), + pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, 1, "all_perm", 1, id="2"), + pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, -1, "all_perm", 16, id="3"), + pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, 30, "all_perm", 16, id="4"), + pytest.param(_2x2_PARAMS, {}, -1, "all_perm", 4, id="5"), + pytest.param({}, _2x2_EXE_ARG, -1, "all_perm", 4, id="6"), + pytest.param({}, {}, -1, "all_perm", 1, id="7"), + pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, 2, "step", 2, id="8"), + pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, 1, "step", 1, id="9"), + pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, -1, "step", 2, id="10"), + pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, 30, "step", 2, id="11"), + pytest.param(_2x2_PARAMS, {}, -1, "step", 1, id="12"), + pytest.param({}, _2x2_EXE_ARG, -1, "step", 1, id="13"), + pytest.param({}, {}, -1, "step", 1, id="14"), + pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, 8, "random", 8, id="15"), + pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, 1, "random", 1, id="16"), + pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, -1, "random", 16, id="17"), + pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, 30, "random", 16, id="18"), + pytest.param(_2x2_PARAMS, {}, -1, "random", 4, id="19"), + pytest.param({}, _2x2_EXE_ARG, -1, "random", 4, id="20"), + pytest.param({}, {}, -1, "random", 1, id="21"), + ), +) +def test_permutate_config_file( + params, exe_arg_params, max_perms, strategy, expected_combinations +): + ensemble = Ensemble( + "name", + "echo", + exe_arg_parameters=exe_arg_params, + permutation_strategy=strategy, + max_permutations=max_perms, + ) + permutation_strategy = strategies.resolve(strategy) + config_file = EnsembleConfigureOperation( + src=pathlib.Path("/src"), file_parameters=params + ) + file_set_list = ensemble._permutate_config_file(config_file, permutation_strategy) + assert len(file_set_list) == expected_combinations + + +def test_cartesian_values(): + ensemble = Ensemble( + "name", + "echo", + exe_arg_parameters={"-N": ["1", "2"]}, + permutation_strategy="step", + ) + permutation_strategy = strategies.resolve("all_perm") + config_file_1 = EnsembleConfigureOperation( + src=pathlib.Path("/src_1"), file_parameters={"SPAM": ["a"]} + ) + config_file_2 = EnsembleConfigureOperation( + src=pathlib.Path("/src_2"), file_parameters={"EGGS": ["b"]} + ) + file_set_list = [] + file_set_list.append( + ensemble._permutate_config_file(config_file_1, permutation_strategy) + ) + file_set_list.append( + ensemble._permutate_config_file(config_file_2, permutation_strategy) + ) + file_set_tuple = ensemble._cartesian_values(file_set_list) + assert len(file_set_tuple) == 4 + for tup in file_set_tuple: + assert len(tup) == 2 + + +def test_ensemble_type_build_jobs(): + ensemble = Ensemble("ensemble-name", "echo", replicas=2) + with pytest.raises(TypeError): + ensemble.build_jobs("invalid") + + +@pytest.mark.parametrize( + "bad_settings", + [pytest.param(None, id="Nullish"), pytest.param("invalid", id="String")], +) +def test_ensemble_incorrect_launch_settings_type(bad_settings): + """test starting an ensemble with invalid launch settings""" + ensemble = Ensemble("ensemble-name", "echo", replicas=2) + with pytest.raises(TypeError): + ensemble.build_jobs(bad_settings) + + +@pytest.mark.parametrize( + "bad_settings", + [ + pytest.param([1, 2, 3], id="sequence of ints"), + pytest.param(0, id="null"), + pytest.param({"foo": "bar"}, id="dict"), + ], +) +def test_ensemble_type_exe_args(bad_settings): + ensemble = Ensemble( + "ensemble-name", + exe="echo", + ) + with pytest.raises( + TypeError, match="exe_args argument was not of type sequence of str" + ): + ensemble.exe_args = bad_settings diff --git a/tests/test_ensemble_operations.py b/tests/test_ensemble_operations.py index 4c1f306235..aa15f5b0dd 100644 --- a/tests/test_ensemble_operations.py +++ b/tests/test_ensemble_operations.py @@ -4,26 +4,25 @@ import pickle from glob import glob from os import path as osp -import pickle import pytest from smartsim._core.commands import Command from smartsim._core.generation.operations.ensemble_operations import ( + EnsembleConfigureOperation, EnsembleCopyOperation, + EnsembleFileSysOperationSet, EnsembleSymlinkOperation, - EnsembleConfigureOperation, - EnsembleFileSysOperationSet ) from smartsim._core.generation.operations.operations import default_tag from smartsim.builders import Ensemble from smartsim.builders.utils import strategies - pytestmark = pytest.mark.group_a # TODO missing test for _filter + @pytest.fixture def ensemble_copy_operation(mock_src: pathlib.Path, mock_dest: pathlib.Path): """Fixture to create a EnsembleCopyOperation object.""" @@ -51,12 +50,16 @@ def ensemble_file_system_operation_set( ensemble_configure_operation: EnsembleConfigureOperation, ): """Fixture to create a FileSysOperationSet object.""" - return EnsembleFileSysOperationSet([ensemble_copy_operation, ensemble_symlink_operation, ensemble_configure_operation]) + return EnsembleFileSysOperationSet( + [ + ensemble_copy_operation, + ensemble_symlink_operation, + ensemble_configure_operation, + ] + ) -def test_init_ensemble_copy_operation( - mock_src: pathlib.Path, mock_dest: pathlib.Path -): +def test_init_ensemble_copy_operation(mock_src: pathlib.Path, mock_dest: pathlib.Path): """Validate EnsembleCopyOperation init""" ensemble_copy_operation = EnsembleCopyOperation(mock_src, mock_dest) assert isinstance(ensemble_copy_operation, EnsembleCopyOperation) @@ -64,9 +67,7 @@ def test_init_ensemble_copy_operation( assert ensemble_copy_operation.dest == mock_dest -def test_init_ensemble_symlink_operation( - mock_src: str, mock_dest: str -): +def test_init_ensemble_symlink_operation(mock_src: str, mock_dest: str): """Validate EnsembleSymlinkOperation init""" ensemble_symlink_operation = EnsembleSymlinkOperation(mock_src, mock_dest) assert isinstance(ensemble_symlink_operation, EnsembleSymlinkOperation) @@ -74,11 +75,11 @@ def test_init_ensemble_symlink_operation( assert ensemble_symlink_operation.dest == mock_dest -def test_init_ensemble_configure_operation( - mock_src: str -): +def test_init_ensemble_configure_operation(mock_src: str): """Validate EnsembleConfigureOperation init""" - ensemble_configure_operation = EnsembleConfigureOperation(mock_src, file_parameters={"FOO": ["BAR", "TOE"]}) + ensemble_configure_operation = EnsembleConfigureOperation( + mock_src, file_parameters={"FOO": ["BAR", "TOE"]} + ) assert isinstance(ensemble_configure_operation, EnsembleConfigureOperation) assert ensemble_configure_operation.src == mock_src assert ensemble_configure_operation.dest == None @@ -89,25 +90,34 @@ def test_init_ensemble_configure_operation( def test_init_ensemble_file_sys_operation_set( copy_operation: EnsembleCopyOperation, symlink_operation: EnsembleSymlinkOperation, - configure_operation: EnsembleConfigureOperation): + configure_operation: EnsembleConfigureOperation, +): """Test initialize EnsembleFileSysOperationSet""" - ensemble_fs_op_set = EnsembleFileSysOperationSet([copy_operation, symlink_operation, configure_operation]) + ensemble_fs_op_set = EnsembleFileSysOperationSet( + [copy_operation, symlink_operation, configure_operation] + ) assert isinstance(ensemble_fs_op_set.operations, list) assert len(ensemble_fs_op_set.operations) == 3 -def test_add_ensemble_copy_operation(ensemble_file_system_operation_set: EnsembleFileSysOperationSet): +def test_add_ensemble_copy_operation( + ensemble_file_system_operation_set: EnsembleFileSysOperationSet, +): """Test EnsembleFileSysOperationSet.add_copy""" orig_num_ops = len(ensemble_file_system_operation_set.copy_operations) ensemble_file_system_operation_set.add_copy(src=pathlib.Path("/src")) assert len(ensemble_file_system_operation_set.copy_operations) == orig_num_ops + 1 -def test_add_ensemble_symlink_operation(ensemble_file_system_operation_set: EnsembleFileSysOperationSet): +def test_add_ensemble_symlink_operation( + ensemble_file_system_operation_set: EnsembleFileSysOperationSet, +): """Test EnsembleFileSysOperationSet.add_symlink""" orig_num_ops = len(ensemble_file_system_operation_set.symlink_operations) ensemble_file_system_operation_set.add_symlink(src=pathlib.Path("/src")) - assert len(ensemble_file_system_operation_set.symlink_operations) == orig_num_ops + 1 + assert ( + len(ensemble_file_system_operation_set.symlink_operations) == orig_num_ops + 1 + ) def test_add_ensemble_configure_operation( @@ -115,8 +125,12 @@ def test_add_ensemble_configure_operation( ): """Test FileSystemOperationSet.add_configuration""" orig_num_ops = len(ensemble_file_system_operation_set.configure_operations) - ensemble_file_system_operation_set.add_configuration(src=pathlib.Path("/src"), file_parameters={"FOO": "BAR"}) - assert len(ensemble_file_system_operation_set.configure_operations) == orig_num_ops + 1 + ensemble_file_system_operation_set.add_configuration( + src=pathlib.Path("/src"), file_parameters={"FOO": "BAR"} + ) + assert ( + len(ensemble_file_system_operation_set.configure_operations) == orig_num_ops + 1 + ) @pytest.mark.parametrize( @@ -124,7 +138,9 @@ def test_add_ensemble_configure_operation( ( pytest.param(123, TypeError, id="dest as integer"), pytest.param("", TypeError, id="dest as empty str"), - pytest.param(pathlib.Path("/absolute/path"), ValueError, id="dest as absolute str"), + pytest.param( + pathlib.Path("/absolute/path"), ValueError, id="dest as absolute str" + ), ), ) def test_ensemble_copy_files_invalid_dest(dest, error, source): @@ -138,7 +154,9 @@ def test_ensemble_copy_files_invalid_dest(dest, error, source): ( pytest.param(123, TypeError, id="src as integer"), pytest.param("", TypeError, id="src as empty str"), - pytest.param(pathlib.Path("relative/path"), ValueError, id="src as relative str"), + pytest.param( + pathlib.Path("relative/path"), ValueError, id="src as relative str" + ), ), ) def test_ensemble_copy_files_invalid_src(src, error): @@ -152,7 +170,9 @@ def test_ensemble_copy_files_invalid_src(src, error): ( pytest.param(123, TypeError, id="dest as integer"), pytest.param("", TypeError, id="dest as empty str"), - pytest.param(pathlib.Path("/absolute/path"), ValueError, id="dest as absolute str"), + pytest.param( + pathlib.Path("/absolute/path"), ValueError, id="dest as absolute str" + ), ), ) def test_ensemble_symlink_files_invalid_dest(dest, error, source): @@ -166,7 +186,9 @@ def test_ensemble_symlink_files_invalid_dest(dest, error, source): ( pytest.param(123, TypeError, id="src as integer"), pytest.param("", TypeError, id="src as empty str"), - pytest.param(pathlib.Path("relative/path"), ValueError, id="src as relative str"), + pytest.param( + pathlib.Path("relative/path"), ValueError, id="src as relative str" + ), ), ) def test_ensemble_symlink_files_invalid_src(src, error): @@ -180,14 +202,18 @@ def test_ensemble_symlink_files_invalid_src(src, error): ( pytest.param(123, TypeError, id="dest as integer"), pytest.param("", TypeError, id="dest as empty str"), - pytest.param(pathlib.Path("/absolute/path"), ValueError, id="dest as absolute str"), + pytest.param( + pathlib.Path("/absolute/path"), ValueError, id="dest as absolute str" + ), ), ) def test_ensemble_configure_files_invalid_dest(dest, error, source): """Test invalid configure destination""" with pytest.raises(error): _ = [ - EnsembleConfigureOperation(src=file, dest=dest, file_parameters={"FOO": "BAR"}) + EnsembleConfigureOperation( + src=file, dest=dest, file_parameters={"FOO": "BAR"} + ) for file in source ] @@ -197,10 +223,12 @@ def test_ensemble_configure_files_invalid_dest(dest, error, source): ( pytest.param(123, TypeError, id="src as integer"), pytest.param("", TypeError, id="src as empty str"), - pytest.param(pathlib.Path("relative/path"), ValueError, id="src as relative str"), + pytest.param( + pathlib.Path("relative/path"), ValueError, id="src as relative str" + ), ), ) def test_ensemble_configure_files_invalid_src(src, error): """Test invalid configure source""" with pytest.raises(error): - _ = EnsembleConfigureOperation(src=src, file_parameters={"FOO":["BAR", "TOE"]}) + _ = EnsembleConfigureOperation(src=src, file_parameters={"FOO": ["BAR", "TOE"]}) From 27aa710f419c25c6b357f4bb4eb82599fa4f1b16 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Tue, 15 Oct 2024 15:37:07 -0700 Subject: [PATCH 29/35] remove unused EntityFile imports --- smartsim/_core/generation/operations/operations.py | 6 ------ smartsim/entity/application.py | 1 - tests/test_application.py | 1 - 3 files changed, 8 deletions(-) diff --git a/smartsim/_core/generation/operations/operations.py b/smartsim/_core/generation/operations/operations.py index 54cb18a0b3..918f30e5d0 100644 --- a/smartsim/_core/generation/operations/operations.py +++ b/smartsim/_core/generation/operations/operations.py @@ -48,11 +48,6 @@ def _check_run_path(run_path: pathlib.Path) -> None: ) if not run_path.is_absolute(): raise ValueError(f"The Job's run path must be absolute.") - # TODO - # if not run_path.is_dir(): - # raise ValueError( - # "The Job's run path must be a directory." - # ) class GenerationContext: @@ -208,7 +203,6 @@ def format(self, context: GenerationContext) -> Command: class FileSysOperationSet: """Dataclass to represent a set of file system operation objects""" - # TODO disallow modification - dunder function (post ticket to reevaluate API objects) operations: t.List[GenerationProtocol] = field(default_factory=list) """Set of file system objects that match the GenerationProtocol""" diff --git a/smartsim/entity/application.py b/smartsim/entity/application.py index a9afa5f7cf..501279c85f 100644 --- a/smartsim/entity/application.py +++ b/smartsim/entity/application.py @@ -36,7 +36,6 @@ from .._core.utils.helpers import expand_exe_path from ..log import get_logger from .entity import SmartSimEntity -from .files import EntityFiles logger = get_logger(__name__) diff --git a/tests/test_application.py b/tests/test_application.py index cd5cc5b3da..54a02c5b4d 100644 --- a/tests/test_application.py +++ b/tests/test_application.py @@ -30,7 +30,6 @@ import pytest from smartsim.entity.application import Application -from smartsim.entity.files import EntityFiles from smartsim.settings.launch_settings import LaunchSettings pytestmark = pytest.mark.group_a From 7bc70f201f1bd01c0ca4a5923a285896b9e82968 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Tue, 15 Oct 2024 15:38:07 -0700 Subject: [PATCH 30/35] remove EntityFiles --- smartsim/entity/files.py | 144 --------------------------------------- 1 file changed, 144 deletions(-) delete mode 100644 smartsim/entity/files.py diff --git a/smartsim/entity/files.py b/smartsim/entity/files.py deleted file mode 100644 index 42586f153e..0000000000 --- a/smartsim/entity/files.py +++ /dev/null @@ -1,144 +0,0 @@ -# BSD 2-Clause License -# -# Copyright (c) 2021-2024, Hewlett Packard Enterprise -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# 1. Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import typing as t -from os import path - -from tabulate import tabulate - - -# TODO remove when Ensemble is addressed -class EntityFiles: - """EntityFiles are the files a user wishes to have available to - applications and nodes within SmartSim. Each entity has a method - `entity.attach_generator_files()` that creates one of these - objects such that at generation time, each file type will be - present within the generated application or node directory. - - Tagged files are the configuration files for a application that - can be searched through and edited by the ApplicationWriter. - - Copy files are files that a user wants to copy into the - application or node directory without searching through and - editing them for tags. - - Lastly, symlink can be used for big datasets or input - files that a user just wants to have present in the directory - without necessary having to copy the entire file. - """ - - def __init__( - self, - tagged: t.Optional[t.List[str]] = None, - copy: t.Optional[t.List[str]] = None, - symlink: t.Optional[t.List[str]] = None, - ) -> None: - """Initialize an EntityFiles instance - - :param tagged: tagged files for application configuration - :param copy: files or directories to copy into application - or node directories - :param symlink: files to symlink into application or node - directories - """ - self.tagged = tagged or [] - self.copy = copy or [] - self.link = symlink or [] - self._check_files() - - def _check_files(self) -> None: - """Ensure the files provided by the user are of the correct - type and actually exist somewhere on the filesystem. - - :raises SSConfigError: If a user provides a directory within - the tagged files. - """ - - # type check all files provided by user - self.tagged = self._type_check_files(self.tagged, "Tagged") - self.copy = self._type_check_files(self.copy, "Copyable") - self.link = self._type_check_files(self.link, "Symlink") - - for i, value in enumerate(self.copy): - self.copy[i] = self._check_path(value) - - for i, value in enumerate(self.link): - self.link[i] = self._check_path(value) - - @staticmethod - def _type_check_files( - file_list: t.Union[t.List[str], None], file_type: str - ) -> t.List[str]: - """Check the type of the files provided by the user. - - :param file_list: either tagged, copy, or symlink files - :param file_type: name of the file type e.g. "tagged" - :raises TypeError: if incorrect type is provided by user - :return: file list provided - """ - if file_list: - if not isinstance(file_list, list): - if isinstance(file_list, str): - file_list = [file_list] - else: - raise TypeError( - f"{file_type} files given were not of type list or str" - ) - else: - if not all(isinstance(f, str) for f in file_list): - raise TypeError(f"Not all {file_type} files were of type str") - return file_list or [] - - @staticmethod - def _check_path(file_path: str) -> str: - """Given a user provided path-like str, find the actual path to - the directory or file and create a full path. - - :param file_path: path to a specific file or directory - :raises FileNotFoundError: if file or directory does not exist - :return: full path to file or directory - """ - full_path = path.abspath(file_path) - if path.isfile(full_path): - return full_path - if path.isdir(full_path): - return full_path - raise FileNotFoundError(f"File or Directory {file_path} not found") - - def __str__(self) -> str: - """Return table summarizing attached files.""" - values = [] - - if self.copy: - values.append(["Copy", "\n".join(self.copy)]) - if self.link: - values.append(["Symlink", "\n".join(self.link)]) - if self.tagged: - values.append(["Configure", "\n".join(self.tagged)]) - - if not values: - return "No file attached to this entity." - - return tabulate(values, headers=["Strategy", "Files"], tablefmt="grid") From b71871e7f3c4b049dd89c4d1deedca241cfe8487 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Tue, 15 Oct 2024 15:42:08 -0700 Subject: [PATCH 31/35] remove dead fixture --- tests/test_ensemble.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/test_ensemble.py b/tests/test_ensemble.py index 32099b8d18..0869b07606 100644 --- a/tests/test_ensemble.py +++ b/tests/test_ensemble.py @@ -48,11 +48,6 @@ _2x2_EXE_ARG = {"EXE": [["a"], ["b", "c"]], "ARGS": [["d"], ["e", "f"]]} -@pytest.fixture -def get_gen_configure_dir(fileutils): - yield fileutils.get_test_conf_path(osp.join("generator_files", "tag_dir_template")) - - def user_created_function( file_params: t.Mapping[str, t.Sequence[str]], exe_arg_params: t.Mapping[str, t.Sequence[t.Sequence[str]]], From 87372e15deacf35d049409494858509dfa4b15b1 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Tue, 15 Oct 2024 15:44:10 -0700 Subject: [PATCH 32/35] changing types fro t.List to list --- smartsim/_core/generation/generator.py | 6 +++--- smartsim/_core/generation/operations/operations.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/smartsim/_core/generation/generator.py b/smartsim/_core/generation/generator.py index 08a6653b6b..1cc1670655 100644 --- a/smartsim/_core/generation/generator.py +++ b/smartsim/_core/generation/generator.py @@ -281,7 +281,7 @@ def _mkdir_file(file_path: pathlib.Path) -> Command: @staticmethod def _copy_files( - files: t.List[CopyOperation], context: GenerationContext + files: list[CopyOperation], context: GenerationContext ) -> CommandList: """Build commands to copy files/directories from specified source paths to an optional destination in the run directory. @@ -294,7 +294,7 @@ def _copy_files( @staticmethod def _symlink_files( - files: t.List[SymlinkOperation], context: GenerationContext + files: list[SymlinkOperation], context: GenerationContext ) -> CommandList: """Build commands to symlink files/directories from specified source paths to an optional destination in the run directory. @@ -307,7 +307,7 @@ def _symlink_files( @staticmethod def _configure_files( - files: t.List[ConfigureOperation], + files: list[ConfigureOperation], context: GenerationContext, ) -> CommandList: """Build commands to configure files/directories from specified source paths diff --git a/smartsim/_core/generation/operations/operations.py b/smartsim/_core/generation/operations/operations.py index 918f30e5d0..83609b0710 100644 --- a/smartsim/_core/generation/operations/operations.py +++ b/smartsim/_core/generation/operations/operations.py @@ -203,7 +203,7 @@ def format(self, context: GenerationContext) -> Command: class FileSysOperationSet: """Dataclass to represent a set of file system operation objects""" - operations: t.List[GenerationProtocol] = field(default_factory=list) + operations: list[GenerationProtocol] = field(default_factory=list) """Set of file system objects that match the GenerationProtocol""" def add_copy( From bd2ac509c105ac0aae31e13417e5df688e13749a Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Tue, 15 Oct 2024 15:45:56 -0700 Subject: [PATCH 33/35] changing t.List to list --- .../_core/generation/operations/ensemble_operations.py | 10 +++++----- smartsim/builders/ensemble.py | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/smartsim/_core/generation/operations/ensemble_operations.py b/smartsim/_core/generation/operations/ensemble_operations.py index 8bf72cbc19..cb4f8b61a4 100644 --- a/smartsim/_core/generation/operations/ensemble_operations.py +++ b/smartsim/_core/generation/operations/ensemble_operations.py @@ -90,7 +90,7 @@ def __init__( class EnsembleFileSysOperationSet: """Dataclass to represent a set of Ensemble file system operation objects""" - operations: t.List[EnsembleGenerationProtocol] = field(default_factory=list) + operations: list[EnsembleGenerationProtocol] = field(default_factory=list) """Set of Ensemble file system objects that match the EnsembleGenerationProtocol""" def add_copy( @@ -132,7 +132,7 @@ def add_configuration( ) @property - def copy_operations(self) -> t.List[EnsembleCopyOperation]: + def copy_operations(self) -> list[EnsembleCopyOperation]: """Property to get the list of copy files. :return: List of EnsembleCopyOperation objects @@ -140,7 +140,7 @@ def copy_operations(self) -> t.List[EnsembleCopyOperation]: return self._filter(EnsembleCopyOperation) @property - def symlink_operations(self) -> t.List[EnsembleSymlinkOperation]: + def symlink_operations(self) -> list[EnsembleSymlinkOperation]: """Property to get the list of symlink files. :return: List of EnsembleSymlinkOperation objects @@ -148,7 +148,7 @@ def symlink_operations(self) -> t.List[EnsembleSymlinkOperation]: return self._filter(EnsembleSymlinkOperation) @property - def configure_operations(self) -> t.List[EnsembleConfigureOperation]: + def configure_operations(self) -> list[EnsembleConfigureOperation]: """Property to get the list of configure files. :return: List of EnsembleConfigureOperation objects @@ -157,7 +157,7 @@ def configure_operations(self) -> t.List[EnsembleConfigureOperation]: def _filter( self, type: t.Type[EnsembleGenerationProtocolT] - ) -> t.List[EnsembleGenerationProtocolT]: + ) -> list[EnsembleGenerationProtocolT]: """Filters the operations list to include only instances of the specified type. diff --git a/smartsim/builders/ensemble.py b/smartsim/builders/ensemble.py index 2ad9a171b4..86294383c7 100644 --- a/smartsim/builders/ensemble.py +++ b/smartsim/builders/ensemble.py @@ -177,7 +177,7 @@ def exe(self, value: str | os.PathLike[str]) -> None: self._exe = os.fspath(value) @property - def exe_args(self) -> t.List[str]: + def exe_args(self) -> list[str]: """Return attached list of executable arguments. :return: the executable arguments @@ -322,7 +322,7 @@ def _permutate_config_file( ], list[ParamSet], ], - ) -> t.List[FileSet]: + ) -> list[FileSet]: """Generate all possible permutations of file parameters using the given strategy, and create corresponding FileSet objects. From 6198a141a9f42231b773caf2bb3a3b51f66511b1 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Wed, 16 Oct 2024 14:15:39 -0700 Subject: [PATCH 34/35] address one comment from Matt --- smartsim/_core/generation/operations/operations.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/smartsim/_core/generation/operations/operations.py b/smartsim/_core/generation/operations/operations.py index 83609b0710..48ccc6c7b2 100644 --- a/smartsim/_core/generation/operations/operations.py +++ b/smartsim/_core/generation/operations/operations.py @@ -9,17 +9,21 @@ from ...commands import Command from .utils.helpers import check_src_and_dest_path -# pylint: disable=invalid-name +# pylint: disable-next=invalid-name entry_point_path = "smartsim._core.entrypoints.file_operations" """Path to file operations module""" +# pylint: disable-next=invalid-name copy_cmd = "copy" """Copy file operation command""" +# pylint: disable-next=invalid-name symlink_cmd = "symlink" """Symlink file operation command""" +# pylint: disable-next=invalid-name configure_cmd = "configure" """Configure file operation command""" +# pylint: disable-next=invalid-name default_tag = ";" """Default configure tag""" From c03968051e7107b5339a60e4cdf8f220eb986871 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Wed, 16 Oct 2024 15:58:22 -0700 Subject: [PATCH 35/35] failing --- .../operations/ensemble_operations.py | 2 - smartsim/builders/ensemble.py | 79 ++-- smartsim/builders/utils/strategies.py | 19 - tests/test_ensemble.py | 384 +++++++++--------- 4 files changed, 247 insertions(+), 237 deletions(-) diff --git a/smartsim/_core/generation/operations/ensemble_operations.py b/smartsim/_core/generation/operations/ensemble_operations.py index cb4f8b61a4..5e1d0038d1 100644 --- a/smartsim/_core/generation/operations/ensemble_operations.py +++ b/smartsim/_core/generation/operations/ensemble_operations.py @@ -6,8 +6,6 @@ from .utils.helpers import check_src_and_dest_path -# TODO do we need to add check for tags? -# TODO do I need to add checks for file_params? class EnsembleGenerationProtocol(t.Protocol): """Protocol for Ensemble Generation Operations.""" diff --git a/smartsim/builders/ensemble.py b/smartsim/builders/ensemble.py index 86294383c7..efbf197468 100644 --- a/smartsim/builders/ensemble.py +++ b/smartsim/builders/ensemble.py @@ -33,7 +33,7 @@ import os.path import random import typing as t -from dataclasses import dataclass, field +from dataclasses import dataclass from smartsim._core.generation.operations.ensemble_operations import ( EnsembleConfigureOperation, @@ -52,11 +52,13 @@ @dataclass(frozen=True) class FileSet: """ - Represents a set of file parameters and execution arguments as parameters. + Represents a relationship between a parameterized set of arguments and the configuration file. """ file: EnsembleConfigureOperation - combinations: ParamSet + """The configuration file associated with the parameter set""" + combination: ParamSet + """The set of parameters""" class Ensemble(entity.CompoundEntity): @@ -311,30 +313,23 @@ def replicas(self, value: int) -> None: self._replicas = value - def _permutate_config_file( + def _permutate_file_parameters( self, file: EnsembleConfigureOperation, - permutation_strategy: t.Callable[ - [ - t.Mapping[str, t.Sequence[str]], - t.Mapping[str, t.Sequence[t.Sequence[str]]], - int, - ], - list[ParamSet], - ], + permutation_strategy: strategies.PermutationStrategyType, ) -> list[FileSet]: - """Generate all possible permutations of file parameters using the given strategy, - and create corresponding FileSet objects. + """Generate all possible permutations of file parameters using the provided strategy, + and create FileSet objects. This method applies the provided permutation strategy to the file's parameters, along with execution argument parameters and a maximum permutation limit. It returns a list of FileSet objects, each containing one of the generated - ParamSets and an instance of the ConfigurationObject. + ParamSets and an instance of the EnsembleConfigurationObject. - :param file: The configuration file to be permuted + :param file: The configuration file :param permutation_strategy: A function that generates permutations - of the file parameters - :returns: list[FileSet] + of file parameters + :returns: a list of FileSet objects """ combinations = permutation_strategy( file.file_parameters, self.exe_arg_parameters, self.max_permutations @@ -344,11 +339,11 @@ def _permutate_config_file( def _cartesian_values(self, ls: list[list[FileSet]]) -> list[tuple[FileSet, ...]]: """Generate the Cartesian product of a list of lists of FileSets. - This method takes a list of lists of FileSets and returns a list of tuples, - where each tuple contains one FileSet from each inner list. + This method takes a list of lists of FileSet objects and returns a list of tuples, + where each tuple contains one FileSet from each sublist. :param ls: A list of lists of FileSets - :returns: A list of tuples, each containing one FileSet from each inner list + :returns: A list of tuples, each containing one FileSet from each sublist """ return list(itertools.product(*ls)) @@ -362,27 +357,47 @@ def _create_applications(self) -> tuple[Application, ...]: :return: A tuple of Application instances """ permutation_strategy = strategies.resolve(self.permutation_strategy) - perm_list: list[list[FileSet]] = [ - self._permutate_config_file(config_file, permutation_strategy) + file_set_list: list[list[FileSet]] = [ + self._permutate_file_parameters(config_file, permutation_strategy) for config_file in self.files.configure_operations ] - val: list[tuple[FileSet, ...]] = self._cartesian_values(perm_list) + file_set_tuple: list[tuple[FileSet, ...]] = self._cartesian_values( + file_set_list + ) permutations_ = itertools.chain.from_iterable( - itertools.repeat(permutation, self.replicas) for permutation in val + itertools.repeat(permutation, self.replicas) + for permutation in file_set_tuple ) - all_apps = [] + app_list = [] for i, item in enumerate(permutations_, start=1): app = Application( name=f"{self.name}-{i}", exe=self.exe, exe_args=self.exe_args, ) - for file_set in item: - app.files.add_configuration( - src=file_set.file.src, file_parameters=file_set.combinations.params - ) - all_apps.append(app) - return tuple(all_apps) + self._attach_files(app, item) + app_list.append(app) + return tuple(app_list) + + def _attach_files( + self, app: Application, file_set_tuple: tuple[FileSet, ...] + ) -> None: + """Attach files to an Application. + + :param app: The Application to attach files to + :param file_set_tuple: A tuple containing FileSet objects, each representing a configuration file + """ + for config_file in file_set_tuple: + app.files.add_configuration( + src=config_file.file.src, + dest=config_file.file.dest, + file_parameters=config_file.combination.params, + tag=config_file.file.tag, + ) + for copy_file in self.files.copy_operations: + app.files.add_copy(src=copy_file.src, dest=copy_file.dest) + for sym_file in self.files.symlink_operations: + app.files.add_symlink(src=sym_file.src, dest=sym_file.dest) def build_jobs(self, settings: LaunchSettings) -> tuple[Job, ...]: """Expand an Ensemble into a list of deployable Jobs and apply diff --git a/smartsim/builders/utils/strategies.py b/smartsim/builders/utils/strategies.py index 9c3b2b55c5..e3a2527a52 100644 --- a/smartsim/builders/utils/strategies.py +++ b/smartsim/builders/utils/strategies.py @@ -260,22 +260,3 @@ def random_permutations( if 0 <= n_permutations < len(permutations): permutations = random.sample(permutations, n_permutations) return permutations - - -# def create_combos(file_sets: List[FileSet]) -> List[List[Combo]]: -# # Extract the combinations from each FileSet -# all_combinations = [file_set.combinations for file_set in file_sets] - -# # Generate the Cartesian product of all combinations -# product_combinations = itertools.product(*all_combinations) - -# # Create Combo instances for each combination in the product -# combo_lists = [] -# for combination_tuple in product_combinations: -# combo_list = [ -# Combo(file=file_set.file, combination=combination) -# for file_set, combination in zip(file_sets, combination_tuple) -# ] -# combo_lists.append(combo_list) - -# return combo_lists diff --git a/tests/test_ensemble.py b/tests/test_ensemble.py index 0869b07606..683c4de01a 100644 --- a/tests/test_ensemble.py +++ b/tests/test_ensemble.py @@ -38,6 +38,7 @@ EnsembleSymlinkOperation, ) from smartsim.builders.ensemble import Ensemble, FileSet +from smartsim.entity import Application from smartsim.builders.utils import strategies from smartsim.builders.utils.strategies import ParamSet from smartsim.settings.launch_settings import LaunchSettings @@ -81,14 +82,14 @@ def test_ensemble_init(): assert ensemble.exe == os.fspath("python") -def test_ensemble_init_empty_params(test_dir: str) -> None: - """Ensemble created without required args""" +def test_ensemble_init_empty_params() -> None: + """Invalid Ensemble init""" with pytest.raises(TypeError): Ensemble() def test_exe_property(ensemble): - """Validate Ensemble property""" + """Validate Ensemble exe property""" exe = ensemble.exe assert exe == ensemble.exe @@ -100,7 +101,7 @@ def test_exe_property(ensemble): pytest.param(None, TypeError, id="exe as None"), ), ) -def test_exe_set_invalid(ensemble, exe, error): +def test_set_exe_invalid(ensemble, exe, error): """Validate Ensemble exe setter throws""" with pytest.raises(error): ensemble.exe = exe @@ -113,13 +114,14 @@ def test_exe_set_invalid(ensemble, exe, error): pytest.param("this/is/path", id="exe as str"), ), ) -def test_exe_set_valid(ensemble, exe): +def test_set_exe_valid(ensemble, exe): """Validate Ensemble exe setter sets""" ensemble.exe = exe assert ensemble.exe == str(exe) def test_exe_args_property(ensemble): + """Validate Ensemble exe_args property""" exe_args = ensemble.exe_args assert exe_args == ensemble.exe_args @@ -132,7 +134,7 @@ def test_exe_args_property(ensemble): pytest.param(["script.py", 123], TypeError, id="exe_args as None"), ), ) -def test_exe_args_set_invalid(ensemble, exe_args, error): +def test_set_exe_args_invalid(ensemble, exe_args, error): """Validate Ensemble exe_arg setter throws""" with pytest.raises(error): ensemble.exe_args = exe_args @@ -145,18 +147,35 @@ def test_exe_args_set_invalid(ensemble, exe_args, error): pytest.param([], id="exe_args as str"), ), ) -def test_exe_args_set_valid(ensemble, exe_args): +def test_set_exe_args_valid(ensemble, exe_args): """Validate Ensemble exe_args setter sets""" ensemble.exe_args = exe_args assert ensemble.exe_args == exe_args def test_exe_arg_parameters_property(ensemble): + """Validate Ensemble exe_arg_parameters property""" exe_arg_parameters = ensemble.exe_arg_parameters assert exe_arg_parameters == ensemble.exe_arg_parameters -# TODO need a valid test for exe args as params +@pytest.mark.parametrize( + "exe_arg_parameters", + ( + pytest.param( + {"key": [["test"]]}, + id="Value is a sequence of sequence of str", + ), + pytest.param( + {"key": [["test"], ["test"]]}, + id="Value is a sequence of sequence of str", + ), + ), +) +def test_set_exe_arg_parameters_valid(exe_arg_parameters, ensemble): + """Validate Ensemble exe_arg_parameters setter sets""" + ensemble.exe_arg_parameters = exe_arg_parameters + assert ensemble.exe_arg_parameters == exe_arg_parameters @pytest.mark.parametrize( @@ -177,12 +196,8 @@ def test_exe_arg_parameters_property(ensemble): pytest.param({1: 2}, id="Values not mapping of str and str"), ), ) -def test_exe_arg_parameters_set_invalid(exe_arg_params): - ensemble = Ensemble( - "ensemble-name", - exe="echo", - exe_args=["spam", "eggs"], - ) +def test_set_exe_arg_parameters_invalid(exe_arg_params, ensemble): + """Validate Ensemble exe_arg_parameters setter throws""" with pytest.raises( TypeError, match="exe_arg_parameters argument was not of type mapping " @@ -192,11 +207,13 @@ def test_exe_arg_parameters_set_invalid(exe_arg_params): def test_permutation_strategy_property(ensemble): + """Validate Ensemble permutation_strategy property""" permutation_strategy = ensemble.permutation_strategy assert permutation_strategy == ensemble.permutation_strategy def test_permutation_strategy_set_invalid(ensemble): + """Validate Ensemble permutation_strategy setter throws""" with pytest.raises( TypeError, match="permutation_strategy argument was not of " @@ -205,21 +222,23 @@ def test_permutation_strategy_set_invalid(ensemble): ensemble.permutation_strategy = 2 -# TODO add user created strategy @pytest.mark.parametrize( "strategy", ( pytest.param("all_perm", id="strategy as all_perm"), pytest.param("step", id="strategy as step"), pytest.param("random", id="strategy as random"), + pytest.param(user_created_function, id="strategy as user_created_function"), ), ) def test_permutation_strategy_set_valid(ensemble, strategy): + """Validate Ensemble permutation_strategy setter sets""" ensemble.permutation_strategy = strategy assert ensemble.permutation_strategy == strategy def test_max_permutations_property(ensemble): + """Validate Ensemble max_permutations property""" max_permutations = ensemble.max_permutations assert max_permutations == ensemble.max_permutations @@ -252,6 +271,7 @@ def test_max_permutations_set_invalid(ensemble, max_permutations, error): def test_replicas_property(ensemble): + """Validate Ensemble replicas property""" replicas = ensemble.replicas assert replicas == ensemble.replicas @@ -281,80 +301,140 @@ def test_replicas_set_invalid(ensemble, replicas, error): ensemble.replicas = replicas -# END OF PROPERTY TESTS - - -def test_ensemble_user_created_strategy(mock_launcher_settings, test_dir): - jobs = Ensemble( +@pytest.mark.parametrize( + "file_parameters", + ( + pytest.param({"SPAM": ["eggs"]}, id="Non-Empty Params"), + pytest.param({}, id="Empty Params"), + pytest.param(None, id="Nullish Params"), + ), +) +def test_replicated_applications_have_eq_deep_copies_of_parameters( + file_parameters +): + e = Ensemble( "test_ensemble", "echo", - ("hello", "world"), - permutation_strategy=user_created_function, - ).build_jobs(mock_launcher_settings) - assert len(jobs) == 1 + ("hello",), + replicas=4, + ) + e.files.add_configuration(pathlib.Path("/src"), file_parameters=file_parameters) + apps = list(e._create_applications()) + + assert len(apps) >= 2 # Sanity check to make sure the test is valid + assert all( + app_1.file_parameters == app_2.file_parameters + for app_1 in apps + for app_2 in apps + ) + assert all( + app_1.file_parameters is not app_2.file_parameters + for app_1 in apps + for app_2 in apps + if app_1 is not app_2 + ) -def test_ensemble_without_any_members_raises_when_cast_to_jobs( - mock_launcher_settings, test_dir +@pytest.mark.parametrize( + " params, exe_arg_params, max_perms, strategy, expected_combinations", + ( + pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, 8, "all_perm", 8, id="Limit number of perms - 8 : all_perm"), + pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, 1, "all_perm", 1, id="Limit number of perms - 1 : all_perm"), + pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, -1, "all_perm", 16, id="All permutations : all_perm"), + pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, 30, "all_perm", 16, id="Greater number of perms : all_perm"), + pytest.param(_2x2_PARAMS, {}, -1, "all_perm", 4, id="Empty exe args params : all_perm"), + pytest.param({}, _2x2_EXE_ARG, -1, "all_perm", 4, id="Empty file params : all_perm"), + pytest.param({}, {}, -1, "all_perm", 1, id="Empty exe args params and file params : all_perm"), + pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, 2, "step", 2, id="Limit number of perms - 2 : step"), + pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, 1, "step", 1, id="Limit number of perms - 1 : step"), + pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, -1, "step", 2, id="All permutations : step"), + pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, 30, "step", 2, id="Greater number of perms : step"), + pytest.param(_2x2_PARAMS, {}, -1, "step", 1, id="Empty exe args params : step"), + pytest.param({}, _2x2_EXE_ARG, -1, "step", 1, id="Empty file params : step"), + pytest.param({}, {}, -1, "step", 1, id="Empty exe args params and file params : step"), + pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, 8, "random", 8, id="Limit number of perms - 8 : random"), + pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, 1, "random", 1, id="Limit number of perms - 1 : random"), + pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, -1, "random", 16, id="All permutations : random"), + pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, 30, "random", 16, id="Greater number of perms : random"), + pytest.param(_2x2_PARAMS, {}, -1, "random", 4, id="Empty exe args params : random"), + pytest.param({}, _2x2_EXE_ARG, -1, "random", 4, id="Empty file params : random"), + pytest.param({}, {}, -1, "random", 1, id="Empty exe args params and file params : random"), + ), +) +def test_permutate_file_parameters( + params, exe_arg_params, max_perms, strategy, expected_combinations ): - with pytest.raises(ValueError): - Ensemble( - "test_ensemble", - "echo", - ("hello", "world"), - permutation_strategy="random", - max_permutations=30, - replicas=0, - ).build_jobs(mock_launcher_settings) - - -def test_strategy_error_raised_if_a_strategy_that_dne_is_requested(test_dir): - with pytest.raises(ValueError): - Ensemble( - "test_ensemble", - "echo", - ("hello",), - permutation_strategy="THIS-STRATEGY-DNE", - )._create_applications() - + """Test Ensemble._permutate_file_parameters returns expected number of combinations for step, random and all_perm""" + ensemble = Ensemble( + "name", + "echo", + exe_arg_parameters=exe_arg_params, + permutation_strategy=strategy, + max_permutations=max_perms, + ) + permutation_strategy = strategies.resolve(strategy) + config_file = EnsembleConfigureOperation( + src=pathlib.Path("/src"), file_parameters=params + ) + file_set_list = ensemble._permutate_file_parameters( + config_file, permutation_strategy + ) + assert len(file_set_list) == expected_combinations + for file_set in file_set_list: + assert isinstance(file_set, FileSet) + assert file_set.file == config_file -# @pytest.mark.parametrize( -# "file_parameters", -# ( -# pytest.param({"SPAM": ["eggs"]}, id="Non-Empty Params"), -# pytest.param({}, id="Empty Params"), -# pytest.param(None, id="Nullish Params"), -# ), -# ) -# def test_replicated_applications_have_eq_deep_copies_of_parameters( -# file_parameters -# ): -# apps = list( -# Ensemble( -# "test_ensemble", -# "echo", -# ("hello",), -# replicas=4, -# file_parameters=file_parameters, -# )._create_applications() -# ) -# assert len(apps) >= 2 # Sanitiy check to make sure the test is valid -# assert all( -# app_1.file_parameters == app_2.file_parameters -# for app_1 in apps -# for app_2 in apps -# ) -# assert all( -# app_1.file_parameters is not app_2.file_parameters -# for app_1 in apps -# for app_2 in apps -# if app_1 is not app_2 -# ) +def test_cartesian_values(): + """Test Ensemble._cartesian_values returns expected number of combinations""" + ensemble = Ensemble( + "name", + "echo", + exe_arg_parameters={"-N": ["1", "2"]}, + ) + permutation_strategy = strategies.resolve("all_perm") + config_file_1 = EnsembleConfigureOperation( + src=pathlib.Path("/src_1"), file_parameters={"SPAM": ["a"]} + ) + config_file_2 = EnsembleConfigureOperation( + src=pathlib.Path("/src_2"), file_parameters={"EGGS": ["b"]} + ) + file_set_list = [] + file_set_list.append( + ensemble._permutate_file_parameters(config_file_1, permutation_strategy) + ) + file_set_list.append( + ensemble._permutate_file_parameters(config_file_2, permutation_strategy) + ) + file_set_tuple = ensemble._cartesian_values(file_set_list) + assert isinstance(file_set_tuple, list) + assert len(file_set_tuple) == 4 + for tup in file_set_tuple: + assert isinstance(tup, tuple) + assert len(tup) == 2 + files = [fs.file for fs in tup] + # Validate that each config file is in the tuple of FileSets + assert config_file_1 in files + assert config_file_2 in files + +def test_attach_files(): + ensemble = Ensemble("mock_ensemble", "echo") + ensemble.files.add_copy(src=pathlib.Path("/copy")) + ensemble.files.add_symlink(src=pathlib.Path("/symlink")) + app = Application("mock_app", "echo") + file_set_1 = FileSet(EnsembleConfigureOperation(src=pathlib.Path("/src_1"), file_parameters={"FOO": "TOE"}), ParamSet({}, {})) + file_set_2 = FileSet(EnsembleConfigureOperation(src=pathlib.Path("/src_2"), file_parameters={"FOO": "TOE"}), ParamSet({}, {})) + ensemble._attach_files(app, (file_set_1, file_set_2)) + assert len(app.files.copy_operations) == 1 + assert len(app.files.symlink_operations) == 1 + assert len(app.files.configure_operations) == 2 + srcs = [file.src for file in app.files.configure_operations] + assert file_set_1.file.src in srcs + assert file_set_2.file.src in srcs # fmt: off @pytest.mark.parametrize( - " params, exe_arg_params, max_perms, replicas, expected_num_jobs", + " params, exe_arg_params, max_perms, replicas, expected_num_jobs", (pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, 30, 1, 16 , id="Set max permutation high"), pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, -1, 1, 16 , id="Set max permutation negative"), pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, 0, 1, 1 , id="Set max permutation zero"), @@ -365,16 +445,13 @@ def test_strategy_error_raised_if_a_strategy_that_dne_is_requested(test_dir): pytest.param( {}, {}, 6, 2, 2 , id="Set params as dict, set max permutations and replicas") )) # fmt: on -def test_all_perm_strategy( +def test_all_perm_strategy_create_application( # Parameterized params, exe_arg_params, max_perms, replicas, expected_num_jobs, - # Other fixtures - mock_launcher_settings, - test_dir, ): e = Ensemble( "test_ensemble", @@ -386,8 +463,8 @@ def test_all_perm_strategy( replicas=replicas, ) e.files.add_configuration(src=pathlib.Path("/src_1"), file_parameters=params) - jobs = e.build_jobs(mock_launcher_settings) - assert len(jobs) == expected_num_jobs + apps = e._create_applications() + assert len(apps) == expected_num_jobs # fmt: off @@ -403,16 +480,13 @@ def test_all_perm_strategy( pytest.param( {}, {}, 6, 2, 2 , id="Set params as dict, set max permutations and replicas") )) # fmt: on -def test_step_strategy( +def test_step_strategy_create_application( # Parameterized params, exe_arg_params, max_perms, replicas, expected_num_jobs, - # Other fixtures - mock_launcher_settings, - test_dir, ): e = Ensemble( "test_ensemble", @@ -424,8 +498,8 @@ def test_step_strategy( replicas=replicas, ) e.files.add_configuration(src=pathlib.Path("/src_1"), file_parameters=params) - jobs = e.build_jobs(mock_launcher_settings) - assert len(jobs) == expected_num_jobs + apps = e._create_applications() + assert len(apps) == expected_num_jobs # fmt: off @@ -441,15 +515,13 @@ def test_step_strategy( pytest.param( {}, {}, 6, 2, 2 , id="Set params as dict, set max permutations and replicas") )) # fmt: on -def test_random_strategy( +def test_random_strategy_create_application( # Parameterized params, exe_arg_params, max_perms, replicas, expected_num_jobs, - # Other fixtures - mock_launcher_settings, ): e = Ensemble( "test_ensemble", @@ -461,79 +533,8 @@ def test_random_strategy( replicas=replicas, ) e.files.add_configuration(src=pathlib.Path("/src_1"), file_parameters=params) - jobs = e.build_jobs(mock_launcher_settings) - assert len(jobs) == expected_num_jobs - - -@pytest.mark.parametrize( - " params, exe_arg_params, max_perms, strategy, expected_combinations", - ( - pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, 8, "all_perm", 8, id="1"), - pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, 1, "all_perm", 1, id="2"), - pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, -1, "all_perm", 16, id="3"), - pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, 30, "all_perm", 16, id="4"), - pytest.param(_2x2_PARAMS, {}, -1, "all_perm", 4, id="5"), - pytest.param({}, _2x2_EXE_ARG, -1, "all_perm", 4, id="6"), - pytest.param({}, {}, -1, "all_perm", 1, id="7"), - pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, 2, "step", 2, id="8"), - pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, 1, "step", 1, id="9"), - pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, -1, "step", 2, id="10"), - pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, 30, "step", 2, id="11"), - pytest.param(_2x2_PARAMS, {}, -1, "step", 1, id="12"), - pytest.param({}, _2x2_EXE_ARG, -1, "step", 1, id="13"), - pytest.param({}, {}, -1, "step", 1, id="14"), - pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, 8, "random", 8, id="15"), - pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, 1, "random", 1, id="16"), - pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, -1, "random", 16, id="17"), - pytest.param(_2x2_PARAMS, _2x2_EXE_ARG, 30, "random", 16, id="18"), - pytest.param(_2x2_PARAMS, {}, -1, "random", 4, id="19"), - pytest.param({}, _2x2_EXE_ARG, -1, "random", 4, id="20"), - pytest.param({}, {}, -1, "random", 1, id="21"), - ), -) -def test_permutate_config_file( - params, exe_arg_params, max_perms, strategy, expected_combinations -): - ensemble = Ensemble( - "name", - "echo", - exe_arg_parameters=exe_arg_params, - permutation_strategy=strategy, - max_permutations=max_perms, - ) - permutation_strategy = strategies.resolve(strategy) - config_file = EnsembleConfigureOperation( - src=pathlib.Path("/src"), file_parameters=params - ) - file_set_list = ensemble._permutate_config_file(config_file, permutation_strategy) - assert len(file_set_list) == expected_combinations - - -def test_cartesian_values(): - ensemble = Ensemble( - "name", - "echo", - exe_arg_parameters={"-N": ["1", "2"]}, - permutation_strategy="step", - ) - permutation_strategy = strategies.resolve("all_perm") - config_file_1 = EnsembleConfigureOperation( - src=pathlib.Path("/src_1"), file_parameters={"SPAM": ["a"]} - ) - config_file_2 = EnsembleConfigureOperation( - src=pathlib.Path("/src_2"), file_parameters={"EGGS": ["b"]} - ) - file_set_list = [] - file_set_list.append( - ensemble._permutate_config_file(config_file_1, permutation_strategy) - ) - file_set_list.append( - ensemble._permutate_config_file(config_file_2, permutation_strategy) - ) - file_set_tuple = ensemble._cartesian_values(file_set_list) - assert len(file_set_tuple) == 4 - for tup in file_set_tuple: - assert len(tup) == 2 + apps = e._create_applications() + assert len(apps) == expected_num_jobs def test_ensemble_type_build_jobs(): @@ -553,20 +554,35 @@ def test_ensemble_incorrect_launch_settings_type(bad_settings): ensemble.build_jobs(bad_settings) -@pytest.mark.parametrize( - "bad_settings", - [ - pytest.param([1, 2, 3], id="sequence of ints"), - pytest.param(0, id="null"), - pytest.param({"foo": "bar"}, id="dict"), - ], -) -def test_ensemble_type_exe_args(bad_settings): - ensemble = Ensemble( - "ensemble-name", - exe="echo", - ) - with pytest.raises( - TypeError, match="exe_args argument was not of type sequence of str" - ): - ensemble.exe_args = bad_settings +def test_ensemble_user_created_strategy(mock_launcher_settings, test_dir): + jobs = Ensemble( + "test_ensemble", + "echo", + ("hello", "world"), + permutation_strategy=user_created_function, + ).build_jobs(mock_launcher_settings) + assert len(jobs) == 1 + + +def test_ensemble_without_any_members_raises_when_cast_to_jobs( + mock_launcher_settings +): + with pytest.raises(ValueError): + Ensemble( + "test_ensemble", + "echo", + ("hello", "world"), + permutation_strategy="random", + max_permutations=30, + replicas=0, + ).build_jobs(mock_launcher_settings) + + +def test_strategy_error_raised_if_a_strategy_that_dne_is_requested(test_dir): + with pytest.raises(ValueError): + Ensemble( + "test_ensemble", + "echo", + ("hello",), + permutation_strategy="THIS-STRATEGY-DNE", + )._create_applications()