From 05f428335c8a3405087a95687c93b6fd8243e735 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Wed, 17 Apr 2024 14:52:24 +0000 Subject: [PATCH 1/5] Add check that input j2yaml file exists --- src/wxflow/yaml_file.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/wxflow/yaml_file.py b/src/wxflow/yaml_file.py index a0d0ec6..251e733 100644 --- a/src/wxflow/yaml_file.py +++ b/src/wxflow/yaml_file.py @@ -177,4 +177,7 @@ def parse_j2yaml(path: str, data: Dict, searchpath: Union[str, List] = '/') -> D the dict configuration """ + if not os.path.exists(path): + raise FileNotFoundError(f"Input j2yaml file {path} does not exist!") + return YAMLFile(data=Jinja(path, data, searchpath=searchpath).render) From 3bc72b61ae9ab6b5446a03ecd9f7fa40a005f56c Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Wed, 17 Apr 2024 14:52:46 +0000 Subject: [PATCH 2/5] Add relpath jinja2 filter. --- src/wxflow/jinja.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/wxflow/jinja.py b/src/wxflow/jinja.py index b4e5738..ef1a789 100644 --- a/src/wxflow/jinja.py +++ b/src/wxflow/jinja.py @@ -106,6 +106,7 @@ def get_set_env(self, loader: jinja2.BaseLoader, filters: Dict[str, callable] = to_YMD: convert a datetime object to a YYYYMMDD string to_julian: convert a datetime object to a julian day to_f90bool: convert a boolean to a fortran boolean + relpath: convert a full path to a relative path based on an input root_path getenv: read variable from environment if defined, else UNDEFINED Parameters @@ -129,6 +130,7 @@ def get_set_env(self, loader: jinja2.BaseLoader, filters: Dict[str, callable] = env.filters["to_julian"] = lambda dt: to_julian(dt) if not isinstance(dt, SilentUndefined) else dt env.filters["to_f90bool"] = lambda bool: ".true." if bool else ".false." env.filters['getenv'] = lambda name, default='UNDEFINED': os.environ.get(name, default) + env.filters["relpath"] = lambda pathname, start=os.curdir: os.path.relpath(pathname,start) # Add any additional filters if filters is not None: From 1f23a2bc01fb3640cb43c2dd7cd4336c97f60a56 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Wed, 17 Apr 2024 14:53:26 +0000 Subject: [PATCH 3/5] Add dereference option to htar. --- src/wxflow/htar.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/wxflow/htar.py b/src/wxflow/htar.py index 03fdbe8..ebee134 100644 --- a/src/wxflow/htar.py +++ b/src/wxflow/htar.py @@ -54,7 +54,8 @@ def _htar(self, arg_list: list, silent: bool = False) -> str: return output - def create(self, tarball: str, fileset: Union[List, str], opts: Union[List, str] = "-P") -> str: + def create(self, tarball: str, fileset: Union[List, str], + dereference: bool = False, opts: Union[List, str] = "-P") -> str: """ Method to write an archive to HPSS Parameters @@ -68,6 +69,9 @@ def create(self, tarball: str, fileset: Union[List, str], opts: Union[List, str] fileset : list | str List containing filenames, patterns, or directories to archive + dereference : bool + Whether to dereference symbolic links (archive the pointed-to files instead). + Returns ------- output : str @@ -75,6 +79,9 @@ def create(self, tarball: str, fileset: Union[List, str], opts: Union[List, str] """ arg_list = ["-c"] + if dereference: + arg_list.append("-h") + # Parse any htar options arg_list.extend(Htar._split_opts(opts)) @@ -90,7 +97,7 @@ def create(self, tarball: str, fileset: Union[List, str], opts: Union[List, str] return output - def cvf(self, tarball: str, fileset: Union[List, str]) -> str: + def cvf(self, tarball: str, fileset: Union[List, str], dereference: bool = False) -> str: """ Method to write an archive to HPSS verbosely (without options). Parameters @@ -101,12 +108,15 @@ def cvf(self, tarball: str, fileset: Union[List, str]) -> str: fileset : list | str List containing filenames, patterns, or directories to archive + dereference : bool + Whether to dereference symbolic links (archive the pointed-to files instead). + Returns ------- output : str Concatenated output and error from the htar command """ - output = self.create(tarball, fileset, opts="-v -P") + output = self.create(tarball, fileset, dereference = dereference, opts="-v -P") return output From 745c1bfac218c18a25e483f7de69d2ea9c148fd4 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Wed, 17 Apr 2024 14:54:09 +0000 Subject: [PATCH 4/5] Offer a better use example of executable. --- src/wxflow/executable.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/wxflow/executable.py b/src/wxflow/executable.py index f1ce0c7..af3e396 100644 --- a/src/wxflow/executable.py +++ b/src/wxflow/executable.py @@ -15,12 +15,12 @@ class Executable: -------- >>> from wxflow.executable import Executable - >>> cmd = Executable('srun') # Lets say we need to run command e.g. "srun" - >>> cmd.add_default_arg('my_exec.x') # Lets say we need to run the executable "my_exec.x" - >>> cmd.add_default_arg('my_arg.yaml') # Lets say we need to pass an argument to this executable e.g. "my_arg.yaml" - >>> cmd.add_default_env('OMP_NUM_THREADS', 4) # Lets say we want to run w/ 4 threads in the environment - >>> cmd(output='stdout', error='stderr') # Run the command and capture the stdout and stderr in files named similarly. - + >>> cmd = Executable('srun') # Let's say we need to run command e.g. "srun" + >>> arg_list = ['my_exec.x'] # Let's say we need to run the executable "my_exec.x" + >>> arg_list.append('my_arg.yaml') # Let's say we need to pass an argument to this executable e.g. "my_arg.yaml" + >>> env = os.environ.copy(); env['OMP_NUM_THREADS'] = 4 # Let's say we want to run w/ 4 threads in the environment + >>> # Run the command with the arguments and environment and capture the stdout and stderr in files named similarly. + >>> cmd(*arg_list, env = env, output='stdout', error='stderr') `cmd` line above will translate to: $ export OMP_NUM_THREADS=4 From 534fe44da4b88cd099a9220c654681ee3c1aaaa6 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Wed, 17 Apr 2024 16:43:19 +0000 Subject: [PATCH 5/5] Address PEP8 issues. --- src/wxflow/htar.py | 2 +- src/wxflow/jinja.py | 2 +- tests/test_htar.py | 7 ++++++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/wxflow/htar.py b/src/wxflow/htar.py index ebee134..a5ef0e4 100644 --- a/src/wxflow/htar.py +++ b/src/wxflow/htar.py @@ -116,7 +116,7 @@ def cvf(self, tarball: str, fileset: Union[List, str], dereference: bool = False output : str Concatenated output and error from the htar command """ - output = self.create(tarball, fileset, dereference = dereference, opts="-v -P") + output = self.create(tarball, fileset, dereference=dereference, opts="-v -P") return output diff --git a/src/wxflow/jinja.py b/src/wxflow/jinja.py index ef1a789..89d96e6 100644 --- a/src/wxflow/jinja.py +++ b/src/wxflow/jinja.py @@ -130,7 +130,7 @@ def get_set_env(self, loader: jinja2.BaseLoader, filters: Dict[str, callable] = env.filters["to_julian"] = lambda dt: to_julian(dt) if not isinstance(dt, SilentUndefined) else dt env.filters["to_f90bool"] = lambda bool: ".true." if bool else ".false." env.filters['getenv'] = lambda name, default='UNDEFINED': os.environ.get(name, default) - env.filters["relpath"] = lambda pathname, start=os.curdir: os.path.relpath(pathname,start) + env.filters["relpath"] = lambda pathname, start=os.curdir: os.path.relpath(pathname, start) # Add any additional filters if filters is not None: diff --git a/tests/test_htar.py b/tests/test_htar.py index 387dd57..7127dff 100644 --- a/tests/test_htar.py +++ b/tests/test_htar.py @@ -40,10 +40,14 @@ def test_cvf_xvf_tell(tmp_path): f.touch() f.write_text("Some contents") + # Create a symlink, add it to tmp_files + os.symlink("a.txt", input_dir_path / "ln_a.txt") + in_tmp_files.append(input_dir_path / "ln_a.txt") + test_tarball = test_path + "/test.tar" # Create the archive file - output = htar.cvf(test_tarball, in_tmp_files) + output = htar.cvf(test_tarball, in_tmp_files, dereference=True) assert "a.txt" in output assert hsi.exists(test_tarball) @@ -52,6 +56,7 @@ def test_cvf_xvf_tell(tmp_path): output = htar.xvf(test_tarball, in_tmp_files) assert "a.txt" in output + assert "ln_a.txt" in output # List the contents of the test archive output = htar.tell(test_tarball)