From 26eb81d38e00cccb29eee8b5c43903e733453e2b Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Wed, 22 May 2024 18:24:03 -0500 Subject: [PATCH 01/15] excludes updating config dic with values that are in enviroment and in config files when parcing EXPDIR config files --- src/wxflow/configuration.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/wxflow/configuration.py b/src/wxflow/configuration.py index 77cf009..9cb1e2a 100644 --- a/src/wxflow/configuration.py +++ b/src/wxflow/configuration.py @@ -1,6 +1,7 @@ import glob import os import random +from typing import Optional import subprocess from pathlib import Path from pprint import pprint @@ -79,7 +80,8 @@ def parse_config(self, files: Union[str, bytes, list]) -> Dict[str, Any]: if isinstance(files, (str, bytes)): files = [files] files = [self.find_config(file) for file in files] - return cast_strdict_as_dtypedict(self._get_script_env(files)) + config_dict = self._get_script_env(files) # get the config dictionary from the config files + return cast_strdict_as_dtypedict(self._get_script_env(files, config_dict)) def print_config(self, files: Union[str, bytes, list]) -> None: """ @@ -93,9 +95,11 @@ def print_config(self, files: Union[str, bytes, list]) -> None: pprint(config, width=4) @classmethod - def _get_script_env(cls, scripts: List) -> Dict[str, Any]: + def _get_script_env(cls, scripts: List, config_dict: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: default_env = cls._get_shell_env([]) and_script_env = cls._get_shell_env(scripts) + if config_dict is not None: # only update the environment with the config dictionary if it is provided + default_env = {k: v for k, v in default_env.items() if k in config_dict} vars_just_in_script = set(and_script_env) - set(default_env) union_env = dict(default_env) union_env.update(and_script_env) From 90b2d796646b231133beafe27020fc64b0226698 Mon Sep 17 00:00:00 2001 From: tmcguinness Date: Fri, 31 May 2024 05:45:53 +0000 Subject: [PATCH 02/15] added bash as shell when geting config env --- src/wxflow/configuration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wxflow/configuration.py b/src/wxflow/configuration.py index 9cb1e2a..532d16b 100644 --- a/src/wxflow/configuration.py +++ b/src/wxflow/configuration.py @@ -112,7 +112,7 @@ def _get_shell_env(scripts: List) -> Dict[str, Any]: magic = f'--- ENVIRONMENT BEGIN {random.randint(0,64**5)} ---' runme += f'/bin/echo -n "{magic}" ; /usr/bin/env -0' with open('/dev/null', 'w') as null: - env = subprocess.Popen(runme, shell=True, stdin=null.fileno(), + env = subprocess.Popen(runme, shell=True, executable='/bin/bash' stdin=null.fileno(), stdout=subprocess.PIPE) (out, err) = env.communicate() out = out.decode() From 2e91edcc8662a223fa9e872a61b56e80b94e5fa6 Mon Sep 17 00:00:00 2001 From: tmcguinness Date: Fri, 31 May 2024 05:46:51 +0000 Subject: [PATCH 03/15] missed comma after shell in sub --- src/wxflow/configuration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wxflow/configuration.py b/src/wxflow/configuration.py index 532d16b..2a4395e 100644 --- a/src/wxflow/configuration.py +++ b/src/wxflow/configuration.py @@ -112,7 +112,7 @@ def _get_shell_env(scripts: List) -> Dict[str, Any]: magic = f'--- ENVIRONMENT BEGIN {random.randint(0,64**5)} ---' runme += f'/bin/echo -n "{magic}" ; /usr/bin/env -0' with open('/dev/null', 'w') as null: - env = subprocess.Popen(runme, shell=True, executable='/bin/bash' stdin=null.fileno(), + env = subprocess.Popen(runme, shell=True, executable='/bin/bash', stdin=null.fileno(), stdout=subprocess.PIPE) (out, err) = env.communicate() out = out.decode() From bbeefc343233f7b1d06ef767c9e58514aa5b7d9e Mon Sep 17 00:00:00 2001 From: tmcguinness Date: Fri, 31 May 2024 06:18:54 +0000 Subject: [PATCH 04/15] made sure bash can be anywere for subprocess in configure getting configs --- src/wxflow/configuration.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/wxflow/configuration.py b/src/wxflow/configuration.py index 2a4395e..633f6fe 100644 --- a/src/wxflow/configuration.py +++ b/src/wxflow/configuration.py @@ -3,6 +3,7 @@ import random from typing import Optional import subprocess +import shutil from pathlib import Path from pprint import pprint from typing import Any, Dict, List, Union @@ -111,8 +112,9 @@ def _get_shell_env(scripts: List) -> Dict[str, Any]: runme = ''.join([f'source {s} ; ' for s in scripts]) magic = f'--- ENVIRONMENT BEGIN {random.randint(0,64**5)} ---' runme += f'/bin/echo -n "{magic}" ; /usr/bin/env -0' + bash_path = shutil.which('bash') with open('/dev/null', 'w') as null: - env = subprocess.Popen(runme, shell=True, executable='/bin/bash', stdin=null.fileno(), + env = subprocess.Popen(runme, shell=True, executable=bash_path, stdin=null.fileno(), stdout=subprocess.PIPE) (out, err) = env.communicate() out = out.decode() From b918dc4fa62391264d238bebfbaf89d12d8d63e0 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Wed, 5 Jun 2024 13:25:45 -0400 Subject: [PATCH 05/15] Update src/wxflow/configuration.py --- src/wxflow/configuration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wxflow/configuration.py b/src/wxflow/configuration.py index 633f6fe..417b52f 100644 --- a/src/wxflow/configuration.py +++ b/src/wxflow/configuration.py @@ -100,7 +100,7 @@ def _get_script_env(cls, scripts: List, config_dict: Optional[Dict[str, Any]] = default_env = cls._get_shell_env([]) and_script_env = cls._get_shell_env(scripts) if config_dict is not None: # only update the environment with the config dictionary if it is provided - default_env = {k: v for k, v in default_env.items() if k in config_dict} + default_env = {kk: vv for kk, vv in default_env.items() if kk in config_dict} vars_just_in_script = set(and_script_env) - set(default_env) union_env = dict(default_env) union_env.update(and_script_env) From 9d874ac5679b976c4d223eda2bad6a83ef9bd6b6 Mon Sep 17 00:00:00 2001 From: tmcguinness Date: Wed, 5 Jun 2024 18:03:41 +0000 Subject: [PATCH 06/15] unskip in runner for configuration tests --- tests/test_configuration.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 80ed966..d1077f2 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -144,7 +144,7 @@ def test_configuration_config_dir(tmp_path, create_configs): assert cfg.config_dir == tmp_path -@pytest.mark.skip(reason="fails in GH runner, passes on localhost") +#@pytest.mark.skip(reason="fails in GH runner, passes on localhost") def test_configuration_config_files(tmp_path, create_configs): cfg = Configuration(tmp_path) config_files = [str(tmp_path / 'config.file0'), str(tmp_path / 'config.file1')] @@ -157,14 +157,14 @@ def test_find_config(tmp_path, create_configs): assert str(tmp_path / 'config.file0') == file0 -@pytest.mark.skip(reason="fails in GH runner, passes on localhost") +#@pytest.mark.skip(reason="fails in GH runner, passes on localhost") def test_parse_config1(tmp_path, create_configs): cfg = Configuration(tmp_path) f0 = cfg.parse_config('config.file0') assert file0_dict == f0 -@pytest.mark.skip(reason="fails in GH runner, passes on localhost") +#@pytest.mark.skip(reason="fails in GH runner, passes on localhost") def test_parse_config2(tmp_path, create_configs): cfg = Configuration(tmp_path) ff = cfg.parse_config(['config.file0', 'config.file1']) From 98a8ef0d4f4b9494700ac0e35c4a99325ea46c98 Mon Sep 17 00:00:00 2001 From: tmcguinness Date: Wed, 5 Jun 2024 22:14:34 +0000 Subject: [PATCH 07/15] removed skip configuration tests and used new method that checks for changes in values between env and parm --- src/wxflow/configuration.py | 10 ++++------ tests/test_configuration.py | 7 +++---- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/wxflow/configuration.py b/src/wxflow/configuration.py index 633f6fe..2c22885 100644 --- a/src/wxflow/configuration.py +++ b/src/wxflow/configuration.py @@ -81,8 +81,7 @@ def parse_config(self, files: Union[str, bytes, list]) -> Dict[str, Any]: if isinstance(files, (str, bytes)): files = [files] files = [self.find_config(file) for file in files] - config_dict = self._get_script_env(files) # get the config dictionary from the config files - return cast_strdict_as_dtypedict(self._get_script_env(files, config_dict)) + return cast_strdict_as_dtypedict(self._get_script_env(files)) def print_config(self, files: Union[str, bytes, list]) -> None: """ @@ -96,12 +95,11 @@ def print_config(self, files: Union[str, bytes, list]) -> None: pprint(config, width=4) @classmethod - def _get_script_env(cls, scripts: List, config_dict: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + def _get_script_env(cls, scripts: List) -> Dict[str, Any]: default_env = cls._get_shell_env([]) and_script_env = cls._get_shell_env(scripts) - if config_dict is not None: # only update the environment with the config dictionary if it is provided - default_env = {k: v for k, v in default_env.items() if k in config_dict} - vars_just_in_script = set(and_script_env) - set(default_env) + diff_values_dict = {k: and_script_env[k] for k in default_env if k in and_script_env and default_env[k] != and_script_env[k]} + vars_just_in_script = set(set(and_script_env) - set(default_env)).union(set(diff_values_dict)) union_env = dict(default_env) union_env.update(and_script_env) return dict([(v, union_env[v]) for v in vars_just_in_script]) diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 80ed966..7e67f60 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -144,7 +144,6 @@ def test_configuration_config_dir(tmp_path, create_configs): assert cfg.config_dir == tmp_path -@pytest.mark.skip(reason="fails in GH runner, passes on localhost") def test_configuration_config_files(tmp_path, create_configs): cfg = Configuration(tmp_path) config_files = [str(tmp_path / 'config.file0'), str(tmp_path / 'config.file1')] @@ -157,14 +156,14 @@ def test_find_config(tmp_path, create_configs): assert str(tmp_path / 'config.file0') == file0 -@pytest.mark.skip(reason="fails in GH runner, passes on localhost") def test_parse_config1(tmp_path, create_configs): cfg = Configuration(tmp_path) f0 = cfg.parse_config('config.file0') - assert file0_dict == f0 + print(f0) + assert 1 == 2 + #assert file0_dict == f0 -@pytest.mark.skip(reason="fails in GH runner, passes on localhost") def test_parse_config2(tmp_path, create_configs): cfg = Configuration(tmp_path) ff = cfg.parse_config(['config.file0', 'config.file1']) From 07aa4c07513e88d8346e7d5f7ca3ae91f2cbb672 Mon Sep 17 00:00:00 2001 From: tmcguinness Date: Wed, 5 Jun 2024 22:24:19 +0000 Subject: [PATCH 08/15] restore correct assertion in config1 parse test --- tests/test_configuration.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 7e67f60..fefc4f9 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -159,9 +159,7 @@ def test_find_config(tmp_path, create_configs): def test_parse_config1(tmp_path, create_configs): cfg = Configuration(tmp_path) f0 = cfg.parse_config('config.file0') - print(f0) - assert 1 == 2 - #assert file0_dict == f0 + assert file0_dict == f0 def test_parse_config2(tmp_path, create_configs): From 404cf0a60ab35027aef03233dcd89a69afef0042 Mon Sep 17 00:00:00 2001 From: tmcguinness Date: Thu, 6 Jun 2024 13:41:19 +0000 Subject: [PATCH 09/15] added new solution that checkes in the scripts for name clashes and then usese the key set xor as before --- src/wxflow/configuration.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/wxflow/configuration.py b/src/wxflow/configuration.py index 2c22885..8e5efc0 100644 --- a/src/wxflow/configuration.py +++ b/src/wxflow/configuration.py @@ -98,8 +98,11 @@ def print_config(self, files: Union[str, bytes, list]) -> None: def _get_script_env(cls, scripts: List) -> Dict[str, Any]: default_env = cls._get_shell_env([]) and_script_env = cls._get_shell_env(scripts) - diff_values_dict = {k: and_script_env[k] for k in default_env if k in and_script_env and default_env[k] != and_script_env[k]} - vars_just_in_script = set(set(and_script_env) - set(default_env)).union(set(diff_values_dict)) + keys_in_scripts = set() + for script in scripts: + result = subprocess.run(['grep', '-o', '-E', '\\b(' + '|'.join(default_env.keys()) + ')\\b', script], stdout=subprocess.PIPE) + keys_in_scripts.update(result.stdout.decode().split()) + vars_just_in_script = set(and_script_env) - set(default_env) | keys_in_scripts union_env = dict(default_env) union_env.update(and_script_env) return dict([(v, union_env[v]) for v in vars_just_in_script]) From 64d73f8c912708b60628b326adf2af81a58a39d3 Mon Sep 17 00:00:00 2001 From: tmcguinness Date: Thu, 6 Jun 2024 14:37:11 +0000 Subject: [PATCH 10/15] added the key search to include = so USER was not picked up --- src/wxflow/configuration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wxflow/configuration.py b/src/wxflow/configuration.py index 8e5efc0..a108377 100644 --- a/src/wxflow/configuration.py +++ b/src/wxflow/configuration.py @@ -100,7 +100,7 @@ def _get_script_env(cls, scripts: List) -> Dict[str, Any]: and_script_env = cls._get_shell_env(scripts) keys_in_scripts = set() for script in scripts: - result = subprocess.run(['grep', '-o', '-E', '\\b(' + '|'.join(default_env.keys()) + ')\\b', script], stdout=subprocess.PIPE) + result = subprocess.run(['grep', '-o', '-P', '\\b(' + '|'.join(default_env.keys()) + ')(?==)', script], stdout=subprocess.PIPE) keys_in_scripts.update(result.stdout.decode().split()) vars_just_in_script = set(and_script_env) - set(default_env) | keys_in_scripts union_env = dict(default_env) From 9740026293e42da4bf1f5823d30b70b5e8f4555f Mon Sep 17 00:00:00 2001 From: tmcguinness Date: Thu, 6 Jun 2024 14:41:31 +0000 Subject: [PATCH 11/15] removed Optoinal from import list --- src/wxflow/configuration.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/wxflow/configuration.py b/src/wxflow/configuration.py index a108377..c9a5aa9 100644 --- a/src/wxflow/configuration.py +++ b/src/wxflow/configuration.py @@ -1,7 +1,6 @@ import glob import os import random -from typing import Optional import subprocess import shutil from pathlib import Path From b8dd227cebf5468e854992b141ea838d730e11f3 Mon Sep 17 00:00:00 2001 From: tmcguinness Date: Thu, 6 Jun 2024 14:46:48 +0000 Subject: [PATCH 12/15] changed order of imports in configureation file due to unkown pynorms fail --- src/wxflow/configuration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wxflow/configuration.py b/src/wxflow/configuration.py index c9a5aa9..2a527d1 100644 --- a/src/wxflow/configuration.py +++ b/src/wxflow/configuration.py @@ -1,8 +1,8 @@ import glob import os import random -import subprocess import shutil +import subprocess from pathlib import Path from pprint import pprint from typing import Any, Dict, List, Union From 86e31dbe7ef9d07a852f331a728ab565e2b3b168 Mon Sep 17 00:00:00 2001 From: tmcguinness Date: Tue, 11 Jun 2024 19:51:27 +0000 Subject: [PATCH 13/15] added dynamic regex with export --- src/wxflow/configuration.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/wxflow/configuration.py b/src/wxflow/configuration.py index 2a527d1..d2b6c81 100644 --- a/src/wxflow/configuration.py +++ b/src/wxflow/configuration.py @@ -1,5 +1,6 @@ import glob import os +import re import random import shutil import subprocess @@ -98,8 +99,9 @@ def _get_script_env(cls, scripts: List) -> Dict[str, Any]: default_env = cls._get_shell_env([]) and_script_env = cls._get_shell_env(scripts) keys_in_scripts = set() + regex_pattern = 'export\\s+\\b(' + '|'.join(map(re.escape, default_env.keys())) + ')(?==)' for script in scripts: - result = subprocess.run(['grep', '-o', '-P', '\\b(' + '|'.join(default_env.keys()) + ')(?==)', script], stdout=subprocess.PIPE) + result = subprocess.run(['grep', '-o', '-P', regex_pattern, script], stdout=subprocess.PIPE) keys_in_scripts.update(result.stdout.decode().split()) vars_just_in_script = set(and_script_env) - set(default_env) | keys_in_scripts union_env = dict(default_env) From 57af2f5f53747cf60d6d366fd82dd6fbd0749df0 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Tue, 25 Jun 2024 19:44:45 -0400 Subject: [PATCH 14/15] updated regex in config --- src/wxflow/configuration.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/wxflow/configuration.py b/src/wxflow/configuration.py index d2b6c81..2d01c4f 100644 --- a/src/wxflow/configuration.py +++ b/src/wxflow/configuration.py @@ -99,7 +99,8 @@ def _get_script_env(cls, scripts: List) -> Dict[str, Any]: default_env = cls._get_shell_env([]) and_script_env = cls._get_shell_env(scripts) keys_in_scripts = set() - regex_pattern = 'export\\s+\\b(' + '|'.join(map(re.escape, default_env.keys())) + ')(?==)' + #regex_pattern = 'export\\s+\\b(' + '|'.join(map(re.escape, default_env.keys())) + ')(?==)' + regex_pattern = '\\b(' + '|'.join(map(re.escape, default_env.keys())) + ')(?==)' for script in scripts: result = subprocess.run(['grep', '-o', '-P', regex_pattern, script], stdout=subprocess.PIPE) keys_in_scripts.update(result.stdout.decode().split()) From 5382a0e8afa26c9c036ffcc93113fef6d4d08e1a Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Wed, 26 Jun 2024 10:55:55 -0400 Subject: [PATCH 15/15] simplfied version of _get_script_env classmethond in Configuration Class --- src/wxflow/configuration.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/wxflow/configuration.py b/src/wxflow/configuration.py index 2d01c4f..e9d0cb3 100644 --- a/src/wxflow/configuration.py +++ b/src/wxflow/configuration.py @@ -96,18 +96,14 @@ def print_config(self, files: Union[str, bytes, list]) -> None: @classmethod def _get_script_env(cls, scripts: List) -> Dict[str, Any]: + varbles = dict() default_env = cls._get_shell_env([]) and_script_env = cls._get_shell_env(scripts) - keys_in_scripts = set() - #regex_pattern = 'export\\s+\\b(' + '|'.join(map(re.escape, default_env.keys())) + ')(?==)' - regex_pattern = '\\b(' + '|'.join(map(re.escape, default_env.keys())) + ')(?==)' - for script in scripts: - result = subprocess.run(['grep', '-o', '-P', regex_pattern, script], stdout=subprocess.PIPE) - keys_in_scripts.update(result.stdout.decode().split()) - vars_just_in_script = set(and_script_env) - set(default_env) | keys_in_scripts - union_env = dict(default_env) - union_env.update(and_script_env) - return dict([(v, union_env[v]) for v in vars_just_in_script]) + + for key, value in and_script_env.items(): + if key not in default_env or default_env[key] != value: + varbles[key] = value + return varbles @staticmethod def _get_shell_env(scripts: List) -> Dict[str, Any]: