From baa881bf68564368da57f6887f3fe8724ab0fec1 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Mon, 12 Dec 2022 16:54:59 -0500 Subject: [PATCH 1/7] Add a logger decorator and its test. --- ush/python/pygw/src/pygw/logger.py | 42 ++++++++++++++++++++++++ ush/python/pygw/src/tests/test_logger.py | 26 +++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/ush/python/pygw/src/pygw/logger.py b/ush/python/pygw/src/pygw/logger.py index 5807d5cd94..f39ba3108a 100644 --- a/ush/python/pygw/src/pygw/logger.py +++ b/ush/python/pygw/src/pygw/logger.py @@ -3,6 +3,7 @@ """ import sys +from functools import wraps from pathlib import Path from typing import Union, List import logging @@ -221,3 +222,44 @@ def add_file_handler(cls, logfile_path: Union[str, Path], handler.setFormatter(logging.Formatter(_format)) return handler + + +def logit(logger, name=None, message=None): + """ + Add logging to a function. + Parameters + ---------- + logger: Logger + name: str + message: str + """ + + def decorate(func): + + log_name = name if name else func.__module__ + log_msg = message if message else log_name + "." + func.__name__ + + @wraps(func) + def wrapper(*args, **kwargs): + + passed_args = [repr(aa) for aa in args] + passed_kwargs = [f"{kk}={repr(vv)}" for kk, vv in list(kwargs.items())] + call_msg = 'BEGIN: ' + log_msg + f"( {', '.join(passed_args + passed_kwargs)} )" + + # Begin the logging with printing input arguments + logger.debug(call_msg) + + # Call the function + retval = func(*args, **kwargs) + + # Close the logging with printing the return val + ret_msg = ' END: ' + log_msg + f" returning {retval}" + logger.debug(ret_msg) + + return retval + + return wrapper + + return decorate + + diff --git a/ush/python/pygw/src/tests/test_logger.py b/ush/python/pygw/src/tests/test_logger.py index 2cce0f7449..3d5f56fb5a 100644 --- a/ush/python/pygw/src/tests/test_logger.py +++ b/ush/python/pygw/src/tests/test_logger.py @@ -1,4 +1,5 @@ from pygw.logger import Logger +from pygw.logger import logit level = 'debug' number_of_log_msgs = 5 @@ -40,3 +41,28 @@ def test_logger(tmp_path): lev = line.split('-')[3].strip().lower() message = line.split(':')[-1].strip() assert reference[lev] == message + + +def test_logit(tmp_path): + + logger = Logger('test_logger', level=level, colored_log=True) + print('logger at', logger) + + @logit(logger) + def add(x, y): + return x + y + + @logit(logger) + def usedict(n, j=0, k=1): + return n + j + k + + @logit(logger, 'example') + def spam(): + print('Spam!') + + add(2, 3) + usedict(2, 3) + usedict(2, k=3) + spam() + + assert True \ No newline at end of file From ae2f16048559d032a623c20f201ad9cea512be4d Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Tue, 13 Dec 2022 15:13:16 -0500 Subject: [PATCH 2/7] Add a test for yaml_file --- ush/python/pygw/src/pygw/fsutils.py | 2 +- ush/python/pygw/src/pygw/yaml_file.py | 14 ++++---- ush/python/pygw/src/tests/test_logger.py | 5 ++- ush/python/pygw/src/tests/test_yaml_file.py | 39 +++++++++++++++++++++ 4 files changed, 50 insertions(+), 10 deletions(-) create mode 100644 ush/python/pygw/src/tests/test_yaml_file.py diff --git a/ush/python/pygw/src/pygw/fsutils.py b/ush/python/pygw/src/pygw/fsutils.py index 66a62455f5..ebd763984e 100644 --- a/ush/python/pygw/src/pygw/fsutils.py +++ b/ush/python/pygw/src/pygw/fsutils.py @@ -33,7 +33,7 @@ def chdir(path): os.chdir(path) yield finally: - print(f"WARNING: Unable to chdir({path})") # TODO: use logging + print(f"WARNING: Unable to chdir({path})") os.chdir(cwd) diff --git a/ush/python/pygw/src/pygw/yaml_file.py b/ush/python/pygw/src/pygw/yaml_file.py index e25c18619f..cdb62eac1a 100644 --- a/ush/python/pygw/src/pygw/yaml_file.py +++ b/ush/python/pygw/src/pygw/yaml_file.py @@ -17,14 +17,17 @@ class YAMLFile(AttrDict): def __init__(self, path=None, data=None): super().__init__() + if path and data: - print("Ignoring 'data' and using 'path' argument") # TODO: use logging - if path: + print("Ignoring 'data' and using 'path' argument") + + config = None + if path is not None: config = parse_yaml(path=path) - elif data: + elif data is not None: config = parse_yaml(data=data) - if config: + if config is not None: self.update(config) def save(self, target): @@ -76,7 +79,7 @@ def parse_yaml(path=None, data=None, envtag = '!ENV' inctag = '!INC' # pattern for global vars: look for ${word} - pattern = re.compile('.*?\${(\w+)}.*?') + pattern = re.compile(r".*?\${(\w+)}.*?") loader = loader or yaml.SafeLoader # the envtag will be used to mark where to start searching for the pattern @@ -91,7 +94,6 @@ def expand_env_variables(line): for g in match: full_value = full_value.replace( f'${{{g}}}', os.environ.get(g, f'${{{g}}}') - #f'${{{g}}}', os.environ.get(g, g) ) return full_value return line diff --git a/ush/python/pygw/src/tests/test_logger.py b/ush/python/pygw/src/tests/test_logger.py index 3d5f56fb5a..a9b4504d57 100644 --- a/ush/python/pygw/src/tests/test_logger.py +++ b/ush/python/pygw/src/tests/test_logger.py @@ -45,8 +45,7 @@ def test_logger(tmp_path): def test_logit(tmp_path): - logger = Logger('test_logger', level=level, colored_log=True) - print('logger at', logger) + logger = Logger('test_logit', level=level, colored_log=True) @logit(logger) def add(x, y): @@ -65,4 +64,4 @@ def spam(): usedict(2, k=3) spam() - assert True \ No newline at end of file + assert True diff --git a/ush/python/pygw/src/tests/test_yaml_file.py b/ush/python/pygw/src/tests/test_yaml_file.py new file mode 100644 index 0000000000..6b4667aca4 --- /dev/null +++ b/ush/python/pygw/src/tests/test_yaml_file.py @@ -0,0 +1,39 @@ +import os +from pygw.yaml_file import YAMLFile + +host_yaml = """ +host: + hostname: test_host + host_user: !ENV ${USER} +""" + +conf_yaml = """ +config: + config_file: !ENV ${TMP_PATH}/config.yaml + user: !ENV ${USER} + host_file: !INC ${TMP_PATH}/host.yaml +""" + + +def test_yaml_file(tmp_path): + + # Create temporary yaml files w/ tags + config_file_path = tmp_path / 'config.yaml' + with open(config_file_path, 'w') as conf_file: + conf_file.write(conf_yaml) + + with open(tmp_path / 'host.yaml', 'w') as host_file: + host_file.write(host_yaml) + + # Set env. variable + os.environ['TMP_PATH'] = str(tmp_path) + conf = YAMLFile(path=config_file_path) + + # Write out yaml file + yaml_out = tmp_path / 'output.yaml' + conf.save(yaml_out) + + # Read in the yaml file and compare w/ conf + yaml_in = YAMLFile(path=str(yaml_out)) + + assert yaml_in == conf From 85d434f7589c1bf2ea8f4f63c99ea4c154b8097a Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Tue, 13 Dec 2022 15:21:39 -0500 Subject: [PATCH 3/7] beef up the logit docu --- ush/python/pygw/src/pygw/logger.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/ush/python/pygw/src/pygw/logger.py b/ush/python/pygw/src/pygw/logger.py index 77409c061d..25b0a24e83 100644 --- a/ush/python/pygw/src/pygw/logger.py +++ b/ush/python/pygw/src/pygw/logger.py @@ -230,9 +230,14 @@ def logit(logger, name=None, message=None): Add logging to a function. Parameters ---------- - logger: Logger - name: str - message: str + logger : Logger + Logger object + name : str + Name of the module to be logged + default: __module__ + message : str + Name of the function to be logged + default: __name__ """ def decorate(func): From 07df8c46c0d26f56276a2def308ee537e972833a Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Tue, 13 Dec 2022 15:26:03 -0500 Subject: [PATCH 4/7] remove extra line --- ush/python/pygw/src/pygw/logger.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ush/python/pygw/src/pygw/logger.py b/ush/python/pygw/src/pygw/logger.py index 25b0a24e83..776dc89e0c 100644 --- a/ush/python/pygw/src/pygw/logger.py +++ b/ush/python/pygw/src/pygw/logger.py @@ -268,4 +268,3 @@ def wrapper(*args, **kwargs): return decorate - From 1ac9f2e95414ee533cd841c4adbbf23ee144f9c3 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Tue, 13 Dec 2022 15:28:00 -0500 Subject: [PATCH 5/7] remove another extra line --- ush/python/pygw/src/pygw/logger.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ush/python/pygw/src/pygw/logger.py b/ush/python/pygw/src/pygw/logger.py index 776dc89e0c..30d0210665 100644 --- a/ush/python/pygw/src/pygw/logger.py +++ b/ush/python/pygw/src/pygw/logger.py @@ -267,4 +267,3 @@ def wrapper(*args, **kwargs): return wrapper return decorate - From 3686cc9e01f6ffc98ccf2189f46164323c67cf32 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Tue, 13 Dec 2022 15:34:41 -0500 Subject: [PATCH 6/7] retain TODO --- ush/python/pygw/src/pygw/fsutils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygw/src/pygw/fsutils.py b/ush/python/pygw/src/pygw/fsutils.py index 16084d0c0b..f0d41fb54f 100644 --- a/ush/python/pygw/src/pygw/fsutils.py +++ b/ush/python/pygw/src/pygw/fsutils.py @@ -33,7 +33,7 @@ def chdir(path): os.chdir(path) yield finally: - print(f"WARNING: Unable to chdir({path})") + print(f"WARNING: Unable to chdir({path})") # TODO: use logging os.chdir(cwd) From 3bf19f12b0722b9faa18e8520e2f41c0372240c7 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Tue, 13 Dec 2022 16:32:27 -0500 Subject: [PATCH 7/7] add yaml_file test w/ templates --- ush/python/pygw/src/tests/test_yaml_file.py | 35 +++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/ush/python/pygw/src/tests/test_yaml_file.py b/ush/python/pygw/src/tests/test_yaml_file.py index 6b4667aca4..276eb2c548 100644 --- a/ush/python/pygw/src/tests/test_yaml_file.py +++ b/ush/python/pygw/src/tests/test_yaml_file.py @@ -14,6 +14,17 @@ host_file: !INC ${TMP_PATH}/host.yaml """ +tmpl_yaml = """ +config: + config_file: !ENV ${TMP_PATH}/config.yaml + user: !ENV ${USER} + host_file: !INC ${TMP_PATH}/host.yaml +tmpl: + cdate: '{{PDY}}{{cyc}}' + homedir: $(user) +""" +# Note the quotes ' ' around {{ }}. These quotes are necessary otherwise YAMLFile will fail parsing + def test_yaml_file(tmp_path): @@ -37,3 +48,27 @@ def test_yaml_file(tmp_path): yaml_in = YAMLFile(path=str(yaml_out)) assert yaml_in == conf + + +def test_yaml_file_with_templates(tmp_path): + + # Create temporary yaml files w/ tags + tmpl_file_path = tmp_path / 'tmpl.yaml' + with open(tmpl_file_path, 'w') as tmpl_file: + tmpl_file.write(tmpl_yaml) + + with open(tmp_path / 'host.yaml', 'w') as host_file: + host_file.write(host_yaml) + + # Set env. variable + os.environ['TMP_PATH'] = str(tmp_path) + conf = YAMLFile(path=tmpl_file_path) + + # Write out yaml file + yaml_out = tmp_path / 'tmpl_output.yaml' + conf.save(yaml_out) + + # Read in the yaml file and compare w/ conf + yaml_in = YAMLFile(path=yaml_out) + + assert yaml_in == conf