From 801c7617a171be091cb410c7361af5719280bb3c Mon Sep 17 00:00:00 2001 From: bikegeek <3753118+bikegeek@users.noreply.github.com> Date: Wed, 26 Jun 2024 12:36:10 -0600 Subject: [PATCH] Feature 255 reformat mpr (#312) * Added logging * Added MPR to list of supported linetypes * Added test for MPR reformatting * Added logging and support to reformat MPR linetype * Used for logging * update to use util module's logger utility * Config file for testing MPR reformatting * Remove unused lines and fix name of yaml file for testing MPR reformatting * use the config file for line plot (no original columns) in testing MPR reformatter * regression data from MET used for testing MPR reformatting * Added MPR specific code * using util logger rather than original logging * Use util module logger * set the keep_all_mpr_cols to True to match what the test is checking * Refactor the reformatting for scatter plot-only need all column headers present, no additional columns required. * fix keep_all_mpr_cols to False because this is reformatting for line plots * clean up mpr reformatting test for scatter plot, use the appropriate yaml file * Config file for testing MPR reformatting for scatter plot * Clean up grammar, add more content for reformatting MPR for scatter plot, reformat some sections for better reading * Clean up grammar and formatting. Replace confusing instructions. * Add the keep_all_mpr_cols setting * Remove debugging where intermediate files are written * Changes based on sonarqube findings * Remove unused variables identified by sonarqube * Resolved conflicts --- METdbLoad/ush/read_load_xml.py | 45 ++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/METdbLoad/ush/read_load_xml.py b/METdbLoad/ush/read_load_xml.py index c23f52ac..fccf9b61 100644 --- a/METdbLoad/ush/read_load_xml.py +++ b/METdbLoad/ush/read_load_xml.py @@ -21,17 +21,18 @@ import logging import pandas as pd from lxml import etree - +import METreformat.util as util from METdbLoad.ush import constants as CN + class XmlLoadFile: """! Class to read in load_spec xml file Returns: N/A """ - def __init__(self, xmlfile): + def __init__(self, xmlfile, logger=None): # set the defaults self.xmlfilename = xmlfile @@ -65,6 +66,13 @@ def __init__(self, xmlfile): self.load_files = [] self.line_types = [] + if logger is None: + log_filename = os.path.join(os.getcwd(), __name__ + "_log.txt") + self.logger = util.get_common_logger('DEBUG', log_filename) + else: + self.logger = logger + + def read_xml(self): """! Read in load_spec xml file, store values as class attributes Returns: @@ -80,13 +88,13 @@ def read_xml(self): sys.exit("*** XML file " + self.xmlfilename + " can not be found!") # parse the XML file - logging.info('Reading XML Load file') + self.logging.info('Reading XML Load file') parser = etree.XMLParser(remove_comments=True, resolve_entities=False) tree = etree.parse(self.xmlfilename, parser=parser) root = tree.getroot() except (RuntimeError, TypeError, NameError, KeyError): - logging.error("*** %s in read_xml ***", sys.exc_info()[0]) + self.logging.error("*** %s in read_xml ***", sys.exc_info()[0]) sys.exit("*** Parsing error(s) in XML file!") # Extract values from load_spec XML tags, store in attributes of class XmlLoadFile @@ -95,7 +103,7 @@ def read_xml(self): # Extract values for connecting to database if root.xpath("connection"): self.read_db_connect(root) - logging.info("Database name is: %s", self.connection['db_database']) + self.logging.info("Database name is: %s", self.connection['db_database']) # group and description for putting databases into groups/categories if root.xpath("group"): @@ -142,7 +150,7 @@ def read_xml(self): self.read_file_info(root) except (RuntimeError, TypeError, NameError, KeyError): - logging.error("*** %s in read_xml ***", sys.exc_info()[0]) + self.logging.error("*** %s in read_xml ***", sys.exc_info()[0]) sys.exit("*** Error(s) found while reading XML file!") # This removes duplicate file names. do we want that? @@ -152,9 +160,9 @@ def read_xml(self): # Remove directory names self.load_files = [lf for lf in self.load_files if '.' in lf.split('/')[-1]] - logging.info("Initial number of files: %s", str(len(self.load_files))) + self.logging.info("Initial number of files: %s", str(len(self.load_files))) - logging.debug("[--- End read_xml ---]") + self.logging.debug("[--- End read_xml ---]") def read_file_info(self, root): """! Gather info on file template, fill-in values, and dates @@ -205,7 +213,7 @@ def read_file_info(self, root): self.load_files = self.filenames_from_template(folder_template, template_fills) except (RuntimeError, TypeError, NameError, KeyError): - logging.error("*** %s in read_xml read_file_info ***", sys.exc_info()[0]) + self.logging.error("*** %s in read_xml read_file_info ***", sys.exc_info()[0]) sys.exit("*** Error(s) found while reading XML file info!") def read_db_connect(self, root): @@ -224,18 +232,18 @@ def read_db_connect(self, root): else: self.connection['db_port'] = CN.SQL_PORT else: - logging.error("!!! XML must include host tag") + self.logging.error("!!! XML must include host tag") raise NameError("Missing required host tag") if root.xpath('connection')[0].xpath('database'): self.connection['db_database'] = \ root.xpath('connection')[0].xpath('database')[0].text else: - logging.error("!!! XML must include database tag") + self.logging.error("!!! XML must include database tag") raise NameError("Missing required database tag") if not self.connection['db_database'].startswith("mv_"): - logging.warning("!!! Database not visible unless name starts with mv_") + self.logging.warning("!!! Database not visible unless name starts with mv_") self.connection['db_user'] = \ root.xpath('connection')[0].xpath('user')[0].text @@ -243,7 +251,7 @@ def read_db_connect(self, root): root.xpath('connection')[0].xpath('password')[0].text if not self.connection['db_user']: - logging.warning("!!! XML expecting user tag") + self.logging.warning("!!! XML expecting user tag") raise NameError("Missing required user tag") if root.xpath('connection')[0].xpath('management_system'): @@ -251,7 +259,7 @@ def read_db_connect(self, root): root.xpath('connection')[0].xpath('management_system')[0].text except (RuntimeError, TypeError, NameError, KeyError): - logging.error("*** %s in read_xml read_db_connect ***", sys.exc_info()[0]) + self.logging.error("*** %s in read_xml read_db_connect ***", sys.exc_info()[0]) sys.exit("*** Error(s) found while reading XML file connection tag!") def flag_default_true(self, root, default_true): @@ -298,14 +306,14 @@ def filenames_from_date(date_list): date_start = date_start + pd.Timedelta(seconds=date_inc) all_dates.append(date_end.strftime(date_format)) else: - logging.error("*** date_list tag has unknown characters ***") + logger.error("*** date_list tag has unknown characters ***") except ValueError as value_error: - logging.error("*** %s in filenames_from_date ***", sys.exc_info()[0]) - logging.error(value_error) + logger.error("*** %s in filenames_from_date ***", sys.exc_info()[0]) + logger.error(value_error) sys.exit("*** Value Error found while expanding XML date format!") except (RuntimeError, TypeError, NameError, KeyError): - logging.error("*** %s in filenames_from_date ***", sys.exc_info()[0]) + logger.error("*** %s in filenames_from_date ***", sys.exc_info()[0]) sys.exit("*** Error found while expanding XML date format!") return all_dates @@ -316,6 +324,7 @@ def filenames_from_template(folder_template, template_fills): Returns: list of filenames """ + logging.debug(f"folder template is: {folder_template}") try: