Feature 255 reformat mpr (#312)

* Added logging

* Added MPR to list of supported linetypes

* Added test for MPR reformatting

* Added logging and support to reformat MPR linetype

* Used for logging

* update to use util module's logger utility

* Config file for testing MPR reformatting

* Remove unused lines and fix name of yaml file for testing MPR reformatting

* use the config file for line plot (no original columns) in testing MPR reformatter

* regression data from MET used for testing MPR reformatting

* Added MPR specific code

* using util logger rather than original logging

* Use util module logger

* set the keep_all_mpr_cols to True to match what the test is checking

* Refactor the reformatting for scatter plot-only need all column headers present, no additional columns required.

* fix keep_all_mpr_cols to False because this is reformatting for line plots

* clean up mpr reformatting test for scatter plot, use the appropriate yaml file

* Config file for testing MPR reformatting for scatter plot

* Clean up grammar, add more content for reformatting MPR for scatter plot, reformat some sections for better reading

* Clean up grammar and formatting.  Replace confusing instructions.

* Add the keep_all_mpr_cols setting

* Remove debugging where intermediate files are written

* Changes based on sonarqube findings

* Remove unused variables identified by sonarqube

* Resolved conflicts

METdbLoad/ush/read_load_xml.py

@@ -21,17 +21,18 @@
 import logging
 import pandas as pd
 from lxml import etree
-
+import METreformat.util as util
 from METdbLoad.ush import constants as CN
 
 
+
 class XmlLoadFile:
     """! Class to read in load_spec xml file
         Returns:
            N/A
     """
 
-    def __init__(self, xmlfile):
+    def __init__(self, xmlfile, logger=None):
         # set the defaults
         self.xmlfilename = xmlfile
 
@@ -65,6 +66,13 @@ def __init__(self, xmlfile):
         self.load_files = []
         self.line_types = []
 
+        if logger is None:
+            log_filename = os.path.join(os.getcwd(), __name__ + "_log.txt")
+            self.logger = util.get_common_logger('DEBUG', log_filename)
+        else:
+            self.logger = logger
+
+
     def read_xml(self):
         """! Read in load_spec xml file, store values as class attributes
             Returns:
@@ -80,13 +88,13 @@ def read_xml(self):
                 sys.exit("*** XML file " + self.xmlfilename + " can not be found!")
 
             # parse the XML file
-            logging.info('Reading XML Load file')
+            self.logging.info('Reading XML Load file')
             parser = etree.XMLParser(remove_comments=True, resolve_entities=False)
             tree = etree.parse(self.xmlfilename, parser=parser)
             root = tree.getroot()
 
         except (RuntimeError, TypeError, NameError, KeyError):
-            logging.error("*** %s in read_xml ***", sys.exc_info()[0])
+            self.logging.error("*** %s in read_xml ***", sys.exc_info()[0])
             sys.exit("*** Parsing error(s) in XML file!")
 
         # Extract values from load_spec XML tags, store in attributes of class XmlLoadFile
@@ -95,7 +103,7 @@ def read_xml(self):
             # Extract values for connecting to database
             if root.xpath("connection"):
                 self.read_db_connect(root)
-                logging.info("Database name is: %s", self.connection['db_database'])
+                self.logging.info("Database name is: %s", self.connection['db_database'])
 
             # group and description for putting databases into groups/categories
             if root.xpath("group"):
@@ -142,7 +150,7 @@ def read_xml(self):
                 self.read_file_info(root)
 
         except (RuntimeError, TypeError, NameError, KeyError):
-            logging.error("*** %s in read_xml ***", sys.exc_info()[0])
+            self.logging.error("*** %s in read_xml ***", sys.exc_info()[0])
             sys.exit("*** Error(s) found while reading XML file!")
 
         # This removes duplicate file names. do we want that?
@@ -152,9 +160,9 @@ def read_xml(self):
         # Remove directory names
         self.load_files = [lf for lf in self.load_files if '.' in lf.split('/')[-1]]
 
-        logging.info("Initial number of files: %s", str(len(self.load_files)))
+        self.logging.info("Initial number of files: %s", str(len(self.load_files)))
 
-        logging.debug("[--- End read_xml ---]")
+        self.logging.debug("[--- End read_xml ---]")
 
     def read_file_info(self, root):
         """! Gather info on file template, fill-in values, and dates
@@ -205,7 +213,7 @@ def read_file_info(self, root):
                 self.load_files = self.filenames_from_template(folder_template, template_fills)
 
         except (RuntimeError, TypeError, NameError, KeyError):
-            logging.error("*** %s in read_xml read_file_info ***", sys.exc_info()[0])
+            self.logging.error("*** %s in read_xml read_file_info ***", sys.exc_info()[0])
             sys.exit("*** Error(s) found while reading XML file info!")
 
     def read_db_connect(self, root):
@@ -224,34 +232,34 @@ def read_db_connect(self, root):
                 else:
                     self.connection['db_port'] = CN.SQL_PORT
             else:
-                logging.error("!!! XML must include host tag")
+                self.logging.error("!!! XML must include host tag")
                 raise NameError("Missing required host tag")
 
             if root.xpath('connection')[0].xpath('database'):
                 self.connection['db_database'] = \
                     root.xpath('connection')[0].xpath('database')[0].text
             else:
-                logging.error("!!! XML must include database tag")
+                self.logging.error("!!! XML must include database tag")
                 raise NameError("Missing required database tag")
 
             if not self.connection['db_database'].startswith("mv_"):
-                logging.warning("!!! Database not visible unless name starts with mv_")
+                self.logging.warning("!!! Database not visible unless name starts with mv_")
 
             self.connection['db_user'] = \
                 root.xpath('connection')[0].xpath('user')[0].text
             self.connection['db_password'] = \
                 root.xpath('connection')[0].xpath('password')[0].text
 
             if not self.connection['db_user']:
-                logging.warning("!!! XML expecting user tag")
+                self.logging.warning("!!! XML expecting user tag")
                 raise NameError("Missing required user tag")
 
             if root.xpath('connection')[0].xpath('management_system'):
                 self.connection['db_management_system'] = \
                     root.xpath('connection')[0].xpath('management_system')[0].text
 
         except (RuntimeError, TypeError, NameError, KeyError):
-            logging.error("*** %s in read_xml read_db_connect ***", sys.exc_info()[0])
+            self.logging.error("*** %s in read_xml read_db_connect ***", sys.exc_info()[0])
             sys.exit("*** Error(s) found while reading XML file connection tag!")
 
     def flag_default_true(self, root, default_true):
@@ -298,14 +306,14 @@ def filenames_from_date(date_list):
                     date_start = date_start + pd.Timedelta(seconds=date_inc)
                 all_dates.append(date_end.strftime(date_format))
             else:
-                logging.error("*** date_list tag has unknown characters ***")
+                logger.error("*** date_list tag has unknown characters ***")
 
         except ValueError as value_error:
-            logging.error("*** %s in filenames_from_date ***", sys.exc_info()[0])
-            logging.error(value_error)
+            logger.error("*** %s in filenames_from_date ***", sys.exc_info()[0])
+            logger.error(value_error)
             sys.exit("*** Value Error found while expanding XML date format!")
         except (RuntimeError, TypeError, NameError, KeyError):
-            logging.error("*** %s in filenames_from_date ***", sys.exc_info()[0])
+            logger.error("*** %s in filenames_from_date ***", sys.exc_info()[0])
             sys.exit("*** Error found while expanding XML date format!")
 
         return all_dates
@@ -316,6 +324,7 @@ def filenames_from_template(folder_template, template_fills):
             Returns:
                list of filenames
         """
+
         logging.debug(f"folder template is: {folder_template}")
 
         try: