50: replace logging with common_logger

METdbLoad/ush/met_db_load.py

@@ -261,7 +261,8 @@ def main(args):
                                                file_data.stat_data,
                                                tmp_dir,
                                                sql_run.cur,
-                                               sql_run.local_infile)
+                                               sql_run.local_infile,
+                                               logger)
 
                 if (not file_data.mode_cts_data.empty) or (not file_data.mode_obj_data.empty):
                     cts_lines = WriteModeSql()
@@ -271,7 +272,8 @@ def main(args):
                                               file_data.mode_obj_data,
                                               tmp_dir,
                                               sql_run.cur,
-                                              sql_run.local_infile)
+                                              sql_run.local_infile,
+                                              logger)
 
                 if not file_data.tcst_data.empty:
                     tcst_lines = WriteTcstSql()
@@ -280,7 +282,8 @@ def main(args):
                                                file_data.tcst_data,
                                                tmp_dir,
                                                sql_run.cur,
-                                               sql_run.local_infile)
+                                               sql_run.local_infile,
+                                               logger)
 
                 if (not file_data.mtd_2d_data.empty) or (not file_data.mtd_3d_single_data.empty) \
                         or (not file_data.mtd_3d_pair_data.empty):
@@ -292,7 +295,8 @@ def main(args):
                                              file_data.mtd_3d_pair_data,
                                              tmp_dir,
                                              sql_run.cur,
-                                             sql_run.local_infile)
+                                             sql_run.local_infile,
+                                             logger)
 
                 # Processing for the last set of data
                 if mid_file >= last_file:

METdbLoad/ush/read_data_files.py

@@ -18,7 +18,6 @@
 import sys
 import os
 from pathlib import Path
-import logging
 import time
 from datetime import timedelta
 from datetime import datetime

METdbLoad/ush/run_sql.py

@@ -17,7 +17,6 @@
 
 import sys
 import os
-import logging
 import time
 from datetime import timedelta
 import pymysql

METdbLoad/ush/write_file_sql.py

@@ -16,25 +16,27 @@
 # constants exist in constants.py
 
 import sys
-import logging
 import time
 from datetime import timedelta
 import getpass
 import pandas as pd
 
 from METdbLoad.ush import constants as CN
-
 from METdbLoad.ush.run_sql import RunSql
-
+from METreformat.util import get_common_logger
 
 class WriteFileSql:
     """ Class to write data_file records to a SQL database
         Returns:
            N/A
     """
 
-    def __init__(self):
+    def __init__(self, logger=None):
         self.sql_met = RunSql()
+        if logger is None:
+            self.logger = get_common_logger('DEBUG', 'stdout')
+        else:
+            self.logger = logger
 
     def write_file_sql(self, load_flags, data_files, stat_data, mode_cts_data,
                        mode_obj_data, tcst_data, mtd_2d_data, mtd_3d_single_data,
@@ -44,7 +46,7 @@ def write_file_sql(self, load_flags, data_files, stat_data, mode_cts_data,
                N/A
         """
 
-        logging.debug("[--- Start write_file_sql ---]")
+        self.logger.debug("[--- Start write_file_sql ---]")
 
         write_time_start = time.perf_counter()
 
@@ -55,7 +57,7 @@ def write_file_sql(self, load_flags, data_files, stat_data, mode_cts_data,
             # --------------------
 
             # get next valid data file id. data files start counting from 1
-            next_file_id = self.sql_met.get_next_id(CN.DATA_FILE, CN.DATA_FILE_ID, sql_cur, logging)
+            next_file_id = self.sql_met.get_next_id(CN.DATA_FILE, CN.DATA_FILE_ID, sql_cur, self.logger)
             if next_file_id == 0:
                 next_file_id = 1
 
@@ -73,12 +75,12 @@ def write_file_sql(self, load_flags, data_files, stat_data, mode_cts_data,
                 if sql_cur.rowcount > 0:
                     list_dupes = list_dupes + [file_line[CN.FILE_ROW]]
                     if not load_flags['force_dup_file']:
-                        logging.warning("!!! Duplicate file %s without FORCE_DUP_FILE tag",
+                        self.logger.warning("!!! Duplicate file %s without FORCE_DUP_FILE tag",
                                         file_line[CN.FULL_FILE])
                     else:
                         # With duplicate files allowed, save the existing id for the file
                         data_files.loc[data_files.index[row_num], CN.DATA_FILE_ID] = result[0]
-                        logging.warning("Duplicate file %s already in data_file",
+                        self.logger.warning("Duplicate file %s already in data_file",
                                         file_line[CN.FULL_FILE])
                 # Not a duplicate - give it a new id
                 else:
@@ -170,17 +172,17 @@ def write_file_sql(self, load_flags, data_files, stat_data, mode_cts_data,
                 if not new_files.empty:
                     self.sql_met.write_to_sql(new_files, CN.DATA_FILE_FIELDS, CN.DATA_FILE,
                                               CN.INS_DATA_FILES, tmp_dir, sql_cur, local_infile,
-                                              logging)
+                                              self.logger)
 
         except (RuntimeError, TypeError, NameError, KeyError):
-            logging.error("*** %s in write_file_sql ***", sys.exc_info()[0])
+            self.logger.error("*** %s in write_file_sql ***", sys.exc_info()[0])
 
         write_time_end = time.perf_counter()
         write_time = timedelta(seconds=write_time_end - write_time_start)
 
-        logging.info("    >>> Write time File: %s", str(write_time))
+        self.logger.info("    >>> Write time File: %s", str(write_time))
 
-        logging.debug("[--- End write_file_sql ---]")
+        self.logger.debug("[--- End write_file_sql ---]")
 
         return data_files, stat_data, mode_cts_data, mode_obj_data, tcst_data, \
             mtd_2d_data, mtd_3d_single_data, mtd_3d_pair_data
@@ -192,7 +194,7 @@ def write_metadata_sql(self, load_flags, data_files, group, description,
                N/A
         """
 
-        logging.debug("[--- Start write_metadata_sql ---]")
+        self.logger.debug("[--- Start write_metadata_sql ---]")
 
         write_time_start = time.perf_counter()
 
@@ -216,7 +218,7 @@ def write_metadata_sql(self, load_flags, data_files, group, description,
                     new_metadata = pd.DataFrame([[group, description]],
                                                 columns=['category', 'description'])
                     self.sql_met.write_to_sql(new_metadata, ['category', 'description'], 'metadata',
-                                              CN.INS_METADATA, tmp_dir, sql_cur, local_infile, logging)
+                                              CN.INS_METADATA, tmp_dir, sql_cur, local_infile, self.logger)
 
             # --------------------
             # Write Instance Info
@@ -225,16 +227,16 @@ def write_metadata_sql(self, load_flags, data_files, group, description,
             if load_flags['load_xml'] and not data_files.empty:
                 update_date = data_files[CN.LOAD_DATE].iloc[0]
                 next_instance_id = self.sql_met.get_next_id(CN.INSTANCE_INFO, CN.INSTANCE_INFO_ID,
-                                                            sql_cur, logging)
+                                                            sql_cur, self.logger)
                 sql_cur.execute(CN.INS_INSTANCE, [next_instance_id, getpass.getuser(), update_date,
                                                   load_note, xml_str])
 
         except (RuntimeError, TypeError, NameError, KeyError):
-            logging.error("*** %s in write_metadata_sql ***", sys.exc_info()[0])
+            self.logger.error("*** %s in write_metadata_sql ***", sys.exc_info()[0])
 
         write_time_end = time.perf_counter()
         write_time = timedelta(seconds=write_time_end - write_time_start)
 
-        logging.info("    >>> Write time Metadata: %s", str(write_time))
+        self.logger.info("    >>> Write time Metadata: %s", str(write_time))
 
-        logging.debug("[--- End write_metadata_sql ---]")
+        self.logger.debug("[--- End write_metadata_sql ---]")

METdbLoad/ush/write_mode_sql.py

@@ -16,7 +16,6 @@
 # constants exist in constants.py
 
 import sys
-import logging
 import time
 from datetime import timedelta
 import pandas as pd
@@ -32,13 +31,13 @@ class WriteModeSql:
            N/A
     """
     @staticmethod
-    def write_mode_data(load_flags, cts_data, obj_data, tmp_dir, sql_cur, local_infile):
+    def write_mode_data(load_flags, cts_data, obj_data, tmp_dir, sql_cur, local_infile, logger):
         """ write mode files (cts and object) to a SQL database.
             Returns:
                N/A
         """
 
-        logging.debug("[--- Start write_mode_sql ---]")
+        logger.debug("[--- Start write_mode_sql ---]")
 
         write_time_start = time.perf_counter()
 
@@ -68,7 +67,7 @@ def write_mode_data(load_flags, cts_data, obj_data, tmp_dir, sql_cur, local_infi
             mode_headers[CN.MODE_HEADER_ID] = CN.NO_KEY
 
             # get the next valid mode header id. Set it to zero (first valid id) if no records yet
-            next_header_id = sql_met.get_next_id(CN.MODE_HEADER, CN.MODE_HEADER_ID, sql_cur, logging)
+            next_header_id = sql_met.get_next_id(CN.MODE_HEADER, CN.MODE_HEADER_ID, sql_cur, logger)
 
             # if the flag is set to check for duplicate headers, get ids from existing headers
             if load_flags["mode_header_db_check"]:
@@ -102,12 +101,12 @@ def write_mode_data(load_flags, cts_data, obj_data, tmp_dir, sql_cur, local_infi
 
             # get just the new headers with their keys
             new_headers = mode_headers[mode_headers[CN.MODE_HEADER_ID] > (next_header_id - 1)]
-            logging.info("New mode headers: %s rows", str(len(new_headers.index)))
+            logger.info("New mode headers: %s rows", str(len(new_headers.index)))
 
             # Write any new headers out to the sql database
             if not new_headers.empty:
                 sql_met.write_to_sql(new_headers, CN.MODE_HEADER_FIELDS, CN.MODE_HEADER,
-                                     CN.INS_MHEADER, tmp_dir, sql_cur, local_infile, logging)
+                                     CN.INS_MHEADER, tmp_dir, sql_cur, local_infile, logger)
                 new_headers = new_headers.iloc[0:0]
 
             # --------------------
@@ -121,7 +120,7 @@ def write_mode_data(load_flags, cts_data, obj_data, tmp_dir, sql_cur, local_infi
                 cts_data = pd.merge(left=mode_headers, right=cts_data, on=CN.MODE_HEADER_KEYS)
 
                 sql_met.write_to_sql(cts_data, CN.MODE_CTS_FIELDS, CN.MODE_CTS_T,
-                                     CN.INS_CHEADER, tmp_dir, sql_cur, local_infile, logging)
+                                     CN.INS_CHEADER, tmp_dir, sql_cur, local_infile, logger)
                 cts_data = cts_data.iloc[0:0]
 
             if not obj_data.empty:
@@ -147,7 +146,7 @@ def write_mode_data(load_flags, cts_data, obj_data, tmp_dir, sql_cur, local_infi
                 obj_data.reset_index(drop=True, inplace=True)
 
                 # get next valid mode object id. Set it to zero (first valid id) if no records yet
-                next_line_id = sql_met.get_next_id(CN.MODE_SINGLE_T, CN.MODE_OBJ_ID, sql_cur, logging)
+                next_line_id = sql_met.get_next_id(CN.MODE_SINGLE_T, CN.MODE_OBJ_ID, sql_cur, logger)
 
                 # create the mode_obj_ids using the dataframe index and next valid id
                 obj_data[CN.MODE_OBJ_ID] = obj_data.index + next_line_id
@@ -176,7 +175,7 @@ def write_mode_data(load_flags, cts_data, obj_data, tmp_dir, sql_cur, local_infi
 
                 # write out the mode single objects
                 sql_met.write_to_sql(obj_data, CN.MODE_SINGLE_FIELDS, CN.MODE_SINGLE_T,
-                                     CN.INS_SHEADER, tmp_dir, sql_cur, local_infile, logging)
+                                     CN.INS_SHEADER, tmp_dir, sql_cur, local_infile, logger)
 
             if not all_pair.empty:
 
@@ -228,16 +227,15 @@ def write_mode_data(load_flags, cts_data, obj_data, tmp_dir, sql_cur, local_infi
 
                 # write out the mode pair objects
                 sql_met.write_to_sql(all_pair, CN.MODE_PAIR_FIELDS, CN.MODE_PAIR_T,
-                                     CN.INS_PHEADER, tmp_dir, sql_cur, local_infile, logging)
+                                     CN.INS_PHEADER, tmp_dir, sql_cur, local_infile, logger)
                 all_pair = all_pair.iloc[0:0]
 
-        except (RuntimeError, TypeError, NameError, KeyError) as e:
-            raise e
-            logging.error("*** %s in write_mode_sql ***", sys.exc_info()[0])
+        except (RuntimeError, TypeError, NameError, KeyError):
+            logger.error("*** %s in write_mode_sql ***", sys.exc_info()[0])
 
         write_time_end = time.perf_counter()
         write_time = timedelta(seconds=write_time_end - write_time_start)
 
-        logging.info("    >>> Write time Mode: %s", str(write_time))
+        logger.info("    >>> Write time Mode: %s", str(write_time))
 
-        logging.debug("[--- End write_mode_sql ---]")
+        logger.debug("[--- End write_mode_sql ---]")

METdbLoad/ush/write_mtd_sql.py

@@ -16,7 +16,6 @@
 # constants exist in constants.py
 
 import sys
-import logging
 import time
 from datetime import timedelta
 import pandas as pd
@@ -33,13 +32,13 @@ class WriteMtdSql:
     """
     @staticmethod
     def write_mtd_data(load_flags, m_2d_data, m_3d_single_data, m_3d_pair_data,
-                       tmp_dir, sql_cur, local_infile):
+                       tmp_dir, sql_cur, local_infile, logger):
         """ write mtd files to a SQL database.
             Returns:
                N/A
         """
 
-        logging.debug("[--- Start write_mtd_sql ---]")
+        logger.debug("[--- Start write_mtd_sql ---]")
 
         write_time_start = time.perf_counter()
 
@@ -78,7 +77,7 @@ def write_mtd_data(load_flags, m_2d_data, m_3d_single_data, m_3d_pair_data,
             mtd_headers[CN.MTD_HEADER_ID] = CN.NO_KEY
 
             # get the next valid MTD header id. Set it to zero (first valid id) if no records yet
-            next_header_id = sql_met.get_next_id(CN.MTD_HEADER, CN.MTD_HEADER_ID, sql_cur, logging)
+            next_header_id = sql_met.get_next_id(CN.MTD_HEADER, CN.MTD_HEADER_ID, sql_cur, logger)
 
             # if the flag is set to check for duplicate headers, get ids from existing headers
             if load_flags["mtd_header_db_check"]:
@@ -122,26 +121,26 @@ def write_mtd_data(load_flags, m_2d_data, m_3d_single_data, m_3d_pair_data,
             # get just the new headers with their keys
             new_headers = mtd_headers[mtd_headers[CN.MTD_HEADER_ID] > (next_header_id - 1)]
             new_headers.obs_valid = pd.to_datetime(new_headers.obs_valid, errors='coerce')
-            logging.info("New MTD headers: %s rows", str(len(new_headers.index)))
+            logger.info("New MTD headers: %s rows", str(len(new_headers.index)))
 
             # Write any new headers out to the sql database
             if not new_headers.empty:
                 # If there are any 2D revision files
                 if new_headers[CN.REVISION_ID].ne(CN.MV_NULL).any():
                     # numbered revision ids must have max revision id added to be unique
-                    next_rev_id = sql_met.get_next_id(CN.MTD_HEADER, CN.REVISION_ID, sql_cur, logging)
+                    next_rev_id = sql_met.get_next_id(CN.MTD_HEADER, CN.REVISION_ID, sql_cur, logger)
                     new_headers.loc[new_headers.revision_id != CN.MV_NULL, CN.REVISION_ID] = \
                         new_headers.loc[new_headers.revision_id != CN.MV_NULL, CN.REVISION_ID] + \
                         next_rev_id
                 new_headers.loc[new_headers.obs_valid.isnull(), CN.OBS_VALID] = CN.MV_NULL
                 sql_met.write_to_sql(new_headers, CN.MTD_HEADER_FIELDS, CN.MTD_HEADER,
-                                     CN.INS_MTDHEADER, tmp_dir, sql_cur, local_infile, logging)
+                                     CN.INS_MTDHEADER, tmp_dir, sql_cur, local_infile, logger)
                 new_headers = new_headers.iloc[0:0]
 
             mtd_headers.obs_valid = pd.to_datetime(mtd_headers.obs_valid, errors='coerce')
 
         except (RuntimeError, TypeError, NameError, KeyError):
-            logging.error("*** %s in write_mtd_sql write MTD headers ***", sys.exc_info()[0])
+            logger.error("*** %s in write_mtd_sql write MTD headers ***", sys.exc_info()[0])
 
         try:
             # --------------------
@@ -180,7 +179,7 @@ def write_mtd_data(load_flags, m_2d_data, m_3d_single_data, m_3d_pair_data,
                                   CN.MATCHED_FLAG] = 1
 
                 sql_met.write_to_sql(m_2d_data, CN.MTD_2D_OBJ_FIELDS, CN.MTD_2D_T,
-                                     CN.INS_M2HEADER, tmp_dir, sql_cur, local_infile, logging)
+                                     CN.INS_M2HEADER, tmp_dir, sql_cur, local_infile, logger)
                 m_2d_data = m_2d_data.iloc[0:0]
 
             if not m_3d_single_data.empty:
@@ -218,7 +217,7 @@ def write_mtd_data(load_flags, m_2d_data, m_3d_single_data, m_3d_pair_data,
                                              CN.MATCHED_FLAG] = 1
 
                 sql_met.write_to_sql(m_3d_single_data, CN.MTD_3D_OBJ_SINGLE_FIELDS, CN.MTD_SINGLE_T,
-                                     CN.INS_M3SHEADER, tmp_dir, sql_cur, local_infile, logging)
+                                     CN.INS_M3SHEADER, tmp_dir, sql_cur, local_infile, logger)
                 m_3d_single_data = m_3d_single_data.iloc[0:0]
 
             if not m_3d_pair_data.empty:
@@ -249,15 +248,15 @@ def write_mtd_data(load_flags, m_2d_data, m_3d_single_data, m_3d_pair_data,
                                        CN.MATCHED_FLAG] = 1
 
                 sql_met.write_to_sql(m_3d_pair_data, CN.MTD_3D_OBJ_PAIR_FIELDS, CN.MTD_PAIR_T,
-                                     CN.INS_M3PHEADER, tmp_dir, sql_cur, local_infile, logging)
+                                     CN.INS_M3PHEADER, tmp_dir, sql_cur, local_infile, logger)
                 m_3d_pair_data = m_3d_pair_data.iloc[0:0]
 
         except (RuntimeError, TypeError, NameError, KeyError):
-            logging.error("*** %s in write_mtd_sql write line data ***", sys.exc_info()[0])
+            logger.error("*** %s in write_mtd_sql write line data ***", sys.exc_info()[0])
 
         write_time_end = time.perf_counter()
         write_time = timedelta(seconds=write_time_end - write_time_start)
 
-        logging.info("    >>> Write time MTD: %s", str(write_time))
+        logger.info("    >>> Write time MTD: %s", str(write_time))
 
-        logging.debug("[--- End write_mtd_sql ---]")
+        logger.debug("[--- End write_mtd_sql ---]")