From df1f83e6c8572f9deffb5d48ac3bde5f41a7c1fc Mon Sep 17 00:00:00 2001 From: JP Swinski Date: Mon, 11 Dec 2023 13:54:44 +0000 Subject: [PATCH] updated to logging configuration user interface to make verbose setting much more convenient --- clients/python/sliderule/earthdata.py | 5 +- clients/python/sliderule/gedi.py | 5 +- clients/python/sliderule/h5.py | 4 +- clients/python/sliderule/icesat2.py | 5 +- clients/python/sliderule/io.py | 22 +++---- clients/python/sliderule/ipxapi.py | 10 +-- clients/python/sliderule/ipysliderule.py | 34 +++++----- clients/python/sliderule/raster.py | 3 - clients/python/sliderule/sliderule.py | 83 ++++++++++++++---------- clients/python/sliderule/swot.py | 6 +- clients/python/utils/benchmark.py | 12 +--- 11 files changed, 84 insertions(+), 105 deletions(-) diff --git a/clients/python/sliderule/earthdata.py b/clients/python/sliderule/earthdata.py index 87b8ce9ba..b8b303111 100644 --- a/clients/python/sliderule/earthdata.py +++ b/clients/python/sliderule/earthdata.py @@ -34,12 +34,12 @@ import ssl import urllib.request import requests -import logging import numpy import geopandas from datetime import datetime from shapely.geometry.multipolygon import MultiPolygon from shapely.geometry import Polygon +from sliderule import logger import sliderule @@ -47,9 +47,6 @@ # GLOBALS ############################################################################### -# create logger -logger = logging.getLogger(__name__) - # profiling times for each major function profiles = {} diff --git a/clients/python/sliderule/gedi.py b/clients/python/sliderule/gedi.py index 62b16cfeb..ca5a1211d 100644 --- a/clients/python/sliderule/gedi.py +++ b/clients/python/sliderule/gedi.py @@ -32,15 +32,12 @@ import numpy import geopandas import sliderule -from sliderule import earthdata +from sliderule import earthdata, logger ############################################################################### # GLOBALS ############################################################################### -# create logger -logger = logging.getLogger(__name__) - # profiling times for each major function profiles = {} diff --git a/clients/python/sliderule/h5.py b/clients/python/sliderule/h5.py index edd575f5a..34e4b0324 100644 --- a/clients/python/sliderule/h5.py +++ b/clients/python/sliderule/h5.py @@ -28,16 +28,14 @@ # ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import time -import logging import numpy import sliderule +from sliderule import logger ############################################################################### # GLOBALS ############################################################################### -logger = logging.getLogger(__name__) - profiles = {} ALL_ROWS = -1 diff --git a/clients/python/sliderule/icesat2.py b/clients/python/sliderule/icesat2.py index 115f9307c..8029d4f9a 100644 --- a/clients/python/sliderule/icesat2.py +++ b/clients/python/sliderule/icesat2.py @@ -32,15 +32,12 @@ import numpy import geopandas import sliderule -from sliderule import earthdata +from sliderule import earthdata, logger ############################################################################### # GLOBALS ############################################################################### -# create logger -logger = logging.getLogger(__name__) - # profiling times for each major function profiles = {} diff --git a/clients/python/sliderule/io.py b/clients/python/sliderule/io.py index 19fe61021..91f19133f 100644 --- a/clients/python/sliderule/io.py +++ b/clients/python/sliderule/io.py @@ -29,11 +29,11 @@ import sys import json -import logging import warnings import datetime import geopandas import numpy as np +from sliderule import logger # imports with warnings if not present try: @@ -404,8 +404,8 @@ def to_json(filename, **kwargs): with open(filename, 'w') as fid: json.dump(output, fid) # print the filename and dictionary structure - logging.info(filename) - logging.info(list(output.keys())) + logger.info(filename) + logger.info(list(output.keys())) # read request parameters and regions from JSON def from_json(filename, **kwargs): @@ -413,8 +413,8 @@ def from_json(filename, **kwargs): with open(filename, 'r') as fid: attributes = json.load(fid) # print the filename and dictionary structure - logging.info(filename) - logging.info(list(attributes.keys())) + logger.info(filename) + logger.info(list(attributes.keys())) # try to get the sliderule adjustable parameters SRparams = ['H_min_win', 'atl08_class', 'atl03_quality', 'ats', 'cnf', 'cnt', 'len', 'maxi', 'res', 'sigma_r_max', 'srt', 'yapc'] @@ -527,8 +527,8 @@ def to_nc(gdf, filename, **kwargs): setattr(fileID, 'poly{0:d}_x'.format(i), json.dumps(lon)) setattr(fileID, 'poly{0:d}_y'.format(i), json.dumps(lat)) # Output netCDF structure information - logging.info(filename) - logging.info(list(fileID.variables.keys())) + logger.info(filename) + logger.info(list(fileID.variables.keys())) # Closing the netCDF file fileID.close() warnings.filterwarnings("default") @@ -709,8 +709,8 @@ def write_pytables(df, filename, attributes, **kwargs): setattr(fileID.root._v_attrs, f'poly{i:d}_x', json.dumps(lon)) setattr(fileID.root._v_attrs, f'poly{i:d}_y', json.dumps(lat)) # Output HDF5 structure information - logging.info(filename) - logging.info(fileID.get_storer('sliderule_segments').non_index_axes[0][1]) + logger.info(filename) + logger.info(fileID.get_storer('sliderule_segments').non_index_axes[0][1]) # Closing the HDF5 file fileID.close() @@ -790,8 +790,8 @@ def write_h5py(df, filename, attributes, **kwargs): fileID.attrs[f'poly{i:d}_x'] = json.dumps(lon) fileID.attrs[f'poly{i:d}_y'] = json.dumps(lat) # Output HDF5 structure information - logging.info(filename) - logging.info(list(fileID.keys())) + logger.info(filename) + logger.info(list(fileID.keys())) # Closing the HDF5 file fileID.close() diff --git a/clients/python/sliderule/ipxapi.py b/clients/python/sliderule/ipxapi.py index 821973229..f44976ae5 100644 --- a/clients/python/sliderule/ipxapi.py +++ b/clients/python/sliderule/ipxapi.py @@ -27,15 +27,7 @@ # OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF # ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from sliderule import icesat2, sliderule -import logging - -############################################################################### -# GLOBALS -############################################################################### - -# create logger -logger = logging.getLogger(__name__) +from sliderule import icesat2, sliderule, logger ############################################################################### # APIs diff --git a/clients/python/sliderule/ipysliderule.py b/clients/python/sliderule/ipysliderule.py index eb4809fa5..b1e6c0e21 100644 --- a/clients/python/sliderule/ipysliderule.py +++ b/clients/python/sliderule/ipysliderule.py @@ -31,7 +31,6 @@ import io import sys import copy -import logging import datetime import traceback import numpy as np @@ -43,6 +42,7 @@ import matplotlib.pyplot as plt import matplotlib.colors as colors from traitlets.utils.bunch import Bunch +from sliderule import logger import sliderule.io # imports with warnings if not present @@ -1334,13 +1334,13 @@ def RGT(self): try: rgt = int(self.rgt.value) except: - logging.critical(f"RGT {self.rgt.value} is invalid") + logger.critical(f"RGT {self.rgt.value} is invalid") return "0" # verify ground track values if (rgt >= 1) and (rgt <= 1387): return self.rgt.value else: - logging.critical(f"RGT {self.rgt.value} is outside available range") + logger.critical(f"RGT {self.rgt.value} is outside available range") return "0" @property @@ -1384,7 +1384,7 @@ def orbital_cycle(self): if (self.cycle.value in all_cycles): return self.cycle.value else: - logging.critical(f"Cycle {self.cycle.value} is outside available range") + logger.critical(f"Cycle {self.cycle.value} is outside available range") return "0" @property @@ -2071,14 +2071,14 @@ def add_layer(self, **kwargs): # simply attempt to add the layer or control self.map.add(layer) except ipyleaflet.LayerException as e: - logging.info(f"Layer {layer} already on map") + logger.info(f"Layer {layer} already on map") pass except ipyleaflet.ControlException as e: - logging.info(f"Control {layer} already on map") + logger.info(f"Control {layer} already on map") pass except Exception as e: - logging.critical(f"Could add layer {layer}") - logging.error(traceback.format_exc()) + logger.critical(f"Could add layer {layer}") + logger.error(traceback.format_exc()) pass # remove map layers @@ -2126,14 +2126,14 @@ def remove_layer(self, **kwargs): # simply attempt to remove the layer or control self.map.remove(layer) except ipyleaflet.LayerException as e: - logging.info(f"Layer {layer} already removed from map") + logger.info(f"Layer {layer} already removed from map") pass except ipyleaflet.ControlException as e: - logging.info(f"Control {layer} already removed from map") + logger.info(f"Control {layer} already removed from map") pass except Exception as e: - logging.critical(f"Could not remove layer {layer}") - logging.error(traceback.format_exc()) + logger.critical(f"Could not remove layer {layer}") + logger.error(traceback.format_exc()) pass # handle cursor movements for label @@ -2214,7 +2214,7 @@ def GeoData(self, gdf, **kwargs): kwargs.setdefault('colorbar', True) kwargs.setdefault('position', 'topright') # add warning that function is deprecated - logging.critical(f"Deprecated. Will be removed in a future release") + logger.critical(f"Deprecated. Will be removed in a future release") # remove any prior instances of a data layer if self.geojson is not None: self.map.remove(self.geojson) @@ -2609,14 +2609,14 @@ def remove(self, layer): try: self.map.remove(layer) except ipyleaflet.LayerException as e: - logging.info(f"Layer {layer} already removed from map") + logger.info(f"Layer {layer} already removed from map") pass except ipyleaflet.ControlException as e: - logging.info(f"Control {layer} already removed from map") + logger.info(f"Control {layer} already removed from map") pass except Exception as e: - logging.critical(f"Could not remove layer {layer}") - logging.error(traceback.format_exc()) + logger.critical(f"Could not remove layer {layer}") + logger.error(traceback.format_exc()) pass # add colorbar widget to leaflet map diff --git a/clients/python/sliderule/raster.py b/clients/python/sliderule/raster.py index d0057f780..ca70ae9fd 100644 --- a/clients/python/sliderule/raster.py +++ b/clients/python/sliderule/raster.py @@ -27,7 +27,6 @@ # OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF # ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import logging import numpy import base64 import sliderule @@ -38,8 +37,6 @@ # GLOBALS ############################################################################### -logger = logging.getLogger(__name__) - profiles = {} ############################################################################### diff --git a/clients/python/sliderule/sliderule.py b/clients/python/sliderule/sliderule.py index b718c73b0..1e9f35ef4 100644 --- a/clients/python/sliderule/sliderule.py +++ b/clients/python/sliderule/sliderule.py @@ -40,7 +40,7 @@ import numpy import geopandas from shapely.geometry import Polygon -from datetime import datetime, timedelta +from datetime import datetime from sliderule import version ############################################################################### @@ -70,11 +70,10 @@ MAX_PS_CLUSTER_WAIT_SECS = 600 -verbose = False - request_timeout = (10, 120) # (connection, read) in seconds logger = logging.getLogger(__name__) +console = None clustering_enabled = False try: @@ -437,18 +436,16 @@ def __override_getaddrinfo(*args): # __logeventrec # def __logeventrec(rec): - if verbose: - eventlogger[rec['level']]('%s' % (rec["attr"])) + eventlogger[rec['level']]('%s' % (rec["attr"])) # # __exceptrec # def __exceptrec(rec): - if verbose: - if rec["code"] >= 0: - eventlogger[rec["level"]]("Exception <%d>: %s", rec["code"], rec["text"]) - else: - eventlogger[rec["level"]]("%s", rec["text"]) + if rec["code"] >= 0: + eventlogger[rec["level"]]("Exception <%d>: %s", rec["code"], rec["text"]) + else: + eventlogger[rec["level"]]("%s", rec["text"]) # # _arrowrec @@ -645,7 +642,7 @@ def init (url=PUBLIC_URL, verbose=False, loglevel=logging.INFO, organization=0, url: str the IP address or hostname of the SlideRule service (slidereearth.io by default) verbose: bool - whether or not user level log messages received from SlideRule generate a Python log message + sets up console logger as a convenience to user so all logs are printed to screen loglevel: int minimum severity of log message to output organization: str @@ -669,11 +666,12 @@ def init (url=PUBLIC_URL, verbose=False, loglevel=logging.INFO, organization=0, >>> import sliderule >>> sliderule.init() ''' - set_verbose(verbose) - logger.setLevel(loglevel) - set_url(url) # configure domain + # massage function parameters if organization == 0: organization = PUBLIC_ORG + # configure client + set_verbose(verbose, loglevel) + set_url(url) # configure domain authenticate(organization) # configure credentials (if any) for organization scaleout(desired_nodes, time_to_live, bypass_dns) # set cluster to desired number of nodes (if permitted based on credentials) return check_version(plugins=plugins) # verify compatibility between client and server versions @@ -812,36 +810,53 @@ def set_url (url): # # set_verbose # -def set_verbose (enable): +def set_verbose (enable, loglevel=logging.INFO): ''' - Configure sliderule package for verbose logging + Sets up a console logger to print log messages to screen + + If you want more control over the behavior of the log messages being captured, do not call this function but instead + create and configure a Python log handler of your own and attach it to `sliderule.logger`. Parameters ---------- enable: bool - whether or not user level log messages received from SlideRule generate a Python log message + True: creates console logger if it doesn't exist, False: destroys console logger if it does exist + + loglevel: int + minimum severity of log message to output Examples -------- >>> import sliderule - >>> sliderule.set_verbose(True) - - The default behavior of Python log messages is for them to be displayed to standard output. - If you want more control over the behavior of the log messages being display, create and configure a Python log handler as shown below: - - >>> # import packages - >>> import logging - >>> from sliderule import sliderule - >>> # Configure Logging - >>> sliderule_logger = logging.getLogger("sliderule.sliderule") - >>> sliderule_logger.setLevel(logging.INFO) - >>> # Create Console Output - >>> ch = logging.StreamHandler() - >>> ch.setLevel(logging.INFO) - >>> sliderule_logger.addHandler(ch) + >>> sliderule.set_verbose(True, loglevel=logging.INFO) ''' - global verbose - verbose = (enable == True) + global console, logger + # massage loglevel parameter if passed in as a string + if loglevel == "DEBUG": + loglevel = logging.DEBUG + elif loglevel == "INFO": + loglevel = logging.INFO + elif loglevel == "WARNING": + loglevel = logging.WARNING + elif loglevel == "WARN": + loglevel = logging.WARN + elif loglevel == "ERROR": + loglevel = logging.ERROR + elif loglevel == "FATAL": + loglevel = logging.FATAL + elif loglevel == "CRITICAL": + loglevel = logging.CRITICAL + # enable/disable logging to console + if (enable == True) and (console == None): + console = logging.StreamHandler() + logger.addHandler(console) + elif (enable == False) and (console != None): + logger.removeHandler(console) + console = None + # always set level to requested + logger.setLevel(loglevel) + if console != None: + console.setLevel(loglevel) # # set_rqst_timeout diff --git a/clients/python/sliderule/swot.py b/clients/python/sliderule/swot.py index 7f8dc8ed4..fc3116844 100644 --- a/clients/python/sliderule/swot.py +++ b/clients/python/sliderule/swot.py @@ -30,17 +30,13 @@ import time import logging import numpy -import geopandas import sliderule -from sliderule import earthdata +from sliderule import earthdata, logger ############################################################################### # GLOBALS ############################################################################### -# create logger -logger = logging.getLogger(__name__) - # profiling times for each major function profiles = {} diff --git a/clients/python/utils/benchmark.py b/clients/python/utils/benchmark.py index 8575c15f7..6ed69d9f2 100644 --- a/clients/python/utils/benchmark.py +++ b/clients/python/utils/benchmark.py @@ -56,18 +56,8 @@ args.desired_nodes = None args.time_to_live = None -# Initialize Log Level -loglevel = logging.CRITICAL -if args.loglvl == "ERROR": - loglevel = logging.ERROR -elif args.loglvl == "INFO": - loglevel = logging.INFO -elif args.loglvl == "DEBUG": - loglevel = logging.DEBUG -logging.basicConfig(level=loglevel) - # Initialize SlideRule Client -sliderule.init(args.domain, verbose=args.verbose, organization=args.organization, desired_nodes=args.desired_nodes, time_to_live=args.time_to_live) +sliderule.init(args.domain, verbose=args.verbose, loglevel=args.loglvl, organization=args.organization, desired_nodes=args.desired_nodes, time_to_live=args.time_to_live) # Generate Region Polygon region = sliderule.toregion(args.aoi)