From 657ef573dd0e74b1090785090b2cb31d0d05d381 Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Mon, 3 Jun 2024 11:59:23 -0400 Subject: [PATCH 01/11] CODE: base_ref property & setter --- podpac/core/node.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/podpac/core/node.py b/podpac/core/node.py index 739ab0ea..ee0423e9 100644 --- a/podpac/core/node.py +++ b/podpac/core/node.py @@ -187,6 +187,8 @@ def _cache_ctrl_default(self): def __init__(self, **kwargs): """Do not overwrite me""" + + self._base_ref = self.__class__.__name__ # Shortcut for users to make setting the cache_ctrl simpler: if "cache_ctrl" in kwargs and isinstance(kwargs["cache_ctrl"], list): @@ -477,7 +479,11 @@ def base_ref(self): str Name of the node in node definitions """ - return self.__class__.__name__ + return self._base_ref + + @base_ref.setter + def base_ref(self, value): + self._base_ref = value @property def _base_definition(self): From fdc0694464aa064f4a02f4c37fa1360e52c65f3e Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Wed, 5 Jun 2024 10:17:59 -0400 Subject: [PATCH 02/11] CODE: base_ref to traitlet --- podpac/core/algorithm/reprojection.py | 4 ++-- podpac/core/algorithm/stats.py | 8 ++++---- podpac/core/data/reprojection.py | 4 ++-- podpac/core/node.py | 13 +++++-------- 4 files changed, 13 insertions(+), 16 deletions(-) diff --git a/podpac/core/algorithm/reprojection.py b/podpac/core/algorithm/reprojection.py index 091c79e5..247a8d38 100644 --- a/podpac/core/algorithm/reprojection.py +++ b/podpac/core/algorithm/reprojection.py @@ -95,6 +95,6 @@ def _source_eval(self, coordinates, selector, output=None): else: return self.source.eval(coords, output=output, _selector=selector) - @property - def base_ref(self): + @tl.default('base_ref') + def _default_base_ref(self): return "{}_reprojected".format(self.source.base_ref) diff --git a/podpac/core/algorithm/stats.py b/podpac/core/algorithm/stats.py index 5cc8f970..2d3ee902 100644 --- a/podpac/core/algorithm/stats.py +++ b/podpac/core/algorithm/stats.py @@ -919,8 +919,8 @@ def _eval(self, coordinates, output=None, _selector=None): return output - @property - def base_ref(self): + @tl.default('base_ref') + def _default_base_ref(self): """ Default node reference/name in node definitions @@ -1011,8 +1011,8 @@ def _eval(self, coordinates, output=None, _selector=None): return output - @property - def base_ref(self): + @tl.default('base_ref') + def _default_base_ref(self): """ Default node reference/name in node definitions diff --git a/podpac/core/data/reprojection.py b/podpac/core/data/reprojection.py index 53f5b53a..0fd65bc9 100644 --- a/podpac/core/data/reprojection.py +++ b/podpac/core/data/reprojection.py @@ -102,6 +102,6 @@ def get_data(self, coordinates, coordinates_index): coordinates.drop(drop_dims) return data - @property - def base_ref(self): + @tl.default('base_ref') + def _default_base_ref(self): return "{}_reprojected".format(self.source.base_ref) diff --git a/podpac/core/node.py b/podpac/core/node.py index ee0423e9..1d8d7434 100644 --- a/podpac/core/node.py +++ b/podpac/core/node.py @@ -139,6 +139,8 @@ class Node(tl.HasTraits): cache_output = tl.Bool() force_eval = tl.Bool(False) cache_ctrl = tl.Instance(CacheCtrl, allow_none=True) + + base_ref = tl.Unicode() # list of attribute names, used by __repr__ and __str__ to display minimal info about the node # e.g. data sources use ['source'] @@ -188,7 +190,6 @@ def _cache_ctrl_default(self): def __init__(self, **kwargs): """Do not overwrite me""" - self._base_ref = self.__class__.__name__ # Shortcut for users to make setting the cache_ctrl simpler: if "cache_ctrl" in kwargs and isinstance(kwargs["cache_ctrl"], list): @@ -469,8 +470,8 @@ def probe(self, lat=None, lon=None, time=None, alt=None, crs=None): # Serialization # ----------------------------------------------------------------------------------------------------------------- - @property - def base_ref(self): + @tl.default('base_ref') + def _default_base_ref(self): """ Default reference/name in node definitions @@ -479,11 +480,7 @@ def base_ref(self): str Name of the node in node definitions """ - return self._base_ref - - @base_ref.setter - def base_ref(self, value): - self._base_ref = value + return self.__class__.__name__ @property def _base_definition(self): From 70e72259ad08a9ae8e89a5dbb777a459c6ee3593 Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Thu, 6 Jun 2024 11:14:33 -0400 Subject: [PATCH 03/11] FIX: Style_Class names --- podpac/core/node.py | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/podpac/core/node.py b/podpac/core/node.py index 1d8d7434..836515f8 100644 --- a/podpac/core/node.py +++ b/podpac/core/node.py @@ -534,6 +534,8 @@ def _base_definition(self): if inputs: d["inputs"] = inputs + if not type(self.style) is Style and isinstance(self.style, Style): + d["style_class"] = self.style.__class__.__module__ + "." + self.style.__class__.__name__ # style if self.style.definition: d["style"] = self.style.definition @@ -646,6 +648,8 @@ def hash(self): for k in d: if "style" in d[k]: del d[k]["style"] + if "style_class" in d[k]: + del d[k]["style_class"] s = json.dumps(d, separators=(",", ":"), cls=JSONEncoder) return hash_alg(s.encode("utf-8")).hexdigest() @@ -1313,7 +1317,26 @@ def _process_kwargs(name, d, definition, nodes): kwargs[k] = _lookup_attr(nodes, name, v) if "style" in d: - style_class = getattr(node_class, "style", Style) + if "style_class" in d: + style_root = module_root + # style_string = "%s.%s" % (style_root, d["style_class"]) + style_string = d["style_class"] + module_style_name, style_name = style_string.rsplit(".", 1) + + try: + style_module = importlib.import_module(module_style_name) + print("imported style_module {}".format(style_module)) + except ImportError: + raise ValueError("Invalid definition for style module '%s': no module found '%s'" % (name, module_style_name)) + try: + style_class = getattr(style_module, style_name) + print("getattr style_class {}".format(style_class)) + except AttributeError: + raise ValueError( + "Invalid definition for style '%s': style class '%s' not found in style module '%s'" % (name, style_name, module_style_name) + ) + else: + style_class = getattr(node_class, "style", Style) if isinstance(style_class, tl.TraitType): # Now we actually have to look through the class to see # if there is a custom initializer for style @@ -1336,11 +1359,13 @@ def _process_kwargs(name, d, definition, nodes): kwargs["style"] = Style.from_definition(d["style"]) # print ("couldn't make style from inferred style class", e) + for k in d: - if k not in ["node", "inputs", "attrs", "lookup_attrs", "plugin", "style"]: + if k not in ["node", "inputs", "attrs", "lookup_attrs", "plugin", "style", "style_class"]: raise ValueError("Invalid definition for node '%s': unexpected property '%s'" % (name, k)) nodes[name] = node_class(**kwargs) + print(nodes[name].style.__class__) # --------------------------------------------------------# From a212b663af62eb68da469ac83e50d8eb1d860203 Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Thu, 6 Jun 2024 11:14:57 -0400 Subject: [PATCH 04/11] DEL: Print Statement --- podpac/core/node.py | 1 - 1 file changed, 1 deletion(-) diff --git a/podpac/core/node.py b/podpac/core/node.py index 836515f8..337b8e32 100644 --- a/podpac/core/node.py +++ b/podpac/core/node.py @@ -1365,7 +1365,6 @@ def _process_kwargs(name, d, definition, nodes): raise ValueError("Invalid definition for node '%s': unexpected property '%s'" % (name, k)) nodes[name] = node_class(**kwargs) - print(nodes[name].style.__class__) # --------------------------------------------------------# From 5b253aa0f9bdde403b0e24769b2c31f7a4baa33a Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Mon, 10 Jun 2024 09:57:12 -0400 Subject: [PATCH 05/11] CODE: Remove print statements --- podpac/core/node.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/podpac/core/node.py b/podpac/core/node.py index 337b8e32..1e20f262 100644 --- a/podpac/core/node.py +++ b/podpac/core/node.py @@ -1325,12 +1325,10 @@ def _process_kwargs(name, d, definition, nodes): try: style_module = importlib.import_module(module_style_name) - print("imported style_module {}".format(style_module)) except ImportError: raise ValueError("Invalid definition for style module '%s': no module found '%s'" % (name, module_style_name)) try: style_class = getattr(style_module, style_name) - print("getattr style_class {}".format(style_class)) except AttributeError: raise ValueError( "Invalid definition for style '%s': style class '%s' not found in style module '%s'" % (name, style_name, module_style_name) @@ -1347,25 +1345,26 @@ def _process_kwargs(name, d, definition, nodes): try: style_class = atr(node_class) except Exception as e: - # print ("couldn't make style from class", e) try: style_class = atr(node_class()) except: - # print ("couldn't make style from class instance", e) style_class = style_class.klass try: kwargs["style"] = style_class.from_definition(d["style"]) except Exception as e: kwargs["style"] = Style.from_definition(d["style"]) - # print ("couldn't make style from inferred style class", e) for k in d: if k not in ["node", "inputs", "attrs", "lookup_attrs", "plugin", "style", "style_class"]: raise ValueError("Invalid definition for node '%s': unexpected property '%s'" % (name, k)) - nodes[name] = node_class(**kwargs) + for k in kwargs.keys(): + if not (hasattr(node_class, k) and isinstance(getattr(node_class, k), tl.TraitType)): + logging.warn("Node definition has key '{}' that will not be set at node creation: attribute is not of type tl.TraitType".format(k)) + nodes[name] = node_class(**kwargs) + # --------------------------------------------------------# # Mixins From 422ba82db630ff710d7f04d459e9b38e4ad0852e Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Wed, 19 Jun 2024 10:29:04 -0400 Subject: [PATCH 06/11] CODE: Parse coordinate objects in json --- podpac/core/node.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/podpac/core/node.py b/podpac/core/node.py index 1e20f262..961106b9 100644 --- a/podpac/core/node.py +++ b/podpac/core/node.py @@ -1308,7 +1308,10 @@ def _process_kwargs(name, d, definition, nodes): kwargs = {} for k, v in d.get("attrs", {}).items(): - kwargs[k] = v + if isinstance(getattr(node_class, k), tl.TraitType) and hasattr(getattr(node_class, k), "klass") and isinstance(v, OrderedDict) and getattr(node_class, k).klass == Coordinates: + kwargs[k] = Coordinates.from_definition(v) + else: + kwargs[k] = v for k, v in d.get("inputs", {}).items(): kwargs[k] = _lookup_input(nodes, name, v, definition) From ecbe801625e803d1704d9d001922db98decec790 Mon Sep 17 00:00:00 2001 From: David Hinckley Date: Tue, 11 Jun 2024 13:34:13 -0400 Subject: [PATCH 07/11] FIX: corrected issue where window did not fully contain evaluation coordinates --- podpac/core/data/rasterio_source.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/podpac/core/data/rasterio_source.py b/podpac/core/data/rasterio_source.py index f52c03e3..68c27a90 100644 --- a/podpac/core/data/rasterio_source.py +++ b/podpac/core/data/rasterio_source.py @@ -212,15 +212,11 @@ def get_data_overviews(self, coordinates, coordinates_index): try: # read data within coordinates_index window at the resolution of the overview # Rasterio will then automatically pull from the overview - window = ( - ((inds[0].min() // overview), int(np.ceil(inds[0].max() / overview) + 1)), - ((inds[1].min() // overview), int(np.ceil(inds[1].max() / overview) + 1)), - ) - slc = (slice(window[0][0], window[0][1], 1), slice(window[1][0], window[1][1], 1)) new_coords = Coordinates.from_geotransform( dataset.transform.to_gdal(), dataset.shape, crs=self.coordinates.crs ) - new_coords = new_coords[slc] + new_coords,slc = new_coords.intersect(coordinates,return_index=True,outer=True) + window = ((slc[0].start,slc[0].stop),(slc[1].start,slc[1].stop)) missing_coords = self.coordinates.drop(["lat", "lon"]) new_coords = merge_dims([new_coords, missing_coords]) new_coords = new_coords.transpose(*self.coordinates.dims) From 31b93056321bec26effe3eb0b378c4bea7ec11af Mon Sep 17 00:00:00 2001 From: David Hinckley Date: Tue, 25 Jun 2024 09:22:27 -0400 Subject: [PATCH 08/11] FIX: removed unused parameters --- podpac/core/data/rasterio_source.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/podpac/core/data/rasterio_source.py b/podpac/core/data/rasterio_source.py index 68c27a90..477af615 100644 --- a/podpac/core/data/rasterio_source.py +++ b/podpac/core/data/rasterio_source.py @@ -151,7 +151,7 @@ def get_coordinates(self): def get_data(self, coordinates, coordinates_index): """{get_data}""" if self.prefer_overviews: - return self.get_data_overviews(coordinates, coordinates_index) + return self.get_data_overviews(coordinates) data = self.create_output_array(coordinates) slc = coordinates_index @@ -169,7 +169,12 @@ def get_data(self, coordinates, coordinates_index): data.data.ravel()[:] = raster_data.ravel() return data - def get_data_overviews(self, coordinates, coordinates_index): + def _get_window_coords(self,coordinates,new_coords): + new_coords,slc = new_coords.intersect(coordinates,return_index=True,outer=True) + window = ((slc[0].start,slc[0].stop),(slc[1].start,slc[1].stop)) + return window,new_coords + + def get_data_overviews(self, coordinates): # Figure out how much coarser the request is than the actual data reduction_factor = np.inf for c in ["lat", "lon"]: @@ -204,7 +209,6 @@ def get_data_overviews(self, coordinates, coordinates_index): overview = self.overviews[np.argmin(diffs)] # Now read the data - inds = coordinates_index if overview_level is None: dataset = self.dataset else: @@ -215,8 +219,7 @@ def get_data_overviews(self, coordinates, coordinates_index): new_coords = Coordinates.from_geotransform( dataset.transform.to_gdal(), dataset.shape, crs=self.coordinates.crs ) - new_coords,slc = new_coords.intersect(coordinates,return_index=True,outer=True) - window = ((slc[0].start,slc[0].stop),(slc[1].start,slc[1].stop)) + window,new_coords = self._get_window_coords_slc(self,coordinates,new_coords) missing_coords = self.coordinates.drop(["lat", "lon"]) new_coords = merge_dims([new_coords, missing_coords]) new_coords = new_coords.transpose(*self.coordinates.dims) From e549e27f8eead7a05a2182e44df849513b4ea19e Mon Sep 17 00:00:00 2001 From: David Hinckley Date: Thu, 27 Jun 2024 10:29:32 -0400 Subject: [PATCH 09/11] FIX: added unit test for rasterio fix --- podpac/core/data/test/test_rasterio.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/podpac/core/data/test/test_rasterio.py b/podpac/core/data/test/test_rasterio.py index e770e132..282c0788 100644 --- a/podpac/core/data/test/test_rasterio.py +++ b/podpac/core/data/test/test_rasterio.py @@ -9,6 +9,7 @@ from podpac.core.coordinates import Coordinates from podpac.core.units import UnitsDataArray from podpac.core.data.rasterio_source import Rasterio +from podpac import clinspace class TestRasterio(object): @@ -74,3 +75,26 @@ def test_get_band_numbers(self): node = Rasterio(source=self.source) numbers = node.get_band_numbers("STATISTICS_MINIMUM", "0") np.testing.assert_array_equal(numbers, [1, 2, 3]) + + def test_get_window_coords(self): + """test get_window_coords method""" + c1 = Coordinates([clinspace(31,30,16,"lat"),clinspace(-0.25,1.5,64,"lon")]) + c2 = Coordinates([clinspace(30.75,30.25,16,"lat"),clinspace(-0.0,1.25,64,"lon")]) + + c3 = Coordinates([clinspace(31,30,16,"lat"),clinspace(-0.5,1.25,64,"lon")]) + + node = Rasterio() + window_1,new_coords_1 = node._get_window_coords(c2,c1) # tests when 1 coord completely contains the other + window_2,new_coords_2 = node._get_window_coords(c3,c1) # tests when 1 coord does not completely contian the other + + expected_values = {0:{'lon':46, + 'lat':10}, + 1:{'lon':55, + 'lat':16}} + for i,data in enumerate([new_coords_1,new_coords_2]) : + for a in ['lon','lat'] : + assert(np.isnan(data[a].coordinates).sum()==0) # nan check + assert(len(data[a])==expected_values[i][a]) # guard against old issue of return being trimmed + + + \ No newline at end of file From ff38c25e1f33dda474174b594ebd362a2f9f5f17 Mon Sep 17 00:00:00 2001 From: rparker Date: Fri, 28 Jun 2024 13:57:26 -0400 Subject: [PATCH 10/11] FIX: unit tests --- podpac/core/data/test/test_datasource.py | 4 ++-- podpac/core/test/test_node.py | 8 +------- podpac/core/utils.py | 4 ++-- setup.py | 2 +- 4 files changed, 6 insertions(+), 12 deletions(-) diff --git a/podpac/core/data/test/test_datasource.py b/podpac/core/data/test/test_datasource.py index ddd8ec1f..3b62f7d5 100644 --- a/podpac/core/data/test/test_datasource.py +++ b/podpac/core/data/test/test_datasource.py @@ -109,7 +109,7 @@ def get_coordinates(self): assert node.get_coordinates_called == 1 # can't set coordinates attribute - with pytest.raises(AttributeError, match="can't set attribute"): + with pytest.raises(AttributeError): node.coordinates = Coordinates([]) def test_dims(self): @@ -169,7 +169,7 @@ def get_coordinates(self): def test_set_coordinates(self): node = MockDataSource() - node.set_coordinates(Coordinates([])) + node.set_coordinates(Coordinates([]), force=True) assert node.coordinates == Coordinates([]) assert node.coordinates != node.get_coordinates() diff --git a/podpac/core/test/test_node.py b/podpac/core/test/test_node.py index 055c5972..1dc4a5c9 100644 --- a/podpac/core/test/test_node.py +++ b/podpac/core/test/test_node.py @@ -957,14 +957,8 @@ def test_from_url_with_plugin_style_params(self): r"&CRS=EPSG%3A3857&BBOX=-20037508.342789244,10018754.171394618,-10018754.171394622,20037508.34278071&" r'PARAMS={"plugin": "podpac.algorithm"}' ) - url1 = ( - r"https://mobility-devel.crearecomputing.com/geowatch?&SERVICE=WMS&REQUEST=GetMap&VERSION=1.3.0&" - r"LAYERS=datalib.terraintiles.TerrainTiles&STYLES=&FORMAT=image%2Fpng&TRANSPARENT=true&HEIGHT=256&WIDTH=256&" - r"TIME=2021-03-01T12%3A00%3A00.000Z&CRS=EPSG%3A3857&BBOX=-10018754.171394622,5009377.08569731,-9392582.035682458,5635549.221409475" - r'&PARAMS={"style": {"name": "Aspect (Composited 30-90 m)","units": "radians","colormap": "hsv","clim": [0,6.283185307179586]}}' - ) + node = Node.from_url(url0) - node = Node.from_url(url1) def test_from_name_params(self): # Normal diff --git a/podpac/core/utils.py b/podpac/core/utils.py index 82ded19f..3c451797 100644 --- a/podpac/core/utils.py +++ b/podpac/core/utils.py @@ -157,9 +157,9 @@ def __init__(self, ndim=None, shape=None, dtype=None, dtypes=None, default_value if ndim is not None and shape is not None and len(shape) != ndim: raise ValueError("Incompatible ndim and shape (ndim=%d, shape=%s)" % (ndim, shape)) if dtype is not None and not isinstance(dtype, type): - if dtype not in np.typeDict: + if dtype not in np.sctypeDict: raise ValueError("Unknown dtype '%s'" % dtype) - dtype = np.typeDict[dtype] + dtype = np.sctypeDict[dtype] self.ndim = ndim self.shape = shape self.dtype = dtype diff --git a/setup.py b/setup.py index 0db210fa..d1fa1733 100644 --- a/setup.py +++ b/setup.py @@ -48,7 +48,7 @@ ], "aws": ["awscli>=1.16", "boto3>=1.9.200", "s3fs>=0.4"], "algorithms": ["numexpr>=2.6"], - "datalib": ["podpacdatalib"], + # "datalib": ["podpacdatalib"], "notebook": [ "jupyterlab", "ipyleaflet", From 2dfe687aaedc6cac2ad822e579c0878072184cb0 Mon Sep 17 00:00:00 2001 From: rparker Date: Fri, 28 Jun 2024 14:16:02 -0400 Subject: [PATCH 11/11] MAINT: bumped version, updated changelog --- CHANGELOG.md | 10 ++++++++++ podpac/version.py | 4 ++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 44be66d9..39a6cdd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## 3.5.0 + +### Features +* Allows users to specify the name of a node when serializing to json #517 +* Allows users to specify custom style class in json node definition #517 + +### Bugfixes +* Matplotlib colormap deprecation fix #518 +* Fixed incorrect overview selection in rasterio #519 + ## 3.4.1 Point Probe Value Format for Enumerated Legends HOTFIX ### Hotfix diff --git a/podpac/version.py b/podpac/version.py index 050a0bc1..73840495 100644 --- a/podpac/version.py +++ b/podpac/version.py @@ -16,8 +16,8 @@ ## UPDATE VERSION HERE ############## MAJOR = 3 -MINOR = 4 -HOTFIX = 1 +MINOR = 5 +HOTFIX = 0 ##############