Merge branch 'release/3.5.0' into main-3.X

CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## 3.5.0
+
+### Features
+* Allows users to specify the name of a node when serializing to json #517
+* Allows users to specify custom style class in json node definition #517
+
+### Bugfixes
+* Matplotlib colormap deprecation fix #518
+* Fixed incorrect overview selection in rasterio #519
+
 ## 3.4.1 Point Probe Value Format for Enumerated Legends HOTFIX
 
 ### Hotfix

podpac/core/algorithm/reprojection.py

@@ -95,6 +95,6 @@ def _source_eval(self, coordinates, selector, output=None):
         else:
             return self.source.eval(coords, output=output, _selector=selector)
 
-    @property
-    def base_ref(self):
+    @tl.default('base_ref')
+    def _default_base_ref(self):
         return "{}_reprojected".format(self.source.base_ref)

podpac/core/algorithm/stats.py

@@ -919,8 +919,8 @@ def _eval(self, coordinates, output=None, _selector=None):
 
         return output
 
-    @property
-    def base_ref(self):
+    @tl.default('base_ref')
+    def _default_base_ref(self):
         """
         Default node reference/name in node definitions
 
@@ -1011,8 +1011,8 @@ def _eval(self, coordinates, output=None, _selector=None):
 
         return output
 
-    @property
-    def base_ref(self):
+    @tl.default('base_ref')
+    def _default_base_ref(self):
         """
         Default node reference/name in node definitions
 

podpac/core/data/rasterio_source.py

@@ -151,7 +151,7 @@ def get_coordinates(self):
     def get_data(self, coordinates, coordinates_index):
         """{get_data}"""
         if self.prefer_overviews:
-            return self.get_data_overviews(coordinates, coordinates_index)
+            return self.get_data_overviews(coordinates)
 
         data = self.create_output_array(coordinates)
         slc = coordinates_index
@@ -169,7 +169,12 @@ def get_data(self, coordinates, coordinates_index):
         data.data.ravel()[:] = raster_data.ravel()
         return data
 
-    def get_data_overviews(self, coordinates, coordinates_index):
+    def _get_window_coords(self,coordinates,new_coords):
+        new_coords,slc = new_coords.intersect(coordinates,return_index=True,outer=True)
+        window = ((slc[0].start,slc[0].stop),(slc[1].start,slc[1].stop))
+        return window,new_coords
+
+    def get_data_overviews(self, coordinates):
         # Figure out how much coarser the request is than the actual data
         reduction_factor = np.inf
         for c in ["lat", "lon"]:
@@ -204,23 +209,17 @@ def get_data_overviews(self, coordinates, coordinates_index):
             overview = self.overviews[np.argmin(diffs)]
 
         # Now read the data
-        inds = coordinates_index
         if overview_level is None:
             dataset = self.dataset
         else:
             dataset = self.open_dataset(self.source, overview_level)
         try:
             # read data within coordinates_index window at the resolution of the overview
             # Rasterio will then automatically pull from the overview
-            window = (
-                ((inds[0].min() // overview), int(np.ceil(inds[0].max() / overview) + 1)),
-                ((inds[1].min() // overview), int(np.ceil(inds[1].max() / overview) + 1)),
-            )
-            slc = (slice(window[0][0], window[0][1], 1), slice(window[1][0], window[1][1], 1))
             new_coords = Coordinates.from_geotransform(
                 dataset.transform.to_gdal(), dataset.shape, crs=self.coordinates.crs
             )
-            new_coords = new_coords[slc]
+            window,new_coords = self._get_window_coords_slc(self,coordinates,new_coords)
             missing_coords = self.coordinates.drop(["lat", "lon"])
             new_coords = merge_dims([new_coords, missing_coords])
             new_coords = new_coords.transpose(*self.coordinates.dims)

podpac/core/data/reprojection.py

@@ -102,6 +102,6 @@ def get_data(self, coordinates, coordinates_index):
         coordinates.drop(drop_dims)
         return data
 
-    @property
-    def base_ref(self):
+    @tl.default('base_ref')
+    def _default_base_ref(self):
         return "{}_reprojected".format(self.source.base_ref)

podpac/core/data/test/test_datasource.py

@@ -109,7 +109,7 @@ def get_coordinates(self):
         assert node.get_coordinates_called == 1
 
         # can't set coordinates attribute
-        with pytest.raises(AttributeError, match="can't set attribute"):
+        with pytest.raises(AttributeError):
             node.coordinates = Coordinates([])
 
     def test_dims(self):
@@ -169,7 +169,7 @@ def get_coordinates(self):
 
     def test_set_coordinates(self):
         node = MockDataSource()
-        node.set_coordinates(Coordinates([]))
+        node.set_coordinates(Coordinates([]), force=True)
         assert node.coordinates == Coordinates([])
         assert node.coordinates != node.get_coordinates()
 

podpac/core/data/test/test_rasterio.py

@@ -9,6 +9,7 @@
 from podpac.core.coordinates import Coordinates
 from podpac.core.units import UnitsDataArray
 from podpac.core.data.rasterio_source import Rasterio
+from podpac import clinspace
 
 
 class TestRasterio(object):
@@ -74,3 +75,26 @@ def test_get_band_numbers(self):
         node = Rasterio(source=self.source)
         numbers = node.get_band_numbers("STATISTICS_MINIMUM", "0")
         np.testing.assert_array_equal(numbers, [1, 2, 3])
+
+    def test_get_window_coords(self):
+        """test get_window_coords method"""
+        c1 = Coordinates([clinspace(31,30,16,"lat"),clinspace(-0.25,1.5,64,"lon")])
+        c2 = Coordinates([clinspace(30.75,30.25,16,"lat"),clinspace(-0.0,1.25,64,"lon")])
+
+        c3 = Coordinates([clinspace(31,30,16,"lat"),clinspace(-0.5,1.25,64,"lon")])
+
+        node = Rasterio()
+        window_1,new_coords_1 = node._get_window_coords(c2,c1) # tests when 1 coord completely contains the other
+        window_2,new_coords_2 = node._get_window_coords(c3,c1) # tests when 1 coord does not completely contian the other
+
+        expected_values = {0:{'lon':46,
+                              'lat':10},
+                           1:{'lon':55,
+                              'lat':16}}
+        for i,data in enumerate([new_coords_1,new_coords_2]) :
+            for a in ['lon','lat'] :
+                assert(np.isnan(data[a].coordinates).sum()==0) # nan check
+                assert(len(data[a])==expected_values[i][a]) # guard against old issue of return being trimmed
+
+
+

podpac/core/node.py

@@ -139,6 +139,8 @@ class Node(tl.HasTraits):
     cache_output = tl.Bool()
     force_eval = tl.Bool(False)
     cache_ctrl = tl.Instance(CacheCtrl, allow_none=True)
+
+    base_ref = tl.Unicode()
 
     # list of attribute names, used by __repr__ and __str__ to display minimal info about the node
     # e.g. data sources use ['source']
@@ -187,6 +189,7 @@ def _cache_ctrl_default(self):
 
     def __init__(self, **kwargs):
         """Do not overwrite me"""
+
 
         # Shortcut for users to make setting the cache_ctrl simpler:
         if "cache_ctrl" in kwargs and isinstance(kwargs["cache_ctrl"], list):
@@ -467,8 +470,8 @@ def probe(self, lat=None, lon=None, time=None, alt=None, crs=None):
     # Serialization
     # -----------------------------------------------------------------------------------------------------------------
 
-    @property
-    def base_ref(self):
+    @tl.default('base_ref')
+    def _default_base_ref(self):
         """
         Default reference/name in node definitions
 
@@ -477,7 +480,7 @@ def base_ref(self):
         str
             Name of the node in node definitions
         """
-        return self.__class__.__name__
+        return self.__class__.__name__ 
 
     @property
     def _base_definition(self):
@@ -531,6 +534,8 @@ def _base_definition(self):
         if inputs:
             d["inputs"] = inputs
 
+        if not type(self.style) is Style and isinstance(self.style, Style):
+            d["style_class"] = self.style.__class__.__module__ + "." + self.style.__class__.__name__
         # style
         if self.style.definition:
             d["style"] = self.style.definition
@@ -643,6 +648,8 @@ def hash(self):
         for k in d:
             if "style" in d[k]:
                 del d[k]["style"]
+            if "style_class" in d[k]:
+                del d[k]["style_class"]
 
         s = json.dumps(d, separators=(",", ":"), cls=JSONEncoder)
         return hash_alg(s.encode("utf-8")).hexdigest()
@@ -1301,7 +1308,10 @@ def _process_kwargs(name, d, definition, nodes):
 
     kwargs = {}
     for k, v in d.get("attrs", {}).items():
-        kwargs[k] = v
+        if isinstance(getattr(node_class, k), tl.TraitType) and hasattr(getattr(node_class, k), "klass") and isinstance(v, OrderedDict) and  getattr(node_class, k).klass == Coordinates:
+            kwargs[k] = Coordinates.from_definition(v)
+        else:
+            kwargs[k] = v
 
     for k, v in d.get("inputs", {}).items():
         kwargs[k] = _lookup_input(nodes, name, v, definition)
@@ -1310,7 +1320,24 @@ def _process_kwargs(name, d, definition, nodes):
         kwargs[k] = _lookup_attr(nodes, name, v)
 
     if "style" in d:
-        style_class = getattr(node_class, "style", Style)
+        if "style_class" in d:
+            style_root = module_root
+            # style_string  = "%s.%s" % (style_root, d["style_class"])
+            style_string  = d["style_class"]
+            module_style_name, style_name = style_string.rsplit(".", 1)
+
+            try:
+                style_module = importlib.import_module(module_style_name)
+            except ImportError:
+                raise ValueError("Invalid definition for style module '%s': no module found '%s'" % (name, module_style_name))
+            try:
+                style_class = getattr(style_module, style_name)
+            except AttributeError:
+                raise ValueError(
+                    "Invalid definition for style '%s': style class '%s' not found in style module '%s'" % (name, style_name, module_style_name)
+                )
+        else:  
+            style_class = getattr(node_class, "style", Style)
         if isinstance(style_class, tl.TraitType):
             # Now we actually have to look through the class to see
             # if there is a custom initializer for style
@@ -1321,24 +1348,26 @@ def _process_kwargs(name, d, definition, nodes):
                 try:
                     style_class = atr(node_class)
                 except Exception as e:
-                    # print ("couldn't make style from class", e)
                     try:
                         style_class = atr(node_class())
                     except:
-                        # print ("couldn't make style from class instance", e)
                         style_class = style_class.klass
         try:
             kwargs["style"] = style_class.from_definition(d["style"])
         except Exception as e:
             kwargs["style"] = Style.from_definition(d["style"])
-            # print ("couldn't make style from inferred style class", e)
+
 
     for k in d:
-        if k not in ["node", "inputs", "attrs", "lookup_attrs", "plugin", "style"]:
+        if k not in ["node", "inputs", "attrs", "lookup_attrs", "plugin", "style", "style_class"]:
             raise ValueError("Invalid definition for node '%s': unexpected property '%s'" % (name, k))
 
-    nodes[name] = node_class(**kwargs)
+    for k in kwargs.keys():
+        if not (hasattr(node_class, k) and isinstance(getattr(node_class, k), tl.TraitType)):
+            logging.warn("Node definition has key '{}' that will not be set at node creation: attribute is not of type tl.TraitType".format(k))
 
+    nodes[name] = node_class(**kwargs)
+
 
 # --------------------------------------------------------#
 #  Mixins

podpac/core/test/test_node.py

@@ -957,14 +957,8 @@ def test_from_url_with_plugin_style_params(self):
             r"&CRS=EPSG%3A3857&BBOX=-20037508.342789244,10018754.171394618,-10018754.171394622,20037508.34278071&"
             r'PARAMS={"plugin": "podpac.algorithm"}'
         )
-        url1 = (
-            r"https://mobility-devel.crearecomputing.com/geowatch?&SERVICE=WMS&REQUEST=GetMap&VERSION=1.3.0&"
-            r"LAYERS=datalib.terraintiles.TerrainTiles&STYLES=&FORMAT=image%2Fpng&TRANSPARENT=true&HEIGHT=256&WIDTH=256&"
-            r"TIME=2021-03-01T12%3A00%3A00.000Z&CRS=EPSG%3A3857&BBOX=-10018754.171394622,5009377.08569731,-9392582.035682458,5635549.221409475"
-            r'&PARAMS={"style": {"name": "Aspect (Composited 30-90 m)","units": "radians","colormap": "hsv","clim": [0,6.283185307179586]}}'
-        )
+
         node = Node.from_url(url0)
-        node = Node.from_url(url1)
 
     def test_from_name_params(self):
         # Normal

podpac/core/utils.py

@@ -157,9 +157,9 @@ def __init__(self, ndim=None, shape=None, dtype=None, dtypes=None, default_value
         if ndim is not None and shape is not None and len(shape) != ndim:
             raise ValueError("Incompatible ndim and shape (ndim=%d, shape=%s)" % (ndim, shape))
         if dtype is not None and not isinstance(dtype, type):
-            if dtype not in np.typeDict:
+            if dtype not in np.sctypeDict:
                 raise ValueError("Unknown dtype '%s'" % dtype)
-            dtype = np.typeDict[dtype]
+            dtype = np.sctypeDict[dtype]
         self.ndim = ndim
         self.shape = shape
         self.dtype = dtype

podpac/version.py

@@ -16,8 +16,8 @@
 ## UPDATE VERSION HERE
 ##############
 MAJOR = 3
-MINOR = 4
-HOTFIX = 1
+MINOR = 5
+HOTFIX = 0
 ##############
 
 

setup.py

@@ -48,7 +48,7 @@
     ],
     "aws": ["awscli>=1.16", "boto3>=1.9.200", "s3fs>=0.4"],
     "algorithms": ["numexpr>=2.6"],
-    "datalib": ["podpacdatalib"],
+    # "datalib": ["podpacdatalib"],
     "notebook": [
         "jupyterlab",
         "ipyleaflet",