diff --git a/podpac/core/data/datasource.py b/podpac/core/data/datasource.py index d07d130ef..db0142103 100644 --- a/podpac/core/data/datasource.py +++ b/podpac/core/data/datasource.py @@ -152,8 +152,8 @@ class DataSource(Node): Custom DataSource Nodes must implement the :meth:`get_data` and :meth:`get_native_coordinates` methods. """ - source = tl.Any() - native_coordinates = tl.Instance(Coordinates) + source = tl.Any().tag(readonly=True) + native_coordinates = tl.Instance(Coordinates).tag(readonly=True) interpolation = interpolation_trait() coordinate_index_type = tl.Enum(["list", "numpy", "xarray", "pandas"], default_value="numpy") nan_vals = tl.List(allow_none=True) @@ -170,8 +170,7 @@ class DataSource(Node): # when native_coordinates is not defined, default calls get_native_coordinates @tl.default("native_coordinates") def _default_native_coordinates(self): - self.native_coordinates = self.get_native_coordinates() - return self.native_coordinates + return self.get_native_coordinates() # this adds a more helpful error message if user happens to try an inspect _interpolation before evaluate @tl.default("_interpolation") diff --git a/podpac/core/data/test/test_interpolate.py b/podpac/core/data/test/test_interpolate.py index 1250e63ef..5249fedae 100644 --- a/podpac/core/data/test/test_interpolate.py +++ b/podpac/core/data/test/test_interpolate.py @@ -167,7 +167,8 @@ def test_init_interpolators(self): Interpolation({"default": {"method": "nearest", "params": {"spatial_tolerance": "tol"}}}) # should not allow undefined params - interp = Interpolation({"default": {"method": "nearest", "params": {"myarg": 1}}}) + with pytest.warns(DeprecationWarning): # eventually, Traitlets will raise an exception here + interp = Interpolation({"default": {"method": "nearest", "params": {"myarg": 1}}}) with pytest.raises(AttributeError): assert interp.config[("default",)]["interpolators"][0].myarg == "tol" diff --git a/podpac/core/data/test/test_types.py b/podpac/core/data/test/test_types.py index 850791b89..fc3031b48 100644 --- a/podpac/core/data/test/test_types.py +++ b/podpac/core/data/test/test_types.py @@ -172,29 +172,10 @@ def test_auth_session(self): assert node.auth_session is None def test_dataset(self): - """test dataset attribute and traitlet default """ + """test dataset trait """ self.mock_pydap() node = PyDAP(source=self.source, datakey=self.datakey) - - # override/reset source on dataset opening - node._open_dataset(source="newsource") - assert node.source == "newsource" - assert isinstance(node.dataset, pydap.model.DatasetType) - - def test_source(self): - """test source attribute and trailet observer """ - self.mock_pydap() - - node = PyDAP(source=self.source, datakey=self.datakey, native_coordinates=self.coordinates) - - # observe source - node._update_dataset(change={"old": None}) - assert node.source == self.source - - output = node._update_dataset(change={"new": "newsource", "old": "oldsource"}) - assert node.source == "newsource" - assert node.native_coordinates == self.coordinates assert isinstance(node.dataset, pydap.model.DatasetType) def test_get_data(self): @@ -298,13 +279,6 @@ def test_dataset(self): RasterReader = rasterio.io.DatasetReader # Rasterio >= v1.0 assert isinstance(node.dataset, RasterReader) - # update source when asked - with pytest.raises(rasterio.errors.RasterioIOError): - node.source = "assets/not-tiff" - node._open_dataset() - - assert node.source == "assets/not-tiff" - node.close_dataset() def test_default_native_coordinates(self): @@ -349,19 +323,6 @@ def test_get_band_numbers(self): assert isinstance(numbers, np.ndarray) np.testing.assert_array_equal(numbers, np.arange(3) + 1) - def test_source(self): - """test source attribute and trailets observe""" - - node = Rasterio(source=self.source) - assert node.source == self.source - - def test_change_source(self): - node = Rasterio(source=self.source) - assert node.band_count == 3 - - node.source = self.source.replace("RGB.byte.tif", "h5raster.hdf5") - assert node.band_count == 1 - class TestH5PY(object): source = os.path.join(os.path.dirname(__file__), "assets/h5raster.hdf5") diff --git a/podpac/core/data/types.py b/podpac/core/data/types.py index 0a425eb94..c9cd87b52 100644 --- a/podpac/core/data/types.py +++ b/podpac/core/data/types.py @@ -74,7 +74,7 @@ class Array(DataSource): `native_coordinates` need to supplied by the user when instantiating this node. """ - source = ArrayTrait() + source = ArrayTrait().tag(readonly=True) @tl.validate("source") def _validate_source(self, d): @@ -123,16 +123,17 @@ class PyDAP(DataSource): auth_session instead if you have security concerns. """ - # required inputs - source = tl.Unicode(default_value="") + source = tl.Unicode().tag(readonly=True) + dataset = tl.Instance("pydap.model.DatasetType").tag(readonly=True) + + # node attrs datakey = tl.Unicode().tag(attr=True) - # optional inputs and later defined traits - auth_session = tl.Instance(authentication.Session, allow_none=True) + # optional inputs auth_class = tl.Type(authentication.Session) - username = tl.Unicode(None, allow_none=True) - password = tl.Unicode(None, allow_none=True) - dataset = tl.Instance("pydap.model.DatasetType") + auth_session = tl.Instance(authentication.Session, allow_none=True) + username = tl.Unicode(default_value=None, allow_none=True) + password = tl.Unicode(default_value=None, allow_none=True) @tl.default("auth_session") def _auth_session_default(self): @@ -157,7 +158,7 @@ def _auth_session_default(self): return session @tl.default("dataset") - def _open_dataset(self, source=None): + def _open_dataset(self): """Summary Parameters @@ -170,23 +171,17 @@ def _open_dataset(self, source=None): TYPE Description """ - # TODO: is source ever None? - # TODO: enforce string source - if source is None: - source = self.source - else: - self.source = source # auth session # if self.auth_session: try: - dataset = pydap.client.open_url(source, session=self.auth_session) + dataset = self._open_url() except Exception: # TODO handle a 403 error # TODO: Check Url (probably inefficient...) try: self.auth_session.get(self.source + ".dds") - dataset = pydap.client.open_url(source, session=self.auth_session) + dataset = self._open_url() except Exception: # TODO: handle 403 error print ("Warning, dataset could not be opened. Check login credentials.") @@ -194,22 +189,8 @@ def _open_dataset(self, source=None): return dataset - @tl.observe("source") - def _update_dataset(self, change=None): - if change is None: - return - - if change["old"] == None or change["old"] == "": - return - - if self.dataset is not None and "new" in change: - self.dataset = self._open_dataset(source=change["new"]) - - try: - if self.native_coordinates is not None: - self.native_coordinates = self.get_native_coordinates() - except NotImplementedError: - pass + def _open_url(self): + return pydap.client.open_url(self.source, session=self.auth_session) @common_doc(COMMON_DATA_DOC) def get_native_coordinates(self): @@ -279,14 +260,16 @@ class CSV(DataSource): Raw Pandas DataFrame used to read the data """ - source = tl.Unicode() + source = tl.Unicode().tag(readonly=True) + dataset = tl.Instance(pd.DataFrame).tag(readonly=True) + + # node attrs + dims = tl.List(default_value=["alt", "lat", "lon", "time"]).tag(attr=True) alt_col = tl.Union([tl.Unicode(), tl.Int()]).tag(attr=True) lat_col = tl.Union([tl.Unicode(), tl.Int()]).tag(attr=True) lon_col = tl.Union([tl.Unicode(), tl.Int()]).tag(attr=True) time_col = tl.Union([tl.Unicode(), tl.Int()]).tag(attr=True) data_col = tl.Union([tl.Unicode(), tl.Int()]).tag(attr=True) - dims = tl.List(default_value=["alt", "lat", "lon", "time"]).tag(attr=True) - dataset = tl.Instance(pd.DataFrame) def _first_init(self, **kwargs): # First part of if tests to make sure this is the CSV parent class @@ -397,9 +380,10 @@ class Rasterio(DataSource): * Linux: export CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt """ - source = tl.Union([tl.Unicode(), tl.Instance(BytesIO)]) + source = tl.Union([tl.Unicode(), tl.Instance(BytesIO)]).tag(readonly=True) + dataset = tl.Any().tag(readonly=True) - dataset = tl.Any(allow_none=True) + # node attrs band = tl.CInt(1).tag(attr=True) @tl.default("dataset") @@ -431,25 +415,6 @@ def close_dataset(self): """ self.dataset.close() - @tl.observe("source") - def _update_dataset(self, change): - if hasattr(self, "_band_count"): - delattr(self, "_band_count") - - if hasattr(self, "_band_descriptions"): - delattr(self, "_band_descriptions") - - if hasattr(self, "_band_keys"): - delattr(self, "_band_keys") - - # only update dataset if dataset trait has been defined the first time - if trait_is_defined(self, "dataset"): - self.dataset = self._open_dataset() - - # update native_coordinates if they have been defined - if trait_is_defined(self, "native_coordinates"): - self.native_coordinates = self.get_native_coordinates() - @common_doc(COMMON_DATA_DOC) def get_native_coordinates(self): """{get_native_coordinates} @@ -672,13 +637,15 @@ class H5PY(DatasetCoordinatedMixin, DataSource): Default is 'r'. The mode used to open the HDF5 file. Options are r, r+, w, w- or x, a (see h5py.File). """ - source = tl.Unicode() - dataset = tl.Any(allow_none=True) + source = tl.Unicode().tag(readonly=True) + dataset = tl.Any().tag(readonly=True) + file_mode = tl.Unicode(default_value="r").tag(readonly=True) + + # node attrs datakey = tl.Unicode().tag(attr=True) - file_mode = tl.Unicode(default_value="r") @tl.default("dataset") - def _open_dataset(self, source=None): + def _open_dataset(self): """Opens the data source Parameters @@ -691,30 +658,16 @@ def _open_dataset(self, source=None): Any raster.open(source) """ - # TODO: update this to remove block (see Rasterio) - if source is None: - source = self.source - else: - self.source = source # TODO: dataset should not open by default # prefer with as: syntax - return h5py.File(source, self.file_mode) + return h5py.File(self.source, self.file_mode) def close_dataset(self): """Closes the file for the datasource """ self.dataset.close() - @tl.observe("source") - def _update_dataset(self, change): - # TODO: update this to look like Rasterio - if self.dataset is not None: - self.close_dataset() - self.dataset = self._open_dataset(change["new"]) - if trait_is_defined(self, "native_coordinates"): - self.native_coordinates = self.get_native_coordinates() - @common_doc(COMMON_DATA_DOC) def get_data(self, coordinates, coordinates_index): """{get_data} @@ -749,10 +702,13 @@ def _find_h5py_keys(obj, keys=[]): class Zarr(DatasetCoordinatedMixin, DataSource): - source = tl.Unicode(allow_none=True) - dataset = tl.Any() + source = tl.Unicode(default_value=None, allow_none=True).tag(readonly=True) + dataset = tl.Any().tag(readonly=True) + + # node attrs datakey = tl.Unicode().tag(attr=True) + # optional inputs access_key_id = tl.Unicode() secret_access_key = tl.Unicode() region_name = tl.Unicode() @@ -852,11 +808,13 @@ class WCS(DataSource): The coordinates of the WCS source """ - source = tl.Unicode() + source = tl.Unicode().tag(readonly=True) + wcs_coordinates = tl.Instance(Coordinates).tag(readonly=True) # default below + + # node attrs layer_name = tl.Unicode().tag(attr=True) version = tl.Unicode(WCS_DEFAULT_VERSION).tag(attr=True) crs = tl.Unicode(WCS_DEFAULT_CRS).tag(attr=True) - wcs_coordinates = tl.Instance(Coordinates) # default below _get_capabilities_qs = tl.Unicode("SERVICE=WCS&REQUEST=DescribeCoverage&" "VERSION={version}&COVERAGE={layer}") _get_data_qs = tl.Unicode( @@ -1188,7 +1146,9 @@ class ReprojectedSource(DataSource): Coordinates where the source node should be evaluated. """ - source = NodeTrait() + source = NodeTrait().tag(readonly=True) + + # node attrs source_interpolation = interpolation_trait().tag(attr=True) reprojected_coordinates = tl.Instance(Coordinates).tag(attr=True) @@ -1270,9 +1230,12 @@ class Dataset(DataSource): For example, if the data contains ['lat', 'lon', 'channel'], the second channel can be selected using `extra_dim=dict(channel=1)` """ + source = tl.Any(allow_none=True).tag(readonly=True) + dataset = tl.Instance(xr.Dataset).tag(readonly=True) + + # node attrs extra_dim = tl.Dict({}).tag(attr=True) datakey = tl.Unicode().tag(attr=True) - dataset = tl.Instance(xr.Dataset) @tl.default("dataset") def _dataset_default(self): diff --git a/podpac/core/node.py b/podpac/core/node.py index 637fc0111..6390bc602 100644 --- a/podpac/core/node.py +++ b/podpac/core/node.py @@ -139,8 +139,18 @@ def _validate_units(self, d): def __init__(self, **kwargs): """ Do not overwrite me """ + tkwargs = self._first_init(**kwargs) + # make tagged "readonly" and "attr" traits read_only, and set them using set_trait + # NOTE: The set_trait is required because this sets the traits read_only at the *class* level; + # on subsequent initializations, they will already be read_only. + for name, trait in self.traits().items(): + if trait.metadata.get("readonly") or trait.metadata.get("attr"): + if name in tkwargs: + self.set_trait(name, tkwargs.pop(name)) + trait.read_only = True + # Call traitlest constructor super(Node, self).__init__(**tkwargs) self.init() diff --git a/podpac/core/test/test_node.py b/podpac/core/test/test_node.py index 0d17c3111..dfcdce9f2 100644 --- a/podpac/core/test/test_node.py +++ b/podpac/core/test/test_node.py @@ -314,9 +314,9 @@ def d2(self): assert t2.c2() == 2 assert t2.d2() == 2 - t.a = 2 + t.set_trait("a", 2) assert t.a2() == 4 - t.b = 2 + t.set_trait("b", 2) assert t.b2() == 4 # This happens because the node definition changed t.rem_cache(key="*", coordinates="*") assert t.c2() == 2 # This forces the cache to update based on the new node definition @@ -332,9 +332,9 @@ def d2(self): assert t2.c2() == 2 assert t2.d2() == 2 - t2.a = 2 + t2.set_trait("a", 2) assert t2.get_cache("a2") == 4 # This was cached by t - t2.b = 2 + t2.set_trait("b", 2) assert t2.get_cache("c2") == 4 # This was cached by t assert t2.get_cache("d2") == 2 # This was cached by t @@ -385,9 +385,9 @@ def d2(self): assert t2.c2() == 2 assert t2.d2() == 2 - t.a = 2 + t.set_trait("a", 2) assert t.a2() == 4 - t.b = 2 + t.set_trait("b", 2) assert t.b2() == 4 # This happens because the node definition changed t.rem_cache(key="*", coordinates="*") assert t.c2() == 2 # This forces the cache to update based on the new node definition diff --git a/podpac/datalib/gfs.py b/podpac/datalib/gfs.py index f6fa3d00a..4b49c4ef5 100644 --- a/podpac/datalib/gfs.py +++ b/podpac/datalib/gfs.py @@ -31,7 +31,6 @@ class GFSSource(Rasterio): date = tl.Unicode().tag(attr=True) hour = tl.Unicode().tag(attr=True) forecast = tl.Unicode().tag(attr=True) - dataset = tl.Any() def init(self): self._logger = logging.getLogger(__name__) diff --git a/podpac/datalib/intake.py b/podpac/datalib/intake.py index 723caff7a..0be222f4b 100644 --- a/podpac/datalib/intake.py +++ b/podpac/datalib/intake.py @@ -57,8 +57,8 @@ class IntakeCatalog(podpac.data.DataSource): """ # input parameters + source = tl.Unicode().tag(readonly=True) uri = tl.Unicode() - source = tl.Unicode() # optional input parameters field = tl.Unicode(default_value=None, allow_none=True) diff --git a/podpac/datalib/smap.py b/podpac/datalib/smap.py index 20cf30c4e..572c8eab1 100644 --- a/podpac/datalib/smap.py +++ b/podpac/datalib/smap.py @@ -445,7 +445,7 @@ class SMAPProperties(SMAPSource): file_url_re = re.compile(r"SMAP.*_[0-9]{8}T[0-9]{6}_.*\.h5") - source = tl.Unicode() + source = tl.Unicode().tag(readonly=True) @tl.default("source") def _property_source_default(self): diff --git a/podpac/datalib/terraintiles.py b/podpac/datalib/terraintiles.py index 9321bcbb2..6fe53b21f 100644 --- a/podpac/datalib/terraintiles.py +++ b/podpac/datalib/terraintiles.py @@ -76,14 +76,12 @@ class TerrainTilesSource(Rasterio): """ # parameters - source = tl.Unicode() + source = tl.Unicode().tag(readonly=True) # attributes interpolation = interpolation_trait( default_value={"method": "nearest", "interpolators": [RasterioInterpolator, ScipyGrid, ScipyPoint]} - ) - - dataset = tl.Any() + ).tag(readonly=True) @tl.default("dataset") def open_dataset(self):