973 deprecate dfmtopen dataset extra (#974)

* deprecated open_dataset_extra and renamed to open_prepare_dataset

* updated whatsnew

dfm_tools/deprecated.py

@@ -1,56 +1,32 @@
 def get_ncmodeldata(file_nc=None, varname=None, timestep=None, layer=None, depth=None, station=None, multipart=None, get_linkedgridinfo=False, silent=False):
-    """
-    deprecated
-    """
     raise DeprecationWarning('dfmt.get_ncmodeldata() is deprecated, use `uds = dfmt.open_partitioned_dataset(file_nc)` instead: https://github.com/Deltares/dfm_tools/blob/main/notebooks/postprocessing_example.ipynb')
 
 
 def get_netdata(file_nc=None, multipart=None):
-    """
-    deprecated
-    """
     raise DeprecationWarning('dfmt.get_netdata() is deprecated, use `uds = dfmt.open_partitioned_dataset(file_nc)` instead: https://github.com/Deltares/dfm_tools/blob/main/notebooks/postprocessing_example.ipynb')
 
 
 def plot_netmapdata(verts=None, values=None, ax=None, **kwargs):
-    """
-    deprecated
-    """
     raise DeprecationWarning('dfmt.plot_netmapdata() is deprecated, use `uds = dfmt.open_partitioned_dataset(file_nc); uds.mesh2d_s1.isel(time=0).ugrid.plot()` instead: https://github.com/Deltares/dfm_tools/blob/main/notebooks/postprocessing_example.ipynb')
 
 
 def plot_background(ax=None, projection=None, google_style='satellite', resolution=1, features=None, nticks=6, latlon_format=False, gridlines=False, **kwargs):
-    """
-    deprecated
-    """
     raise DeprecationWarning('dfmt.plot_background() is deprecated, use contextily instead')
 
 
 def get_ugrid_verts(data_xr_map=None):
-    """
-    deprecated
-    """
     raise DeprecationWarning('dfmt.get_ugrid_verts() is deprecated, use uds.grid.face_node_coordinates instead (https://github.com/Deltares/xugrid/issues/48)')
 
 
 def scatter_to_regulargrid(xcoords=None, ycoords=None, values=None, ncellx=None, ncelly=None, reg_x_vec=None, reg_y_vec=None, method='nearest', maskland_dist=None):
-    """
-    deprecated
-    """
     raise DeprecationWarning('dfm_tools.regulargrid.scatter_to_regulargrid() is deprecated, use ds = dfmt.rasterize_ugrid(uds) instead')
 
 
 def get_varnamefromattrs(data_xr=None, varname=None):
-    """
-    deprecated
-    """
     raise DeprecationWarning('dfmt.get_varnamefromattrs() will be deprecated in a future version of dfm_tools, ds=dfmt.rename_waqvars(ds) is a more convenient alternative')
 
 
 class Polygon:
-    """
-    deprecated
-    """
     def __init__(self, data=None, name=None, comments=None):
         raise DeprecationWarning('the function dfm_tools.polygon.Polygon() is deprecated, please use the new hydrolib alternative.')
 
@@ -59,51 +35,33 @@ def fromfile(self, file_pol=None, pd_output=False, tekmap_output=False):
 
 
 def write_bcfile(filename=None, datablocks=None, metadatas=None, refdate=None, tzone=0, float_format='%6.2f'):
-    """
-    deprecated
-    """
     raise DeprecationWarning('the function dfm_tools.io.bc.write_bcfile() is deprecated, please use the new hydrolib alternative. Example script: dfm_tools/tests/examples/CMEMS_interpolate_example.py')
 
 
 def read_bcfile(filename=None, converttime=False):
-    """
-    deprecated
-    """
     raise DeprecationWarning('the function dfm_tools.io.bc.read_bcfile() is deprecated, please use the new hydrolib alternative. Example script: dfm_tools/tests/examples/hydrolib_readbc.py')
 
 
 def write_timfile(filename=None, datablock=None, header=None, converttime=False, refdate=None, float_format='%6.2f'):
-    """
-    deprecated
-    """
     raise DeprecationWarning('the function dfm_tools.write_timfile() is deprecated, please use the new hydrolib alternative: https://github.com/Deltares/dfm_tools/blob/301-convert-timmodel-to-pandasdataframe/tests/examples/preprocess_hydrolib_readtim.py.')
 
 
 def read_timfile(filename=None, converttime=False, refdate=None):
-    """
-    deprecated
-    """
     raise DeprecationWarning('the function dfm_tools.read_timfile() is deprecated, please use the new hydrolib alternative: https://github.com/Deltares/dfm_tools/blob/301-convert-timmodel-to-pandasdataframe/tests/examples/preprocess_hydrolib_readtim.py.')
 
 
 def generate_bndpli(**kwargs):
-    """
-    deprecated
-    """
     raise DeprecationWarning('the function dfmt.generate_bndpli() is deprecated, please use dfmt.generate_bndpli_cutland() instead.')
 
 
 def preprocess_hirlam(ds):
-    """
-    deprecated
-    """
     raise DeprecationWarning('the function dfmt.preprocess_hirlam() is deprecated, xarray now supports datasets with multidimensional coordinates.')
 
 
 def interp_regularnc_to_plipoints(ds):
-    """
-    deprecated
-    """
     raise DeprecationWarning('the function dfmt.interp_regularnc_to_plipoints() is deprecated, '
                              'use dfmt.interp_regularnc_to_plipointsDataset() instead with gdf_points '
                              'as in https://github.com/Deltares/dfm_tools/issues/938')
+
+def open_dataset_extra(**kwargs):
+    raise DeprecationWarning('the function dfmt.open_dataset_extra() is deprecated, dfmt.open_prepare_dataset() is similar but does not support multiple quantities at once')

dfm_tools/interpolate_grid2bnd.py

@@ -362,7 +362,7 @@ def ds_apply_conversion_dict(data_xr, conversion_dict, quantity):
     return data_xr
 
 
-def open_dataset_extra(dir_pattern, quantity, tstart, tstop, conversion_dict=None, refdate_str=None, chunks=None):
+def open_prepare_dataset(dir_pattern, quantity, tstart, tstop, conversion_dict=None, refdate_str=None, chunks=None):
     """
     empty docstring
     """

dfm_tools/modelbuilder.py

@@ -9,7 +9,7 @@
 from hydrolib.core.utils import get_path_style_for_current_operating_system
 from dfm_tools.hydrolib_helpers import get_ncbnd_construct
 from dfm_tools.interpolate_grid2bnd import (ext_add_boundary_object_per_polyline,
-                                            open_dataset_extra,
+                                            open_prepare_dataset,
                                             ds_apply_conversion_dict,
                                             )
 
@@ -64,10 +64,11 @@ def cmems_nc_to_bc(ext_bnd, list_quantities, tstart, tstop, file_pli, dir_patter
             ncvarname = get_ncvarname(quantity=quantity_key, conversion_dict=conversion_dict)
             dir_pattern_one = str(dir_pattern).format(ncvarname=ncvarname)
             #open regulargridDataset and do some basic stuff (time selection, renaming depth/lat/lon/varname, converting units, etc)
-            data_xr_onevar = open_dataset_extra(dir_pattern=dir_pattern_one, quantity=quantity_key,
-                                                tstart=tstart, tstop=tstop,
-                                                conversion_dict=conversion_dict,
-                                                refdate_str=refdate_str)
+            data_xr_onevar = open_prepare_dataset(dir_pattern=dir_pattern_one, 
+                                                  quantity=quantity_key,
+                                                  tstart=tstart, tstop=tstop,
+                                                  conversion_dict=conversion_dict,
+                                                  refdate_str=refdate_str)
             if quantity_key == quantity_list[0]:
                 data_xr_vars = data_xr_onevar
             else: # only relevant in case of ux/uy, others all have only one quantity

dfm_tools/xarray_helpers.py

@@ -186,7 +186,7 @@ def merge_meteofiles(file_nc:str, preprocess=None,
     print(f'{(dt.datetime.now()-dtstart).total_seconds():.2f} sec')
 
     # rename variables
-    # TODO: make generic, comparable rename in rename_dims_dict in dfmt.interpolate_grid2bnd.open_dataset_extra()
+    # TODO: make generic, comparable rename in rename_dims_dict in dfmt.interpolate_grid2bnd.open_prepare_dataset()
     if 'longitude' not in data_xr.variables:
         if 'lon' in data_xr.variables:
             data_xr = data_xr.rename({'lon':'longitude', 'lat':'latitude'})

docs/whats-new.md

@@ -6,6 +6,7 @@
 - properly assigning units attribute in `ds_apply_conversion_dict()` (it did not always stick) in [#965](https://github.com/Deltares/dfm_tools/pull/965)
 - skipping initialwaterlevel in `dfmt.cmems_nc_to_ini()` [#970](https://github.com/Deltares/dfm_tools/pull/970)
 - update to cdsapi 0.7.2 and properly catching error for dummy dataset in [#972](https://github.com/Deltares/dfm_tools/pull/972)
+- deprecated `dfmt.open_dataset_extra()` (partly replaced by `dfmt.open_prepare_dataset()`) in [#974](https://github.com/Deltares/dfm_tools/pull/974)
 
 
 ## 0.25.0 (2024-08-16)

tests/test_get_nc.py

@@ -28,6 +28,7 @@ def test_polyline_mapslice():
     assert np.isclose(uds_crs.grid.node_y.max(), 0.9261683648147339)
 
 
+@pytest.mark.unittest
 def test_get_dataset_atdepths_hisfile():
 
     file_nc = dfmt.data.fm_grevelingen_his(return_filepath=True)

tests/test_interpolate_grid2bnd.py

@@ -21,7 +21,7 @@
                                             ext_add_boundary_object_per_polyline,
                                             ds_apply_conventions,
                                             ds_apply_conversion_dict,
-                                            open_dataset_extra,
+                                            open_prepare_dataset,
                                             )
 from dfm_tools.hydrolib_helpers import get_ncbnd_construct
 import hydrolib.core.dflowfm as hcdfm
@@ -189,7 +189,7 @@ def test_interpolate_nc_to_bc():
     dir_pattern = os.path.join(r'p:\1204257-dcsmzuno\data\CMEMS\nc\DCSM_allAvailableTimes',f'{ncvarname}_2012-12*.nc')
 
     #open regulargridDataset and do some basic stuff (time selection, renaming depth/lat/lon/varname, converting units, etc)
-    data_xr_vars = open_dataset_extra(dir_pattern=dir_pattern, quantity='salinitybnd', tstart=tstart, tstop=tstop)
+    data_xr_vars = open_prepare_dataset(dir_pattern=dir_pattern, quantity='salinitybnd', tstart=tstart, tstop=tstop)
     #interpolate regulargridDataset to plipointsDataset
     data_interp = dfmt.interp_regularnc_to_plipointsDataset(data_xr_reg=data_xr_vars, gdf_points=gdf_points)
 
@@ -248,16 +248,16 @@ def test_plipointsDataset_to_ForcingModel_drop_allnan_points():
 
 
 @pytest.mark.systemtest
-def test_open_dataset_extra_correctdepths(tmp_path):
+def test_open_prepare_dataset_correctdepths(tmp_path):
     """
-    to validate open_dataset_extra behaviour for depths, in the past the depth values got lost and replaced by depth idx
+    to validate open_prepare_dataset behaviour for depths, in the past the depth values got lost and replaced by depth idx
     """
 
     ds_moretime = cmems_dataset_4times()
     file_nc = tmp_path / 'temp_cmems_dummydata.nc'
     ds_moretime.to_netcdf(file_nc)
 
-    ds_moretime_import = open_dataset_extra(dir_pattern=file_nc, quantity='salinitybnd', tstart='2020-01-01 12:00:00', tstop='2020-01-02 12:00:00')
+    ds_moretime_import = open_prepare_dataset(dir_pattern=file_nc, quantity='salinitybnd', tstart='2020-01-01 12:00:00', tstop='2020-01-02 12:00:00')
     assert len(ds_moretime_import.time) == 2
 
 
@@ -304,7 +304,7 @@ def test_ds_apply_conversion_dict_rename_and_factor():
 
 
 @pytest.mark.unittest
-def test_open_dataset_extra_slightly_different_latlons(tmp_path):
+def test_open_prepare_dataset_slightly_different_latlons(tmp_path):
     """
     to check whether an error is raised when trying to combine datasets with slightly 
     different coordinates: https://github.com/Deltares/dfm_tools/issues/574
@@ -325,7 +325,7 @@ def test_open_dataset_extra_slightly_different_latlons(tmp_path):
     file_nc = tmp_path / 'temp_cmems_2day_*.nc'
 
     with pytest.raises(ValueError) as e:
-        open_dataset_extra(file_nc, quantity='salinitybnd', tstart='2020-01-01 12:00:00', tstop='2020-01-02 12:00:00')
+        open_prepare_dataset(file_nc, quantity='salinitybnd', tstart='2020-01-01 12:00:00', tstop='2020-01-02 12:00:00')
 
     # ValueError: cannot align objects with join='exact' where index/labels/sizes are not equal along these coordinates (dimensions): 'longitude' ('longitude',)
     assert "cannot align objects with join='exact' " in str(e.value)
@@ -628,3 +628,7 @@ def test_ext_add_boundary_object_per_polyline_wrong_name(tmp_path):
         ext_add_boundary_object_per_polyline(ext_new=ext_new, boundary_object=boundary_object)
     assert "The names of one of the polylines in the polyfile is the same as the polyfilename" in str(e.value)
 
+
+def test_open_dataset_extra_deprecated():
+    with pytest.raises(DeprecationWarning):
+        dfmt.open_dataset_extra()