From bd08720f5a67ae15c5a2a6a1a7708889d498e0b4 Mon Sep 17 00:00:00 2001 From: Christian Jauvin Date: Wed, 10 Mar 2021 12:20:59 -0500 Subject: [PATCH 01/10] Fix gis_import_error_message in utilities --- ravenpy/utilities/__init__.py | 2 +- ravenpy/utilities/analysis.py | 4 ++-- ravenpy/utilities/checks.py | 4 ++-- ravenpy/utilities/forecasting.py | 8 +++----- ravenpy/utilities/geo.py | 4 ++-- ravenpy/utilities/geoserver.py | 8 +++----- ravenpy/utilities/io.py | 4 ++-- 7 files changed, 15 insertions(+), 19 deletions(-) diff --git a/ravenpy/utilities/__init__.py b/ravenpy/utilities/__init__.py index fd99d1bd..bc921492 100644 --- a/ravenpy/utilities/__init__.py +++ b/ravenpy/utilities/__init__.py @@ -1,6 +1,6 @@ from .regionalization import read_gauged_params, read_gauged_properties, regionalize -gis_error_message = ( +gis_import_error_message = ( "`{}` requires installation of the RavenPy GIS libraries. These can be installed using the" " `pip install ravenpy[gis]` recipe or via Anaconda (`conda env -n ravenpy-env -f environment.yml`)" " from the RavenPy repository source files." diff --git a/ravenpy/utilities/analysis.py b/ravenpy/utilities/analysis.py index 11f0704c..d05cfa16 100644 --- a/ravenpy/utilities/analysis.py +++ b/ravenpy/utilities/analysis.py @@ -6,14 +6,14 @@ import numpy as np -from . import gis_error_message +from . import gis_import_error_message try: import rasterio from osgeo.gdal import Dataset, DEMProcessing from shapely.geometry import GeometryCollection, MultiPolygon, Polygon, shape except (ImportError, ModuleNotFoundError) as e: - msg = gis_error_message.format(Path(__file__).stem) + msg = gis_import_error_message.format(Path(__file__).stem) raise ImportError(msg) from e from ravenpy.utilities.geo import generic_raster_clip diff --git a/ravenpy/utilities/checks.py b/ravenpy/utilities/checks.py index 2f1ed982..610722ee 100644 --- a/ravenpy/utilities/checks.py +++ b/ravenpy/utilities/checks.py @@ -8,7 +8,7 @@ from pathlib import Path from typing import Any, List, Sequence, Tuple, Union -from . import gis_error_message +from . import gis_import_error_message try: import fiona @@ -17,7 +17,7 @@ from pyproj.exceptions import CRSError from shapely.geometry import GeometryCollection, MultiPolygon, Point, shape except (ImportError, ModuleNotFoundError) as e: - msg = gis_error_message.format(Path(__file__).stem) + msg = gis_import_error_message.format(Path(__file__).stem) raise ImportError(msg) from e import ravenpy.utilities.io as io diff --git a/ravenpy/utilities/forecasting.py b/ravenpy/utilities/forecasting.py index e374c14b..fbca56b4 100644 --- a/ravenpy/utilities/forecasting.py +++ b/ravenpy/utilities/forecasting.py @@ -19,15 +19,13 @@ import xarray as xr from climpred import HindcastEnsemble +from . import gis_import_error_message + try: import rioxarray from clisops.core import subset except (ImportError, ModuleNotFoundError) as e: - msg = ( - f"`{Path(__file__).stem}` requires installation of the RavenPy GIS libraries. These can be installed using the" - " `pip install ravenpy[gis]` recipe or via Anaconda (`conda env -n ravenpy-env -f environment.yml`)" - " from the RavenPy repository source files." - ) + msg = gis_import_error_message.format(Path(__file__).stem) raise ImportError(msg) from e from ravenpy.models import get_model diff --git a/ravenpy/utilities/geo.py b/ravenpy/utilities/geo.py index 3bb13875..7a8f0da0 100644 --- a/ravenpy/utilities/geo.py +++ b/ravenpy/utilities/geo.py @@ -8,7 +8,7 @@ from pathlib import Path from typing import List, Union -from . import gis_error_message +from . import gis_import_error_message try: import fiona @@ -26,7 +26,7 @@ ) from shapely.ops import transform except (ImportError, ModuleNotFoundError) as e: - msg = gis_error_message.format(Path(__file__).stem) + msg = gis_import_error_message.format(Path(__file__).stem) raise ImportError(msg) from e RASTERIO_TIFF_COMPRESSION = "lzw" diff --git a/ravenpy/utilities/geoserver.py b/ravenpy/utilities/geoserver.py index b920f878..564205d7 100644 --- a/ravenpy/utilities/geoserver.py +++ b/ravenpy/utilities/geoserver.py @@ -15,6 +15,8 @@ from requests import Request +from . import gis_import_error_message + try: import fiona import pandas as pd @@ -24,11 +26,7 @@ from owslib.wfs import WebFeatureService from shapely.geometry import Point, shape except (ImportError, ModuleNotFoundError) as e: - msg = ( - f"`{Path(__file__).stem}` requires installation of the RavenPy GIS libraries. These can be installed using the" - " `pip install ravenpy[gis]` recipe or via Anaconda (`conda env -n ravenpy-env -f environment.yml`)" - " from the RavenPy repository source files." - ) + msg = gis_import_error_message.format(Path(__file__).stem) raise ImportError(msg) from e # Do not remove the trailing / otherwise `urljoin` will remove the geoserver path. diff --git a/ravenpy/utilities/io.py b/ravenpy/utilities/io.py index 1f0081c1..35d442d6 100644 --- a/ravenpy/utilities/io.py +++ b/ravenpy/utilities/io.py @@ -10,7 +10,7 @@ from re import search from typing import Iterable, List, Optional, Sequence, Union -from . import gis_error_message +from . import gis_import_error_message try: import fiona @@ -18,7 +18,7 @@ from pyproj import CRS from shapely.geometry import shape except (ImportError, ModuleNotFoundError) as e: - msg = gis_error_message.format(Path(__file__).stem) + msg = gis_import_error_message.format(Path(__file__).stem) raise ImportError(msg) from e LOGGER = logging.getLogger("RavenPy") From cd901de135bdb6f838779c6ee8481b238eab785f Mon Sep 17 00:00:00 2001 From: Christian Jauvin Date: Wed, 10 Mar 2021 12:21:21 -0500 Subject: [PATCH 02/10] Add pre-commit to installation docs --- docs/installation.rst | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/installation.rst b/docs/installation.rst index d94ed8e0..6ff80a96 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -111,7 +111,13 @@ Then clone the Raven Test Data repo somewhere on your disk: (ravenpy-env) $ git clone git@github.com:Ouranosinc/raven-testdata.git -You can then run the test suite by doing: +Install the pre-commit hook (to make sure that any code you contribute is properly formatted): + +.. code-block:: console + + (ravenpy-env) $ pre-commit install + +If everything was properly installed the test suite should run successfully: .. code-block:: console From 31e054819617b71aec33c05eac81f977e739f1c1 Mon Sep 17 00:00:00 2001 From: Christian Jauvin Date: Wed, 10 Mar 2021 12:21:36 -0500 Subject: [PATCH 03/10] Fix 2020 -> 2021 copyright dates --- LICENSE | 2 +- docs/conf.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LICENSE b/LICENSE index 720d8ac6..9b804ef9 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2020, Canadian Society for Hydrological Sciences +Copyright (c) 2021, Canadian Society for Hydrological Sciences Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/docs/conf.py b/docs/conf.py index c22881ef..178b1a45 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -83,7 +83,7 @@ # General information about the project. project = "RavenPy" -copyright = "2020, David Huard" +copyright = "2021, David Huard" author = "David Huard" # The version info for the project you're documenting, acts as replacement From 505c8ac5e2825d1733854b50bfff4a8f60feb38c Mon Sep 17 00:00:00 2001 From: Christian Jauvin Date: Wed, 10 Mar 2021 12:57:40 -0500 Subject: [PATCH 04/10] Update HISTORY for 0.3.0 --- HISTORY.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index de67ab4a..60699b56 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,6 +2,16 @@ History ======= +0.3.0 +----- + +* Migration and refactoring of GIS and IO utilities (`utils.py`, `utilities/gis.py`) from RavenWPS to RavenPy. +* RavenPy can now be installed fropm PyPI without GIS dependencies (limited functionality). +* Hydro routing product is now supported from `geoserver.py` (a notebook has been added to demonstrate the new functions). +* New script `ravenpy aggregate-forcings-to-hrus` to aggregate NetCDF files and compute updated grid weights. +* Add the basis for a new routing emulator option (WIP). +* Add climpred verification capabilities. + 0.2.3 ----- From 88a467ec54a48b45159fd954cc5863ae2139fbbc Mon Sep 17 00:00:00 2001 From: Christian Jauvin Date: Wed, 10 Mar 2021 12:59:39 -0500 Subject: [PATCH 05/10] Bumpversion to 0.3.0 (minor) --- .cruft.json | 2 +- ravenpy/__version__.py | 2 +- setup.cfg | 2 +- setup.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.cruft.json b/.cruft.json index 3f67ccb5..62144273 100644 --- a/.cruft.json +++ b/.cruft.json @@ -10,7 +10,7 @@ "project_slug": "ravenpy", "project_short_description": "A Python wrapper to setup and run the hydrologic modelling framework Raven.", "pypi_username": "CSHS-CWRA", - "version": "0.2.3", + "version": "0.3.0", "use_pytest": "y", "use_pypi_deployment_with_travis": "y", "add_pyup_badge": "y", diff --git a/ravenpy/__version__.py b/ravenpy/__version__.py index 83f6834b..00f9ece5 100644 --- a/ravenpy/__version__.py +++ b/ravenpy/__version__.py @@ -6,4 +6,4 @@ __author__ = """David Huard""" __email__ = "huard.david@ouranos.ca" -__version__ = "0.2.3" +__version__ = "0.3.0" diff --git a/setup.cfg b/setup.cfg index a8eedc91..9d5d75d2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.2.3 +current_version = 0.3.0 commit = False tag = False diff --git a/setup.py b/setup.py index 80344286..af098c25 100644 --- a/setup.py +++ b/setup.py @@ -204,7 +204,7 @@ def run(self): gis=gis_requirements, ), url="https://github.com/CSHS-CWRA/ravenpy", - version="0.2.3", + version="0.3.0", zip_safe=False, cmdclass={ "install": create_external_deps_install_class(install), From 7a5b87c2e78d8f3256a79ad0eb4bc7316fe59e2c Mon Sep 17 00:00:00 2001 From: Christian Jauvin Date: Wed, 10 Mar 2021 13:17:05 -0500 Subject: [PATCH 06/10] Update HISTORY.rst Co-authored-by: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> --- HISTORY.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 60699b56..e8339d7e 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -6,7 +6,7 @@ History ----- * Migration and refactoring of GIS and IO utilities (`utils.py`, `utilities/gis.py`) from RavenWPS to RavenPy. -* RavenPy can now be installed fropm PyPI without GIS dependencies (limited functionality). +* RavenPy can now be installed from PyPI without GIS dependencies (limited functionality). * Hydro routing product is now supported from `geoserver.py` (a notebook has been added to demonstrate the new functions). * New script `ravenpy aggregate-forcings-to-hrus` to aggregate NetCDF files and compute updated grid weights. * Add the basis for a new routing emulator option (WIP). From 420b2b24c47f0c82291879f4132a51742792f628 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 10 Mar 2021 13:22:51 -0500 Subject: [PATCH 07/10] reduce precision needed for mean aspect, slope, elevation. --- tests/test_geo_utilities.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_geo_utilities.py b/tests/test_geo_utilities.py index 3ef143c4..ed3f945f 100644 --- a/tests/test_geo_utilities.py +++ b/tests/test_geo_utilities.py @@ -159,18 +159,18 @@ def test_gdal_slope_not_projected(self, tmp_path): # Slope values are high due to data values using Geographic CRS def test_dem_properties(self): dem_properties = self.analysis.dem_prop(self.raster_file) - np.testing.assert_almost_equal(dem_properties["aspect"], 10.9119033) - np.testing.assert_almost_equal(dem_properties["elevation"], 79.0341721) - np.testing.assert_almost_equal(dem_properties["slope"], 64.4365427) + np.testing.assert_almost_equal(dem_properties["aspect"], 10.911, 3) + np.testing.assert_almost_equal(dem_properties["elevation"], 79.0341, 4) + np.testing.assert_almost_equal(dem_properties["slope"], 64.43654, 5) with self.fiona.open(self.geojson_file) as gj: feature = next(iter(gj)) geom = self.sgeo.shape(feature["geometry"]) region_dem_properties = self.analysis.dem_prop(self.raster_file, geom=geom) - np.testing.assert_almost_equal(region_dem_properties["aspect"], 280.6814208) - np.testing.assert_almost_equal(region_dem_properties["elevation"], 145.8899082) - np.testing.assert_almost_equal(region_dem_properties["slope"], 61.2650882) + np.testing.assert_almost_equal(region_dem_properties["aspect"], 280.681, 3) + np.testing.assert_almost_equal(region_dem_properties["elevation"], 145.8899, 4) + np.testing.assert_almost_equal(region_dem_properties["slope"], 61.26508, 5) # Slope values are high due to data values using Geographic CRS def test_geom_properties(self): From f2bca9905ed6ae0ecea68b55f867a6ef40a31ad8 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 10 Mar 2021 13:48:01 -0500 Subject: [PATCH 08/10] reduce precision needed for EPSG:3348 transformation --- tests/test_geo_utilities.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/test_geo_utilities.py b/tests/test_geo_utilities.py index ed3f945f..ffaf6a55 100644 --- a/tests/test_geo_utilities.py +++ b/tests/test_geo_utilities.py @@ -243,12 +243,13 @@ def test_raster_warp(self, tmp_path): self.raster_file, output=reproj_file, target_crs="EPSG:3348" ) + # EPSG:3348 is a very general transformation; Some tolerance should be allowed. with self.rasterio.open(reproj_file) as gt: assert gt.crs.to_epsg() == 3348 - np.testing.assert_almost_equal(gt.bounds.left, -2077535.25979486) - np.testing.assert_almost_equal(gt.bounds.right, 15591620.75098695) - np.testing.assert_almost_equal(gt.bounds.bottom, -4167898.76317739) - np.testing.assert_almost_equal(gt.bounds.top, 5817014.91999878) + np.testing.assert_allclose(gt.bounds.left, -2077535, atol=1.25) + np.testing.assert_allclose(gt.bounds.right, 15591620, atol=1.25) + np.testing.assert_allclose(gt.bounds.bottom, -4167898, atol=1.25) + np.testing.assert_allclose(gt.bounds.top, 5817014, atol=1.25) data = gt.read(1) # read band 1 (red) assert data.min() == 0 @@ -278,7 +279,7 @@ def test_warped_raster_aspect(self, tmp_path): aspect_grid = self.analysis.gdal_aspect_analysis(reproj_file) np.testing.assert_almost_equal( - self.analysis.circular_mean_aspect(aspect_grid), 7.7805879 + self.analysis.circular_mean_aspect(aspect_grid), 7.780, decimal=3 ) def test_raster_clip(self, tmp_path): From b991dfa98415092d63eb44bdc59a6f83622ec58a Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 10 Mar 2021 14:03:59 -0500 Subject: [PATCH 09/10] bump atol to 3m for test_raster_warp --- tests/test_geo_utilities.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_geo_utilities.py b/tests/test_geo_utilities.py index ffaf6a55..1d2af511 100644 --- a/tests/test_geo_utilities.py +++ b/tests/test_geo_utilities.py @@ -246,10 +246,10 @@ def test_raster_warp(self, tmp_path): # EPSG:3348 is a very general transformation; Some tolerance should be allowed. with self.rasterio.open(reproj_file) as gt: assert gt.crs.to_epsg() == 3348 - np.testing.assert_allclose(gt.bounds.left, -2077535, atol=1.25) - np.testing.assert_allclose(gt.bounds.right, 15591620, atol=1.25) - np.testing.assert_allclose(gt.bounds.bottom, -4167898, atol=1.25) - np.testing.assert_allclose(gt.bounds.top, 5817014, atol=1.25) + np.testing.assert_allclose(gt.bounds.left, -2077535, atol=3) + np.testing.assert_allclose(gt.bounds.right, 15591620, atol=3) + np.testing.assert_allclose(gt.bounds.bottom, -4167898, atol=3) + np.testing.assert_allclose(gt.bounds.top, 5817014, atol=3) data = gt.read(1) # read band 1 (red) assert data.min() == 0 From 976307e74a081e1d38eb65c906b5697a2ef85378 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Thu, 11 Mar 2021 09:24:16 -0500 Subject: [PATCH 10/10] reduce precision for `test_raster_warp` --- tests/test_geo_utilities.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_geo_utilities.py b/tests/test_geo_utilities.py index 1d2af511..4f0fc99e 100644 --- a/tests/test_geo_utilities.py +++ b/tests/test_geo_utilities.py @@ -254,7 +254,7 @@ def test_raster_warp(self, tmp_path): data = gt.read(1) # read band 1 (red) assert data.min() == 0 assert data.max() == 255 - np.testing.assert_almost_equal(data.mean(), 60.7291936) + np.testing.assert_almost_equal(data.mean(), 60.729, 3) def test_warped_raster_slope(self, tmp_path): reproj_file = tempfile.NamedTemporaryFile(