diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..7bd4262c --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,18 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file + +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + open-pull-requests-limit: 10 + + - package-ecosystem: pip + directory: / + schedule: + interval: weekly + open-pull-requests-limit: 10 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index aa37e332..da4fa5be 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -9,51 +9,62 @@ on: env: RAVEN_TESTING_DATA_BRANCH: master +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} + +permissions: + contents: read + jobs: - black: + lint: name: Code linting runs-on: ubuntu-latest - concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: - python-version: "3.9" + python-version: "3.x" - name: Install tox and setuptools run: | pip install tox - pip install --upgrade "setuptools<65.6" - name: Run linting suite run: tox -e black pip: - name: Python${{ matrix.python-version }} (${{ matrix.os }}) - needs: black + name: Python${{ matrix.python-version }} (${{ matrix.os }}, upstream=${{ matrix.upstream-branch }}) + needs: lint runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: include: - - tox-env: py38-linux - python-version: "3.8" - os: ubuntu-latest - - tox-env: py38-macos - python-version: "3.8" - os: macos-latest - tox-env: py39-linux python-version: "3.9" os: ubuntu-latest - - tox-env: py310-linux + - tox-env: py39-macos + python-version: "3.9" + os: macos-latest + - tox-env: py310-linux-upstream python-version: "3.10" os: ubuntu-latest + upstream-branch: "main" + - tox-env: py310-macos-upstream + python-version: "3.10" + os: macos-latest + upstream-branch: "main" - tox-env: py311-linux python-version: "3.11" os: ubuntu-latest - tox-env: py311-macos python-version: "3.11" os: macos-latest + - tox-env: py312-linux + python-version: "3.12" + os: ubuntu-latest + - tox-env: py312-macos + python-version: "3.12" + os: macos-latest steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} @@ -83,7 +94,7 @@ jobs: - name: Install tox run: | - pip install tox~=4.0 + python3 -m pip install tox~=4.5.0 - name: Test with tox and report coverage run: | python3 -m tox -e ${{ matrix.tox-env }} @@ -92,23 +103,17 @@ jobs: COVERALLS_FLAG_NAME: run-${{ matrix.tox-env }} COVERALLS_PARALLEL: true COVERALLS_SERVICE_NAME: github + UPSTREAM_BRANCH: ${{ matrix.upstream-branch }} conda: name: Python${{ matrix.python-version }} (${{ matrix.os }}) (Conda) - needs: black + needs: lint runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - include: - - os: ubuntu-latest - python-version: "3.9" - - os: ubuntu-latest - python-version: "3.10" - - os: ubuntu-latest - python-version: "3.11" - - os: macos-latest - python-version: "3.9" + os: [ubuntu-latest, macos-latest] # windows-latest # disabled until xesmf is available + python-version: ["3.9", "3.10", "3.11", "3.12"] defaults: run: shell: bash -l {0} @@ -134,15 +139,17 @@ jobs: echo "micromamba: $(micromamba --version)" - name: Install RavenPy run: | - pip install -e ".[dev]" + python -m pip install -e ".[dev]" - name: List installed packages run: | conda list + python -m pip check || true - name: Test RavenPy run: | - pytest --cov --numprocesses=logical + python -m pytest --cov --numprocesses=logical - name: Report coverage - run: coveralls --service=github + run: | + python -m coveralls env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} COVERALLS_FLAG_NAME: run-conda_${{ matrix.python-version }}_${{ matrix.os }} @@ -158,8 +165,8 @@ jobs: steps: - name: Coveralls Finished run: | - pip install --upgrade coveralls - coveralls --finish + python -m pip install --upgrade coveralls + python -m coveralls --finish env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} COVERALLS_SERVICE_NAME: github diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index 5e5aea84..0c1230a7 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -5,25 +5,36 @@ on: types: - published +permissions: + contents: read + jobs: build-n-publish-pypi: name: Build and publish Python 🐍 distributions 📦 to PyPI runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.8"] + environment: production + permissions: + # IMPORTANT: this permission is mandatory for trusted publishing + id-token: write steps: + - name: Harden Runner + uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0 + with: + disable-sudo: true + egress-policy: block + allowed-endpoints: > + files.pythonhosted.org:443 + github.com:443 + pypi.org:443 + upload.pypi.org:443 - uses: actions/checkout@v3 - name: Set up Python uses: actions/setup-python@v4 with: - python-version: ${{ matrix.python-version }} + python-version: "3.x" - name: Install packaging libraries run: pip install flit - name: Build a binary wheel and a source tarball run: flit build - name: Publish distribution 📦 to PyPI uses: pypa/gh-action-pypi-publish@release/v1 - with: - user: __token__ - password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.github/workflows/remove-obsolete-cache.yml b/.github/workflows/remove-obsolete-cache.yml index fdbabca6..dbce6f82 100644 --- a/.github/workflows/remove-obsolete-cache.yml +++ b/.github/workflows/remove-obsolete-cache.yml @@ -6,10 +6,23 @@ on: types: - closed +permissions: + contents: read + jobs: cleanup: runs-on: ubuntu-latest steps: + - name: Harden Runner + uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0 + with: + disable-sudo: true + egress-policy: block + allowed-endpoints: > + api.github.com:443 + github.com:443 + objects.githubusercontent.com:443 + - name: Check out code uses: actions/checkout@v3 diff --git a/.github/workflows/tag-testpypi.yml b/.github/workflows/tag-testpypi.yml index 6e24e670..e1d9ef0a 100644 --- a/.github/workflows/tag-testpypi.yml +++ b/.github/workflows/tag-testpypi.yml @@ -5,19 +5,33 @@ on: tags: - '*' +permissions: + contents: read + jobs: build-n-publish-testpypi: name: Build and publish Python 🐍 distributions 📦 to TestPyPI runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.8"] + environment: staging + permissions: + # IMPORTANT: this permission is mandatory for trusted publishing + id-token: write steps: + - name: Harden Runner + uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0 + with: + disable-sudo: true + egress-policy: block + allowed-endpoints: > + files.pythonhosted.org:443 + github.com:443 + pypi.org:443 + test.pypi.org:443 - uses: actions/checkout@v3 - name: Set up Python uses: actions/setup-python@v4 with: - python-version: ${{ matrix.python-version }} + python-version: "3.x" - name: Install packaging libraries run: pip install flit - name: Build a binary wheel and a source tarball @@ -27,5 +41,5 @@ jobs: with: user: __token__ password: ${{ secrets.TEST_PYPI_API_TOKEN }} - repository_url: https://test.pypi.org/legacy/ - skip_existing: true + repository-url: https://test.pypi.org/legacy/ + skip-existing: true diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 80c806ba..e99b5e26 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,7 @@ default_language_version: repos: - repo: https://github.com/asottile/pyupgrade - rev: v3.15.0 + rev: v3.15.1 hooks: - id: pyupgrade args: [ '--py38-plus' ] @@ -32,7 +32,7 @@ repos: hooks: - id: toml-sort-fix - repo: https://github.com/psf/black-pre-commit-mirror - rev: 24.1.1 + rev: 24.2.0 hooks: - id: black exclude: ^docs/ @@ -46,13 +46,13 @@ repos: hooks: - id: isort - repo: https://github.com/nbQA-dev/nbQA - rev: 1.7.1 + rev: 1.8.3 hooks: - id: nbqa-pyupgrade args: [ '--py38-plus' ] - additional_dependencies: [ 'pyupgrade==3.15.0' ] + additional_dependencies: [ 'pyupgrade==3.15.1' ] - id: nbqa-black - additional_dependencies: [ 'black==24.1.1' ] + additional_dependencies: [ 'black==24.2.0' ] - id: nbqa-isort additional_dependencies: [ 'isort==5.13.2' ] - repo: https://github.com/kynan/nbstripout @@ -60,7 +60,7 @@ repos: hooks: - id: nbstripout files: ".ipynb" - args: [ '--extra-keys', 'metadata.kernelspec' ] + args: [ '--extra-keys=metadata.kernelspec' ] - repo: https://github.com/pycqa/pydocstyle rev: 6.3.0 hooks: @@ -70,9 +70,9 @@ repos: rev: v0.3.9 hooks: - id: blackdoc - additional_dependencies: [ 'black==24.1.1' ] + additional_dependencies: [ 'black==24.2.0' ] - repo: https://github.com/adrienverge/yamllint.git - rev: v1.34.0 + rev: v1.35.1 hooks: - id: yamllint args: [ '--config-file=.yamllint.yaml' ] diff --git a/HISTORY.rst b/HISTORY.rst index c3ae05f8..40003a14 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,22 +2,36 @@ History ======= -0.13 (2023-01-10) ------------------ -* Fixed problem with scalar elevation in netCDF files parsed with `nc_specs` (issue #279, PR #323) -* Added notebook on sensitivity analysis (PR #320) -* Updated Notebooks 03 and 04 (PR #319) -* Upgrade to pydantic v2 (PR #326) -* Pin cf-xarray for Python3.8 (PR #325) -* Fix Coveralls Workflows (PR #328) -* Fix notebook execution (PR #329) -* Refactor and simplify testing data fetching (PR #332) +0.14.0 (2024--soon) +------------------- +* Added support for Python 3.12 and dropped support for Python3.8. +* Upgraded `raven-hydro` to v0.3.0 and `RavenHydroFramework` to v3.8. +* `ravenpy` now requires `xclim` >= v0.48.2, `xarray` >= v2023.11.0, and `pandas` >= 2.2.0. + +Internal changes +^^^^^^^^^^^^^^^^ +* Updated GitHub publishing workflows to use Trusted Publisher for TestPyPI/PyPI releases. +* Added Dependabot to keep dependencies up-to-date. +* Now using step-security/harden-runner action to harden GitHub Actions runners. +* Adjusted GitHub Workflows to test against Python 3.9, 3.10, 3.11, and 3.12. +* Updated the build-system requirements when testing with `tox` to use newer `setuptools` and `wheel` versions when building `gdal`. + +0.13.0 (2024-01-10) +------------------- +* Fixed problem with scalar elevation in netCDF files parsed with `nc_specs`. (issue #279, PR #323) +* Added notebook on sensitivity analysis. (PR #320) +* Updated Notebooks 03 and 04. (PR #319) +* Upgrade to `pydantic` v2.0. (PR #326) +* Pin `cf-xarray` for Python3.8. (PR #325) +* Fix `Coveralls` Workflows. (PR #328) +* Fix notebook execution. (PR #329) +* Refactor and simplify testing data fetching. (PR #332) Breaking changes ^^^^^^^^^^^^^^^^ -* Update to Pydantic v2. -* Added `h5netcdf` as a core dependency to provide a stabler backend for `xarray.open_dataset`. -* Switched from `autodoc_pydantic` to `autodoc-pydantic` for `pydantic` v2.0+ support in documentation. +* Update to `pydantic` v2.0. (PR #326) +* Added `h5netcdf` as a core dependency to provide a stabler backend for `xarray.open_dataset`. (PR #332) +* Switched from `autodoc_pydantic` to `autodoc-pydantic` for `pydantic` v2.0+ support in documentation. (PR #326) Internal changes ^^^^^^^^^^^^^^^^ diff --git a/environment-rtd.yml b/environment-rtd.yml index c3f9b26f..8331da1e 100644 --- a/environment-rtd.yml +++ b/environment-rtd.yml @@ -4,7 +4,7 @@ channels: - defaults dependencies: - python >=3.9,<3.10 # fixed to reduce solver time - - raven-hydro ==0.2.4 + - raven-hydro ==0.2.4 # FIXME: Update when raven-hydro 0.3.0 is available on conda-forge - autodoc-pydantic - click # - clisops # mocked diff --git a/environment.yml b/environment.yml index ea552a02..6f8dc0a0 100644 --- a/environment.yml +++ b/environment.yml @@ -3,11 +3,11 @@ channels: - conda-forge - defaults dependencies: - - python >=3.8,<3.12 - - raven-hydro ==0.2.4 + - python >=3.9,<3.13 + - raven-hydro ==0.2.4 # FIXME: Update when raven-hydro 0.3.0 is available on conda-forge - libgcc # for mixing raven-hydro from PyPI with conda environments - affine - - black >=24.1.1 + - black >=24.2.0 - cftime - cf_xarray - click @@ -26,9 +26,9 @@ dependencies: - lxml - matplotlib - netcdf4 - - numpy <1.25 + - numpy - owslib <0.29.0 # see: https://github.com/geopython/OWSLib/issues/871 - - pandas <2.2.0 # xclim <0.48.0 is incompatible with pandas >=2.2.0 + - pandas >=2.2.0 - pint >=0.20 - platformdirs - pre-commit @@ -44,7 +44,7 @@ dependencies: - spotpy - statsmodels - typing_extensions - - xarray >=2022.12.0,<2023.9.0 # xarray v2023.9.0 is incompatible with xclim<=0.45.0 - - xclim >=0.43,<0.48 + - xarray >=2023.11.0 # xarray v2023.9.0 is incompatible with xclim<=0.45.0 + - xclim >=0.48.2 - xesmf - xskillscore diff --git a/pyproject.toml b/pyproject.toml index 698eecee..7935fab4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ maintainers = [ {name = "Trevor James Smith", email = "smith.trevorj@ouranos.ca"} ] readme = {file = "README.rst", content-type = "text/x-rst"} -requires-python = ">=3.8.0" +requires-python = ">=3.9.0" keywords = ["raven", "raven-hydro", "hydrology", "gis", "analysis", "modelling"] license = {file = "LICENSE"} classifiers = [ @@ -23,12 +23,12 @@ classifiers = [ "License :: OSI Approved :: MIT License", "Natural Language :: English", "Operating System :: OS Independent", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", - "Programming Language :: Python", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: Implementation :: CPython", "Topic :: Scientific/Engineering :: Atmospheric Science", "Topic :: Scientific/Engineering :: GIS", "Topic :: Scientific/Engineering :: Hydrology" @@ -37,52 +37,52 @@ dynamic = ["description", "version"] dependencies = [ "cftime", # cf-xarray is differently named on conda-forge - "cf-xarray[all]<0.8.5; python_version == '3.8'", - "cf-xarray[all]; python_version >= '3.9'", - "climpred>=2.4.0", + "cf-xarray[all]", + "climpred >=2.4.0", "dask", "haversine", "h5netcdf", "matplotlib", "netCDF4", - "numpy<1.25", - "owslib>=0.24.1,<0.29", # see: https://github.com/geopython/OWSLib/issues/871 - "pandas<2.0.0; python_version == '3.8'", - "pandas<2.2.0; python_version >= '3.9'", - "pint>=0.20", + "numpy", + "owslib >=0.24.1,<0.29", # see: https://github.com/geopython/OWSLib/issues/871 + "pandas >=2.2.0", + "pint >=0.20", "platformdirs", - "pydantic>=2.0", + "pydantic >=2.0", "pymbolic", - "raven-hydro==0.2.4", + "raven-hydro ==0.3.0", "requests", "scipy", "spotpy", "statsmodels", "typing-extensions", - "xarray>=2022.12.0,<2023.9.0", # xarray v2023.9.0 is incompatible with xclim<=0.45.0 - "xclim>=0.43.0,<0.48.0", + "xarray >=2023.11.0", + "xclim >=0.48.2", "xskillscore" ] [project.optional-dependencies] dev = [ - "black>=24.1.1", + "black >=24.2.0", "bump2version", "coverage", "coveralls", "filelock", - "flake8>=7.0.0", + "flake8 >=7.0.0", "flit", "holoviews", "hvplot", - "isort>=5.13.2", + "isort >=5.13.2", "mypy", "pre-commit", "pytest", "pytest-cov", - "pytest-xdist>=3.2.0", + "pytest-xdist >=3.2.0", + "setuptools >=68.0", "tox", - "watchdog" + "watchdog", + "wheel >=0.42.0" ] docs = [ "autodoc-pydantic", @@ -93,7 +93,7 @@ docs = [ # Needed for notebooks/HydroShare_integration.ipynb # See: https://github.com/CSHS-CWRA/RavenPy/pull/326 # "hsclient", - "intake", + "intake <2.0.0", "intake-esm", "intake-xarray", "ipykernel", @@ -113,17 +113,17 @@ docs = [ "sphinx-click", "sphinx-codeautolink", "sphinx-copybutton", - "sphinx-rtd-theme>=1.0", + "sphinx-rtd-theme >=1.0", "xesmf" ] gis = [ "affine", - "fiona>=1.9", - "geopandas>=0.13.0", + "fiona >=1.9", + "geopandas >=0.13.0", "gdal", "lxml", "pyogrio", - "pyproj>=3.0.0", + "pyproj >=3.0.0", "rasterio", "rioxarray", "shapely" diff --git a/ravenpy/utilities/coords.py b/ravenpy/utilities/coords.py index ec3632ae..d9454042 100644 --- a/ravenpy/utilities/coords.py +++ b/ravenpy/utilities/coords.py @@ -97,7 +97,7 @@ def infer_scale_and_offset( multi, base, start_anchor, _ = parse_offset(freq) if base in ["M", "Q", "A"]: raise ValueError(f"Irregular time frequency for input data {da}") - real_source = source / multi / units(FREQ_UNITS[base]) + real_source = source / multi / units(FREQ_UNITS.get(base, base)) scale, offset = units_transform(real_source, target) else: raise NotImplementedError(f"data_type: {data_type}") diff --git a/tests/test_emulators.py b/tests/test_emulators.py index c6b73018..3e525492 100644 --- a/tests/test_emulators.py +++ b/tests/test_emulators.py @@ -3,6 +3,8 @@ import numpy as np import pytest import xarray as xr +from packaging.version import Version +from raven_hydro import __raven_version__ from ravenpy import Emulator from ravenpy.config import commands as rc @@ -15,7 +17,9 @@ "HMETS": -3.0132, "Mohyse": 0.194612, "HBVEC": 0.0186633, - "CanadianShield": 0.3968, # 0.39602 <- this is the original value for 3.6 + # "CanadianShield": 0.39602, <- This is the original value for CanadianShield with RavenHydroFramework v3.6 + # "CanadianShield": 0.3968, <- This is the value for CanadianShield with RavenHydroFramework v3.7 + # "CanadianShield": 0.4001, <- This is the new value for CanadianShield with RavenHydroFramework v3.8 "HYPR": 0.685188, "SACSMA": -0.0382907, "Blended": -0.913785, @@ -38,11 +42,21 @@ def test_run(numeric_config, tmp_path): out = e.run() d = out.diagnostics - np.testing.assert_almost_equal(d["DIAG_NASH_SUTCLIFFE"], NSE[name], 4) if name == "CanadianShield": + # FIXME: The CanadianShield values all need to be verified. + if Version(__raven_version__) == Version("3.8"): + np.testing.assert_almost_equal(d["DIAG_NASH_SUTCLIFFE"], 0.4001, 4) + elif Version(__raven_version__) == Version("3.7"): + np.testing.assert_almost_equal(d["DIAG_NASH_SUTCLIFFE"], 0.3968, 4) + elif Version(__raven_version__) == Version("3.6"): + np.testing.assert_almost_equal(d["DIAG_NASH_SUTCLIFFE"], 0.39602, 4) + else: + raise NotImplementedError("めんどくさい") pytest.skip("Missing solution due to SuppressOutput.") + np.testing.assert_almost_equal(d["DIAG_NASH_SUTCLIFFE"], NSE[name], 4) + # Start new simulation with final state from initial run. new = e.resume() assert new.start_date == conf.end_date diff --git a/tox.ini b/tox.ini index 89d6a601..15119254 100644 --- a/tox.ini +++ b/tox.ini @@ -1,12 +1,14 @@ [tox] -min_version = 4.0 +min_version = 4.5 envlist = black - py{38,39,310,311}-{linux,macos} + py{38,39,310,311,312}-{linux,macos} docs requires = - pip >=21.0 - setuptools >=63.0 + flit + pip >=23.0 + setuptools >=68.0 + wheel >=0.42.0 opts = -vv @@ -14,7 +16,7 @@ opts = skip_install = True deps = flake8 >=7.0.0 - black >=24.1.1 + black >=24.2.0 isort >=5.13.2 commands = make lint @@ -44,6 +46,7 @@ passenv = LD_LIBRARY_PATH LD_PRELOAD RAVENPY_* + UPSTREAM_BRANCH extras = dev gis @@ -53,13 +56,18 @@ install_command = deps = # numpy must be present in python env before GDAL is installed numpy - gdal == {env:GDAL_VERSION} + gdal[numpy] == {env:GDAL_VERSION} +commands_pre = + python -m pip list + - python -m pip check commands = # Regenerate NetCDF4-Python wheel from source files # Rebuild netcdf4 on Linux due to errors in new version (https://github.com/Unidata/netcdf4-python/issues/1192) # linux: python -m pip install --upgrade --force-reinstall --no-deps --no-cache-dir netcdf4==1.6.4 --no-binary netcdf4 # Rebuild GDAL in order to gain access to GDAL system-level objects - python -m pip install --upgrade --force-reinstall --no-deps --no-cache-dir gdal=={env:GDAL_VERSION}.* --no-binary gdal + python -m pip install --upgrade --force-reinstall --no-deps --no-cache-dir --no-build-isolation gdal[numpy]=={env:GDAL_VERSION}.* + # Install raven-hydro from the upstream branch + upstream: python -m pip install --upgrade --force-reinstall --no-deps --no-cache-dir git+https://github.com/Ouranosinc/raven-hydro.git@{env:UPSTREAM_BRANCH} # Run tests pytest --cov # Coveralls requires access to a repo token set in .coveralls.yml in order to report stats