GridModel; HydroMT 0.9 compliant; Deprecated PCR; Ubuntu testing (#211)

* Switched to gridmodel; deprecated pcr methods

* Update the docs workflow

* Update environment on linux

* Rely on main branch for building docs

* Fixed PCR stuff

* Small rewrite

* Added data for linux

* Added linux test model results

* Order of dir

* Extra linux testdata

* Flexible spatial dims

* Linting..

* Linux data full; dont rely on core for clipping

* Run on Ubuntu!

* Added asterisk

* Small fix in data_libs

* fix prepare_ldd docs

make code more robust for different CRS

* Updated docs a biut

* Added older function to api

* small fixes and remove deprecation warnings from core

* update example notebooks and ini

* improve install guide

* update docs

* small fixes for docs

* Fix comments; improved reading PCR files

* Updated caching of workflows; inline with core; clock on dev test

* Added extra dev py version

* Updated mode to 'w+'

* Use core main branch to build docs

---------

Co-authored-by: JoostBuitink <44062204+JoostBuitink@users.noreply.github.com>

.act_event.json

@@ -0,0 +1,3 @@
+{
+  "act": true
+}

.actrc

@@ -0,0 +1,5 @@
+-P ubuntu-latest=catthehacker/ubuntu:full-latest
+-P ubuntu-latest=catthehacker/ubuntu:full-20.04
+-P ubuntu-18.04=catthehacker/ubuntu:full-18.04
+
+-e .act_event.json

.github/workflows/cache.yml

@@ -0,0 +1,103 @@
+
+name: Refrech cache
+
+on:
+  schedule:
+    - cron: 0 0 * * Sun # run on main every sunday at 00:00
+  workflow_dispatch:
+
+jobs:
+  clear-cache:
+    uses: ./.github/workflows/purge_cache.yml
+  rebuild-cache:
+    needs: clear-cache
+    defaults:
+      run:
+        shell: bash -l {0}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ['3.10','3.11']
+    name: py ${{ matrix.python-version }}
+    runs-on: ${{ matrix.os }}
+    # note absence of cucurrency, this one should only be run one at a time
+    steps:
+      - name: checkout code
+        uses: actions/checkout@v3
+
+      - name: Setup Miniforge3
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+            python-version: ${{ matrix.python-version }}
+            miniforge-variant: Miniforge3
+            miniforge-version: latest
+            use-mamba: true
+
+      - name: Setup env
+        run: |
+          pip install tomli
+          python make_env.py test -p ${{ matrix.python-version}}.* -n hydromt_wflow
+          mamba env create -f environment.yml
+          mamba run -n hydromt_wflow pip install -e .
+
+      # run tests first so that we can also cache all of the artefacts
+      - name: Test
+        run: |
+          export NUMBA_DISABLE_JIT=1
+          PYTHONPYCACHEPREFIX=~/pycache mamba run -n hydromt_wflow python -m pytest --verbose --cov=hydromt_wflow --cov-report xml
+
+      - name: Upload cache
+        uses: actions/cache/save@v3
+        if: always()
+        with:
+          path: |
+            /usr/share/miniconda3
+            ~/pycache
+          key: test-py${{ matrix.python-version }}-${{ hashFiles('environment.yml')  }}
+        id: test-cache
+
+  docs-cache:
+    needs: clear-cache
+    defaults:
+      run:
+        shell: bash -l {0}
+    strategy:
+        fail-fast: false
+        matrix:
+            os: [ubuntu-latest]
+            python-version: ['3.11']
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: checkout code
+        uses: actions/checkout@v3
+
+      - name: Setup Miniforge3
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+            python-version: ${{ matrix.python-version }}
+            miniforge-variant: Miniforge3
+            miniforge-version: latest
+            use-mamba: true
+
+      - name: Setup env
+        run: |
+          pip install tomli
+          python make_env.py doc -p ${{ matrix.python-version }}.* -n hydromt_wflow
+          mamba env create -f environment.yml
+          mamba run -n hydromt_wflow pip install -e .
+
+      # run tests first so that we can also cache all of the artefacts
+      - name: Generate docs
+        run: PYTHONPYCACHEPREFIX=~/pycache mamba run -n hydromt_wflow sphinx-build -M html ./docs ./docs/_build
+
+      - name: Upload cache
+        uses: actions/cache/save@v3
+        if: always()
+        with:
+          path: |
+            /usr/share/miniconda3
+            ~/pycache
+            ./docs/_build
+          key: docs-${{ hashFiles('environment.yml')  }}
+        id: docs-cache

.github/workflows/docs.yml

@@ -1,50 +1,99 @@
-name: Build Docs
+name: Build Documentation
 
-# We don't want pushes (or PRs) to gh-pages to kick anything off
 on:
+  workflow_dispatch:
   push:
-    branches: [ main ]
+    branches: [main]
+    paths:
+      - tests/*
+      - hydromt_wflow/*
+      - docs/*
+      - examples/*
+      - pyproject.toml
   pull_request:
-    branches: [ main ]
+    branches: [main]
+    paths:
+      - tests/*
+      - hydromt_wflow/*
+      - docs/*
+      - examples/*
+      - pyproject.toml
 
 jobs:
   # Build docs on Linux
   Docs:
-    name: linux docs
-    runs-on: ubuntu-latest
-    env:
-      DOC_VERSION: dev
     defaults:
       run:
         shell: bash -l {0}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ["ubuntu-latest" ] #, "macos-latest", "windows-latest"]
+        python-version: ['3.11']
+        include:
+        - os: ubuntu-latest
+          label: linux-64
+          prefix: /usr/share/miniconda3/envs/hydromt_wflow
+
+    name: ${{ matrix.label }}
+    runs-on: ${{ matrix.os }}
+
     steps:
       - name: Checkout source
         uses: actions/checkout@v3
-      - name: install tomli
-        run: pip install tomli
-      - name: Generate env spec
-        run: python make_env.py doc
-      - name: Setup environment
-        uses: conda-incubator/setup-miniconda@v2
+
+      - name: Generate env yaml
+        run: |
+          pip install tomli
+          python make_env.py doc -p ${{ matrix.python-version }}.* -n hydromt_wflow
+
+      - name: Load cache
+        id: cache
+        uses: actions/cache/restore@v3
         with:
-          python-version: "3.9"
-          miniforge-variant: Mambaforge
-          channels: conda-forge,defaults
-          channel-priority: strict
-          environment-file: environment.yml
-          activate-environment: hydromt-wflow
+          path: |
+            /usr/share/miniconda3
+            ~/pycache
+            ./docs/_build
+          # the below two settings mean we'll alway srestore the cache
+          # but the cache hit output can tell us if we have to update afterwards
+          key: docs-${{ hashFiles('environment.yml')  }}
+          restore-keys: |
+            docs
 
-      - name: Install HydroMT-Wflow plugin
-        run: pip install .
+      - name: Failed cache restore
+        if: steps.cache.outputs.cache-matched-key == ''
+        run: |
+          echo "Failed to restore any cache. exiting..."
+          exit 1
+
+      # by avoiding the mamba setup stage by loading it from cache instead we save
+      # a lot of setup time, but we do have to do our own PATH management
+      # hence the exports
+      - name: Update environment
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: |
+          export PATH=/usr/share/miniconda3/bin:$PATH
+          mamba env update -n hydromt_wflow -f environment.yml
 
-      - name: Build docs
+      - name: Install HydroMT-core dev
         run: |
-          pushd docs
-          make html
-          popd
+          export PATH=/usr/share/miniconda3/bin:$PATH
+          mamba run -n hydromt_wflow python -m pip install git+https://github.com/Deltares/hydromt.git@main --user
 
-      - name: Set doc version
-        run: echo "DOC_VERSION=$(python -c 'from hydromt_wflow import __version__ as v; print("dev" if "dev" in v else "v"+v.replace(".dev",""))')" >> $GITHUB_ENV
+      - name: Generate dummy docs
+        if: ${{ github.event_name == 'pull_request' }}
+        run: |
+          export PATH=/usr/share/miniconda3/bin:$PATH
+          PYTHONPYCACHEPREFIX=~/pycache mamba run -n hydromt_wflow sphinx-build ./docs ./docs/_build -b dummy
+
+      - name: Generate docs
+        if: ${{ github.event_name != 'pull_request' && !github.event.act }}
+        run: |
+          export PATH=/usr/share/miniconda3/bin:$PATH
+          PYTHONPYCACHEPREFIX=~/pycache mamba run -n hydromt_wflow sphinx-build -M html ./docs ./docs/_build
+          echo "DOC_VERSION=$(mamba run -n hydromt_wflow python -c 'from hydromt_wflow import __version__ as v; print("dev" if "dev" in v else "v"+v.replace(".dev",""))')" >> $GITHUB_ENV
 
       - name: Upload to GitHub Pages
         if: ${{ github.event_name != 'pull_request'}}

.github/workflows/purge_cache.yml

@@ -0,0 +1,22 @@
+name: Purge cache
+on:
+  workflow_dispatch:
+  workflow_call:
+
+
+jobs:
+  cleanup:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v3
+
+      - name: Cleanup cache
+        run: |
+          if gh cache list | grep -vq "No cache"; then
+            gh cache delete -a
+          else
+            echo "No caches found. skipping..."
+          fi
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}