Merge branch 'main' into optimize-engine-flox

* main: (64 commits) import `normalize_axis_index` from `numpy.lib` on `numpy>=2` (#364) Optimize `min_count` when `expected_groups` is not provided. (#236) Use threadpool for finding labels in chunk (#327) Manually fuse reindexing intermediates with blockwise reduction for cohorts. (#300) Bump codecov/codecov-action from 4.1.1 to 4.3.1 (#362) Add cubed notebook for hourly climatology example using "map-reduce" method (#356) Optimize bitmask finding for chunk size 1 and single chunk cases (#360) Edits to climatology doc (#361) Fix benchmarks (#358) Trim CI (#355) [pre-commit.ci] pre-commit autoupdate (#350) Initial minimal working Cubed example for "map-reduce" (#352) Bump codecov/codecov-action from 4.1.0 to 4.1.1 (#349) `method` heuristics: Avoid dot product as much as possible (#347) Fix nanlen with strings (#344) Fix direct quantile reduction (#343) Fix upstream-dev CI, silence warnings (#341) Bump codecov/codecov-action from 4.0.0 to 4.1.0 (#338) Fix direct reductions of Xarray objects (#339) Test with py3.12 (#336) ...
xarray-contrib · May 2, 2024 · 11e2c7d · 11e2c7d
2 parents 4ab01d3 + fb180e8
commit 11e2c7d
Show file tree

Hide file tree

Showing 42 changed files with 2,302 additions and 773 deletions.
diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
@@ -41,15 +41,19 @@ jobs:
           OMP_NUM_THREADS: 1
           ASV_FACTOR: 1.5
           ASV_SKIP_SLOW: 1
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          LAST_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+          HEAD_LABEL: ${{ github.event.pull_request.head.label }}
+          BASE_LABEL: ${{ github.event.pull_request.base.label }}
         run: |
           # set -x
           # ID this runner
           asv machine --yes
-          echo "Baseline:  ${{ github.event.pull_request.base.sha }} (${{ github.event.pull_request.base.label }})"
-          echo "Contender: ${GITHUB_SHA} (${{ github.event.pull_request.head.label }})"
+          echo "Baseline:  $LAST_HEAD_SHA ($BASE_LABEL)"
+          echo "Contender: ${GITHUB_SHA} ($HEAD_LABEL)"
           # Run benchmarks for current commit against base
           ASV_OPTIONS="--split --show-stderr --factor $ASV_FACTOR"
-          asv continuous $ASV_OPTIONS ${{ github.event.pull_request.base.sha }} ${GITHUB_SHA} \
+          asv continuous $ASV_OPTIONS $BASE_SHA ${GITHUB_SHA} \
               | sed "/Traceback \|failed$\|PERFORMANCE DECREASED/ s/^/::error::/" \
               | tee benchmarks.log
           # Report and export results for subsequent steps
@@ -64,7 +68,7 @@ jobs:
           cp benchmarks/README_CI.md benchmarks.log .asv/results/
         working-directory: ${{ env.ASV_DIR }}
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         if: always()
         with:
           name: asv-benchmark-results-${{ runner.os }}

diff --git a/.github/workflows/ci-additional.yaml b/.github/workflows/ci-additional.yaml
@@ -41,7 +41,7 @@ jobs:
 
     env:
       CONDA_ENV_FILE: ci/environment.yml
-      PYTHON_VERSION: "3.11"
+      PYTHON_VERSION: "3.12"
 
     steps:
       - uses: actions/checkout@v4
@@ -59,7 +59,7 @@ jobs:
           environment-name: flox-tests
           init-shell: bash
           cache-environment: true
-          cache-env-key: "${{runner.os}}-${{runner.arch}}-py${{env.PYTHON_VERSION}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}"
+          cache-environment-key: "${{runner.os}}-${{runner.arch}}-py${{env.PYTHON_VERSION}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}"
           create-args: |
             python=${{ env.PYTHON_VERSION }}
 
@@ -77,7 +77,7 @@ jobs:
           --ignore flox/tests \
           --cov=./ --cov-report=xml
       - name: Upload code coverage to Codecov
-        uses: codecov/codecov-action@v3.1.4
+        uses: codecov/codecov-action@v4.3.1
         with:
           file: ./coverage.xml
           flags: unittests
@@ -95,7 +95,7 @@ jobs:
         shell: bash -l {0}
     env:
       CONDA_ENV_FILE: ci/environment.yml
-      PYTHON_VERSION: "3.11"
+      PYTHON_VERSION: "3.12"
 
     steps:
       - uses: actions/checkout@v4
@@ -112,7 +112,7 @@ jobs:
           environment-name: flox-tests
           init-shell: bash
           cache-environment: true
-          cache-env-key: "${{runner.os}}-${{runner.arch}}-py${{env.PYTHON_VERSION}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}"
+          cache-environment-key: "${{runner.os}}-${{runner.arch}}-py${{env.PYTHON_VERSION}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}"
           create-args: |
             python=${{ env.PYTHON_VERSION }}
       - name: Install flox
@@ -131,7 +131,7 @@ jobs:
           python -m mypy --install-types --non-interactive --cobertura-xml-report mypy_report
 
       - name: Upload mypy coverage to Codecov
-        uses: codecov/codecov-action@v3.1.4
+        uses: codecov/codecov-action@v4.3.1
         with:
           file: mypy_report/cobertura.xml
           flags: mypy

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -24,8 +24,11 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: ["ubuntu-latest", "windows-latest"]
-        python-version: ["3.9", "3.11"]
+        os: ["ubuntu-latest"]
+        python-version: ["3.9", "3.12"]
+        include:
+          - os: "windows-latest"
+            python-version: "3.12"
     steps:
       - uses: actions/checkout@v4
         with:
@@ -49,7 +52,7 @@ jobs:
         run: |
           pytest -n auto --cov=./ --cov-report=xml
       - name: Upload code coverage to Codecov
-        uses: codecov/codecov-action@v3.1.4
+        uses: codecov/codecov-action@v4.3.1
         with:
           file: ./coverage.xml
           flags: unittests
@@ -66,11 +69,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.11"]
-        env: ["no-xarray", "no-dask"]
+        python-version: ["3.12"]
+        env: ["no-dask"] # "no-xarray", "no-numba"
         include:
-          - env: "no-numba"
-            python-version: "3.12"
           - env: "minimal-requirements"
             python-version: "3.9"
     steps:
@@ -93,7 +94,7 @@ jobs:
         run: |
           python -m pytest -n auto --cov=./ --cov-report=xml
       - name: Upload code coverage to Codecov
-        uses: codecov/codecov-action@v3.1.4
+        uses: codecov/codecov-action@v4.3.1
         with:
           file: ./coverage.xml
           flags: unittests
@@ -119,8 +120,9 @@ jobs:
           environment-name: xarray-tests
           init-shell: bash
           cache-environment: true
-          create-args: |
-            python=3.10
+          create-args: >-
+            python=3.11
+            pint>=0.22
       - name: Install xarray
         run: |
           python -m pip install --no-deps .
@@ -144,4 +146,11 @@ jobs:
         id: status
         run: |
           set -euo pipefail
-          python -m pytest -n auto xarray/tests/test_groupby.py
+          python -m pytest -n auto \
+              xarray/tests/test_groupby.py \
+              xarray/tests/test_units.py::TestDataArray::test_computation_objects \
+              xarray/tests/test_units.py::TestDataArray::test_grouped_operations \
+              xarray/tests/test_units.py::TestDataArray::test_resample \
+              xarray/tests/test_units.py::TestDataset::test_computation_objects \
+              xarray/tests/test_units.py::TestDataset::test_grouped_operations \
+              xarray/tests/test_units.py::TestDataset::test_resample
diff --git a/.github/workflows/pypi.yaml b/.github/workflows/pypi.yaml
@@ -10,7 +10,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: "3.x"
       - name: Install dependencies

diff --git a/.github/workflows/testpypi-release.yaml b/.github/workflows/testpypi-release.yaml
@@ -21,10 +21,10 @@ jobs:
         with:
           fetch-depth: 0
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         name: Install Python
         with:
-          python-version: "3.11"
+          python-version: "3.12"
 
       - name: Install dependencies
         run: |
@@ -53,7 +53,7 @@ jobs:
             echo "✅ Looks good"
           fi
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: releases
           path: dist
@@ -62,11 +62,11 @@ jobs:
     needs: build-artifacts
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         name: Install Python
         with:
-          python-version: "3.11"
-      - uses: actions/download-artifact@v3
+          python-version: "3.12"
+      - uses: actions/download-artifact@v4
         with:
           name: releases
           path: dist

diff --git a/.github/workflows/upstream-dev-ci.yaml b/.github/workflows/upstream-dev-ci.yaml
@@ -7,6 +7,10 @@ on:
     types: [opened, reopened, synchronize, labeled]
     branches:
       - main
+    paths:
+      - ".github/workflows/upstream-dev-ci.yaml"
+      - "ci/upstream-dev-env.yml"
+      - "flox/*"
   schedule:
     - cron: "0 0 * * *" # Daily “At 00:00” UTC
   workflow_dispatch: # allows you to trigger the workflow run manually
@@ -19,14 +23,18 @@ jobs:
   upstream-dev:
     name: upstream-dev
     runs-on: ubuntu-latest
-    if: ${{ (contains(github.event.pull_request.labels.*.name, 'test-upstream') && github.event_name == 'pull_request') || github.event_name == 'workflow_dispatch' }}
+    if: ${{
+      (contains(github.event.pull_request.labels.*.name, 'test-upstream') && github.event_name == 'pull_request')
+      || github.event_name == 'workflow_dispatch'
+      || github.event_name == 'schedule'
+      }}
     defaults:
       run:
         shell: bash -l {0}
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.11"]
+        python-version: ["3.12"]
     steps:
       - uses: actions/checkout@v4
         with:
@@ -37,16 +45,49 @@ jobs:
       - name: Set up conda environment
         uses: mamba-org/setup-micromamba@v1
         with:
-          environment-file: ci/upstream-dev-env.yml
           environment-name: flox-tests
           init-shell: bash
-          cache-environment: true
+          # cache-environment: true
+          # micromamba list does not list pip dependencies, so install mamba
           create-args: >-
+            mamba
+            pip
             python=${{ matrix.python-version }}
             pytest-reportlog
+
+      - name: Install upstream dev dependencies
+        run: |
+          # install cython for building cftime without build isolation
+          micromamba install -f ci/upstream-dev-env.yml
+          micromamba remove --force numpy scipy pandas cftime
+          python -m pip install \
+            -i https://pypi.anaconda.org/scientific-python-nightly-wheels/simple \
+            --no-deps \
+            --pre \
+            --upgrade \
+            numpy \
+            scipy \
+            pandas \
+            xarray
+          # without build isolation for packages compiling against numpy
+          # TODO: remove once there are `numpy>=2.0` builds for cftime
+          python -m pip install \
+            --no-deps \
+            --upgrade \
+            --no-build-isolation \
+            git+https://github.com/Unidata/cftime
+          python -m pip install \
+            git+https://github.com/dask/dask \
+            git+https://github.com/ml31415/numpy-groupies
+
       - name: Install flox
         run: |
           python -m pip install --no-deps -e .
+
+      - name: List deps
+        run: |
+          # micromamba list does not list pip dependencies
+          mamba list
       - name: Run Tests
         if: success()
         id: status

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -4,13 +4,13 @@ ci:
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: "v0.0.292"
+    rev: "v0.3.5"
     hooks:
       - id: ruff
         args: ["--fix", "--show-fixes"]
 
   - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: "v3.0.3"
+    rev: "v4.0.0-alpha.8"
     hooks:
       - id: prettier
 
@@ -23,7 +23,7 @@ repos:
       - id: check-docstring-first
 
   - repo: https://github.com/psf/black-pre-commit-mirror
-    rev: 23.9.1
+    rev: 24.3.0
     hooks:
       - id: black
 
@@ -36,14 +36,14 @@ repos:
           - mdformat-myst
 
   - repo: https://github.com/nbQA-dev/nbQA
-    rev: 1.7.0
+    rev: 1.8.5
     hooks:
       - id: nbqa-black
       - id: nbqa-ruff
         args: [--fix]
 
   - repo: https://github.com/kynan/nbstripout
-    rev: 0.6.1
+    rev: 0.7.1
     hooks:
       - id: nbstripout
         args: [--extra-keys=metadata.kernelspec metadata.language_info.version]
@@ -56,6 +56,25 @@ repos:
           - tomli
 
   - repo: https://github.com/abravalheri/validate-pyproject
-    rev: v0.15
+    rev: v0.16
     hooks:
       - id: validate-pyproject
+
+  - repo: https://github.com/rhysd/actionlint
+    rev: v1.6.27
+    hooks:
+      - id: actionlint
+        files: ".github/workflows/"
+        args:
+          [
+            "-ignore",
+            "SC1090",
+            "-ignore",
+            "SC2046",
+            "-ignore",
+            "SC2086",
+            "-ignore",
+            "SC2129",
+            "-ignore",
+            "SC2155",
+          ]
diff --git a/asv_bench/asv.conf.json b/asv_bench/asv.conf.json
@@ -27,6 +27,11 @@
   //     "python setup.py build",
   //     "PIP_NO_BUILD_ISOLATION=false python -mpip wheel --no-deps --no-index -w {build_cache_dir} {build_dir}"
   // ],
+  //
+  "build_command": [
+    "python setup.py build",
+    "python -mpip wheel --no-deps --no-build-isolation --no-index -w {build_cache_dir} {build_dir}"
+  ],
 
   // List of branches to benchmark. If not provided, defaults to "master"
   // (for git) or "default" (for mercurial).