Merge remote-tracking branch 'origin/main' into read-facets-from-file…

…name
ESMValGroup · Sep 26, 2024 · 690d515 · 690d515
2 parents 52d3951 + 4b0dd41
commit 690d515
Show file tree

Hide file tree

Showing 636 changed files with 54,306 additions and 29,608 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -2,8 +2,8 @@
 version: 2.1
 
 orbs:
-  coverage-reporter: codacy/coverage-reporter@11.4.1
-  codecov: codecov/codecov@1.2.3
+  coverage-reporter: codacy/coverage-reporter@14.0.0
+  codecov: codecov/codecov@4.1.0
 
 commands:
   check_changes:
@@ -28,11 +28,10 @@ commands:
             mkdir -p test-reports
             . /opt/conda/etc/profile.d/conda.sh
             conda activate esmvaltool
-            flake8 -j 4
             pytest -n 4 --junitxml=test-reports/report.xml
             esmvaltool version
       - store_test_results:
-          path: test-reports/
+          path: test-reports/report.xml
       - store_artifacts:
           path: /logs
       - run:
@@ -48,8 +47,15 @@ commands:
   test_installation_from_source:
     parameters:
       extra:
+        description: pip "extra"s to install
         type: string
+        default: "test"
       flags:
+        description: pip install flags
+        type: string
+        default: ""
+      upstream_packages:
+        description: List of packages that will be installed with pip.
         type: string
         default: ""
     steps:
@@ -74,20 +80,25 @@ commands:
               - r-yaml
               - ncl
             " >> environment.yml
+            # Installation of development version of packages requires compilers
+            if [[ "<< parameters.upstream_packages >>" ]]; then
+                echo "  - compilers" >> environment.yml
+            fi
             # Install
             . /opt/conda/etc/profile.d/conda.sh
             mkdir /logs
-            mamba env create >> /logs/conda.txt 2>&1
+            mamba env create |& tee /logs/conda.txt
+            git stash  # Restore repository state to get clean version number.
             conda activate esmvaltool
-            pip install << parameters.flags >> ".[<<parameters.extra>>]"> /logs/install.txt 2>&1
+            pip install << parameters.flags >> ".[<<parameters.extra>>]" << parameters.upstream_packages >> |& tee /logs/install.txt
       - run:
           name: Log versions
           command: |
             . /opt/conda/etc/profile.d/conda.sh
             conda activate esmvaltool
-            dpkg -l > /logs/versions.txt
-            conda env export > /logs/environment.yml
-            pip freeze > /logs/requirements.txt
+            dpkg -l | tee /logs/versions.txt
+            conda env export | tee /logs/environment.yml
+            pip freeze | tee /logs/requirements.txt
       - test_and_report
       - save_cache:
           key: install-<< parameters.extra >>-{{ .Branch }}-{{ checksum "cache_key.txt" }}
@@ -116,14 +127,17 @@ jobs:
             . /opt/conda/etc/profile.d/conda.sh
             mkdir /logs
             conda activate esmvaltool
-            pip install .[test] > /logs/install.txt 2>&1
+            pip install .[test] |& tee -a /logs/install.txt
       - test_and_report
       - save_cache:
           key: test-{{ .Branch }}-{{ checksum "cache_key.txt" }}
           paths:
             - /root/.cache/pip
             - .mypy_cache
             - .pytest_cache
+      - run:
+          name: Install gpg (required by codecov orb)
+          command: apt update && apt install -y gpg
       - codecov/upload:
           when: always
           file: 'test-reports/coverage.xml'
@@ -138,8 +152,7 @@ jobs:
       - image: condaforge/mambaforge
     resource_class: large
     steps:
-      - test_installation_from_source:
-          extra: test
+      - test_installation_from_source
 
   test_installation_from_source_develop_mode:
     # Test development installation
@@ -151,35 +164,53 @@ jobs:
           extra: develop
           flags: "--editable"
 
+  test_with_upstream_developments:
+    # Test with development versions of upstream packages
+    docker:
+      - image: condaforge/mambaforge
+    resource_class: large
+    steps:
+      - test_installation_from_source:
+          upstream_packages: >-
+            git+https://github.com/esgf/esgf-pyclient
+            git+https://github.com/euro-cordex/py-cordex
+            git+https://github.com/SciTools/cartopy
+            git+https://github.com/SciTools/cf-units
+            git+https://github.com/SciTools/iris
+            git+https://github.com/SciTools/iris-grib
+            git+https://github.com/SciTools/nc-time-axis
+            git+https://github.com/SciTools-incubator/iris-esmf-regrid
+            git+https://github.com/SciTools-incubator/python-stratify
+            git+https://github.com/Toblerity/Fiona
+
   test_installation_from_conda:
     # Test conda package installation
     working_directory: /esmvaltool
     docker:
       - image: condaforge/mambaforge
-    resource_class: small
+    resource_class: medium
     steps:
       - run:
           command: |
             . /opt/conda/etc/profile.d/conda.sh
             set -x
             # Install prerequisites
             mkdir /logs
-            # conda update -y conda > /logs/conda.txt 2>&1
             # Create and activate conda environment
-            mamba create -y --name esmvaltool 'python=3.10'
+            mamba create -y --name esmvaltool 'python=3.11'
             set +x; conda activate esmvaltool; set -x
             # Install
             mamba install -y esmvalcore
             # Log versions
-            conda env export > /logs/environment.yml
+            conda env export | tee /logs/environment.yml
             # Test installation
             esmvaltool version
 
   build_documentation:
     # Test building documentation
     docker:
       - image: condaforge/mambaforge
-    resource_class: small
+    resource_class: medium
     steps:
       - checkout
       - run:
@@ -191,9 +222,9 @@ jobs:
             conda activate esmvaltool
             pip install .[doc]
             # Log versions
-            dpkg -l > /logs/versions.txt
-            conda env export > /logs/environment.yml
-            pip freeze > /logs/requirements.txt
+            dpkg -l | tee /logs/versions.txt
+            conda env export | tee /logs/environment.yml
+            pip freeze | tee /logs/requirements.txt
             # Test building documentation
             MPLBACKEND=Agg sphinx-build -W doc doc/build
       - store_artifacts:
@@ -203,8 +234,9 @@ workflows:
   commit:
     jobs:
       - run_tests
-      - test_installation_from_source_test_mode
       - test_installation_from_source_develop_mode
+      - test_installation_from_source_test_mode
+      - test_with_upstream_developments
 
   nightly:
     triggers:
@@ -215,8 +247,9 @@ workflows:
               only:
                 - main
     jobs:
-      - run_tests
-      - test_installation_from_source_test_mode
       - build_documentation
-      - test_installation_from_source_develop_mode
+      - run_tests
       - test_installation_from_conda
+      - test_installation_from_source_develop_mode
+      - test_installation_from_source_test_mode
+      - test_with_upstream_developments
diff --git a/.editorconfig b/.editorconfig
@@ -27,4 +27,3 @@ indent_size = 2
 
 [*.{md,Rmd}]
 trim_trailing_whitespace = false
-
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -1,3 +1 @@
-esmvalcore/cmor @jvegasbsc
 .github/workflows @valeriupredoi
-
diff --git a/.github/workflows/build-and-deploy-on-pypi.yml b/.github/workflows/build-and-deploy-on-pypi.yml
@@ -12,14 +12,20 @@ jobs:
   build-n-publish:
     name: Build and publish ESMValCore on PyPi
     runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/project/ESMValCore/
+    permissions:
+      # IMPORTANT: this permission is mandatory for trusted publishing
+      id-token: write
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v1
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: "3.12"
       - name: Install pep517
         run: >-
           python -m
@@ -34,14 +40,10 @@ jobs:
           --binary
           --out-dir dist/
           .
-      #- name: Publish distribution 📦 to Test PyPI
-      #  uses: pypa/gh-action-pypi-publish@master
-      #  with:
-      #    password: ${{ secrets.test_pypi_password }}
-      #    repository_url: https://test.pypi.org/legacy/
+      # - name: Publish distribution to Test PyPI
+      #   uses: pypa/gh-action-pypi-publish@release/v1
+      #   with:
+      #     repository-url: https://test.pypi.org/legacy/
       - name: Publish distribution 📦 to PyPI
         if: startsWith(github.ref, 'refs/tags')
         uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          user: __token__
-          password: ${{ secrets.pypi_password }}
diff --git a/.github/workflows/citation_file_validator.yml b/.github/workflows/citation_file_validator.yml
@@ -16,7 +16,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out a copy of the repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Check whether the citation metadata from CITATION.cff is valid
         uses: citation-file-format/cffconvert-github-action@2.0.0
         with:

diff --git a/.github/workflows/create-condalock-file.yml b/.github/workflows/create-condalock-file.yml
@@ -10,62 +10,68 @@ on:
   schedule:
     - cron: '0 4 */10 * *'
 
+# Required shell entrypoint to have properly configured bash shell
+defaults:
+  run:
+    shell: bash -l {0}
+
 jobs:
   create-verify-lock-file:
     name: Create and verify conda lock file for latest Python
     runs-on: 'ubuntu-latest'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - uses: conda-incubator/setup-miniconda@v2
+      - uses: conda-incubator/setup-miniconda@v3
         with:
           auto-update-conda: true
-          python-version: "3.10"
-          miniconda-version: "latest"
           activate-environment: esmvaltool-fromlock
-          channels: conda-forge
-      - name: Show conda config
-        shell: bash -l {0}
+          python-version: "3.12"
+          miniforge-version: "latest"
+          miniforge-variant: Mambaforge
+          use-mamba: true
+      - name: Update and show conda config
         run: |
+          conda update -n base -c conda-forge conda
           conda info
           conda list
           conda config --show-sources
           conda config --show
+          # setup-miniconda@v3 installs an old conda and mamba
+          # forcing a modern mamba updates both mamba and conda
+          conda install -c conda-forge "mamba>=1.4.8"
+          conda --version
+          mamba --version
       - name: Gather Python info
-        shell: bash -l {0}
         run: |
           which python
           python --version
       - name: Install conda-lock
-        shell: bash -l {0}
-        run: conda install -y conda-lock
+        run: mamba install -y -c conda-forge conda-lock
+      - name: Check version of conda-lock
+        run: conda-lock --version
       - name: Create conda lock file for linux-64
-        shell: bash -l {0}
         run: conda-lock lock --platform linux-64 -f environment.yml --mamba --kind explicit
       - name: Creating environment from lock file
-        shell: bash -l {0}
         run: conda create --name esmvaltool-fromlock --file conda-linux-64.lock
       - name: Installing pip
-        shell: bash -l {0}
-        run: conda install pip
+        run: mamba install -y pip
       - name: Gather pip info
-        shell: bash -l {0}
         run: pip --version
       - name: Gather Python info again
-        shell: bash -l {0}
         run: |
           which python
           python --version
-      - shell: bash -l {0}
+      - name: Show environment contents
+        run: conda list
+      - name: Install esmvalcore
         run: pip install -e .[develop]
-      - shell: bash -l {0}
-        run: esmvaltool --help
-      - shell: bash -l {0}
-        run: esmvaltool version
-      - shell: bash -l {0}
-        run: flake8
-      - shell: bash -l {0}
+      - name: Run basic commands
+        run: |
+          esmvaltool --help
+          esmvaltool version
+      - name: Run pytests
         run: pytest -n 2 -m "not installation"
       # Automated PR
       # see https://github.com/marketplace/actions/create-pull-request
@@ -88,7 +94,4 @@ jobs:
             automatedPR
           assignees: valeriupredoi
           reviewers: valeriupredoi
-          team-reviewers: |
-            owners
-            maintainers
           draft: false