diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index ad7af9083..dde423557 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -22,11 +22,38 @@ jobs: egress-policy: block allowed-endpoints: > api.deps.dev:443 + api.electricitymap.org:443 api.github.com:443 + api.green-coding.io:443 + api.securityscorecards.dev:443 github.com:443 + ip-api.com:80 + ipapi.co:443 + proxy.golang.org:443 + pypi.org:443 + sum.golang.org:443 + + - name: Start Energy Measurement + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: start-measurement + branch: ${{ github.head_ref || github.ref_name }} - name: Checkout Repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Dependency Review uses: actions/dependency-review-action@4081bf99e2866ebe428fc0477b69eb4fcda7220a # v4.4.0 + + - name: Energy Measurement + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: get-measurement + label: 'Dependency Review' + continue-on-error: true + + - name: Show Energy Results + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: display-results + continue-on-error: true diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5f3affc29..1faca5919 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -45,9 +45,22 @@ jobs: disable-sudo: true egress-policy: block allowed-endpoints: > + api.electricitymap.org:443 + api.github.com:443 + api.green-coding.io:443 files.pythonhosted.org:443 github.com:443 + ip-api.com:80 + ipapi.co:443 + proxy.golang.org:443 pypi.org:443 + sum.golang.org:443 + - name: Start Measurement + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: start-measurement + branch: ${{ github.head_ref || github.ref_name }} + continue-on-error: true - name: Checkout Repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Set up Python3 @@ -58,6 +71,12 @@ jobs: - name: Install CI libraries run: | python -m pip install --require-hashes -r CI/requirements_ci.txt + - name: Setup Python Measurement + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: get-measurement + label: 'Environment setup (Python${{ matrix.python-version }}, lint)' + continue-on-error: true - name: Environment Caching uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 with: @@ -71,6 +90,17 @@ jobs: - name: Run linting suite run: | python -m tox -e lint + - name: Tests measurement + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: get-measurement + label: 'Lint (Python${{ matrix.python-version }})' + continue-on-error: true + - name: Show Energy Results + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: display-results + continue-on-error: true test-preliminary: name: Python${{ matrix.python-version }} (${{ matrix.os }}) @@ -88,12 +118,24 @@ jobs: disable-sudo: true egress-policy: block allowed-endpoints: > + api.electricitymap.org:443 api.github.com:443 - dap.service.does.not.exist:443 + api.green-coding.io:443 + coveralls.io:443 files.pythonhosted.org:443 github.com:443 + ip-api.com:80 + ipapi.co:443 + proxy.golang.org:443 pypi.org:443 raw.githubusercontent.com:443 + sum.golang.org:443 + - name: Start Energy Measurement + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: start-measurement + branch: ${{ github.head_ref || github.ref_name }} + continue-on-error: true - name: Checkout Repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Set up Python${{ matrix.python-version }} @@ -104,6 +146,12 @@ jobs: - name: Install CI libraries run: | python -m pip install --require-hashes -r CI/requirements_ci.txt + - name: Environment Setup Energy Measurement + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: get-measurement + label: 'Environment setup (Python${{ matrix.python-version }}, standard)' + continue-on-error: true - name: Environment Caching uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 with: @@ -116,7 +164,19 @@ jobs: python -m tox -- -m 'not slow' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + COVERALLS_FLAG_NAME: run-${{ matrix.python-version }}-ubuntu-latest-preliminary COVERALLS_PARALLEL: true + - name: Tests measurement + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: get-measurement + label: 'Testing with tox (Python${{ matrix.python-version }})' + continue-on-error: true + - name: Show Energy Results + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: display-results + continue-on-error: true test-pypi: needs: lint @@ -180,18 +240,27 @@ jobs: with: egress-policy: block allowed-endpoints: > + api.electricitymap.org:443 api.github.com:443 + api.green-coding.io:443 azure.archive.ubuntu.com:80 coveralls.io:443 - dap.service.does.not.exist:443 - esm.ubuntu.com:443 files.pythonhosted.org:443 github.com:443 + ip-api.com:80 + ipapi.co:443 motd.ubuntu.com:443 - packages.microsoft.com:443 - ppa.launchpadcontent.net:443 + proxy.golang.org:443 pypi.org:443 raw.githubusercontent.com:443 + sum.golang.org:443 + - name: Start Energy Measurement + if : ${{ matrix.os == 'ubuntu-latest' }} + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: start-measurement + branch: ${{ github.head_ref || github.ref_name }} + continue-on-error: true - name: Checkout Repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Eigen3 (SBCK) @@ -207,6 +276,13 @@ jobs: - name: Install CI libraries run: | python -m pip install --require-hashes -r CI/requirements_ci.txt + - name: Environment Setup Energy Measurement + if : ${{ matrix.os == 'ubuntu-latest' }} + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: get-measurement + label: 'Environment setup (Python${{ matrix.python-version }}, ${{ matrix.tox-env }})' + continue-on-error: true - name: Environment Caching # if prefetch is not in tox-env if: contains(matrix.tox-env, 'prefetch') == false @@ -232,6 +308,19 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} COVERALLS_FLAG_NAME: run-${{ matrix.python-version }}-${{ matrix.os }}-${{ matrix.tox-env }} COVERALLS_PARALLEL: true + - name: Tests measurement + if : ${{ matrix.os == 'ubuntu-latest' }} + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: get-measurement + label: 'Testing with tox (Python${{ matrix.python-version }}, ${{ matrix.tox-env }})' + continue-on-error: true + - name: Show Energy Results + if : ${{ matrix.os == 'ubuntu-latest' }} + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: display-results + continue-on-error: true test-conda: needs: lint @@ -240,7 +329,7 @@ jobs: contains(github.event.pull_request.labels.*.name, 'approved') || (github.event.review.state == 'approved') || (github.event_name == 'push') - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest strategy: matrix: os: [ 'ubuntu-latest' ] @@ -256,16 +345,25 @@ jobs: disable-sudo: true egress-policy: block allowed-endpoints: > + api.electricitymap.org:443 api.github.com:443 - conda.anaconda.org:443 + api.green-coding.io:443 coveralls.io:443 - dap.service.does.not.exist:443 files.pythonhosted.org:443 github.com:443 + ip-api.com:80 + ipapi.co:443 objects.githubusercontent.com:443 + proxy.golang.org:443 pypi.org:443 raw.githubusercontent.com:443 - repo.anaconda.com:443 + sum.golang.org:443 + - name: Start Energy Measurement + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: start-measurement + branch: ${{ github.head_ref || github.ref_name }} + continue-on-error: true - name: Checkout Repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Setup Conda (Micromamba) with Python${{ matrix.python-version }} @@ -276,9 +374,6 @@ jobs: environment-file: environment.yml create-args: >- python=${{ matrix.python-version }} - - name: Micromamba version - run: | - echo "micromamba: $(micromamba --version)" - name: Test Data Caching uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 with: @@ -293,21 +388,15 @@ jobs: micromamba list xclim show_version_info python -m pip check || true + - name: Environment Setup Energy Measurement + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: get-measurement + label: 'Environment setup (conda, Python${{ matrix.python-version }})' + continue-on-error: true - name: Test with pytest run: | python -m pytest --numprocesses=logical --durations=10 --cov=xclim --cov-report=term-missing -# - name: Install tox -# shell: bash -l {0} -# run: | -# mamba install -n xclim39 tox tox-conda -# - name: Test -# shell: bash -l {0} -# run: | -# conda activate xclim39 -# tox -e opt-slow -# env: -# CONDA_EXE: mamba -# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Report coverage run: | coveralls @@ -315,20 +404,64 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} COVERALLS_FLAG_NAME: run-{{ matrix.python-version }}-conda COVERALLS_PARALLEL: true + - name: Tests measurement + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: get-measurement + label: 'Testing with pytest (conda, Python${{ matrix.python-version }})' + continue-on-error: true + - name: Show Energy Results + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: display-results + continue-on-error: true finish: + name: Finish needs: - test-preliminary - test-pypi - test-conda runs-on: ubuntu-latest + permissions: + pull-requests: write steps: - name: Harden Runner uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1 with: disable-sudo: true - egress-policy: audit + egress-policy: block + allowed-endpoints: > + api.electricitymap.org:443 + api.github.com:443 + api.green-coding.io:443 + coveralls.io:443 + files.pythonhosted.org:443 + github.com:443 + ip-api.com:80 + ipapi.co:443 + objects.githubusercontent.com:443 + proxy.golang.org:443 + pypi.org:443 + sum.golang.org:443 + - name: Start Energy Measurement + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: start-measurement + branch: ${{ github.head_ref || github.ref_name }} + continue-on-error: true - name: Coveralls Finished uses: coverallsapp/github-action@cfd0633edbd2411b532b808ba7a8b5e04f76d2c8 # v2.3.4 with: parallel-finished: true + - name: Finish measurement + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: get-measurement + label: 'Finish Run' + continue-on-error: true + - name: Show Energy Results + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: display-results + continue-on-error: true diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 7965189a8..8477d39b1 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -35,12 +35,12 @@ jobs: egress-policy: audit - name: Checkout Repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false - name: Run Analysis - uses: ossf/scorecard-action@62b2cac7ed8198b15735ed49ab1e5cf35480ba46 + uses: ossf/scorecard-action@62b2cac7ed8198b15735ed49ab1e5cf35480ba46 # v2.4.0 with: results_file: results.sarif results_format: sarif @@ -59,7 +59,7 @@ jobs: # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF # format to the repository Actions tab. - name: Upload Artifact - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 with: name: SARIF file path: results.sarif @@ -67,6 +67,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: Upload to code-scanning - uses: github/codeql-action/upload-sarif@e5f05b81d5b6ff8cfa111c80c22c5fd02a384118 # 3.23.0 + uses: github/codeql-action/upload-sarif@e5f05b81d5b6ff8cfa111c80c22c5fd02a384118 # 3.23.0 with: sarif_file: results.sarif diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml index 175c7be54..f5e284a7e 100644 --- a/.github/workflows/upstream.yml +++ b/.github/workflows/upstream.yml @@ -44,15 +44,27 @@ jobs: disable-sudo: true egress-policy: block allowed-endpoints: > + api.electricitymap.org:443 api.github.com:443 + api.green-coding.io:443 + api.securityscorecards.dev:443 conda.anaconda.org:443 dap.service.does.not.exist:443 files.pythonhosted.org:443 github.com:443 + ip-api.com:80 + ipapi.co:443 objects.githubusercontent.com:443 + proxy.golang.org:4433 pypi.org:443 raw.githubusercontent.com:443 repo.anaconda.com:443 + sum.golang.org:443 + - name: Start Measurement + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: start-measurement + branch: ${{ github.head_ref || github.ref_name }} - name: Checkout Repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: @@ -84,6 +96,12 @@ jobs: micromamba list xclim show_version_info python -m pip check || true + - name: Setup Python Measurement + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: get-measurement + label: 'Environment Setup (Upstream, Python${{ matrix.python-version }})' + continue-on-error: true - name: Test Data Caching uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 with: @@ -105,3 +123,14 @@ jobs: with: issue-title: "⚠️ Nightly upstream-dev CI failed for Python${{ matrix.python-version }} ⚠️" log-path: output-${{ matrix.python-version }}-log.jsonl + - name: Tests measurement + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: get-measurement + label: 'Testing and Reporting (Upstream, Python${{ matrix.python-version }})' + continue-on-error: true + - name: Show Energy Results + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: display-results + continue-on-error: true diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 223085d08..a9a00ea1b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,7 +7,7 @@ v0.54.0 (unreleased) Contributors to this version: Trevor James Smith (:user:`Zeitsperre`), Pascal Bourgault (:user:`aulemahal`). Breaking changes ----------------- +^^^^^^^^^^^^^^^^ * The minimum required version of `dask` has been increased to `2024.8.1`. (:issue:`1992`, :pull:`1991`). Bug fixes @@ -21,6 +21,11 @@ Internal changes ^^^^^^^^^^^^^^^^ * Changed french translations with word "pluvieux" to "avec précipitations". (:issue:`1960`, :pull:`1994`). * In order to address 403 (forbidden) request errors when retrieving data from GitHub via ReadTheDocs, the ``nimbus`` class has been modified to use an overloaded `fetch` method that appends a User-Agent header to the request. (:pull:`2001`). +* Addressed a very rare race condition that can happen if `pytest` is tearing down the test environment when running across multiple workers. (:pull:`1863`). + +CI changes +^^^^^^^^^^ +* Added the `green-coding-solutions/eco-ci-energy-estimation` GitHub Action to the workflows to establish energy and carbon usage of CI activity. (:pull:`1863`). v0.53.2 (2024-10-31) -------------------- diff --git a/README.rst b/README.rst index 05d940f5a..f02d9dbff 100644 --- a/README.rst +++ b/README.rst @@ -11,7 +11,7 @@ xclim: Climate services library |logo| |logo-dark| |logo-light| +----------------------------+-----------------------------------------------------+ | Coding Standards | |black| |ruff| |pre-commit| |security| |fossa| | +----------------------------+-----------------------------------------------------+ -| Development Status | |status| |build| |coveralls| | +| Development Status | |status| |build| |coveralls| |energy| | +----------------------------+-----------------------------------------------------+ `xclim` is an operational Python library for climate services, providing numerous climate-related indicator tools @@ -168,6 +168,10 @@ This package was created with Cookiecutter_ and the `audreyfeldroy/cookiecutter- :target: https://bestpractices.coreinfrastructure.org/projects/6041 :alt: Open Source Security Foundation +.. |energy| image:: https://api.green-coding.io/v1/ci/badge/get?repo=Ouranosinc/xclim&branch=main&workflow=47013755 + :target: https://metrics.green-coding.io/ci.html?repo=Ouranosinc/xclim&branch=main&workflow=47013755 + :alt: Energy Consumption + .. |fair| image:: https://img.shields.io/badge/fair--software.eu-%E2%97%8F%20%20%E2%97%8F%20%20%E2%97%8F%20%20%E2%97%8F%20%20%E2%97%8B-yellow :target: https://fair-software.eu :alt: FAIR Software Compliance diff --git a/tests/conftest.py b/tests/conftest.py index ba3597b17..0e3785c93 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,7 @@ # noqa: D104 from __future__ import annotations +import logging import os from functools import partial from pathlib import Path @@ -411,6 +412,12 @@ def remove_data_written_flag(): """Cleanup cache folder once we are finished.""" flag = default_testdata_cache.joinpath(".data_written") if flag.exists(): - flag.unlink() + try: + flag.unlink() + except FileNotFoundError: + logging.info( + "Teardown race condition occurred: .data_written flag already removed. Lucky!" + ) + pass request.addfinalizer(remove_data_written_flag)