From 084554c00a21a7e92ec032ee8afa48f66e1c2d55 Mon Sep 17 00:00:00 2001 From: Agustin Borgna Date: Mon, 30 Oct 2023 16:51:08 +0000 Subject: [PATCH 1/3] ci: Use codecov for coverage reports --- .github/workflows/coverage.yml | 87 +++++++++++----------------------- README.md | 2 + 2 files changed, 30 insertions(+), 59 deletions(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 37cf165bb..853215c38 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -1,68 +1,37 @@ -name: Coverage +name: Coverage check on: - schedule: - # 04:00 daily - - cron: '0 4 * * *' + push: + branches: + - main + pull_request: + branches: + - main + workflow_dispatch: {} jobs: - check-coverage: - runs-on: ubuntu-latest - outputs: - msg: ${{ steps.make_msg.outputs.msg }} + coverage: + runs-on: ${{ matrix.os }}-latest + strategy: + matrix: + os: [ubuntu] steps: - uses: actions/checkout@v4 - - name: Install stable toolchain - uses: dtolnay/rust-toolchain@stable - - name: Install llvm - uses: KyleMayes/install-llvm-action@v1 + - uses: mozilla-actions/sccache-action@v0.0.3 + - uses: dtolnay/rust-toolchain@nightly with: - version: "16.0" # should match version used by rustc - - name: Download previous summary - uses: dawidd6/action-download-artifact@v2 - with: - # Downloads the artifact from the most recent successful run - workflow: 'coverage.yml' - name: coverage-summary - - name: Rename file - run: mv coverage-summary.txt coverage-summary-prev.txt - - name: Generate coverage report and summary - env: - RUSTFLAGS: "-C instrument-coverage" - run: | - TGT=`cargo test --tests 2>&1 | grep Running | awk -F'[()]' '{print $2}'` - llvm-profdata merge -sparse default_*.profraw -o hugr.profdata - llvm-cov show --format=html --ignore-filename-regex='/.cargo/registry' --instr-profile=hugr.profdata --output-dir coverage --object ${TGT} - llvm-cov report --ignore-filename-regex='/.cargo/registry' --instr-profile=hugr.profdata --object ${TGT} | grep TOTAL | awk '{print $10}' | tr -dc '[:digit:].' > coverage-summary.txt - - name: Upload full report - uses: actions/upload-artifact@v3 - with: - name: coverage - path: coverage/ - - name: Upload summary - uses: actions/upload-artifact@v3 - with: - name: coverage-summary - path: coverage-summary.txt - - name: Compare with previous summary and make message - id: make_msg + components: llvm-tools-preview + - name: Install cargo-llvm-cov + uses: taiki-e/install-action@cargo-llvm-cov + - name: Run tests with coverage instrumentation run: | - change="`cat coverage-summary-prev.txt`% --> `cat coverage-summary.txt`%" - if (( $(echo "`cat coverage-summary-prev.txt` < `cat coverage-summary.txt` + 0.04" | bc -l) )) - then - echo "msg=Coverage check for hugr shows no regression (${change}). ✅ ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" >> "$GITHUB_OUTPUT" - else - echo "msg=Coverage check for hugr shows regression (${change}). ❌ ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" >> "$GITHUB_OUTPUT" - fi - notify-slack: - needs: check-coverage - runs-on: ubuntu-latest - steps: - - name: Send notification - uses: slackapi/slack-github-action@v1.24.0 + cargo llvm-cov clean --workspace + cargo llvm-cov --doctests + - name: Generate coverage report + run: cargo llvm-cov report --codecov --output-path coverage.json + - name: Upload coverage to codecov.io + uses: codecov/codecov-action@v3 with: - channel-id: 'C04SHCL4FKP' - slack-message: ${{ needs.check-coverage.outputs.msg }} - env: - SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} - + files: coverage.json + name: ${{ matrix.os }} + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/README.md b/README.md index fa64b9465..9217bf7bb 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ quantinuum-hugr [![build_status][]](https://github.com/CQCL/hugr/actions) [![msrv][]](https://github.com/CQCL/hugr) +[![codecov][]](https://codecov.io/gh/CQCL/hugr) The Hierarchical Unified Graph Representation (HUGR, pronounced _hugger_) is the common representation of quantum circuits and operations in the Quantinuum @@ -38,4 +39,5 @@ This project is licensed under Apache License, Version 2.0 ([LICENSE][] or http: [build_status]: https://github.com/CQCL/hugr/workflows/Continuous%20integration/badge.svg?branch=main [msrv]: https://img.shields.io/badge/rust-1.70.0%2B-blue.svg + [codecov]: https://img.shields.io/codecov/c/gh/CQCL/hugr?logo=codecov [LICENSE]: LICENCE From 240eb0b539ae69c9f3777c72a4736d811fd035ab Mon Sep 17 00:00:00 2001 From: Agustin Borgna Date: Tue, 31 Oct 2023 10:10:02 +0000 Subject: [PATCH 2/3] Only run coverage if tests succeed So that we get test errors on the right place --- .github/workflows/ci.yml | 27 ++++++++++++++++++++++++- .github/workflows/coverage.yml | 37 ---------------------------------- 2 files changed, 26 insertions(+), 38 deletions(-) delete mode 100644 .github/workflows/coverage.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1bca569e4..1a1a119a5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,4 +88,29 @@ jobs: - name: Tests with no features run: cargo test --verbose --no-default-features - name: Tests with all features - run: cargo test --verbose --all-features \ No newline at end of file + run: cargo test --verbose --all-features + + coverage: + if: github.event_name != 'merge_group' + needs: [tests, check] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: mozilla-actions/sccache-action@v0.0.3 + - uses: dtolnay/rust-toolchain@nightly + with: + components: llvm-tools-preview + - name: Install cargo-llvm-cov + uses: taiki-e/install-action@cargo-llvm-cov + - name: Run tests with coverage instrumentation + run: | + cargo llvm-cov clean --workspace + cargo llvm-cov --doctests + - name: Generate coverage report + run: cargo llvm-cov report --codecov --output-path coverage.json + - name: Upload coverage to codecov.io + uses: codecov/codecov-action@v3 + with: + files: coverage.json + name: ubuntu + token: ${{ secrets.CODECOV_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml deleted file mode 100644 index 853215c38..000000000 --- a/.github/workflows/coverage.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: Coverage check - -on: - push: - branches: - - main - pull_request: - branches: - - main - workflow_dispatch: {} - -jobs: - coverage: - runs-on: ${{ matrix.os }}-latest - strategy: - matrix: - os: [ubuntu] - steps: - - uses: actions/checkout@v4 - - uses: mozilla-actions/sccache-action@v0.0.3 - - uses: dtolnay/rust-toolchain@nightly - with: - components: llvm-tools-preview - - name: Install cargo-llvm-cov - uses: taiki-e/install-action@cargo-llvm-cov - - name: Run tests with coverage instrumentation - run: | - cargo llvm-cov clean --workspace - cargo llvm-cov --doctests - - name: Generate coverage report - run: cargo llvm-cov report --codecov --output-path coverage.json - - name: Upload coverage to codecov.io - uses: codecov/codecov-action@v3 - with: - files: coverage.json - name: ${{ matrix.os }} - token: ${{ secrets.CODECOV_TOKEN }} From f78e789f423f960e7b7f2078f8ad2724974901ae Mon Sep 17 00:00:00 2001 From: Agustin Borgna Date: Tue, 31 Oct 2023 11:48:22 +0000 Subject: [PATCH 3/3] Bring back daily coverage message --- .github/workflows/coverage.yml | 64 ++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 .github/workflows/coverage.yml diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 000000000..4a946ef56 --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,64 @@ +name: Coverage + +on: + schedule: + # 04:00 daily + - cron: '0 4 * * *' + workflow_dispatch: {} + +jobs: + check-coverage: + runs-on: ubuntu-latest + outputs: + msg: ${{ steps.make_msg.outputs.msg }} + steps: + - uses: actions/checkout@v4 + - name: Install rust toolchain + uses: dtolnay/rust-toolchain@nightly + with: + components: llvm-tools-preview + - name: Install cargo-llvm-cov + uses: taiki-e/install-action@cargo-llvm-cov + - name: Download previous summary + uses: dawidd6/action-download-artifact@v2 + with: + # Downloads the artifact from the most recent successful run + workflow: 'coverage.yml' + name: coverage-summary + - name: Rename file + run: mv coverage-summary.txt coverage-summary-prev.txt + - name: Run tests with coverage instrumentation + run: | + cargo llvm-cov clean --workspace + cargo llvm-cov --doctests + - name: Generate coverage report and summary + run: | + cargo llvm-cov report --summary-only | grep TOTAL | awk '{print $10}' | tr -dc '[:digit:].' > coverage-summary.txt + - name: Upload summary + uses: actions/upload-artifact@v3 + with: + name: coverage-summary + path: coverage-summary.txt + - name: Compare with previous summary and make message + id: make_msg + run: | + change="`cat coverage-summary-prev.txt`% --> `cat coverage-summary.txt`%" + codecov="https://codecov.io/gh/${{ github.repository }}?search=&trend=7%20days" + if (( $(echo "`cat coverage-summary-prev.txt` < `cat coverage-summary.txt` + 0.04" | bc -l) )) + then + echo "msg=Coverage check for hugr shows no regression (${change}). ✅ ${codecov}" >> "$GITHUB_OUTPUT" + else + echo "msg=Coverage check for hugr shows regression (${change}). ❌ ${codecov}" >> "$GITHUB_OUTPUT" + fi + notify-slack: + needs: check-coverage + runs-on: ubuntu-latest + steps: + - name: Send notification + uses: slackapi/slack-github-action@v1.24.0 + with: + channel-id: 'C04SHCL4FKP' + slack-message: ${{ needs.check-coverage.outputs.msg }} + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} +