From 7a9410d6a124c4eb74f7fc873923c805ce65916d Mon Sep 17 00:00:00 2001 From: Ian Lumsden Date: Fri, 1 Nov 2024 17:39:45 -0400 Subject: [PATCH 1/4] Adds CI to run unit tests against Pandas 3.0 --- .github/workflows/test-for-pandas-3.yml | 82 +++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 .github/workflows/test-for-pandas-3.yml diff --git a/.github/workflows/test-for-pandas-3.yml b/.github/workflows/test-for-pandas-3.yml new file mode 100644 index 00000000..11ff4812 --- /dev/null +++ b/.github/workflows/test-for-pandas-3.yml @@ -0,0 +1,82 @@ +name: Run unit tests with Pandas 3.0 + +on: + push: + branches: [ develop ] + pull_request: + branches: [ develop, releases/** ] + +jobs: + run_pandas_3_unit_tests: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: 3.11 + + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: node + + - name: Install Pandas 3.0 (i.e., Nightly) + run: | + python3 -m pip install --upgrade pip pytest + python3 -m pip install --upgrade --force-reinstall --pre --extra-index https://pypi.anaconda.org/scientific-python-nightly-wheels/simple pandas + + - name: Install Python 3 dependencies + run: | + python3 -m pip install --upgrade --force-reinstall git+https://github.com/LLNL/hatchet.git@develop + python3 -m pip install -r requirements.txt + python3 -m pip install . + python3 -m pip list + + # From here on, I'm making use of the trick from the following + # GitHub repo to not fail the full CI/CD pipeline if this fails: + # https://github.com/burningmantech/ranger-ims-server/pull/1347/files + - name: Basic test with PyTest + id: test + run: | + PYTHONPATH . $(which pytest) + status=$? + if [ $status -ne 0 ]; then + echo "::warning::Optional CI test with Pandas 3 failed" + echo "optional_fail=true" >> "${GITHUB_OUTPUT}" + echo "optional_fail_status=${status}" >> "${GITHUB_OUTPUT}" + fi + exit 0 + + - name: Add comment to PR if test failed + if: ${{ steps.test.outputs.optional_fail == 'true' }} + uses: thollander/actions-comment-pull-request@v3 + with: + comment-tag: "pandas-3.0-test-notice" + message: | + ### Pandas 3.0 Unit Tests Failed! + Due to breaking changes in Pandas 3.0 (namely, copy-on-write), we are performing optional + tests of Thicket against the nightly release of Pandas 3.0. + + This is not a full testing failure, and it will not prevent your PR from being merged + at this time. However, as we prepare for the release of Pandas 3.0, we encourage + all developers to design their code to work with Pandas 3.0, if possible. + + Pytest status code: ${{ steps.test.outputs.optional_fail_status }} + mode: 'upsert' + + - name: Add comment to PR if test passed + if: ${{ steps.test.outputs.optional_fail == 'false' }} + uses: thollander/actions-comment-pull-request@v3 + with: + comment-tag: "pandas-3.0-test-notice" + message: | + ### Pandas 3.0 Unit Tests Passed! + Due to breaking changes in Pandas 3.0 (namely, copy-on-write), we are performing optional + tests of Thicket against the nightly release of Pandas 3.0. + + This PR passed unit tests when run with Pandas 3.0! + mode: 'upsert' \ No newline at end of file From 88d480a46ac0f5c43820777aab935e4fb7a461c6 Mon Sep 17 00:00:00 2001 From: Ian Lumsden Date: Fri, 1 Nov 2024 17:48:10 -0400 Subject: [PATCH 2/4] Adds pyarrow support to Pandas 3 and fixes some pip flags --- .github/workflows/test-for-pandas-3.yml | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test-for-pandas-3.yml b/.github/workflows/test-for-pandas-3.yml index 11ff4812..78fc93e2 100644 --- a/.github/workflows/test-for-pandas-3.yml +++ b/.github/workflows/test-for-pandas-3.yml @@ -1,10 +1,9 @@ name: Run unit tests with Pandas 3.0 -on: +on: + pull_request_target: push: branches: [ develop ] - pull_request: - branches: [ develop, releases/** ] jobs: run_pandas_3_unit_tests: @@ -27,11 +26,12 @@ jobs: - name: Install Pandas 3.0 (i.e., Nightly) run: | python3 -m pip install --upgrade pip pytest - python3 -m pip install --upgrade --force-reinstall --pre --extra-index https://pypi.anaconda.org/scientific-python-nightly-wheels/simple pandas + python3 -m pip install --upgrade --force-reinstall --pre --extra-index \ + https://pypi.anaconda.org/scientific-python-nightly-wheels/simple pandas[pyarrow] - name: Install Python 3 dependencies run: | - python3 -m pip install --upgrade --force-reinstall git+https://github.com/LLNL/hatchet.git@develop + python3 -m pip install git+https://github.com/LLNL/hatchet.git@develop python3 -m pip install -r requirements.txt python3 -m pip install . python3 -m pip list @@ -42,9 +42,10 @@ jobs: - name: Basic test with PyTest id: test run: | - PYTHONPATH . $(which pytest) + set +e # Prevent immediate exit if pytest reports errors + PYTHONPATH=. $(which pytest) status=$? - if [ $status -ne 0 ]; then + if [ ${status} -ne 0 ]; then echo "::warning::Optional CI test with Pandas 3 failed" echo "optional_fail=true" >> "${GITHUB_OUTPUT}" echo "optional_fail_status=${status}" >> "${GITHUB_OUTPUT}" @@ -66,7 +67,9 @@ jobs: all developers to design their code to work with Pandas 3.0, if possible. Pytest status code: ${{ steps.test.outputs.optional_fail_status }} + Action log: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} mode: 'upsert' + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Add comment to PR if test passed if: ${{ steps.test.outputs.optional_fail == 'false' }} @@ -79,4 +82,5 @@ jobs: tests of Thicket against the nightly release of Pandas 3.0. This PR passed unit tests when run with Pandas 3.0! - mode: 'upsert' \ No newline at end of file + mode: 'upsert' + github-token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file From 58edf680238a6542ca65231b84da177d12ab73db Mon Sep 17 00:00:00 2001 From: Ian Lumsden Date: Mon, 4 Nov 2024 12:19:19 -0500 Subject: [PATCH 3/4] Changes commenting to only run off of 'develop' and to write to a new tracking issue (#221) --- .github/workflows/test-for-pandas-3.yml | 36 ++++++++++++++++--------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test-for-pandas-3.yml b/.github/workflows/test-for-pandas-3.yml index 78fc93e2..7331593a 100644 --- a/.github/workflows/test-for-pandas-3.yml +++ b/.github/workflows/test-for-pandas-3.yml @@ -1,9 +1,10 @@ name: Run unit tests with Pandas 3.0 on: - pull_request_target: push: branches: [ develop ] + pull_request: + branches: [ develop, releases/** ] jobs: run_pandas_3_unit_tests: @@ -51,13 +52,23 @@ jobs: echo "optional_fail_status=${status}" >> "${GITHUB_OUTPUT}" fi exit 0 + + - name: Find issue comment + if: github.event_name == 'push' + uses: peter-evans/find-comment@v3 + id: find_comment + with: + issue-number: 221 + comment-author: 'github-actions[bot]' + body-includes: Pandas 3.0 Unit Tests - name: Add comment to PR if test failed - if: ${{ steps.test.outputs.optional_fail == 'true' }} - uses: thollander/actions-comment-pull-request@v3 + if: github.event_name == 'push' && steps.test.outputs.optional_fail == 'true' + uses: peter-evans/create-or-update-comment@v4 with: - comment-tag: "pandas-3.0-test-notice" - message: | + comment-id: ${{ steps.find_comment.outputs.comment-id }} + issue-number: 221 + body: | ### Pandas 3.0 Unit Tests Failed! Due to breaking changes in Pandas 3.0 (namely, copy-on-write), we are performing optional tests of Thicket against the nightly release of Pandas 3.0. @@ -68,19 +79,18 @@ jobs: Pytest status code: ${{ steps.test.outputs.optional_fail_status }} Action log: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - mode: 'upsert' - github-token: ${{ secrets.GITHUB_TOKEN }} + edit-mode: 'replace' - name: Add comment to PR if test passed - if: ${{ steps.test.outputs.optional_fail == 'false' }} - uses: thollander/actions-comment-pull-request@v3 + if: github.event_name == 'push' && steps.test.outputs.optional_fail == 'false' + uses: peter-evans/create-or-update-comment@v4 with: - comment-tag: "pandas-3.0-test-notice" - message: | + comment-id: ${{ steps.find_comment.outputs.comment-id }} + issue-number: 221 + body: | ### Pandas 3.0 Unit Tests Passed! Due to breaking changes in Pandas 3.0 (namely, copy-on-write), we are performing optional tests of Thicket against the nightly release of Pandas 3.0. This PR passed unit tests when run with Pandas 3.0! - mode: 'upsert' - github-token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file + edit-mode: 'replace' \ No newline at end of file From 1f522f27e91f19f15f86bb4acf5bbd87c9ce8333 Mon Sep 17 00:00:00 2001 From: Ian Lumsden Date: Mon, 4 Nov 2024 14:48:36 -0500 Subject: [PATCH 4/4] Adds final printout in Pandas 3.0 test step --- .github/workflows/test-for-pandas-3.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/test-for-pandas-3.yml b/.github/workflows/test-for-pandas-3.yml index 7331593a..8a968d90 100644 --- a/.github/workflows/test-for-pandas-3.yml +++ b/.github/workflows/test-for-pandas-3.yml @@ -47,9 +47,16 @@ jobs: PYTHONPATH=. $(which pytest) status=$? if [ ${status} -ne 0 ]; then + echo "==================================" + echo "Unit tests with Pandas 3.0 FAILED!" + echo "==================================" echo "::warning::Optional CI test with Pandas 3 failed" echo "optional_fail=true" >> "${GITHUB_OUTPUT}" echo "optional_fail_status=${status}" >> "${GITHUB_OUTPUT}" + else + echo "==================================" + echo "Unit tests with Pandas 3.0 PASSED!" + echo "==================================" fi exit 0