Implement persistent bucket fixtures for integration tests #1410
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Continuous Integration | |
| on: | |
| push: | |
| branches: [master] | |
| pull_request: | |
| branches: [master] | |
| env: | |
| PYTHON_DEFAULT_VERSION: "3.12" | |
| jobs: | |
| lint: | |
| timeout-minutes: 60 | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - uses: codespell-project/actions-codespell@2391250ab05295bddd51e36a8c6295edb6343b0e | |
| with: | |
| ignore_words_list: datas re-use | |
| - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_DEFAULT_VERSION }} | |
| - name: Install dependencies | |
| run: python -m pip install --upgrade nox pdm | |
| - name: Run linters | |
| run: nox -vs lint | |
| - name: Validate new changelog entries | |
| if: (contains(github.event.pull_request.labels.*.name, '-changelog') == false) && (github.event.pull_request.base.ref != '') | |
| run: if [ -z "$(git diff --diff-filter=A --name-only origin/${{ github.event.pull_request.base.ref }} changelog.d)" ]; | |
| then echo no changelog item added; exit 1; fi | |
| - name: Changelog validation | |
| run: nox -vs towncrier_check | |
| build: | |
| needs: lint | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_DEFAULT_VERSION }} | |
| - name: Install dependencies | |
| run: python -m pip install --upgrade nox pdm | |
| - name: Build the distribution | |
| run: nox -vs build | |
| cleanup_buckets: | |
| needs: lint | |
| env: | |
| B2_TEST_APPLICATION_KEY: ${{ secrets.B2_TEST_APPLICATION_KEY }} | |
| B2_TEST_APPLICATION_KEY_ID: ${{ secrets.B2_TEST_APPLICATION_KEY_ID }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }} # TODO: skip this whole job instead | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }} | |
| if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }} # TODO: skip this whole job instead | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_DEFAULT_VERSION }} | |
| cache: "pip" | |
| - name: Install dependencies | |
| if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }} # TODO: skip this whole job instead | |
| run: python -m pip install --upgrade nox pdm | |
| - name: Find and remove old buckets | |
| if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }} # TODO: skip this whole job instead | |
| run: nox -vs cleanup_buckets | |
| test: | |
| timeout-minutes: 90 | |
| needs: cleanup_buckets | |
| env: | |
| B2_TEST_APPLICATION_KEY: ${{ secrets.B2_TEST_APPLICATION_KEY }} | |
| B2_TEST_APPLICATION_KEY_ID: ${{ secrets.B2_TEST_APPLICATION_KEY_ID }} | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: ["ubuntu-latest", "macos-latest", "windows-latest"] | |
| python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "pypy3.9", "pypy3.10"] | |
| exclude: | |
| - os: "macos-latest" | |
| python-version: "pypy3.10" | |
| - os: "windows-latest" | |
| python-version: "pypy3.10" | |
| # Workaround for https://github.com/actions/setup-python/issues/696 | |
| - os: "macos-latest" | |
| python-version: 3.8 | |
| - os: "macos-latest" | |
| python-version: 3.9 | |
| include: | |
| # Workaround for https://github.com/actions/setup-python/issues/696 | |
| - os: "macos-13" | |
| python-version: 3.8 | |
| - os: "macos-13" | |
| python-version: 3.9 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install test binary dependencies | |
| if: startsWith(matrix.os, 'ubuntu') | |
| run: | | |
| sudo apt-get -y update | |
| sudo apt-get -y install zsh fish | |
| sudo chmod -R 755 /usr/share/zsh/vendor-completions /usr/share/zsh # Fix permissions for zsh completions | |
| - name: Install test binary dependencies (macOS) | |
| if: startsWith(matrix.os, 'macos') | |
| run: | | |
| brew install fish | |
| - name: Install dependencies | |
| run: python -m pip install --upgrade nox pdm | |
| - name: Run unit tests | |
| run: nox -vs unit -p ${{ matrix.python-version }} | |
| - name: Run integration tests (without secrets) | |
| run: nox -vs integration -p ${{ matrix.python-version }} -- -m "not require_secrets" | |
| - name: Run integration tests (with secrets) | |
| # Limit CI workload by running integration tests with secrets only on edge Python versions. | |
| if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' && contains(fromJSON('["3.7", "pypy3.10", "3.12"]'), matrix.python-version) }} | |
| run: nox -vs integration -p ${{ matrix.python-version }} -- -m "require_secrets" --cleanup | |
| test-docker: | |
| timeout-minutes: 90 | |
| needs: cleanup_buckets | |
| env: | |
| B2_TEST_APPLICATION_KEY: ${{ secrets.B2_TEST_APPLICATION_KEY }} | |
| B2_TEST_APPLICATION_KEY_ID: ${{ secrets.B2_TEST_APPLICATION_KEY_ID }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_DEFAULT_VERSION }} | |
| - name: Install dependencies | |
| run: sudo python -m pip install --upgrade nox pdm | |
| - name: Generate Dockerfile | |
| run: nox -vs generate_dockerfile | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Build Docker | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| load: true | |
| tags: backblazeit/b2:test | |
| platforms: linux/amd64 | |
| - name: Run tests with docker | |
| if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }} | |
| run: nox -vs docker_test -- backblazeit/b2:test | |
| test-linux-bundle: | |
| timeout-minutes: 90 | |
| needs: cleanup_buckets | |
| env: | |
| B2_TEST_APPLICATION_KEY: ${{ secrets.B2_TEST_APPLICATION_KEY }} | |
| B2_TEST_APPLICATION_KEY_ID: ${{ secrets.B2_TEST_APPLICATION_KEY_ID }} | |
| runs-on: ubuntu-latest | |
| container: | |
| image: "python:3.12" # can not use ${{ env.PYTHON_DEFAULT_VERSION }} here | |
| env: | |
| DEBIAN_FRONTEND: noninteractive | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install dependencies | |
| run: | | |
| apt-get -y update | |
| apt-get -y install patchelf | |
| python -m pip install --upgrade nox pdm | |
| git config --global --add safe.directory '*' | |
| - name: Bundle the distribution | |
| id: bundle | |
| run: nox -vs bundle | |
| - name: Generate hashes | |
| id: hashes | |
| run: nox -vs make_dist_digest | |
| - name: Run integration tests (without secrets) | |
| run: nox -vs integration -p ${{ env.PYTHON_DEFAULT_VERSION }} -- --sut=${{ steps.bundle.outputs.sut_path }} -m "not require_secrets" | |
| - name: Run integration tests (with secrets) | |
| if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }} | |
| run: nox -vs integration -p ${{ env.PYTHON_DEFAULT_VERSION }} -- --sut=${{ steps.bundle.outputs.sut_path }} -m "require_secrets" --cleanup | |
| - name: Upload assets | |
| if: failure() | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| path: ${{ steps.bundle.outputs.asset_path }} | |
| if-no-files-found: warn | |
| retention-days: 7 | |
| test-windows-bundle: | |
| timeout-minutes: 90 | |
| needs: cleanup_buckets | |
| env: | |
| B2_TEST_APPLICATION_KEY: ${{ secrets.B2_TEST_APPLICATION_KEY }} | |
| B2_TEST_APPLICATION_KEY_ID: ${{ secrets.B2_TEST_APPLICATION_KEY_ID }} | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [windows-2019, windows-latest] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_DEFAULT_VERSION }} | |
| - name: Install dependencies | |
| run: python -m pip install --upgrade nox pdm | |
| - name: Bundle the distribution | |
| id: bundle | |
| shell: bash | |
| run: nox -vs bundle | |
| - name: Generate hashes | |
| id: hashes | |
| run: nox -vs make_dist_digest | |
| - name: Run integration tests (without secrets) | |
| run: nox -vs integration -p ${{ env.PYTHON_DEFAULT_VERSION }} -- --sut=${{ steps.bundle.outputs.sut_path }} -m "not require_secrets" | |
| - name: Run integration tests (with secrets) | |
| if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }} | |
| run: nox -vs integration -p ${{ env.PYTHON_DEFAULT_VERSION }} -- --sut=${{ steps.bundle.outputs.sut_path }} -m "require_secrets" --cleanup | |
| - name: Upload assets | |
| if: failure() | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| path: ${{ steps.bundle.outputs.asset_path }} | |
| if-no-files-found: warn | |
| retention-days: 7 | |
| doc: | |
| timeout-minutes: 30 | |
| needs: build | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_DEFAULT_VERSION }} | |
| - name: Install dependencies | |
| env: | |
| DEBIAN_FRONTEND: noninteractive | |
| run: | | |
| sudo apt-get update -y | |
| sudo apt-get install -y graphviz plantuml | |
| python -m pip install --upgrade nox pdm | |
| - name: Build the docs | |
| run: nox --non-interactive -vs doc |