From f8b9d0b6dd9203fe11e9140c946c495897628aad Mon Sep 17 00:00:00 2001 From: David Robertson Date: Mon, 3 Oct 2022 14:19:51 +0100 Subject: [PATCH 01/10] WIP: build aarch64 wheels on release branches --- .github/workflows/release-artifacts.yml | 30 +++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index 0708d631cd36..70b22c1fcbec 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -94,29 +94,55 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-20.04, macos-10.15] + # is_pr is a flag used to exclude certain jobs from the matrix on PRs. + # It is not read by the rest of the workflow. is_pr: - ${{ startsWith(github.ref, 'refs/pull/') }} + include: + - os: ubuntu-20.04 + emulate_aarch64: false + - os: ubuntu-20.04 + emulate_aarch64: true + - os: macos-10.15 + emulate_aarch64: false exclude: # Don't build macos wheels on PR CI. - is_pr: true os: "macos-10.15" + # Don't build ARM linux wheels on PR CI either. + - is_pr: true + emulate_aarch64: true steps: + - name: Debug + run: echo '${{ toJSON(matrix) }}' + - uses: actions/checkout@v3 - uses: actions/setup-python@v3 + - name: Set up QEMU to emulate arm64 + if: matrix.emulate_aarch64 + uses: docker/setup-qemu-action@v2 + with: + platforms: arm64 + - name: Install cibuildwheel run: python -m pip install cibuildwheel==2.9.0 poetry==1.2.0 # Only build a single wheel in CI. - - name: Set env vars. + - name: "Env: build single wheel for CI" run: | echo "CIBW_BUILD="cp37-manylinux_x86_64"" >> $GITHUB_ENV if: startsWith(github.ref, 'refs/pull/') + # Only build a single wheel in CI. + - name: "Env: build aarch64 wheels" + run: | + echo "CIBW_ARCHS_LINUX=aarch64 >> $GITHUB_ENV + if: matrix.emulate_aarch64 + - name: Build wheels run: python -m cibuildwheel --output-dir wheelhouse env: From 42040d83a459962eb65e07ef7156ee2e74229ea2 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Mon, 3 Oct 2022 14:21:39 +0100 Subject: [PATCH 02/10] Changelog --- changelog.d/14016.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/14016.misc diff --git a/changelog.d/14016.misc b/changelog.d/14016.misc new file mode 100644 index 000000000000..76082a80f926 --- /dev/null +++ b/changelog.d/14016.misc @@ -0,0 +1 @@ +Build wheels for ARM (aarch64) on release branches. From c22562b916a11479678af153fc818696b4c4363e Mon Sep 17 00:00:00 2001 From: David Robertson Date: Mon, 3 Oct 2022 14:47:40 +0100 Subject: [PATCH 03/10] Try a different matrix syntax --- .github/workflows/release-artifacts.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index 70b22c1fcbec..1aa8f2f345ee 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -94,17 +94,15 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: + os: [ubuntu-20.04, macos-10.15] + emulate_aarch64: [false] # is_pr is a flag used to exclude certain jobs from the matrix on PRs. # It is not read by the rest of the workflow. is_pr: - ${{ startsWith(github.ref, 'refs/pull/') }} include: - - os: ubuntu-20.04 - emulate_aarch64: false - os: ubuntu-20.04 emulate_aarch64: true - - os: macos-10.15 - emulate_aarch64: false exclude: # Don't build macos wheels on PR CI. From d04b8976c751c469efab4901e3cfac696f292815 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Mon, 3 Oct 2022 14:49:22 +0100 Subject: [PATCH 04/10] Fix syntax --- .github/workflows/release-artifacts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index 1aa8f2f345ee..b152975f6466 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -138,7 +138,7 @@ jobs: # Only build a single wheel in CI. - name: "Env: build aarch64 wheels" run: | - echo "CIBW_ARCHS_LINUX=aarch64 >> $GITHUB_ENV + echo "CIBW_ARCHS_LINUX=aarch64" >> $GITHUB_ENV if: matrix.emulate_aarch64 - name: Build wheels From 84850c178777c263b4c8907c264778ef1680f67c Mon Sep 17 00:00:00 2001 From: David Robertson Date: Mon, 3 Oct 2022 14:52:03 +0100 Subject: [PATCH 05/10] Don't run tests while I'm experimenting --- .github/workflows/tests.yml | 508 ------------------------------------ 1 file changed, 508 deletions(-) delete mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml deleted file mode 100644 index 9fe61930a55a..000000000000 --- a/.github/workflows/tests.yml +++ /dev/null @@ -1,508 +0,0 @@ -name: Tests - -on: - push: - branches: ["develop", "release-*"] - pull_request: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - # Job to detect what has changed so we don't run e.g. Rust checks on PRs that - # don't modify Rust code. - changes: - runs-on: ubuntu-latest - outputs: - rust: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.rust }} - steps: - - uses: dorny/paths-filter@v2 - id: filter - # We only check on PRs - if: startsWith(github.ref, 'refs/pull/') - with: - filters: | - rust: - - 'rust/**' - - 'Cargo.toml' - - check-sampleconfig: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - uses: matrix-org/setup-python-poetry@v1 - with: - extras: "all" - - run: poetry run scripts-dev/generate_sample_config.sh --check - - run: poetry run scripts-dev/config-lint.sh - - check-schema-delta: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - run: "pip install 'click==8.1.1' 'GitPython>=3.1.20'" - - run: scripts-dev/check_schema_delta.py --force-colors - - lint: - uses: "matrix-org/backend-meta/.github/workflows/python-poetry-ci.yml@v1" - with: - typechecking-extras: "all" - - lint-crlf: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Check line endings - run: scripts-dev/check_line_terminators.sh - - lint-newsfile: - if: ${{ github.base_ref == 'develop' || contains(github.base_ref, 'release-') }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - ref: ${{ github.event.pull_request.head.sha }} - fetch-depth: 0 - - uses: actions/setup-python@v2 - - run: "pip install 'towncrier>=18.6.0rc1'" - - run: scripts-dev/check-newsfragment.sh - env: - PULL_REQUEST_NUMBER: ${{ github.event.number }} - - lint-pydantic: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - ref: ${{ github.event.pull_request.head.sha }} - - uses: matrix-org/setup-python-poetry@v1 - with: - extras: "all" - - run: poetry run scripts-dev/check_pydantic_models.py - - lint-clippy: - runs-on: ubuntu-latest - needs: changes - if: ${{ needs.changes.outputs.rust == 'true' }} - - steps: - - uses: actions/checkout@v2 - - - name: Install Rust - uses: actions-rs/toolchain@v1 - with: - toolchain: 1.58.1 - override: true - components: clippy - - uses: Swatinem/rust-cache@v2 - - - run: cargo clippy - - lint-rustfmt: - runs-on: ubuntu-latest - needs: changes - if: ${{ needs.changes.outputs.rust == 'true' }} - - steps: - - uses: actions/checkout@v2 - - - name: Install Rust - uses: actions-rs/toolchain@v1 - with: - toolchain: 1.58.1 - override: true - components: rustfmt - - uses: Swatinem/rust-cache@v2 - - - run: cargo fmt --check - - # Dummy step to gate other tests on without repeating the whole list - linting-done: - if: ${{ !cancelled() }} # Run this even if prior jobs were skipped - needs: - - lint - - lint-crlf - - lint-newsfile - - lint-pydantic - - check-sampleconfig - - check-schema-delta - - lint-clippy - - lint-rustfmt - runs-on: ubuntu-latest - steps: - - run: "true" - - calculate-test-jobs: - if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail - needs: linting-done - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - id: get-matrix - run: .ci/scripts/calculate_jobs.py - outputs: - trial_test_matrix: ${{ steps.get-matrix.outputs.trial_test_matrix }} - sytest_test_matrix: ${{ steps.get-matrix.outputs.sytest_test_matrix }} - - trial: - if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail - needs: calculate-test-jobs - runs-on: ubuntu-latest - strategy: - matrix: - job: ${{ fromJson(needs.calculate-test-jobs.outputs.trial_test_matrix) }} - - steps: - - uses: actions/checkout@v2 - - run: sudo apt-get -qq install xmlsec1 - - name: Set up PostgreSQL ${{ matrix.job.postgres-version }} - if: ${{ matrix.job.postgres-version }} - run: | - docker run -d -p 5432:5432 \ - -e POSTGRES_PASSWORD=postgres \ - -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \ - postgres:${{ matrix.job.postgres-version }} - - uses: matrix-org/setup-python-poetry@v1 - with: - python-version: ${{ matrix.job.python-version }} - extras: ${{ matrix.job.extras }} - - name: Await PostgreSQL - if: ${{ matrix.job.postgres-version }} - timeout-minutes: 2 - run: until pg_isready -h localhost; do sleep 1; done - - run: poetry run trial --jobs=2 tests - env: - SYNAPSE_POSTGRES: ${{ matrix.job.database == 'postgres' || '' }} - SYNAPSE_POSTGRES_HOST: localhost - SYNAPSE_POSTGRES_USER: postgres - SYNAPSE_POSTGRES_PASSWORD: postgres - - name: Dump logs - # Logs are most useful when the command fails, always include them. - if: ${{ always() }} - # Note: Dumps to workflow logs instead of using actions/upload-artifact - # This keeps logs colocated with failing jobs - # It also ignores find's exit code; this is a best effort affair - run: >- - find _trial_temp -name '*.log' - -exec echo "::group::{}" \; - -exec cat {} \; - -exec echo "::endgroup::" \; - || true - - trial-olddeps: - # Note: sqlite only; no postgres - if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail - needs: linting-done - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v2 - - - name: Install Rust - uses: actions-rs/toolchain@v1 - with: - toolchain: 1.58.1 - override: true - - uses: Swatinem/rust-cache@v2 - - # There aren't wheels for some of the older deps, so we need to install - # their build dependencies - - run: | - sudo apt-get -qq install build-essential libffi-dev python-dev \ - libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev - - - uses: actions/setup-python@v4 - with: - python-version: '3.7' - - # Calculating the old-deps actually takes a bunch of time, so we cache the - # pyproject.toml / poetry.lock. We need to cache pyproject.toml as - # otherwise the `poetry install` step will error due to the poetry.lock - # file being outdated. - # - # This caches the output of `Prepare old deps`, which should generate the - # same `pyproject.toml` and `poetry.lock` for a given `pyproject.toml` input. - - uses: actions/cache@v3 - id: cache-poetry-old-deps - name: Cache poetry.lock - with: - path: | - poetry.lock - pyproject.toml - key: poetry-old-deps2-${{ hashFiles('pyproject.toml') }} - - name: Prepare old deps - if: steps.cache-poetry-old-deps.outputs.cache-hit != 'true' - run: .ci/scripts/prepare_old_deps.sh - - # We only now install poetry so that `setup-python-poetry` caches the - # right poetry.lock's dependencies. - - uses: matrix-org/setup-python-poetry@v1 - with: - python-version: '3.7' - extras: "all test" - - - run: poetry run trial -j2 tests - - name: Dump logs - # Logs are most useful when the command fails, always include them. - if: ${{ always() }} - # Note: Dumps to workflow logs instead of using actions/upload-artifact - # This keeps logs colocated with failing jobs - # It also ignores find's exit code; this is a best effort affair - run: >- - find _trial_temp -name '*.log' - -exec echo "::group::{}" \; - -exec cat {} \; - -exec echo "::endgroup::" \; - || true - - trial-pypy: - # Very slow; only run if the branch name includes 'pypy' - # Note: sqlite only; no postgres. Completely untested since poetry move. - if: ${{ contains(github.ref, 'pypy') && !failure() && !cancelled() }} - needs: linting-done - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["pypy-3.7"] - extras: ["all"] - - steps: - - uses: actions/checkout@v2 - # Install libs necessary for PyPy to build binary wheels for dependencies - - run: sudo apt-get -qq install xmlsec1 libxml2-dev libxslt-dev - - uses: matrix-org/setup-python-poetry@v1 - with: - python-version: ${{ matrix.python-version }} - extras: ${{ matrix.extras }} - - run: poetry run trial --jobs=2 tests - - name: Dump logs - # Logs are most useful when the command fails, always include them. - if: ${{ always() }} - # Note: Dumps to workflow logs instead of using actions/upload-artifact - # This keeps logs colocated with failing jobs - # It also ignores find's exit code; this is a best effort affair - run: >- - find _trial_temp -name '*.log' - -exec echo "::group::{}" \; - -exec cat {} \; - -exec echo "::endgroup::" \; - || true - - sytest: - if: ${{ !failure() && !cancelled() }} - needs: calculate-test-jobs - runs-on: ubuntu-latest - container: - image: matrixdotorg/sytest-synapse:${{ matrix.job.sytest-tag }} - volumes: - - ${{ github.workspace }}:/src - env: - SYTEST_BRANCH: ${{ github.head_ref }} - POSTGRES: ${{ matrix.job.postgres && 1}} - MULTI_POSTGRES: ${{ (matrix.job.postgres == 'multi-postgres') && 1}} - WORKERS: ${{ matrix.job.workers && 1 }} - BLACKLIST: ${{ matrix.job.workers && 'synapse-blacklist-with-workers' }} - TOP: ${{ github.workspace }} - - strategy: - fail-fast: false - matrix: - job: ${{ fromJson(needs.calculate-test-jobs.outputs.sytest_test_matrix) }} - - steps: - - uses: actions/checkout@v2 - - name: Prepare test blacklist - run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers - - - name: Install Rust - uses: actions-rs/toolchain@v1 - with: - toolchain: 1.58.1 - override: true - - uses: Swatinem/rust-cache@v2 - - - name: Run SyTest - run: /bootstrap.sh synapse - working-directory: /src - - name: Summarise results.tap - if: ${{ always() }} - run: /sytest/scripts/tap_to_gha.pl /logs/results.tap - - name: Upload SyTest logs - uses: actions/upload-artifact@v2 - if: ${{ always() }} - with: - name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.job.*, ', ') }}) - path: | - /logs/results.tap - /logs/**/*.log* - - export-data: - if: ${{ !failure() && !cancelled() }} # Allow previous steps to be skipped, but not fail - needs: [linting-done, portdb] - runs-on: ubuntu-latest - env: - TOP: ${{ github.workspace }} - - services: - postgres: - image: postgres - ports: - - 5432:5432 - env: - POSTGRES_PASSWORD: "postgres" - POSTGRES_INITDB_ARGS: "--lc-collate C --lc-ctype C --encoding UTF8" - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - - steps: - - uses: actions/checkout@v2 - - run: sudo apt-get -qq install xmlsec1 postgresql-client - - uses: matrix-org/setup-python-poetry@v1 - with: - extras: "postgres" - - run: .ci/scripts/test_export_data_command.sh - env: - PGHOST: localhost - PGUSER: postgres - PGPASSWORD: postgres - PGDATABASE: postgres - - - portdb: - if: ${{ !failure() && !cancelled() }} # Allow previous steps to be skipped, but not fail - needs: linting-done - runs-on: ubuntu-latest - strategy: - matrix: - include: - - python-version: "3.7" - postgres-version: "10" - - - python-version: "3.10" - postgres-version: "14" - - services: - postgres: - image: postgres:${{ matrix.postgres-version }} - ports: - - 5432:5432 - env: - POSTGRES_PASSWORD: "postgres" - POSTGRES_INITDB_ARGS: "--lc-collate C --lc-ctype C --encoding UTF8" - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - - steps: - - uses: actions/checkout@v2 - - run: sudo apt-get -qq install xmlsec1 postgresql-client - - uses: matrix-org/setup-python-poetry@v1 - with: - python-version: ${{ matrix.python-version }} - extras: "postgres" - - run: .ci/scripts/test_synapse_port_db.sh - id: run_tester_script - env: - PGHOST: localhost - PGUSER: postgres - PGPASSWORD: postgres - PGDATABASE: postgres - - name: "Upload schema differences" - uses: actions/upload-artifact@v3 - if: ${{ failure() && !cancelled() && steps.run_tester_script.outcome == 'failure' }} - with: - name: Schema dumps - path: | - unported.sql - ported.sql - schema_diff - - complement: - if: "${{ !failure() && !cancelled() }}" - needs: linting-done - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - include: - - arrangement: monolith - database: SQLite - - - arrangement: monolith - database: Postgres - - - arrangement: workers - database: Postgres - - steps: - - name: Run actions/checkout@v2 for synapse - uses: actions/checkout@v2 - with: - path: synapse - - - name: Install Rust - uses: actions-rs/toolchain@v1 - with: - toolchain: 1.58.1 - override: true - - uses: Swatinem/rust-cache@v2 - - - name: Prepare Complement's Prerequisites - run: synapse/.ci/scripts/setup_complement_prerequisites.sh - - - run: | - set -o pipefail - POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | synapse/.ci/scripts/gotestfmt - shell: bash - name: Run Complement Tests - - cargo-test: - if: ${{ needs.changes.outputs.rust == 'true' }} - runs-on: ubuntu-latest - needs: - - linting-done - - changes - - steps: - - uses: actions/checkout@v2 - - - name: Install Rust - uses: actions-rs/toolchain@v1 - with: - toolchain: 1.58.1 - override: true - - uses: Swatinem/rust-cache@v2 - - - run: cargo test - - # a job which marks all the other jobs as complete, thus allowing PRs to be merged. - tests-done: - if: ${{ always() }} - needs: - - trial - - trial-olddeps - - sytest - - export-data - - portdb - - complement - - cargo-test - runs-on: ubuntu-latest - steps: - - uses: matrix-org/done-action@v2 - with: - needs: ${{ toJSON(needs) }} - - # The newsfile lint may be skipped on non PR builds - # Cargo test is skipped if there is no changes on Rust code - skippable: | - lint-newsfile - cargo-test From 3b73e73ca5b22e0aa162b1ede69052f52d54d153 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Mon, 3 Oct 2022 14:56:40 +0100 Subject: [PATCH 06/10] Try another matrix syntax --- .github/workflows/release-artifacts.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index b152975f6466..ce824f9b872e 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -94,15 +94,18 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-20.04, macos-10.15] - emulate_aarch64: [false] # is_pr is a flag used to exclude certain jobs from the matrix on PRs. # It is not read by the rest of the workflow. is_pr: - ${{ startsWith(github.ref, 'refs/pull/') }} + include: + - os: ubuntu-20.04 + emulate_aarch64: false - os: ubuntu-20.04 emulate_aarch64: true + - os: macos-10.15 + emulate_aarch64: false exclude: # Don't build macos wheels on PR CI. From dddc0a6fee72827edb939b7d01dc06932ecedf50 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Mon, 3 Oct 2022 14:58:03 +0100 Subject: [PATCH 07/10] Revert "Don't run tests while I'm experimenting" This reverts commit 84850c178777c263b4c8907c264778ef1680f67c. --- .github/workflows/tests.yml | 508 ++++++++++++++++++++++++++++++++++++ 1 file changed, 508 insertions(+) create mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 000000000000..9fe61930a55a --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,508 @@ +name: Tests + +on: + push: + branches: ["develop", "release-*"] + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + # Job to detect what has changed so we don't run e.g. Rust checks on PRs that + # don't modify Rust code. + changes: + runs-on: ubuntu-latest + outputs: + rust: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.rust }} + steps: + - uses: dorny/paths-filter@v2 + id: filter + # We only check on PRs + if: startsWith(github.ref, 'refs/pull/') + with: + filters: | + rust: + - 'rust/**' + - 'Cargo.toml' + + check-sampleconfig: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - uses: matrix-org/setup-python-poetry@v1 + with: + extras: "all" + - run: poetry run scripts-dev/generate_sample_config.sh --check + - run: poetry run scripts-dev/config-lint.sh + + check-schema-delta: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - run: "pip install 'click==8.1.1' 'GitPython>=3.1.20'" + - run: scripts-dev/check_schema_delta.py --force-colors + + lint: + uses: "matrix-org/backend-meta/.github/workflows/python-poetry-ci.yml@v1" + with: + typechecking-extras: "all" + + lint-crlf: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Check line endings + run: scripts-dev/check_line_terminators.sh + + lint-newsfile: + if: ${{ github.base_ref == 'develop' || contains(github.base_ref, 'release-') }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + - uses: actions/setup-python@v2 + - run: "pip install 'towncrier>=18.6.0rc1'" + - run: scripts-dev/check-newsfragment.sh + env: + PULL_REQUEST_NUMBER: ${{ github.event.number }} + + lint-pydantic: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + ref: ${{ github.event.pull_request.head.sha }} + - uses: matrix-org/setup-python-poetry@v1 + with: + extras: "all" + - run: poetry run scripts-dev/check_pydantic_models.py + + lint-clippy: + runs-on: ubuntu-latest + needs: changes + if: ${{ needs.changes.outputs.rust == 'true' }} + + steps: + - uses: actions/checkout@v2 + + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: 1.58.1 + override: true + components: clippy + - uses: Swatinem/rust-cache@v2 + + - run: cargo clippy + + lint-rustfmt: + runs-on: ubuntu-latest + needs: changes + if: ${{ needs.changes.outputs.rust == 'true' }} + + steps: + - uses: actions/checkout@v2 + + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: 1.58.1 + override: true + components: rustfmt + - uses: Swatinem/rust-cache@v2 + + - run: cargo fmt --check + + # Dummy step to gate other tests on without repeating the whole list + linting-done: + if: ${{ !cancelled() }} # Run this even if prior jobs were skipped + needs: + - lint + - lint-crlf + - lint-newsfile + - lint-pydantic + - check-sampleconfig + - check-schema-delta + - lint-clippy + - lint-rustfmt + runs-on: ubuntu-latest + steps: + - run: "true" + + calculate-test-jobs: + if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail + needs: linting-done + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - id: get-matrix + run: .ci/scripts/calculate_jobs.py + outputs: + trial_test_matrix: ${{ steps.get-matrix.outputs.trial_test_matrix }} + sytest_test_matrix: ${{ steps.get-matrix.outputs.sytest_test_matrix }} + + trial: + if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail + needs: calculate-test-jobs + runs-on: ubuntu-latest + strategy: + matrix: + job: ${{ fromJson(needs.calculate-test-jobs.outputs.trial_test_matrix) }} + + steps: + - uses: actions/checkout@v2 + - run: sudo apt-get -qq install xmlsec1 + - name: Set up PostgreSQL ${{ matrix.job.postgres-version }} + if: ${{ matrix.job.postgres-version }} + run: | + docker run -d -p 5432:5432 \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \ + postgres:${{ matrix.job.postgres-version }} + - uses: matrix-org/setup-python-poetry@v1 + with: + python-version: ${{ matrix.job.python-version }} + extras: ${{ matrix.job.extras }} + - name: Await PostgreSQL + if: ${{ matrix.job.postgres-version }} + timeout-minutes: 2 + run: until pg_isready -h localhost; do sleep 1; done + - run: poetry run trial --jobs=2 tests + env: + SYNAPSE_POSTGRES: ${{ matrix.job.database == 'postgres' || '' }} + SYNAPSE_POSTGRES_HOST: localhost + SYNAPSE_POSTGRES_USER: postgres + SYNAPSE_POSTGRES_PASSWORD: postgres + - name: Dump logs + # Logs are most useful when the command fails, always include them. + if: ${{ always() }} + # Note: Dumps to workflow logs instead of using actions/upload-artifact + # This keeps logs colocated with failing jobs + # It also ignores find's exit code; this is a best effort affair + run: >- + find _trial_temp -name '*.log' + -exec echo "::group::{}" \; + -exec cat {} \; + -exec echo "::endgroup::" \; + || true + + trial-olddeps: + # Note: sqlite only; no postgres + if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail + needs: linting-done + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: 1.58.1 + override: true + - uses: Swatinem/rust-cache@v2 + + # There aren't wheels for some of the older deps, so we need to install + # their build dependencies + - run: | + sudo apt-get -qq install build-essential libffi-dev python-dev \ + libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev + + - uses: actions/setup-python@v4 + with: + python-version: '3.7' + + # Calculating the old-deps actually takes a bunch of time, so we cache the + # pyproject.toml / poetry.lock. We need to cache pyproject.toml as + # otherwise the `poetry install` step will error due to the poetry.lock + # file being outdated. + # + # This caches the output of `Prepare old deps`, which should generate the + # same `pyproject.toml` and `poetry.lock` for a given `pyproject.toml` input. + - uses: actions/cache@v3 + id: cache-poetry-old-deps + name: Cache poetry.lock + with: + path: | + poetry.lock + pyproject.toml + key: poetry-old-deps2-${{ hashFiles('pyproject.toml') }} + - name: Prepare old deps + if: steps.cache-poetry-old-deps.outputs.cache-hit != 'true' + run: .ci/scripts/prepare_old_deps.sh + + # We only now install poetry so that `setup-python-poetry` caches the + # right poetry.lock's dependencies. + - uses: matrix-org/setup-python-poetry@v1 + with: + python-version: '3.7' + extras: "all test" + + - run: poetry run trial -j2 tests + - name: Dump logs + # Logs are most useful when the command fails, always include them. + if: ${{ always() }} + # Note: Dumps to workflow logs instead of using actions/upload-artifact + # This keeps logs colocated with failing jobs + # It also ignores find's exit code; this is a best effort affair + run: >- + find _trial_temp -name '*.log' + -exec echo "::group::{}" \; + -exec cat {} \; + -exec echo "::endgroup::" \; + || true + + trial-pypy: + # Very slow; only run if the branch name includes 'pypy' + # Note: sqlite only; no postgres. Completely untested since poetry move. + if: ${{ contains(github.ref, 'pypy') && !failure() && !cancelled() }} + needs: linting-done + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["pypy-3.7"] + extras: ["all"] + + steps: + - uses: actions/checkout@v2 + # Install libs necessary for PyPy to build binary wheels for dependencies + - run: sudo apt-get -qq install xmlsec1 libxml2-dev libxslt-dev + - uses: matrix-org/setup-python-poetry@v1 + with: + python-version: ${{ matrix.python-version }} + extras: ${{ matrix.extras }} + - run: poetry run trial --jobs=2 tests + - name: Dump logs + # Logs are most useful when the command fails, always include them. + if: ${{ always() }} + # Note: Dumps to workflow logs instead of using actions/upload-artifact + # This keeps logs colocated with failing jobs + # It also ignores find's exit code; this is a best effort affair + run: >- + find _trial_temp -name '*.log' + -exec echo "::group::{}" \; + -exec cat {} \; + -exec echo "::endgroup::" \; + || true + + sytest: + if: ${{ !failure() && !cancelled() }} + needs: calculate-test-jobs + runs-on: ubuntu-latest + container: + image: matrixdotorg/sytest-synapse:${{ matrix.job.sytest-tag }} + volumes: + - ${{ github.workspace }}:/src + env: + SYTEST_BRANCH: ${{ github.head_ref }} + POSTGRES: ${{ matrix.job.postgres && 1}} + MULTI_POSTGRES: ${{ (matrix.job.postgres == 'multi-postgres') && 1}} + WORKERS: ${{ matrix.job.workers && 1 }} + BLACKLIST: ${{ matrix.job.workers && 'synapse-blacklist-with-workers' }} + TOP: ${{ github.workspace }} + + strategy: + fail-fast: false + matrix: + job: ${{ fromJson(needs.calculate-test-jobs.outputs.sytest_test_matrix) }} + + steps: + - uses: actions/checkout@v2 + - name: Prepare test blacklist + run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers + + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: 1.58.1 + override: true + - uses: Swatinem/rust-cache@v2 + + - name: Run SyTest + run: /bootstrap.sh synapse + working-directory: /src + - name: Summarise results.tap + if: ${{ always() }} + run: /sytest/scripts/tap_to_gha.pl /logs/results.tap + - name: Upload SyTest logs + uses: actions/upload-artifact@v2 + if: ${{ always() }} + with: + name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.job.*, ', ') }}) + path: | + /logs/results.tap + /logs/**/*.log* + + export-data: + if: ${{ !failure() && !cancelled() }} # Allow previous steps to be skipped, but not fail + needs: [linting-done, portdb] + runs-on: ubuntu-latest + env: + TOP: ${{ github.workspace }} + + services: + postgres: + image: postgres + ports: + - 5432:5432 + env: + POSTGRES_PASSWORD: "postgres" + POSTGRES_INITDB_ARGS: "--lc-collate C --lc-ctype C --encoding UTF8" + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + steps: + - uses: actions/checkout@v2 + - run: sudo apt-get -qq install xmlsec1 postgresql-client + - uses: matrix-org/setup-python-poetry@v1 + with: + extras: "postgres" + - run: .ci/scripts/test_export_data_command.sh + env: + PGHOST: localhost + PGUSER: postgres + PGPASSWORD: postgres + PGDATABASE: postgres + + + portdb: + if: ${{ !failure() && !cancelled() }} # Allow previous steps to be skipped, but not fail + needs: linting-done + runs-on: ubuntu-latest + strategy: + matrix: + include: + - python-version: "3.7" + postgres-version: "10" + + - python-version: "3.10" + postgres-version: "14" + + services: + postgres: + image: postgres:${{ matrix.postgres-version }} + ports: + - 5432:5432 + env: + POSTGRES_PASSWORD: "postgres" + POSTGRES_INITDB_ARGS: "--lc-collate C --lc-ctype C --encoding UTF8" + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + steps: + - uses: actions/checkout@v2 + - run: sudo apt-get -qq install xmlsec1 postgresql-client + - uses: matrix-org/setup-python-poetry@v1 + with: + python-version: ${{ matrix.python-version }} + extras: "postgres" + - run: .ci/scripts/test_synapse_port_db.sh + id: run_tester_script + env: + PGHOST: localhost + PGUSER: postgres + PGPASSWORD: postgres + PGDATABASE: postgres + - name: "Upload schema differences" + uses: actions/upload-artifact@v3 + if: ${{ failure() && !cancelled() && steps.run_tester_script.outcome == 'failure' }} + with: + name: Schema dumps + path: | + unported.sql + ported.sql + schema_diff + + complement: + if: "${{ !failure() && !cancelled() }}" + needs: linting-done + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + include: + - arrangement: monolith + database: SQLite + + - arrangement: monolith + database: Postgres + + - arrangement: workers + database: Postgres + + steps: + - name: Run actions/checkout@v2 for synapse + uses: actions/checkout@v2 + with: + path: synapse + + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: 1.58.1 + override: true + - uses: Swatinem/rust-cache@v2 + + - name: Prepare Complement's Prerequisites + run: synapse/.ci/scripts/setup_complement_prerequisites.sh + + - run: | + set -o pipefail + POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | synapse/.ci/scripts/gotestfmt + shell: bash + name: Run Complement Tests + + cargo-test: + if: ${{ needs.changes.outputs.rust == 'true' }} + runs-on: ubuntu-latest + needs: + - linting-done + - changes + + steps: + - uses: actions/checkout@v2 + + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: 1.58.1 + override: true + - uses: Swatinem/rust-cache@v2 + + - run: cargo test + + # a job which marks all the other jobs as complete, thus allowing PRs to be merged. + tests-done: + if: ${{ always() }} + needs: + - trial + - trial-olddeps + - sytest + - export-data + - portdb + - complement + - cargo-test + runs-on: ubuntu-latest + steps: + - uses: matrix-org/done-action@v2 + with: + needs: ${{ toJSON(needs) }} + + # The newsfile lint may be skipped on non PR builds + # Cargo test is skipped if there is no changes on Rust code + skippable: | + lint-newsfile + cargo-test From 4fe4d3d548a05c7e8d01bbd27d208a41bf5a3ace Mon Sep 17 00:00:00 2001 From: David Robertson Date: Mon, 3 Oct 2022 14:58:24 +0100 Subject: [PATCH 08/10] Please don't run tests --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9fe61930a55a..0bfc01fdb297 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -3,7 +3,7 @@ name: Tests on: push: branches: ["develop", "release-*"] - pull_request: +# pull_request: concurrency: group: ${{ github.workflow }}-${{ github.ref }} From 68aad3d690870e0f7cea2c730f9c21b589f185bf Mon Sep 17 00:00:00 2001 From: David Robertson Date: Mon, 3 Oct 2022 15:28:10 +0100 Subject: [PATCH 09/10] Yet another matrix syntax --- .github/workflows/release-artifacts.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index ce824f9b872e..fc9d508a4112 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -94,6 +94,8 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: + os: [ubuntu-20.04, macos-10.15] + emulate_aarch64: [false] # is_pr is a flag used to exclude certain jobs from the matrix on PRs. # It is not read by the rest of the workflow. is_pr: @@ -101,11 +103,8 @@ jobs: include: - os: ubuntu-20.04 - emulate_aarch64: false - - os: ubuntu-20.04 + is_pr: ${{ startsWith(github.ref, 'refs/pull/') }} emulate_aarch64: true - - os: macos-10.15 - emulate_aarch64: false exclude: # Don't build macos wheels on PR CI. From f84186eb3ac82d93d6c3bda5550692d601a71cd4 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Mon, 3 Oct 2022 16:05:20 +0100 Subject: [PATCH 10/10] I just want to cry --- .github/workflows/release-artifacts.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index fc9d508a4112..07766f0e79ee 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -95,22 +95,20 @@ jobs: strategy: matrix: os: [ubuntu-20.04, macos-10.15] - emulate_aarch64: [false] + emulate_aarch64: [false, true] # is_pr is a flag used to exclude certain jobs from the matrix on PRs. # It is not read by the rest of the workflow. is_pr: - ${{ startsWith(github.ref, 'refs/pull/') }} - include: - - os: ubuntu-20.04 - is_pr: ${{ startsWith(github.ref, 'refs/pull/') }} - emulate_aarch64: true - exclude: # Don't build macos wheels on PR CI. - is_pr: true os: "macos-10.15" - # Don't build ARM linux wheels on PR CI either. + # Don't build aarch64 wheels on mac. + - os: "macos-10.15" + emulate_aarch64: true + # Don't build aarch64 wheels on PR CI. - is_pr: true emulate_aarch64: true