Move from Buildkite to GitHub Actions (#1127, #1115)

Suggest we leave Buildkite running in parallel for the time being. In particular we want to ensure that the cross-branch synapse-sytest and dendrite-sytest stuff works.

Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com>

.github/workflows/pipeline.yml

@@ -0,0 +1,156 @@
+name: Run sytest
+on:
+  push:
+    branches: ["develop", "release-*"]
+  pull_request:
+
+# Only run this action once per pull request/branch; restart if a new commit arrives.
+# C.f. https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#concurrency
+# and https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#github-context
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  synapse:
+    name: "Synapse: ${{ matrix.label }}"
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - label: Py 3.6, SQLite 3.22.0, Monolith
+            sytest-tag: bionic
+
+          - label: Py 3.6, PG 10, Monolith
+            sytest-tag: bionic
+            postgres: postgres
+
+          - label: Py 3.6, PG 10, Workers
+            sytest-tag: bionic
+            postgres: postgres
+            workers: workers
+
+          - label: debian-testing, Monolith
+            sytest-tag: testing
+            postgres: postgres
+
+          - label: debian-testing, Workers
+            sytest-tag: testing
+            postgres: postgres
+            workers: workers
+
+    container:
+      image: matrixdotorg/sytest-synapse:${{ matrix.sytest-tag }}
+      volumes:
+        # bootstrap.sh expects the sytest source available at /sytest.
+        # TODO Buildkite mounted /sytest as readonly. Can we do this on GHA? Do we need it?
+        - ${{ github.workspace }}/sytest:/sytest
+        # synapse_sytest.sh expects a synapse checkout at /src
+        - ${{ github.workspace }}/synapse:/src
+      env:
+        POSTGRES: ${{ matrix.postgres && 1 }}
+        WORKERS: ${{ matrix.workers && 1 }}
+        BLACKLIST: ${{ (matrix.workers && 'synapse-blacklist-with-workers') || 'sytest-blacklist' }}
+
+    steps:
+      - name: Checkout sytest
+        uses: actions/checkout@v2
+        with:
+          path: sytest
+
+      - name: Fetch corresponding synapse branch
+        shell: bash
+        run: |
+          BRANCH=${GITHUB_REF#refs/heads/}
+          (wget -O - https://github.com/matrix-org/synapse/archive/$BRANCH.tar.gz || wget -O - https://github.com/matrix-org/synapse/archive/develop.tar.gz) \
+            | tar -xz --strip-components=1 -C /src/
+
+      - name: Prepare blacklist file for running with workers
+        if: ${{ matrix.workers }}
+        run: cat /src/sytest-blacklist /src/.ci/worker-blacklist > /src/synapse-blacklist-with-workers
+
+      - name: Run sytest
+        run: |
+          echo POSTGRES=${POSTGRES:-<NOT SET>}
+          echo WORKERS=${WORKERS:-<NOT SET>}
+          echo BLACKLIST=${BLACKLIST:-<NOT SET>}
+          bash -xe /bootstrap.sh synapse
+
+      - name: Summarise results.tap
+        # Use always() to run this step even if previous ones failed.
+        if: ${{ always() }}
+        run:  /sytest/scripts/tap_to_gha.pl /logs/results.tap
+
+      - name: Upload SyTest logs
+        uses: actions/upload-artifact@v2
+        if: ${{ always() }}
+        with:
+          name: Sytest Logs - ${{ job.status }} - (Synapse, ${{ matrix.label }})
+          path: |
+            /logs/results.tap
+            /logs/**/*.log*
+
+  dendrite:
+    runs-on: ubuntu-latest
+    name: "Dendrite: ${{ matrix.label }}"
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - label: SQLite
+
+          - label: SQLite, full HTTP APIs
+            api: full-http
+
+          - label: Postgres
+            postgres: postgres
+
+          - label: Postgres, full HTTP APIs
+            postgres: postgres
+            api: full-http
+
+    container:
+      image: matrixdotorg/sytest-dendrite
+      volumes:
+        # bootstrap.sh expects the sytest source available at /sytest.
+        # TODO Buildkite mounted /sytest as readonly. Can we do this on GHA? Do we need it?
+        - ${{ github.workspace }}/sytest:/sytest
+        # synapse_sytest.sh expects a synapse checkout at /src
+        - ${{ github.workspace }}/dendrite:/src
+      env:
+        POSTGRES: ${{ matrix.postgres && 1 }}
+        API: ${{ matrix.api && 1 }}
+
+    steps:
+      - name: Checkout sytest
+        uses: actions/checkout@v2
+        with:
+          path: sytest
+
+      - name: Fetch corresponding dendrite branch
+        shell: bash
+        run: |
+          BRANCH=${GITHUB_REF#refs/heads/}
+          (wget -O - https://github.com/matrix-org/dendrite/archive/$BRANCH.tar.gz || wget -O - https://github.com/matrix-org/dendrite/archive/master.tar.gz) \
+            | tar -xz --strip-components=1 -C /src/
+
+      - name: Run sytest
+        run: |
+          echo POSTGRES=${POSTGRES:-<NOT SET>}
+          echo API=${API:-<NOT SET>}
+          bash -xe /bootstrap.sh dendrite
+
+      - name: Summarise results.tap
+        if: ${{ always() }}
+        run:  /sytest/scripts/tap_to_gha.pl /logs/results.tap
+
+      - name: Upload SyTest logs
+        uses: actions/upload-artifact@v2
+        if: ${{ always() }}
+        with:
+          name: Sytest Logs - ${{ job.status }} - (Dendrite, ${{ join(matrix.*, ', ') }})
+          path: |
+            /logs/results.tap
+            /logs/**/*.log*

docker/bootstrap.sh

@@ -9,6 +9,7 @@ shift
 
 if [ -d "/sytest" ]; then
     # If the user has mounted in a SyTest checkout, use that.
+    # This is the case for sytest's GitHub Actions.
     echo "Using local sytests"
 else
     echo "--- Trying to get same-named sytest branch..."
@@ -20,8 +21,6 @@ else
         # Otherwise, try and find the branch that the Synapse/Dendrite checkout
         # is using. Fall back to develop if unknown.
         branch_name="$(git --git-dir=/src/.git symbolic-ref HEAD 2>/dev/null)" || branch_name="develop"
-    fi
-
     if [ "$SYTEST_TARGET" == "dendrite" ] && [ "$branch_name" == "master" ]; then
         # Dendrite uses master as its main branch. If the branch is master, we probably want sytest develop
         branch_name="develop"

scripts/dendrite_sytest.sh

@@ -68,7 +68,8 @@ echo >&2 "--- Copying assets"
 # Copy out the logs
 rsync -r --ignore-missing-args --min-size=1B -av /work/server-0 /work/server-1 /logs --include "*/" --include="*.log.*" --include="*.log" --exclude="*"
 
-if [ $TEST_STATUS -ne 0 ]; then
+# Generate annotate.md. This is Buildkite-specific.
+if [ -n "$BUILDKITE_LABEL" ] && [ $TEST_STATUS -ne 0 ]; then
     # Build the annotation
     perl /sytest/scripts/format_tap.pl /logs/results.tap "$BUILDKITE_LABEL" >/logs/annotate.md
     # If show-expected-fail-tests logged something, put it into the annotation

scripts/synapse_sytest.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/bash -xe
 #
 # This script is run by the bootstrap.sh script in the docker image.
 #
@@ -13,7 +13,7 @@ set -e
 
 cd "$(dirname $0)/.."
 
-mkdir /work
+mkdir -p /work
 
 # start the redis server, if desired
 if [ -n "$REDIS" ]; then
@@ -201,7 +201,8 @@ rsync --ignore-missing-args --min-size=1B -av /work/server-0 /work/server-1 /log
 #export TOP=/src
 #/venv/bin/coverage combine
 
-if [ $TEST_STATUS -ne 0 ]; then
+# Generate annotate.md. This is Buildkite-specific.
+if [ -n "$BUILDKITE_LABEL" ] && [ $TEST_STATUS -ne 0 ]; then
     # Build the annotation
     perl /sytest/scripts/format_tap.pl /logs/results.tap "$BUILDKITE_LABEL" >/logs/annotate.md
 fi