autobuild.yml

# This file builds the source and produces artifacts for every supported platform.
# For release tags it creates a Github release and uploads the binaries to that releases.
# The builds are instrumented with CodeQL.

# see analyse_git_reference.py for implementation of the logic:
#     for every push to a branch starting with "autobuild": (can be used during development for tighter supervision of builds)
#          - do CodeQl while building for every platform
#          - publish the created binaries/packs only as artifacts/appendix of the github-action-run (not as release), and only retain those files for limited period
#     for every pull-request to main:
#          - do CodeQl while building for every platform
#          - publish the created binaries/packs only as artifacts/appendix of the github-action-run (not as release), and only retain those files for limited period
#     for every tag that starts with 'r' and has an arbitrary suffix (e.g. beta1, rc1, etc.)
#          - do CodeQl while building for every platform
#          - publish the created binaries/packs only as artifacts/appendix as a prerelease
#     for every tag that starts with 'r' and does not have any suffix:
#          - do CodeQl while building for every platform
#          - publish the created binaries/packs only as artifacts/appendix as a release

on:
  workflow_dispatch:
    inputs:
      build_all_targets:
        type: boolean
        description: 'Build all targets (instead of just the main platforms)'
  push:
    tags:
      - "r*"
    branches:
      # For developers: Branches starting with autobuild will be built and evaluated on each push.
      - "autobuild**"
      # CodeQL requires every branch from on.pull_request to be part of on.push as well in order to run comparisons.
      # We also need main here to trigger builds on PR merge to main and manual pushes (e.g. as part of the release process):
      - "main"
    paths-ignore:
      - '**README.md'
      - 'docs/**'
      - 'SECURITY.md'
      - 'CONTRIBUTING.md'
      - 'COMPILING.md'
      - 'COPYING'
      - 'APPLEAPPSTORE.LICENCE.WAIVER'
      - '.github/ISSUE_TEMPLATE/*'
      - '.github/pull_request_template.md'
  pull_request:
    branches:
      - main
    paths-ignore:
      - '**README.md'
      - 'docs/**'
      - 'SECURITY.md'
      - 'CONTRIBUTING.md'
      - 'COMPILING.md'
      - 'COPYING'
      - 'APPLEAPPSTORE.LICENCE.WAIVER'
      - '.github/ISSUE_TEMPLATE/*'
      - '.github/pull_request_template.md'

name:                               Auto-Build
permissions:                        {}
jobs:
  create_release:
    # Check if we are doing a release or just a normal build.
    # This must be done before actually building the app to find out where to upload the binaries and if we need to create a Github release.
    name:                           Build vars & Github release (if required)
    runs-on:                        ubuntu-20.04
    outputs:
      publish_to_release:           ${{ steps.get-build-vars.outputs.PUBLISH_TO_RELEASE }}
      tag_name:                     ${{ steps.get-build-vars.outputs.RELEASE_TAG }}
      build_version:                ${{ steps.get-build-vars.outputs.BUILD_VERSION }}
      build_all_targets:            ${{ steps.decide-build-targets.outputs.build_all_targets }}
    env:
      release_changelog_path:       ./.github_release_changelog.md
    # Set permissions. We need write permissions to content for creating/removing the release
    permissions:
        contents:                   write
    steps:
      - name:                       Checkout code
        uses:                       actions/checkout@v4

      - name:                       Determine release version, type and prerelease variables
        run:                        ./.github/autobuild/get_build_vars.py
        id:                         get-build-vars

      - name:                       Extract Changelog for the Github release body
        if:                         steps.get-build-vars.outputs.PUBLISH_TO_RELEASE == 'true'
        run:                        ./.github/autobuild/extractVersionChangelog.pl ChangeLog ${{ steps.get-build-vars.outputs.JAMULUS_PRO_VERSION }} > ${{ env.release_changelog_path }}

      - name:                       Create/Recreate Release ${{steps.get-build-vars.outputs.RELEASE_TAG}}  ${{steps.get-build-vars.outputs.RELEASE_TITLE}}
        if:                         steps.get-build-vars.outputs.PUBLISH_TO_RELEASE == 'true'
        id:                         create-release
        run:                        |
          set -eu

          # delete release if existing
          gh release delete "${tag_name}" || true

          RELEASE_PRERELEASE_ARG=""
          if [[ "${prerelease}" == 'true' ]]; then
            RELEASE_PRERELEASE_ARG="--prerelease"
          fi
          # Actually create the release. This will print the release url if successful
          gh release create "${tag_name}" --title "${release_name}" --notes-file "${body_path}" ${RELEASE_PRERELEASE_ARG}

        env:
          GITHUB_TOKEN:             ${{ secrets.GITHUB_TOKEN }}
          tag_name:                 ${{ steps.get-build-vars.outputs.RELEASE_TAG }}
          release_name:             ${{ steps.get-build-vars.outputs.RELEASE_TITLE }}
          body_path:                ${{ env.release_changelog_path }}
          prerelease:               ${{ steps.get-build-vars.outputs.IS_PRERELEASE }}

      - name:                       Decide which targets to build for
        id:                         decide-build-targets
        env:
          GITHUB_TOKEN:             ${{ secrets.GITHUB_TOKEN }}
          INPUT_BUILD_ALL_TARGETS:  ${{ github.event.inputs.build_all_targets }}
          PR_NUMBER:                ${{ github.event.number }}
          PR_BASE:                  ${{ github.event.pull_request.base.sha }}
          PR_HEAD:                  ${{ github.event.pull_request.head.sha }}
        run:                        |
          set -eu
          build_all_targets() {
            echo "build_all_targets=${1}" >> $GITHUB_OUTPUT
            echo "Building for all targets? Result: ${1}"
            exit 0
          }

          handle_push() {
            if [[ "${GITHUB_REF}" == "refs/tags/"* ]]; then
              echo 'Triggered by a tag push, building all targets'
              build_all_targets 'true'
            fi
            if [[ "${GITHUB_REF}" == "refs/heads/autobuild"* ]]; then
              echo 'Triggered by a push to an autobuild* branch, building all targets'
              build_all_targets 'true'
            fi
          }

          handle_workflow_dispatch() {
            if [[ "${INPUT_BUILD_ALL_TARGETS}" == 'true' ]]; then
              echo 'Triggered by manual run with "Build all targets" checkbox set'
              build_all_targets 'true'
            fi
          }

          handle_pull_request() {
            pr_body_contains_magic_string() {
              pr_body=$(gh pr view "${PR_NUMBER}" --json body --jq .body)
              grep -vP '<!--' <<< "$pr_body" | grep -qiF -- 'AUTOBUILD: Please build all targets'
            }
            if pr_body_contains_magic_string; then
              echo 'Triggered by a PR with magic AUTOBUILD: string, building all targets'
              build_all_targets 'true'
            fi

            pr_contains_build_changes() {
              git fetch origin "${PR_BASE}" "${PR_HEAD}"
              git diff --name-only "${PR_BASE}..${PR_HEAD}" |
                grep -qP 'autobuild|windows|linux|mac|ios|android|\.pro'
            }
            if pr_contains_build_changes; then
              echo 'Triggered by a PR with build- or platform-specific changes, building all targets'
              build_all_targets 'true'
            fi
          }

          case "${GITHUB_EVENT_NAME}" in
            push)
              handle_push
              ;;
            workflow_dispatch)
              handle_workflow_dispatch
              ;;
            pull_request)
              handle_pull_request
              ;;
          esac

          echo 'default case, not building all targets'
          build_all_targets 'false'


  release_assets:
    name:                           Build for ${{ matrix.config.config_name }}
    needs:                          create_release
    # Set permissions to allow uploading artifact, uploading to release and allowing CodeQl to set security events
    permissions:
      checks:                       write
      contents:                     write
      security-events:              write
    strategy:
      fail-fast:                    false
      matrix:
        # Think of this like a foreach loop. Basically runs the steps with every combination of
        # the contents of this.
        config:
          - config_name:            Android .apk (artifact+codeQL)
            target_os:              android
            building_on_os:         ubuntu-20.04
            base_command:           ./.github/autobuild/android.sh
            run_codeql:             true
            is_main_build_target:   true
            # Jamulus.pro needs to count git history length for android versioning:
            checkout_fetch_depth:   '0'

          - config_name:            Linux .deb amd64 (artifacts+codeQL)
            target_os:              linux
            building_on_os:         ubuntu-22.04
            building_container:     ubuntu:20.04
            base_command:           ./.github/autobuild/linux_deb.sh
            run_codeql:             true
            is_main_build_target:   true

          - config_name:            Linux .deb armhf (artifacts)
            target_os:              linux
            building_on_os:         ubuntu-22.04
            building_container:     ubuntu:20.04
            base_command:           TARGET_ARCH=armhf ./.github/autobuild/linux_deb.sh
            run_codeql:             false

          - config_name:            Linux .deb arm64 (artifacts)
            target_os:              linux
            building_on_os:         ubuntu-22.04
            building_container:     ubuntu:20.04
            base_command:           TARGET_ARCH=arm64 ./.github/autobuild/linux_deb.sh
            run_codeql:             false

          - config_name:            MacOS (artifacts)
            target_os:              macos
            building_on_os:         macos-14
            base_command:           QT_VERSION=6.6.3 SIGN_IF_POSSIBLE=1 TARGET_ARCHS="x86_64 arm64" ./.github/autobuild/mac.sh
            # Disable CodeQL on mac as it interferes with signing the binaries (signing hangs, see #2563 and #2564)
            run_codeql:             false
            # Latest Xcode which runs on macos-14:
            xcode_version:          15.4.0
            is_main_build_target:   true

          # Reminder: If Legacy is removed, be sure to add a dedicated job for CodeQL again.
          - config_name:            MacOS Legacy (artifacts+CodeQL)
            target_os:              macos
            building_on_os:         macos-12
            base_command:           QT_VERSION=5.15.2 SIGN_IF_POSSIBLE=0 ARTIFACT_SUFFIX=_legacy ./.github/autobuild/mac.sh
            # Enable CodeQL on mac legacy as this version does not get signed
            run_codeql:             true
            # For Qt5 on Mac, we need to use an unsupported SDK version as macOS 12 doesn't
            # support Xcode 12.1 which still ships SDK 10.15.
            # https://developer.apple.com/support/xcode/
            # https://xcodereleases.com/
            xcode_version:          14.2.0
            is_main_build_target:   true

          - config_name:            iOS (artifacts)
            target_os:              ios
            building_on_os:         macos-12
            base_command:           QT_VERSION=5.15.2 ./.github/autobuild/ios.sh
            # Build failed with CodeQL enabled when last tested 03/2022 (#2490).
            # There are no hints that iOS is supposed to be supported by CodeQL.
            # Therefore, disable it:
            run_codeql:             false
            # Unfortunately, more modern Xcode versions no longer seem to support
            # Qt 5.15.2. Therefore upgrading to Qt6 is needed in the medium term (#2711)
            xcode_version:          13.4.1

          - config_name:            Windows (artifact+codeQL)
            target_os:              windows
            building_on_os:         windows-2022
            base_command:           powershell .\.github\autobuild\windows.ps1 -Stage
            run_codeql:             true
            is_main_build_target:   true

          - config_name:            Windows JACK (artifact)
            target_os:              windows
            building_on_os:         windows-2022
            base_command:           powershell .\.github\autobuild\windows.ps1 -BuildOption jackonwindows -Stage
            run_codeql:             false

        # This injects the build_all_targets information into each matrix output:
        build_all_targets:
          - ${{ needs.create_release.outputs.build_all_targets }}

        # Exclude all non-main build targets if we are not building for all targets:
        exclude:
          - build_all_targets:      'false'  # This is based on a script output and is therefore a string
            config:
              is_main_build_target: null

    runs-on:                        ${{ matrix.config.building_on_os }}
    container:                      ${{ matrix.config.building_container }}
    steps:
      - name:                       Select Xcode version for Mac
        if:                         matrix.config.target_os == 'macos' || matrix.config.target_os == 'ios'
        uses:                       maxim-lobanov/setup-xcode@60606e260d2fc5762a71e64e74b2174e8ea3c8bd
        with:
          xcode-version:            ${{ matrix.config.xcode_version }}

      - name:                       Set up base dependencies in container environment
        if:                         matrix.config.building_container == 'ubuntu:20.04'
        run:                        |
          set -eu
          apt-get -qq update
          apt-get install -y software-properties-common sudo curl
          add-apt-repository ppa:git-core/ppa
          curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg
          sudo chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg
          echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null
          apt-get -qq update
          apt-get -qq -y install git-core gh
          # Github Runner environments usually whitelist the default checkout directory for git.
          # However, when using containers, we have to do that manually in order for git commands to work
          # (e.g. in Jamulus.pro's VERSION logic):
          # https://github.com/actions/runner/issues/2033
          # https://github.com/actions/checkout/issues/766
          git config --global --add safe.directory "${GITHUB_WORKSPACE}"

      - name:                       Checkout code
        uses:                       actions/checkout@v4
        with:
          # only Android needs the oboe submodule, so don't fetch it for other builds
          submodules:               ${{ matrix.config.target_os == 'android' }}
          fetch-depth:              ${{ matrix.config.checkout_fetch_depth || '1' }}

      - name:                       Cache Mac dependencies
        if:                         matrix.config.target_os == 'macos'
        uses:                       actions/cache@v4
        with:
          path: |
            /opt/qt
            ~/Library/Cache/jamulus-homebrew-bottles
          key:                      ${{ matrix.config.target_os }}-${{ hashFiles('.github/workflows/autobuild.yml', '.github/autobuild/mac.sh', 'mac/deploy_mac.sh') }}-${{ matrix.config.base_command }}

      - name:                       Cache Windows dependencies
        if:                         matrix.config.target_os == 'windows'
        uses:                       actions/cache@v4
        with:
          path: |
            C:\Qt
            C:\ChocoCache
            C:\AutobuildCache
            ${{ github.workspace }}\libs\NSIS\NSIS-source
            ${{ github.workspace }}\libs\ASIOSDK2
          key:                      ${{ matrix.config.target_os }}-${{ hashFiles('.github/workflows/autobuild.yml', '.github/autobuild/windows.ps1', 'windows/deploy_windows.ps1') }}-${{ matrix.config.base_command }}

      - name:                       Cache Android dependencies
        if:                         matrix.config.target_os == 'android'
        uses:                       actions/cache@v4
        with:
          path: |
            /opt/Qt
            /opt/android/android-sdk
            /opt/android/android-ndk
          key:                      ${{ matrix.config.target_os }}-${{ hashFiles('.github/workflows/autobuild.yml', '.github/autobuild/android.sh') }}-${{ matrix.config.base_command }}

      - name:                       Set up build dependencies for ${{ matrix.config.config_name }}
        run:                        ${{ matrix.config.base_command }} setup
        env:
          JAMULUS_BUILD_VERSION:    ${{ needs.create_release.outputs.build_version }}

      - name:                       Initialize CodeQL
        if:                         matrix.config.run_codeql
        uses:                       github/codeql-action/init@v3
        with:
          languages: 'cpp'

      - name:                       Build for ${{ matrix.config.config_name }}
        id:                         build
        run:                        ${{ matrix.config.base_command }} build
        env:
          JAMULUS_BUILD_VERSION:    ${{ needs.create_release.outputs.build_version }}
          MACOS_CERTIFICATE:        ${{ secrets.MACOS_CERT }} # Base64 encoded Developer ID Application certificate. See https://help.apple.com/xcode/mac/current/#/dev154b28f09
          MACOS_CERTIFICATE_PWD:    ${{ secrets.MACOS_CERT_PWD }} # Password protecting secrets.MACOS_CERTIFICATE
          MACOS_CERTIFICATE_ID:     ${{ secrets.MACOS_CERT_ID }} # Certificate ID of secrets.MACOS_CERTIFICATE. If unknown, import secrets.MACOS_CERT into keychain and get the hash via "security find-identity -v"
          MAC_STORE_APP_CERT:       ${{ secrets.MACAPP_CERT }} # Base64 encoded Mac App Distribution certificate
          MAC_STORE_APP_CERT_PWD:   ${{ secrets.MACAPP_CERT_PWD }}
          MAC_STORE_APP_CERT_ID:    ${{ secrets.MACAPP_CERT_ID }}
          MAC_STORE_INST_CERT:      ${{ secrets.MACAPP_INST_CERT }} # Base64 encoded Mac Installer Distribution certificate
          MAC_STORE_INST_CERT_PWD:  ${{ secrets.MACAPP_INST_CERT_PWD }}
          MAC_STORE_INST_CERT_ID:   ${{ secrets.MACAPP_INST_CERT_ID }}
          NOTARIZATION_PASSWORD:    ${{ secrets.NOTARIZATION_PASSWORD }}
          KEYCHAIN_PASSWORD:        ${{ secrets.KEYCHAIN_PASSWORD }}
          MACOS_CA_PUBLICKEY:       ${{ secrets.MACOS_CA_PUBKEY }}
      - name:                       Post-Build for ${{ matrix.config.config_name }}
        id:                         get-artifacts
        run:                        ${{ matrix.config.base_command }} get-artifacts
        env:
          JAMULUS_BUILD_VERSION:    ${{ needs.create_release.outputs.build_version }}

      - name:                       Upload Artifact 1 to Job
        # Every build job has at least one artifact. Therefore, no `if` here.
        # If the artifact is missing, this should fail.
        uses:                       actions/upload-artifact@v4
        with:
          name:                     ${{ steps.get-artifacts.outputs.artifact_1 }}
          path:                     deploy/${{ steps.get-artifacts.outputs.artifact_1 }}
          retention-days:           31
          if-no-files-found:        error

      - name:                       Upload Artifact 2 to Job
        if:                         steps.get-artifacts.outputs.artifact_2
        uses:                       actions/upload-artifact@v4
        with:
          name:                     ${{ steps.get-artifacts.outputs.artifact_2 }}
          path:                     deploy/${{ steps.get-artifacts.outputs.artifact_2 }}
          retention-days:           31
          if-no-files-found:        error

      - name:                       Notarize macOS Release Build
        if:                         >-
                                    steps.build.outputs.macos_signed == 'true' &&
                                    needs.create_release.outputs.publish_to_release == 'true' &&
                                    steps.build.outputs.macos_notarize == 'true'
        id:                         notarize-macOS-app
        run:                        ${{ matrix.config.base_command }} notarize
        env:
          JAMULUS_BUILD_VERSION:    ${{ needs.create_release.outputs.build_version }}
          ARTIFACT_PATH:            deploy/${{ steps.get-artifacts.outputs.artifact_1 }}
          NOTARIZATION_USERNAME:    ${{ secrets.NOTARIZATION_USERNAME }} # Apple ID for notarization
          NOTARIZATION_PASSWORD:    ${{ secrets.NOTARIZATION_PASSWORD }} # App specific password for Apple ID
          APPLE_TEAM_ID:            ${{ secrets.NOTARIZATION_TEAM_ID }} # Team ID from App Store Connect

      - name:                       Staple macOS Release Build
        if:                         >-
                                    steps.build.outputs.macos_signed == 'true' &&
                                    needs.create_release.outputs.publish_to_release == 'true' &&
                                    steps.build.outputs.macos_notarize == 'true'
        id:                         staple-macOS-app
        run:                        ${{ matrix.config.base_command }} staple
        env:
          JAMULUS_BUILD_VERSION:    ${{ needs.create_release.outputs.build_version }}
          ARTIFACT_PATH:            deploy/${{ steps.get-artifacts.outputs.artifact_1 }}

      - name:                       Deploy Artifacts to Release
        if:                         needs.create_release.outputs.publish_to_release == 'true'
        id:                         upload-release-assets
        run:                        >
          gh release upload "${{ needs.create_release.outputs.tag_name }}" "deploy/${{ steps.get-artifacts.outputs.artifact_1 }}" "deploy/${{ steps.get-artifacts.outputs.artifact_2 }}" --clobber
          ||
          gh release upload "${{ needs.create_release.outputs.tag_name }}" "deploy/${{ steps.get-artifacts.outputs.artifact_1 }}" --clobber
        env:
          GITHUB_TOKEN:             ${{ secrets.GITHUB_TOKEN }}

      - name:                       Perform CodeQL Analysis
        if:                         matrix.config.run_codeql
        uses:                       github/codeql-action/analyze@v3

  create_deb_repo:
    name:                           Create files for .deb repository (if requested)
    runs-on:                        ubuntu-22.04
    needs:                          [create_release, release_assets]
    if:                             needs.create_release.outputs.publish_to_release == 'true'
    # Set permissions to allow uploading artifact, uploading to release
    permissions:
      checks:                       write
      contents:                     write
    steps:
      - name:                       Import GPG key
        env:
          GPG_PRIVATE_KEY:          ${{ secrets.GPG_PRIVATE_KEY }}
        run:                        |
          set -eu

          [[ "${GPG_PRIVATE_KEY:-}" ]] || {
              echo "Missing Github secret GPG_PRIVATE_KEY. Please set it on GitHub to enable deb repository releases. Skipping step..."
              echo "GPG_REPO_KEY_MISSING=true" >> ${GITHUB_ENV}
              exit 0
          }

          echo "GPG_REPO_KEY_MISSING=false" >> ${GITHUB_ENV}
          mkdir -p gpghome
          chmod 700 gpghome
          echo "${GPG_PRIVATE_KEY}" | gpg --homedir gpghome --import -
      # Unfortunately download-artifact action doesn't support wild card downloads. Thus downloading all artifacts
      - name:                       Download all artifacts
        if:                         env.GPG_REPO_KEY_MISSING == 'false'
        uses:                       actions/download-artifact@v4
        with:
          path:                     releasedl/
      - name:                       Create Debian repository
        if:                         env.GPG_REPO_KEY_MISSING == 'false'
        run:                        |
          set -eu

          # Create and cd into repo directory
          mkdir repo
          mv releasedl/*.deb/*.deb repo/
          pushd repo

          # create repo files
          apt-ftparchive packages . > Packages
          apt-ftparchive release . > Release
          gpg --homedir "../gpghome" --armor --yes --clearsign --output InRelease --detach-sign Release
          gpg --homedir "../gpghome" --armor --export > "key.asc"

          # remove .deb files as they have been uploaded to the GitHub release already before
          rm *.deb

          popd
      - name:                       Upload Debian repository files to release
        if:                         env.GPG_REPO_KEY_MISSING == 'false'
        env:
          GITHUB_TOKEN:             ${{ secrets.GITHUB_TOKEN }}
        run:                        |
          gh release upload "${{ needs.create_release.outputs.tag_name }}" "repo/InRelease" "repo/key.asc" "repo/Packages" "repo/Release" --clobber -R "${{ github.repository}}"