#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#

name: Pulsar CI
on:
  pull_request:
    branches:
      - master
  schedule:
    # scheduled job with JDK 17
    - cron: '0 12 * * *'
    # scheduled job with JDK 21
    # if cron expression is changed, make sure to update the expression in jdk_major_version step in preconditions job
    - cron: '0 6 * * *'
  workflow_dispatch:
    inputs:
      collect_coverage:
        description: 'Collect test coverage and upload to Codecov'
        required: true
        type: boolean
        default: true
      jdk_major_version:
        description: 'JDK major version to use for the build'
        required: true
        type: choice
        options:
          - '17'
          - '21'
        default: '17'
      trace_test_resource_cleanup:
        description: 'Collect thread & heap information before exiting a test JVM. When set to "on", thread dump and heap histogram will be collected. When set to "full", a heap dump will also be collected.'
        required: true
        type: choice
        options:
          - 'off'
          - 'on'
          - 'full'
        default: 'off'
      thread_leak_detector_wait_millis:
        description: 'Duration in ms to wait for threads to exit in thread leak detection between test classes. It is necessary to wait for threads to complete before they are determined to be leaked threads.'
        required: true
        type: number
        default: 10000

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}${{ github.event_name == 'workflow_dispatch' && github.event.inputs.jdk_major_version || '' }}
  cancel-in-progress: true

env:
  MAVEN_OPTS: -Xss1500k -Xmx1500m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
  # defines the retention period for the intermediate build artifacts needed for rerunning a failed build job
  # it's possible to rerun individual failed jobs when the build artifacts are available
  # if the artifacts have already been expired, the complete workflow can be rerun by closing and reopening the PR or by rebasing the PR
  ARTIFACT_RETENTION_DAYS: 3

jobs:
  preconditions:
    name: Preconditions
    runs-on: ubuntu-22.04
    outputs:
      docs_only: ${{ steps.check_changes.outputs.docs_only }}
      changed_tests: ${{ steps.changes.outputs.tests_files }}
      need_owasp: ${{ steps.changes.outputs.need_owasp }}
      collect_coverage: ${{ steps.check_coverage.outputs.collect_coverage }}
      jdk_major_version: ${{ steps.jdk_major_version.outputs.jdk_major_version }}

    steps:
      - name: Cancel scheduled jobs in forks by default
        if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'schedule' }}
        uses: actions/github-script@v6
        with:
          script: |
            await github.rest.actions.cancelWorkflowRun({owner: context.repo.owner, repo: context.repo.repo, run_id: context.runId});
            process.exit(1);

      - name: Select JDK major version
        id: jdk_major_version
        run: |
          # use JDK 21 for the scheduled build with cron expression '0 6 * * *'
          if [[ "${{ github.event_name == 'schedule' && github.event.schedule == '0 6 * * *' && 'true' || 'false' }}" == "true" ]]; then
            echo "jdk_major_version=21" >> $GITHUB_OUTPUT
            exit 0
          fi
          # use JDK 17 for build unless overridden with workflow_dispatch input
          echo "jdk_major_version=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.jdk_major_version || '17'}}" >> $GITHUB_OUTPUT

      - name: checkout
        if: ${{ github.event_name == 'pull_request' }}
        uses: actions/checkout@v3

      - name: Detect changed files
        if: ${{ github.event_name == 'pull_request' }}
        id:   changes
        uses: apache/pulsar-test-infra/paths-filter@master
        with:
          filters: .github/changes-filter.yaml
          list-files: csv

      - name: Check changed files
        if: ${{ github.event_name == 'pull_request' }}
        id: check_changes
        run: |
          if [[ "${GITHUB_EVENT_NAME}" != "schedule" && "${GITHUB_EVENT_NAME}" != "workflow_dispatch" ]]; then
            echo "docs_only=${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" >> $GITHUB_OUTPUT
          else
            echo docs_only=false >> $GITHUB_OUTPUT
          fi

      - name: Check if coverage should be collected
        id: check_coverage
        run: |
          echo "collect_coverage=${{ 
          (steps.check_changes.outputs.docs_only != 'true' && github.event_name != 'workflow_dispatch' 
            && (github.base_ref == 'master' || github.ref_name == 'master')) 
          || (github.event_name == 'workflow_dispatch' && github.event.inputs.collect_coverage == 'true')
          }}"  >> $GITHUB_OUTPUT

      - name: Check if the PR has been approved for testing
        if: ${{ steps.check_changes.outputs.docs_only != 'true' && github.repository == 'apache/pulsar' && github.event_name == 'pull_request' }}
        env:
          GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }}
          GITHUB_TOKEN: ${{ github.token }}
        run: |
          build/pulsar_ci_tool.sh check_ready_to_test

  build-and-license-check:
    needs: preconditions
    name: Build and License check
    env:
      JOB_NAME: Build and License check
      GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
    runs-on: ubuntu-22.04
    timeout-minutes: 60
    if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
    steps:
      - name: checkout
        uses: actions/checkout@v3

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Setup ssh access to build runner VM
        # ssh access is enabled for builds in own forks
        if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        uses: ./.github/actions/ssh-access
        continue-on-error: true
        with:
          limit-access-to-actor: true

      - name: Cache local Maven repository
        uses: actions/cache@v3
        timeout-minutes: 5
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/pulsar
          key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-m2-dependencies-core-modules-

      - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
        uses: actions/setup-java@v3
        with:
          distribution: 'temurin'
          java-version: ${{ env.CI_JDK_MAJOR_VERSION }}

      - name: Check source code license headers
        run: mvn -B -T 8 -ntp initialize apache-rat:check license:check

      - name: Check source code style
        run: mvn -B -T 8 -ntp initialize checkstyle:check

      - name: Build core-modules
        run: |
          mvn -B -T 1C -ntp -Pcore-modules,-main clean install -DskipTests -Dlicense.skip=true -Drat.skip=true -Dcheckstyle.skip=true

      - name: Check binary licenses
        run: src/check-binary-license.sh ./distribution/server/target/apache-pulsar-*-bin.tar.gz

      - name: Install gh-actions-artifact-client.js
        uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

      - name: Save maven build results to Github artifact cache so that the results can be reused
        run: |
          cd $HOME
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh store_tar_to_github_actions_artifacts pulsar-maven-repository-binaries \
          tar --exclude '.m2/repository/org/apache/pulsar/pulsar-*-distribution' \
              -I zstd -cf - .m2/repository/org/apache/pulsar
          cd $GITHUB_WORKSPACE
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh store_tar_to_github_actions_artifacts pulsar-server-distribution \
          tar -I zstd -cf - distribution/server/target/apache-pulsar-*-bin.tar.gz

      - name: Wait for ssh connection when build fails
        # ssh access is enabled for builds in own forks
        uses: ./.github/actions/ssh-access
        if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        continue-on-error: true
        with:
          action: wait


  unit-tests:
    name: CI - Unit - ${{ matrix.name }}
    env:
      JOB_NAME: CI - Unit - ${{ matrix.name }}
      COLLECT_COVERAGE: "${{ needs.preconditions.outputs.collect_coverage }}"
      GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
      TRACE_TEST_RESOURCE_CLEANUP: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.trace_test_resource_cleanup || 'off' }}
      TRACE_TEST_RESOURCE_CLEANUP_DIR: ${{ github.workspace }}/target/trace-test-resource-cleanup
      THREAD_LEAK_DETECTOR_WAIT_MILLIS: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.thread_leak_detector_wait_millis || 10000 }}
      THREAD_LEAK_DETECTOR_DIR: ${{ github.workspace }}/target/thread-leak-dumps
    runs-on: ubuntu-22.04
    timeout-minutes: ${{ matrix.timeout || 60 }}
    needs: ['preconditions', 'build-and-license-check']
    if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: Other
            group: OTHER
            timeout: 75
          - name: Brokers - Broker Group 1
            group: BROKER_GROUP_1
          - name: Brokers - Broker Group 2
            group: BROKER_GROUP_2
          - name: Brokers - Broker Group 3
            group: BROKER_GROUP_3
          - name: Brokers - Broker Group 4
            group: BROKER_GROUP_4
          - name: Brokers - Client Api
            group: BROKER_CLIENT_API
          - name: Brokers - Client Impl
            group: BROKER_CLIENT_IMPL
          - name: Proxy
            group: PROXY
          - name: Pulsar IO
            group: PULSAR_IO
            timeout: 75
          - name: Pulsar IO - Elastic Search
            group: PULSAR_IO_ELASTIC
          - name: Pulsar IO - Kafka Connect Adaptor
            group: PULSAR_IO_KAFKA_CONNECT
          - name: Pulsar Client
            group: CLIENT

    steps:
      - name: checkout
        uses: actions/checkout@v3

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Clean Disk when tracing test resource cleanup
        if: ${{ env.TRACE_TEST_RESOURCE_CLEANUP != 'off' }}
        uses: ./.github/actions/clean-disk

      - name: Setup ssh access to build runner VM
        # ssh access is enabled for builds in own forks
        if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        uses: ./.github/actions/ssh-access
        continue-on-error: true
        with:
          limit-access-to-actor: true

      - name: Cache Maven dependencies
        uses: actions/cache@v3
        timeout-minutes: 5
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/pulsar
          key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-m2-dependencies-core-modules-

      - name: Set up JDK ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }}
        uses: actions/setup-java@v3
        with:
          distribution: 'temurin'
          java-version: ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }}

      - name: Install gh-actions-artifact-client.js
        uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

      - name: Restore maven build results from Github artifact cache
        run: |
          cd $HOME
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts

      - name: Run setup commands
        run: |
          ${{ matrix.setup }}

      - name: Run unit test group '${{ matrix.group }}'
        run: |
          CHANGED_TESTS="${{ needs.preconditions.outputs.tests_files }}" ./build/run_unit_group.sh ${{ matrix.group }}

      - name: Upload coverage to build artifacts
        if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }}
        run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh upload_unittest_coverage_files ${{ matrix.group }}

      - name: print JVM thread dumps when cancelled
        if: cancelled()
        run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps

      - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories
        if: ${{ always() }}
        uses: ./.github/actions/copy-test-reports

      - name: Publish Test Report
        uses: apache/pulsar-test-infra/action-junit-report@master
        if: ${{ always() }}
        with:
          report_paths: 'test-reports/TEST-*.xml'
          annotate_only: 'true'

      - name: Report detected thread leaks
        if: ${{ always() }}
        run: |
          if [ -d "$THREAD_LEAK_DETECTOR_DIR" ]; then
            cd "$THREAD_LEAK_DETECTOR_DIR"
            cat threadleak*.txt | awk '/^Summary:/ {print "::warning::" $0 "\n"; next} {print}'
          fi

      - name: Upload Surefire reports
        uses: actions/upload-artifact@v3
        if: ${{ !success() || env.TRACE_TEST_RESOURCE_CLEANUP != 'off' }}
        with:
          name: Unit-${{ matrix.group }}-surefire-reports
          path: surefire-reports
          retention-days: 7

      - name: Upload possible heap dump, core dump or crash files
        uses: actions/upload-artifact@v3
        if: ${{ always() }}
        with:
          name: Unit-${{ matrix.group }}-dumps
          path: |
            /tmp/*.hprof
            **/hs_err_*.log
            **/core.*
            ${{ env.TRACE_TEST_RESOURCE_CLEANUP_DIR }}/*
            ${{ env.THREAD_LEAK_DETECTOR_DIR }}/*
          retention-days: 7
          if-no-files-found: ignore

      - name: Wait for ssh connection when build fails
        # ssh access is enabled for builds in own forks
        uses: ./.github/actions/ssh-access
        if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        continue-on-error: true
        with:
          action: wait


  unit-tests-upload-coverage:
    name: CI - Unit - Upload Coverage
    runs-on: ubuntu-22.04
    timeout-minutes: 30
    needs: ['preconditions', 'unit-tests']
    env:
      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
    if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }}

    steps:
      - name: checkout
        uses: actions/checkout@v3

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Setup ssh access to build runner VM
        # ssh access is enabled for builds in own forks
        if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        uses: ./.github/actions/ssh-access
        continue-on-error: true
        with:
          limit-access-to-actor: true

      - name: Cache Maven dependencies
        uses: actions/cache@v3
        timeout-minutes: 5
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/pulsar
          key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-m2-dependencies-core-modules-

      - name: Set up JDK ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }}
        uses: actions/setup-java@v3
        with:
          distribution: 'temurin'
          java-version: ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }}

      - name: Install gh-actions-artifact-client.js
        uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

      - name: Restore maven build results from Github artifact cache
        run: |
          cd $HOME
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts

      - name: Restore coverage files from build artifacts and create Jacoco reports
        run: |
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_unittest_coverage_files
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_test_coverage_report 
          cd $GITHUB_WORKSPACE/target
          zip -qr jacoco_test_coverage_report_unittests.zip jacoco_test_coverage_report || true

      - name: Upload Jacoco report files to build artifacts
        uses: actions/upload-artifact@v3
        with:
          name: Jacoco-coverage-report-unittests
          path: target/jacoco_test_coverage_report_unittests.zip
          retention-days: 3

      - name: Upload to Codecov
        uses: ./.github/actions/upload-coverage
        with:
          flags: unittests

      - name: Delete coverage files from build artifacts
        run: |
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh delete_unittest_coverage_files

      - name: Wait for ssh connection when build fails
        # ssh access is enabled for builds in own forks
        uses: ./.github/actions/ssh-access
        if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        continue-on-error: true
        with:
          action: wait

  pulsar-java-test-image:
    name: Build Pulsar java-test-image docker image
    runs-on: ubuntu-22.04
    timeout-minutes: 60
    needs: ['preconditions', 'build-and-license-check']
    if: ${{ needs.preconditions.outputs.docs_only != 'true'}}
    env:
      GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
      IMAGE_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
    steps:
      - name: checkout
        uses: actions/checkout@v3

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Setup ssh access to build runner VM
        # ssh access is enabled for builds in own forks
        if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        uses: ./.github/actions/ssh-access
        continue-on-error: true
        with:
          limit-access-to-actor: true

      - name: Cache Maven dependencies
        uses: actions/cache@v3
        timeout-minutes: 5
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/pulsar
          key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-m2-dependencies-core-modules-

      - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
        uses: actions/setup-java@v3
        with:
          distribution: 'temurin'
          java-version: ${{ env.CI_JDK_MAJOR_VERSION }}

      - name: Install gh-actions-artifact-client.js
        uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

      - name: restore maven build results from Github artifact cache
        run: |
          cd $HOME
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries

      - name: Pick ubuntu mirror for the docker image build
        run: |
          # pick the closest ubuntu mirror and set it to UBUNTU_MIRROR environment variable
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh pick_ubuntu_mirror

      - name: Build java-test-image docker image
        run: |
          # build docker image
          mvn -B -am -pl tests/docker-images/java-test-image install -Pcore-modules,-main,integrationTests,docker \
          -Dmaven.test.skip=true -Ddocker.squash=true -DskipSourceReleaseAssembly=true \
          -Dspotbugs.skip=true  -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true

      - name: save docker image apachepulsar/java-test-image:latest to Github artifact cache
        run: |
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_save_image_to_github_actions_artifacts apachepulsar/java-test-image:latest pulsar-java-test-image

      - name: Wait for ssh connection when build fails
        # ssh access is enabled for builds in own forks
        uses: ./.github/actions/ssh-access
        if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        continue-on-error: true
        with:
          action: wait

  integration-tests:
    name: CI - Integration - ${{ matrix.name }}
    runs-on: ubuntu-22.04
    timeout-minutes: ${{ matrix.timeout || 60 }}
    needs: ['preconditions', 'pulsar-java-test-image']
    if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
    env:
      JOB_NAME: CI - Integration - ${{ matrix.name }}
      PULSAR_TEST_IMAGE_NAME: apachepulsar/java-test-image:latest
      GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: Backwards Compatibility
            group: BACKWARDS_COMPAT
            no_coverage: true

          - name: Cli
            group: CLI

          - name: Messaging
            group: MESSAGING

          - name: LoadBalance
            group: LOADBALANCE
            no_coverage: true

          - name: Shade on Java 8
            group: SHADE_RUN
            runtime_jdk: 8
            setup: ./build/run_integration_group.sh SHADE_BUILD
            no_coverage: true

          - name: Shade on Java 11
            group: SHADE_RUN
            runtime_jdk: 11
            setup: ./build/run_integration_group.sh SHADE_BUILD
            no_coverage: true

          - name: Shade on Java 17
            group: SHADE_RUN
            runtime_jdk: 17
            setup: ./build/run_integration_group.sh SHADE_BUILD
            no_coverage: true

          - name: Shade on Java 21
            group: SHADE_RUN
            runtime_jdk: 21
            setup: ./build/run_integration_group.sh SHADE_BUILD
            no_coverage: true

          - name: Standalone
            group: STANDALONE

          - name: Transaction
            group: TRANSACTION

    steps:
      - name: checkout
        uses: actions/checkout@v3

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Setup ssh access to build runner VM
        # ssh access is enabled for builds in own forks
        if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        uses: ./.github/actions/ssh-access
        continue-on-error: true
        with:
          limit-access-to-actor: true

      - name: Cache Maven dependencies
        uses: actions/cache@v3
        timeout-minutes: 5
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/pulsar
          key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-m2-dependencies-core-modules-

      - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
        uses: actions/setup-java@v3
        with:
          distribution: 'temurin'
          java-version: ${{ env.CI_JDK_MAJOR_VERSION }}

      - name: Install gh-actions-artifact-client.js
        uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

      - name: Restore maven build results from Github artifact cache
        run: |
          cd $HOME
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts

      - name: Load docker image apachepulsar/java-test-image:latest from Github artifact cache
        run: |
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-java-test-image

      - name: Run setup commands
        if: ${{ matrix.setup }}
        run: |
          ${{ matrix.setup }}

      - name: Set up runtime JDK ${{ matrix.runtime_jdk }}
        uses: actions/setup-java@v3
        if: ${{ matrix.runtime_jdk }}
        with:
          distribution: 'temurin'
          java-version: ${{ matrix.runtime_jdk }}

      - name: Run integration test group '${{ matrix.group }}'
        run: |
          if [[ "${{ matrix.no_coverage }}" != "true" && "${{ needs.preconditions.outputs.collect_coverage }}" == "true" ]]; then
            coverage_args="--coverage"
          fi
          ./build/run_integration_group.sh ${{ matrix.group }} $coverage_args

      - name: Upload coverage to build artifacts
        if: ${{ !matrix.no_coverage && needs.preconditions.outputs.collect_coverage == 'true' }}
        run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh upload_inttest_coverage_files ${{ matrix.group }}

      - name: print JVM thread dumps when cancelled
        if: cancelled()
        run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps

      - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories
        if: ${{ always() }}
        uses: ./.github/actions/copy-test-reports

      - name: Publish Test Report
        uses: apache/pulsar-test-infra/action-junit-report@master
        if: ${{ always() }}
        with:
          report_paths: 'test-reports/TEST-*.xml'
          annotate_only: 'true'

      - name: Upload Surefire reports
        uses: actions/upload-artifact@v3
        if: ${{ !success() }}
        with:
          name: Integration-${{ matrix.group }}-surefire-reports
          path: surefire-reports
          retention-days: 7

      - name: Upload container logs
        uses: actions/upload-artifact@v3
        if: ${{ !success() }}
        continue-on-error: true
        with:
          name: Integration-${{ matrix.group }}-container-logs
          path: tests/integration/target/container-logs
          retention-days: 7

      - name: Wait for ssh connection when build fails
        # ssh access is enabled for builds in own forks
        uses: ./.github/actions/ssh-access
        if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        continue-on-error: true
        with:
          action: wait

  integration-tests-upload-coverage:
    name: CI - Integration - Upload Coverage
    runs-on: ubuntu-22.04
    timeout-minutes: 30
    needs: ['preconditions', 'integration-tests']
    if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }}
    env:
      PULSAR_TEST_IMAGE_NAME: apachepulsar/java-test-image:latest
      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
    steps:
      - name: checkout
        uses: actions/checkout@v3

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Setup ssh access to build runner VM
        # ssh access is enabled for builds in own forks
        if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        uses: ./.github/actions/ssh-access
        continue-on-error: true
        with:
          limit-access-to-actor: true

      - name: Cache Maven dependencies
        uses: actions/cache@v3
        timeout-minutes: 5
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/pulsar
          key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-m2-dependencies-core-modules-

      - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
        uses: actions/setup-java@v3
        with:
          distribution: 'temurin'
          java-version: ${{ env.CI_JDK_MAJOR_VERSION }}

      - name: Install gh-actions-artifact-client.js
        uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

      - name: Restore maven build results from Github artifact cache
        run: |
          cd $HOME
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts

      - name: Load docker image apachepulsar/java-test-image:latest from Github artifact cache
        run: |
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-java-test-image

      - name: Restore coverage files from build artifacts and create Jacoco reports
        run: |
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_inttest_coverage_files
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_inttest_coverage_report 
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_test_coverage_report
          cd $GITHUB_WORKSPACE/target
          zip -qr jacoco_test_coverage_report_inttests.zip jacoco_test_coverage_report jacoco_inttest_coverage_report || true

      - name: Upload Jacoco report files to build artifacts
        uses: actions/upload-artifact@v3
        with:
          name: Jacoco-coverage-report-inttests
          path: target/jacoco_test_coverage_report_inttests.zip
          retention-days: 3

      - name: Upload to Codecov
        uses: ./.github/actions/upload-coverage
        with:
          flags: inttests

      - name: Delete coverage files from build artifacts
        run: |
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh delete_inttest_coverage_files

      - name: Wait for ssh connection when build fails
        # ssh access is enabled for builds in own forks
        uses: ./.github/actions/ssh-access
        if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        continue-on-error: true
        with:
          action: wait

  delete-integration-test-docker-image-artifact:
    name: "Delete integration test docker image artifact"
    runs-on: ubuntu-22.04
    timeout-minutes: 10
    needs: [
      'preconditions',
      'integration-tests',
      'integration-tests-upload-coverage'
    ]
    if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
    steps:
      - name: checkout
        uses: actions/checkout@v3

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Install gh-actions-artifact-client.js
        uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

      - name: Delete docker image from GitHub Actions Artifacts
        run: |
          gh-actions-artifact-client.js delete pulsar-java-test-image.zst

  pulsar-test-latest-version-image:
    name: Build Pulsar docker image
    runs-on: ubuntu-22.04
    timeout-minutes: 60
    needs: ['preconditions', 'build-and-license-check']
    if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
    env:
      GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
      IMAGE_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
    steps:
      - name: checkout
        uses: actions/checkout@v3

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Setup ssh access to build runner VM
        # ssh access is enabled for builds in own forks
        if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        uses: ./.github/actions/ssh-access
        continue-on-error: true
        with:
          limit-access-to-actor: true

      - name: Clean Disk
        uses: ./.github/actions/clean-disk
        with:
          mode: full

      - name: Cache local Maven repository
        uses: actions/cache@v3
        timeout-minutes: 5
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/pulsar
          key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
            ${{ runner.os }}-m2-dependencies-core-modules-

      - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
        uses: actions/setup-java@v3
        with:
          distribution: 'temurin'
          java-version: ${{ env.CI_JDK_MAJOR_VERSION }}

      - name: Install gh-actions-artifact-client.js
        uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

      - name: restore maven build results from Github artifact cache
        run: |
          cd $HOME
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries

      - name: Pick ubuntu mirror for the docker image build
        run: |
          # pick the closest ubuntu mirror and set it to UBUNTU_MIRROR environment variable
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh pick_ubuntu_mirror

      - name: Build latest-version-image docker image
        run: |
          # build docker image
          # include building of Pulsar SQL, Connectors, Offloaders and server distros
          mvn -B -am -pl pulsar-sql/presto-distribution,distribution/io,distribution/offloaders,distribution/server,distribution/shell,tests/docker-images/latest-version-image install \
          -DUBUNTU_MIRROR="${UBUNTU_MIRROR}" -DUBUNTU_SECURITY_MIRROR="${UBUNTU_SECURITY_MIRROR}" -DIMAGE_JDK_MAJOR_VERSION="${IMAGE_JDK_MAJOR_VERSION}" \
          -Pmain,docker -Dmaven.test.skip=true -Ddocker.squash=true \
          -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true

      # check full build artifacts licenses
      - name: Check binary licenses
        run: src/check-binary-license.sh ./distribution/server/target/apache-pulsar-*-bin.tar.gz && src/check-binary-license.sh ./distribution/shell/target/apache-pulsar-shell-*-bin.tar.gz

      - name: Clean up disk space
        run: |
          # release disk space since saving docker image consumes local disk space
          #
          echo "::group::Available diskspace before cleaning"
          time df -BM / /mnt
          echo "::endgroup::"
          echo "::group::Clean build directory"
          # docker build changes some files to root ownership, fix this before deleting files
          sudo chown -R $USER:$GROUP .
          # clean build directories
          time git clean -fdx
          echo "::endgroup::"
          echo "::group::Available diskspace after cleaning build directory"
          time df -BM / /mnt
          echo "::endgroup::"
          echo "::group::Delete maven repository"
          # delete maven repository
          time rm -rf ~/.m2/repository
          echo "::endgroup::"
          echo "::group::Available diskspace after cleaning maven repository"
          time df -BM / /mnt
          echo "::endgroup::"

      - name: save docker image apachepulsar/pulsar-test-latest-version:latest to Github artifact cache
        run: |
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_save_image_to_github_actions_artifacts apachepulsar/pulsar-test-latest-version:latest pulsar-test-latest-version-image

      - name: Wait for ssh connection when build fails
        # ssh access is enabled for builds in own forks
        uses: ./.github/actions/ssh-access
        if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        continue-on-error: true
        with:
          action: wait

  system-tests:
    name: CI - System - ${{ matrix.name }}
    runs-on: ubuntu-22.04
    timeout-minutes: 60
    needs: ['preconditions', 'pulsar-test-latest-version-image']
    if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
    env:
      JOB_NAME: CI - System - ${{ matrix.name }}
      PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest
      GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: Tiered FileSystem
            group: TIERED_FILESYSTEM

          - name: Tiered JCloud
            group: TIERED_JCLOUD

          - name: Function
            group: FUNCTION

          - name: Schema
            group: SCHEMA

          - name: Pulsar Connectors - Thread
            group: PULSAR_CONNECTORS_THREAD

          - name: Pulsar Connectors - Process
            group: PULSAR_CONNECTORS_PROCESS

          - name: Pulsar IO
            group: PULSAR_IO
            clean_disk: true

          - name: Sql
            group: SQL

    steps:
      - name: checkout
        uses: actions/checkout@v3

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Clean Disk when needed
        if: ${{ matrix.clean_disk }}
        uses: ./.github/actions/clean-disk

      - name: Setup ssh access to build runner VM
        # ssh access is enabled for builds in own forks
        if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        uses: ./.github/actions/ssh-access
        continue-on-error: true
        with:
          limit-access-to-actor: true

      - name: Cache local Maven repository
        uses: actions/cache@v3
        timeout-minutes: 5
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/pulsar
          key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
            ${{ runner.os }}-m2-dependencies-core-modules-

      - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
        uses: actions/setup-java@v3
        with:
          distribution: 'temurin'
          java-version: ${{ env.CI_JDK_MAJOR_VERSION }}

      - name: Install gh-actions-artifact-client.js
        uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

      - name: Restore maven build results from Github artifact cache
        run: |
          cd $HOME
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts

      - name: Load docker image apachepulsar/pulsar-test-latest-version:latest from Github artifact cache
        run: |
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-test-latest-version-image

      - name: Run setup commands
        if: ${{ matrix.setup }}
        run: |
          ${{ matrix.setup }}

      - name: Run system test group '${{ matrix.group }}'
        run: |
          if [[ "${{ matrix.no_coverage }}" != "true" && "${{ needs.preconditions.outputs.collect_coverage }}" == "true" ]]; then
            coverage_args="--coverage"
          fi
          ./build/run_integration_group.sh ${{ matrix.group }} $coverage_args

      - name: Upload coverage to build artifacts
        if: ${{ !matrix.no_coverage && needs.preconditions.outputs.collect_coverage == 'true' }}
        run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh upload_systest_coverage_files ${{ matrix.group }}

      - name: print JVM thread dumps when cancelled
        if: cancelled()
        run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps

      - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories
        if: ${{ always() }}
        uses: ./.github/actions/copy-test-reports

      - name: Publish Test Report
        uses: apache/pulsar-test-infra/action-junit-report@master
        if: ${{ always() }}
        with:
          report_paths: 'test-reports/TEST-*.xml'
          annotate_only: 'true'

      - name: Upload container logs
        uses: actions/upload-artifact@v3
        if: ${{ !success() }}
        continue-on-error: true
        with:
          name: System-${{ matrix.group }}-container-logs
          path: tests/integration/target/container-logs
          retention-days: 7

      - name: Upload Surefire reports
        uses: actions/upload-artifact@v3
        if: ${{ !success() }}
        with:
          name: System-${{ matrix.name }}-surefire-reports
          path: surefire-reports
          retention-days: 7

      - name: Wait for ssh connection when build fails
        # ssh access is enabled for builds in own forks
        uses: ./.github/actions/ssh-access
        if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        continue-on-error: true
        with:
          action: wait

  system-tests-upload-coverage:
    name: CI - System - Upload Coverage
    runs-on: ubuntu-22.04
    timeout-minutes: 30
    needs: ['preconditions', 'system-tests']
    if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }}
    env:
      PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest
      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}

    steps:
      - name: checkout
        uses: actions/checkout@v3

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Setup ssh access to build runner VM
        # ssh access is enabled for builds in own forks
        if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        uses: ./.github/actions/ssh-access
        continue-on-error: true
        with:
          limit-access-to-actor: true

      - name: Cache local Maven repository
        uses: actions/cache@v3
        timeout-minutes: 5
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/pulsar
          key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
            ${{ runner.os }}-m2-dependencies-core-modules-

      - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
        uses: actions/setup-java@v3
        with:
          distribution: 'temurin'
          java-version: ${{ env.CI_JDK_MAJOR_VERSION }}

      - name: Install gh-actions-artifact-client.js
        uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

      - name: Restore maven build results from Github artifact cache
        run: |
          cd $HOME
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries

      - name: Load docker image apachepulsar/pulsar-test-latest-version:latest from Github artifact cache
        run: |
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-test-latest-version-image

      - name: Restore coverage files from build artifacts and create Jacoco reports
        run: |
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_systest_coverage_files
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_inttest_coverage_report
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_test_coverage_report
          cd $GITHUB_WORKSPACE/target
          zip -qr jacoco_test_coverage_report_systests.zip jacoco_test_coverage_report jacoco_inttest_coverage_report || true

      - name: Upload Jacoco report files to build artifacts
        uses: actions/upload-artifact@v3
        with:
          name: Jacoco-coverage-report-systests
          path: target/jacoco_test_coverage_report_systests.zip
          retention-days: 3

      - name: Upload to Codecov
        uses: ./.github/actions/upload-coverage
        with:
          flags: systests

      - name: Delete coverage files from build artifacts
        run: |
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh delete_systest_coverage_files

      - name: Wait for ssh connection when build fails
        # ssh access is enabled for builds in own forks
        uses: ./.github/actions/ssh-access
        if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        continue-on-error: true
        with:
          action: wait

  flaky-system-tests:
    name: CI Flaky - System - ${{ matrix.name }}
    runs-on: ubuntu-22.04
    timeout-minutes: 60
    needs: [ 'preconditions', 'pulsar-test-latest-version-image' ]
    if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
    env:
      JOB_NAME: CI Flaky - System - ${{ matrix.name }}
      PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest
      GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: Plugin
            group: PLUGIN

          - name: Pulsar IO - Oracle
            group: PULSAR_IO_ORA
            clean_disk: true

    steps:
      - name: checkout
        uses: actions/checkout@v3

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Clean Disk when needed
        if: ${{ matrix.clean_disk }}
        uses: ./.github/actions/clean-disk

      - name: Setup ssh access to build runner VM
        # ssh access is enabled for builds in own forks
        if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        uses: ./.github/actions/ssh-access
        continue-on-error: true
        with:
          limit-access-to-actor: true

      - name: Cache local Maven repository
        uses: actions/cache@v3
        timeout-minutes: 5
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/pulsar
          key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
            ${{ runner.os }}-m2-dependencies-core-modules-

      - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
        uses: actions/setup-java@v3
        with:
          distribution: 'temurin'
          java-version: ${{ env.CI_JDK_MAJOR_VERSION }}

      - name: Install gh-actions-artifact-client.js
        uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

      - name: Restore maven build results from Github artifact cache
        run: |
          cd $HOME
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries

      - name: Load docker image apachepulsar/pulsar-test-latest-version:latest from Github artifact cache
        run: |
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-test-latest-version-image

      - name: Run setup commands
        if: ${{ matrix.setup }}
        run: |
          ${{ matrix.setup }}

      - name: Run system test group '${{ matrix.group }}'
        run: |
          ./build/run_integration_group.sh ${{ matrix.group }}

      - name: print JVM thread dumps when cancelled
        if: cancelled()
        run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps

      - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories
        if: ${{ always() }}
        uses: ./.github/actions/copy-test-reports

      - name: Publish Test Report
        uses: apache/pulsar-test-infra/action-junit-report@master
        if: ${{ always() }}
        with:
          report_paths: 'test-reports/TEST-*.xml'
          annotate_only: 'true'

      - name: Upload container logs
        uses: actions/upload-artifact@v3
        if: ${{ !success() }}
        continue-on-error: true
        with:
          name: System-${{ matrix.group }}-container-logs
          path: tests/integration/target/container-logs
          retention-days: 7

      - name: Upload Surefire reports
        uses: actions/upload-artifact@v3
        if: ${{ !success() }}
        with:
          name: System-${{ matrix.name }}-surefire-reports
          path: surefire-reports
          retention-days: 7

      - name: Wait for ssh connection when build fails
        # ssh access is enabled for builds in own forks
        uses: ./.github/actions/ssh-access
        if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        continue-on-error: true
        with:
          action: wait

  delete-system-test-docker-image-artifact:
    name: "Delete system test docker image artifact"
    runs-on: ubuntu-22.04
    timeout-minutes: 10
    needs: [
      'preconditions',
      'system-tests',
      'system-tests-upload-coverage',
      'flaky-system-tests'
    ]
    if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
    steps:
      - name: checkout
        uses: actions/checkout@v3

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Install gh-actions-artifact-client.js
        uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

      - name: Delete docker image from GitHub Actions Artifacts
        run: |
          gh-actions-artifact-client.js delete pulsar-test-latest-version-image.zst

  macos-build:
    name: Build Pulsar on MacOS
    runs-on: macos-11
    timeout-minutes: 120
    needs: ['preconditions', 'integration-tests']
    if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
    env:
      GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
    steps:
      - name: checkout
        uses: actions/checkout@v3

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Cache Maven dependencies
        uses: actions/cache@v3
        timeout-minutes: 5
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/pulsar
          key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-m2-dependencies-all-

      - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
        uses: actions/setup-java@v3
        with:
          distribution: 'temurin'
          java-version: ${{ env.CI_JDK_MAJOR_VERSION }}

      - name: build package
        run: mvn -B clean package -DskipTests -T 1C -ntp

  owasp-dep-check:
    name: OWASP dependency check
    runs-on: ubuntu-22.04
    timeout-minutes: 120
    needs: [ 'preconditions', 'integration-tests' ]
    if: ${{ needs.preconditions.outputs.need_owasp == 'true' }}
    env:
      GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
    steps:
      - name: checkout
        uses: actions/checkout@v3

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Setup ssh access to build runner VM
        # ssh access is enabled for builds in own forks
        if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        uses: ./.github/actions/ssh-access
        continue-on-error: true
        with:
          limit-access-to-actor: true

      - name: Cache Maven dependencies
        uses: actions/cache@v3
        timeout-minutes: 5
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/pulsar
          key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-m2-dependencies-core-modules-
      - name: Set up JDK ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }}
        uses: actions/setup-java@v3
        with:
          distribution: 'temurin'
          java-version: ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }}

      - name: Clean Disk
        uses: ./.github/actions/clean-disk

      - name: Install gh-actions-artifact-client.js
        uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

      - name: Restore maven build results from Github artifact cache
        run: |
          cd $HOME
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
      # Projects dependent on flume, hdfs, hbase, and presto currently excluded from the scan.
      - name: trigger dependency check
        run: |
          mvn -B -ntp verify -PskipDocker,skip-all,owasp-dependency-check -Dcheckstyle.skip=true -DskipTests \
            -pl '!pulsar-sql,!distribution/server,!distribution/io,!distribution/offloaders,!pulsar-sql/presto-distribution,!tiered-storage/file-system,!pulsar-io/flume,!pulsar-io/hbase,!pulsar-io/hdfs2,!pulsar-io/hdfs3,!pulsar-io/docs,!pulsar-io/jdbc/openmldb'

      - name: Upload report
        uses: actions/upload-artifact@v3
        if: ${{ cancelled() || failure() }}
        continue-on-error: true
        with:
          name: dependency report
          path: target/dependency-check-report.html

      - name: Wait for ssh connection when build fails
        # ssh access is enabled for builds in own forks
        uses: ./.github/actions/ssh-access
        if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        continue-on-error: true
        with:
          action: wait

  # This job is required for pulls to be merged.
  # It depends on all other jobs in this workflow.
  # It cleans up the binaries in the same job in order to not spin up another runner for basically doing nothing.
  pulsar-ci-checks-completed:
    name: "Pulsar CI checks completed"
    if: ${{ always() && ((github.event_name != 'schedule') || (github.repository == 'apache/pulsar')) }}
    runs-on: ubuntu-22.04
    timeout-minutes: 10
    needs: [
      'preconditions',
      'unit-tests',
      'integration-tests',
      'system-tests',
      'flaky-system-tests',
      'macos-build',
      'unit-tests-upload-coverage',
      'integration-tests-upload-coverage',
      'system-tests-upload-coverage',
      'owasp-dep-check'
    ]
    steps:
      - name: Check that all required jobs were completed successfully
        if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
        run: |
          if [[ ! ( \
                   "${{ needs.unit-tests.result }}" == "success" \
                && "${{ needs.integration-tests.result }}" == "success" \
                && "${{ needs.system-tests.result }}" == "success" \
                && "${{ needs.macos-build.result }}" == "success" \
               ) ]]; then
            echo "Required jobs haven't been completed successfully."
            exit 1
          fi

      - name: checkout
        if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
        uses: actions/checkout@v3

      - name: Tune Runner VM
        if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
        uses: ./.github/actions/tune-runner-vm

      - name: Install gh-actions-artifact-client.js
        if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
        uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

      - name: Delete maven repository binaries from GitHub Actions Artifacts
        if: ${{ needs.preconditions.outputs.docs_only != 'true' && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }}
        run: |
          gh-actions-artifact-client.js delete pulsar-maven-repository-binaries.tar.zst || true
          gh-actions-artifact-client.js delete pulsar-server-distribution.tar.zst || true