# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # name: Pulsar CI on: pull_request: branches: - master schedule: # scheduled job with JDK 17 - cron: '0 12 * * *' # scheduled job with JDK 21 # if cron expression is changed, make sure to update the expression in jdk_major_version step in preconditions job - cron: '0 6 * * *' workflow_dispatch: inputs: collect_coverage: description: 'Collect test coverage and upload to Codecov' required: true type: boolean default: true jdk_major_version: description: 'JDK major version to use for the build' required: true type: choice options: - '17' - '21' default: '17' trace_test_resource_cleanup: description: 'Collect thread & heap information before exiting a test JVM. When set to "on", thread dump and heap histogram will be collected. When set to "full", a heap dump will also be collected.' required: true type: choice options: - 'off' - 'on' - 'full' default: 'off' thread_leak_detector_wait_millis: description: 'Duration in ms to wait for threads to exit in thread leak detection between test classes. It is necessary to wait for threads to complete before they are determined to be leaked threads.' required: true type: number default: 10000 concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}${{ github.event_name == 'workflow_dispatch' && github.event.inputs.jdk_major_version || '' }} cancel-in-progress: true env: MAVEN_OPTS: -Xss1500k -Xmx1500m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 # defines the retention period for the intermediate build artifacts needed for rerunning a failed build job # it's possible to rerun individual failed jobs when the build artifacts are available # if the artifacts have already been expired, the complete workflow can be rerun by closing and reopening the PR or by rebasing the PR ARTIFACT_RETENTION_DAYS: 3 jobs: preconditions: name: Preconditions runs-on: ubuntu-22.04 outputs: docs_only: ${{ steps.check_changes.outputs.docs_only }} changed_tests: ${{ steps.changes.outputs.tests_files }} need_owasp: ${{ steps.changes.outputs.need_owasp }} collect_coverage: ${{ steps.check_coverage.outputs.collect_coverage }} jdk_major_version: ${{ steps.jdk_major_version.outputs.jdk_major_version }} steps: - name: Cancel scheduled jobs in forks by default if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'schedule' }} uses: actions/github-script@v6 with: script: | await github.rest.actions.cancelWorkflowRun({owner: context.repo.owner, repo: context.repo.repo, run_id: context.runId}); process.exit(1); - name: Select JDK major version id: jdk_major_version run: | # use JDK 21 for the scheduled build with cron expression '0 6 * * *' if [[ "${{ github.event_name == 'schedule' && github.event.schedule == '0 6 * * *' && 'true' || 'false' }}" == "true" ]]; then echo "jdk_major_version=21" >> $GITHUB_OUTPUT exit 0 fi # use JDK 17 for build unless overridden with workflow_dispatch input echo "jdk_major_version=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.jdk_major_version || '17'}}" >> $GITHUB_OUTPUT - name: checkout if: ${{ github.event_name == 'pull_request' }} uses: actions/checkout@v3 - name: Detect changed files if: ${{ github.event_name == 'pull_request' }} id: changes uses: apache/pulsar-test-infra/paths-filter@master with: filters: .github/changes-filter.yaml list-files: csv - name: Check changed files if: ${{ github.event_name == 'pull_request' }} id: check_changes run: | if [[ "${GITHUB_EVENT_NAME}" != "schedule" && "${GITHUB_EVENT_NAME}" != "workflow_dispatch" ]]; then echo "docs_only=${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" >> $GITHUB_OUTPUT else echo docs_only=false >> $GITHUB_OUTPUT fi - name: Check if coverage should be collected id: check_coverage run: | echo "collect_coverage=${{ (steps.check_changes.outputs.docs_only != 'true' && github.event_name != 'workflow_dispatch' && (github.base_ref == 'master' || github.ref_name == 'master')) || (github.event_name == 'workflow_dispatch' && github.event.inputs.collect_coverage == 'true') }}" >> $GITHUB_OUTPUT - name: Check if the PR has been approved for testing if: ${{ steps.check_changes.outputs.docs_only != 'true' && github.repository == 'apache/pulsar' && github.event_name == 'pull_request' }} env: GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} GITHUB_TOKEN: ${{ github.token }} run: | build/pulsar_ci_tool.sh check_ready_to_test build-and-license-check: needs: preconditions name: Build and License check env: JOB_NAME: Build and License check GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} runs-on: ubuntu-22.04 timeout-minutes: 60 if: ${{ needs.preconditions.outputs.docs_only != 'true' }} steps: - name: checkout uses: actions/checkout@v3 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Setup ssh access to build runner VM # ssh access is enabled for builds in own forks if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} uses: ./.github/actions/ssh-access continue-on-error: true with: limit-access-to-actor: true - name: Cache local Maven repository uses: actions/cache@v3 timeout-minutes: 5 with: path: | ~/.m2/repository/*/*/* !~/.m2/repository/org/apache/pulsar key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-m2-dependencies-core-modules- - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }} uses: actions/setup-java@v3 with: distribution: 'temurin' java-version: ${{ env.CI_JDK_MAJOR_VERSION }} - name: Check source code license headers run: mvn -B -T 8 -ntp initialize apache-rat:check license:check - name: Check source code style run: mvn -B -T 8 -ntp initialize checkstyle:check - name: Build core-modules run: | mvn -B -T 1C -ntp -Pcore-modules,-main clean install -DskipTests -Dlicense.skip=true -Drat.skip=true -Dcheckstyle.skip=true - name: Check binary licenses run: src/check-binary-license.sh ./distribution/server/target/apache-pulsar-*-bin.tar.gz - name: Install gh-actions-artifact-client.js uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master - name: Save maven build results to Github artifact cache so that the results can be reused run: | cd $HOME $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh store_tar_to_github_actions_artifacts pulsar-maven-repository-binaries \ tar --exclude '.m2/repository/org/apache/pulsar/pulsar-*-distribution' \ -I zstd -cf - .m2/repository/org/apache/pulsar cd $GITHUB_WORKSPACE $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh store_tar_to_github_actions_artifacts pulsar-server-distribution \ tar -I zstd -cf - distribution/server/target/apache-pulsar-*-bin.tar.gz - name: Wait for ssh connection when build fails # ssh access is enabled for builds in own forks uses: ./.github/actions/ssh-access if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} continue-on-error: true with: action: wait unit-tests: name: CI - Unit - ${{ matrix.name }} env: JOB_NAME: CI - Unit - ${{ matrix.name }} COLLECT_COVERAGE: "${{ needs.preconditions.outputs.collect_coverage }}" GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} TRACE_TEST_RESOURCE_CLEANUP: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.trace_test_resource_cleanup || 'off' }} TRACE_TEST_RESOURCE_CLEANUP_DIR: ${{ github.workspace }}/target/trace-test-resource-cleanup THREAD_LEAK_DETECTOR_WAIT_MILLIS: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.thread_leak_detector_wait_millis || 10000 }} THREAD_LEAK_DETECTOR_DIR: ${{ github.workspace }}/target/thread-leak-dumps runs-on: ubuntu-22.04 timeout-minutes: ${{ matrix.timeout || 60 }} needs: ['preconditions', 'build-and-license-check'] if: ${{ needs.preconditions.outputs.docs_only != 'true' }} strategy: fail-fast: false matrix: include: - name: Other group: OTHER timeout: 75 - name: Brokers - Broker Group 1 group: BROKER_GROUP_1 - name: Brokers - Broker Group 2 group: BROKER_GROUP_2 - name: Brokers - Broker Group 3 group: BROKER_GROUP_3 - name: Brokers - Broker Group 4 group: BROKER_GROUP_4 - name: Brokers - Client Api group: BROKER_CLIENT_API - name: Brokers - Client Impl group: BROKER_CLIENT_IMPL - name: Proxy group: PROXY - name: Pulsar IO group: PULSAR_IO timeout: 75 - name: Pulsar IO - Elastic Search group: PULSAR_IO_ELASTIC - name: Pulsar IO - Kafka Connect Adaptor group: PULSAR_IO_KAFKA_CONNECT - name: Pulsar Client group: CLIENT steps: - name: checkout uses: actions/checkout@v3 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Clean Disk when tracing test resource cleanup if: ${{ env.TRACE_TEST_RESOURCE_CLEANUP != 'off' }} uses: ./.github/actions/clean-disk - name: Setup ssh access to build runner VM # ssh access is enabled for builds in own forks if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} uses: ./.github/actions/ssh-access continue-on-error: true with: limit-access-to-actor: true - name: Cache Maven dependencies uses: actions/cache@v3 timeout-minutes: 5 with: path: | ~/.m2/repository/*/*/* !~/.m2/repository/org/apache/pulsar key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-m2-dependencies-core-modules- - name: Set up JDK ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }} uses: actions/setup-java@v3 with: distribution: 'temurin' java-version: ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }} - name: Install gh-actions-artifact-client.js uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master - name: Restore maven build results from Github artifact cache run: | cd $HOME $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts - name: Run setup commands run: | ${{ matrix.setup }} - name: Run unit test group '${{ matrix.group }}' run: | CHANGED_TESTS="${{ needs.preconditions.outputs.tests_files }}" ./build/run_unit_group.sh ${{ matrix.group }} - name: Upload coverage to build artifacts if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }} run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh upload_unittest_coverage_files ${{ matrix.group }} - name: print JVM thread dumps when cancelled if: cancelled() run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories if: ${{ always() }} uses: ./.github/actions/copy-test-reports - name: Publish Test Report uses: apache/pulsar-test-infra/action-junit-report@master if: ${{ always() }} with: report_paths: 'test-reports/TEST-*.xml' annotate_only: 'true' - name: Report detected thread leaks if: ${{ always() }} run: | if [ -d "$THREAD_LEAK_DETECTOR_DIR" ]; then cd "$THREAD_LEAK_DETECTOR_DIR" cat threadleak*.txt | awk '/^Summary:/ {print "::warning::" $0 "\n"; next} {print}' fi - name: Upload Surefire reports uses: actions/upload-artifact@v3 if: ${{ !success() || env.TRACE_TEST_RESOURCE_CLEANUP != 'off' }} with: name: Unit-${{ matrix.group }}-surefire-reports path: surefire-reports retention-days: 7 - name: Upload possible heap dump, core dump or crash files uses: actions/upload-artifact@v3 if: ${{ always() }} with: name: Unit-${{ matrix.group }}-dumps path: | /tmp/*.hprof **/hs_err_*.log **/core.* ${{ env.TRACE_TEST_RESOURCE_CLEANUP_DIR }}/* ${{ env.THREAD_LEAK_DETECTOR_DIR }}/* retention-days: 7 if-no-files-found: ignore - name: Wait for ssh connection when build fails # ssh access is enabled for builds in own forks uses: ./.github/actions/ssh-access if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} continue-on-error: true with: action: wait unit-tests-upload-coverage: name: CI - Unit - Upload Coverage runs-on: ubuntu-22.04 timeout-minutes: 30 needs: ['preconditions', 'unit-tests'] env: CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }} steps: - name: checkout uses: actions/checkout@v3 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Setup ssh access to build runner VM # ssh access is enabled for builds in own forks if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} uses: ./.github/actions/ssh-access continue-on-error: true with: limit-access-to-actor: true - name: Cache Maven dependencies uses: actions/cache@v3 timeout-minutes: 5 with: path: | ~/.m2/repository/*/*/* !~/.m2/repository/org/apache/pulsar key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-m2-dependencies-core-modules- - name: Set up JDK ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }} uses: actions/setup-java@v3 with: distribution: 'temurin' java-version: ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }} - name: Install gh-actions-artifact-client.js uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master - name: Restore maven build results from Github artifact cache run: | cd $HOME $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts - name: Restore coverage files from build artifacts and create Jacoco reports run: | $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_unittest_coverage_files $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_test_coverage_report cd $GITHUB_WORKSPACE/target zip -qr jacoco_test_coverage_report_unittests.zip jacoco_test_coverage_report || true - name: Upload Jacoco report files to build artifacts uses: actions/upload-artifact@v3 with: name: Jacoco-coverage-report-unittests path: target/jacoco_test_coverage_report_unittests.zip retention-days: 3 - name: Upload to Codecov uses: ./.github/actions/upload-coverage with: flags: unittests - name: Delete coverage files from build artifacts run: | $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh delete_unittest_coverage_files - name: Wait for ssh connection when build fails # ssh access is enabled for builds in own forks uses: ./.github/actions/ssh-access if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} continue-on-error: true with: action: wait pulsar-java-test-image: name: Build Pulsar java-test-image docker image runs-on: ubuntu-22.04 timeout-minutes: 60 needs: ['preconditions', 'build-and-license-check'] if: ${{ needs.preconditions.outputs.docs_only != 'true'}} env: GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} IMAGE_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} steps: - name: checkout uses: actions/checkout@v3 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Setup ssh access to build runner VM # ssh access is enabled for builds in own forks if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} uses: ./.github/actions/ssh-access continue-on-error: true with: limit-access-to-actor: true - name: Cache Maven dependencies uses: actions/cache@v3 timeout-minutes: 5 with: path: | ~/.m2/repository/*/*/* !~/.m2/repository/org/apache/pulsar key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-m2-dependencies-core-modules- - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }} uses: actions/setup-java@v3 with: distribution: 'temurin' java-version: ${{ env.CI_JDK_MAJOR_VERSION }} - name: Install gh-actions-artifact-client.js uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master - name: restore maven build results from Github artifact cache run: | cd $HOME $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries - name: Pick ubuntu mirror for the docker image build run: | # pick the closest ubuntu mirror and set it to UBUNTU_MIRROR environment variable $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh pick_ubuntu_mirror - name: Build java-test-image docker image run: | # build docker image mvn -B -am -pl tests/docker-images/java-test-image install -Pcore-modules,-main,integrationTests,docker \ -Dmaven.test.skip=true -Ddocker.squash=true -DskipSourceReleaseAssembly=true \ -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true - name: save docker image apachepulsar/java-test-image:latest to Github artifact cache run: | $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_save_image_to_github_actions_artifacts apachepulsar/java-test-image:latest pulsar-java-test-image - name: Wait for ssh connection when build fails # ssh access is enabled for builds in own forks uses: ./.github/actions/ssh-access if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} continue-on-error: true with: action: wait integration-tests: name: CI - Integration - ${{ matrix.name }} runs-on: ubuntu-22.04 timeout-minutes: ${{ matrix.timeout || 60 }} needs: ['preconditions', 'pulsar-java-test-image'] if: ${{ needs.preconditions.outputs.docs_only != 'true' }} env: JOB_NAME: CI - Integration - ${{ matrix.name }} PULSAR_TEST_IMAGE_NAME: apachepulsar/java-test-image:latest GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} strategy: fail-fast: false matrix: include: - name: Backwards Compatibility group: BACKWARDS_COMPAT no_coverage: true - name: Cli group: CLI - name: Messaging group: MESSAGING - name: LoadBalance group: LOADBALANCE no_coverage: true - name: Shade on Java 8 group: SHADE_RUN runtime_jdk: 8 setup: ./build/run_integration_group.sh SHADE_BUILD no_coverage: true - name: Shade on Java 11 group: SHADE_RUN runtime_jdk: 11 setup: ./build/run_integration_group.sh SHADE_BUILD no_coverage: true - name: Shade on Java 17 group: SHADE_RUN runtime_jdk: 17 setup: ./build/run_integration_group.sh SHADE_BUILD no_coverage: true - name: Shade on Java 21 group: SHADE_RUN runtime_jdk: 21 setup: ./build/run_integration_group.sh SHADE_BUILD no_coverage: true - name: Standalone group: STANDALONE - name: Transaction group: TRANSACTION steps: - name: checkout uses: actions/checkout@v3 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Setup ssh access to build runner VM # ssh access is enabled for builds in own forks if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} uses: ./.github/actions/ssh-access continue-on-error: true with: limit-access-to-actor: true - name: Cache Maven dependencies uses: actions/cache@v3 timeout-minutes: 5 with: path: | ~/.m2/repository/*/*/* !~/.m2/repository/org/apache/pulsar key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-m2-dependencies-core-modules- - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }} uses: actions/setup-java@v3 with: distribution: 'temurin' java-version: ${{ env.CI_JDK_MAJOR_VERSION }} - name: Install gh-actions-artifact-client.js uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master - name: Restore maven build results from Github artifact cache run: | cd $HOME $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts - name: Load docker image apachepulsar/java-test-image:latest from Github artifact cache run: | $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-java-test-image - name: Run setup commands if: ${{ matrix.setup }} run: | ${{ matrix.setup }} - name: Set up runtime JDK ${{ matrix.runtime_jdk }} uses: actions/setup-java@v3 if: ${{ matrix.runtime_jdk }} with: distribution: 'temurin' java-version: ${{ matrix.runtime_jdk }} - name: Run integration test group '${{ matrix.group }}' run: | if [[ "${{ matrix.no_coverage }}" != "true" && "${{ needs.preconditions.outputs.collect_coverage }}" == "true" ]]; then coverage_args="--coverage" fi ./build/run_integration_group.sh ${{ matrix.group }} $coverage_args - name: Upload coverage to build artifacts if: ${{ !matrix.no_coverage && needs.preconditions.outputs.collect_coverage == 'true' }} run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh upload_inttest_coverage_files ${{ matrix.group }} - name: print JVM thread dumps when cancelled if: cancelled() run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories if: ${{ always() }} uses: ./.github/actions/copy-test-reports - name: Publish Test Report uses: apache/pulsar-test-infra/action-junit-report@master if: ${{ always() }} with: report_paths: 'test-reports/TEST-*.xml' annotate_only: 'true' - name: Upload Surefire reports uses: actions/upload-artifact@v3 if: ${{ !success() }} with: name: Integration-${{ matrix.group }}-surefire-reports path: surefire-reports retention-days: 7 - name: Upload container logs uses: actions/upload-artifact@v3 if: ${{ !success() }} continue-on-error: true with: name: Integration-${{ matrix.group }}-container-logs path: tests/integration/target/container-logs retention-days: 7 - name: Wait for ssh connection when build fails # ssh access is enabled for builds in own forks uses: ./.github/actions/ssh-access if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} continue-on-error: true with: action: wait integration-tests-upload-coverage: name: CI - Integration - Upload Coverage runs-on: ubuntu-22.04 timeout-minutes: 30 needs: ['preconditions', 'integration-tests'] if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }} env: PULSAR_TEST_IMAGE_NAME: apachepulsar/java-test-image:latest CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} steps: - name: checkout uses: actions/checkout@v3 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Setup ssh access to build runner VM # ssh access is enabled for builds in own forks if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} uses: ./.github/actions/ssh-access continue-on-error: true with: limit-access-to-actor: true - name: Cache Maven dependencies uses: actions/cache@v3 timeout-minutes: 5 with: path: | ~/.m2/repository/*/*/* !~/.m2/repository/org/apache/pulsar key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-m2-dependencies-core-modules- - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }} uses: actions/setup-java@v3 with: distribution: 'temurin' java-version: ${{ env.CI_JDK_MAJOR_VERSION }} - name: Install gh-actions-artifact-client.js uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master - name: Restore maven build results from Github artifact cache run: | cd $HOME $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts - name: Load docker image apachepulsar/java-test-image:latest from Github artifact cache run: | $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-java-test-image - name: Restore coverage files from build artifacts and create Jacoco reports run: | $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_inttest_coverage_files $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_inttest_coverage_report $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_test_coverage_report cd $GITHUB_WORKSPACE/target zip -qr jacoco_test_coverage_report_inttests.zip jacoco_test_coverage_report jacoco_inttest_coverage_report || true - name: Upload Jacoco report files to build artifacts uses: actions/upload-artifact@v3 with: name: Jacoco-coverage-report-inttests path: target/jacoco_test_coverage_report_inttests.zip retention-days: 3 - name: Upload to Codecov uses: ./.github/actions/upload-coverage with: flags: inttests - name: Delete coverage files from build artifacts run: | $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh delete_inttest_coverage_files - name: Wait for ssh connection when build fails # ssh access is enabled for builds in own forks uses: ./.github/actions/ssh-access if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} continue-on-error: true with: action: wait delete-integration-test-docker-image-artifact: name: "Delete integration test docker image artifact" runs-on: ubuntu-22.04 timeout-minutes: 10 needs: [ 'preconditions', 'integration-tests', 'integration-tests-upload-coverage' ] if: ${{ needs.preconditions.outputs.docs_only != 'true' }} steps: - name: checkout uses: actions/checkout@v3 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Install gh-actions-artifact-client.js uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master - name: Delete docker image from GitHub Actions Artifacts run: | gh-actions-artifact-client.js delete pulsar-java-test-image.zst pulsar-test-latest-version-image: name: Build Pulsar docker image runs-on: ubuntu-22.04 timeout-minutes: 60 needs: ['preconditions', 'build-and-license-check'] if: ${{ needs.preconditions.outputs.docs_only != 'true' }} env: GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} IMAGE_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} steps: - name: checkout uses: actions/checkout@v3 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Setup ssh access to build runner VM # ssh access is enabled for builds in own forks if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} uses: ./.github/actions/ssh-access continue-on-error: true with: limit-access-to-actor: true - name: Clean Disk uses: ./.github/actions/clean-disk with: mode: full - name: Cache local Maven repository uses: actions/cache@v3 timeout-minutes: 5 with: path: | ~/.m2/repository/*/*/* !~/.m2/repository/org/apache/pulsar key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} ${{ runner.os }}-m2-dependencies-core-modules- - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }} uses: actions/setup-java@v3 with: distribution: 'temurin' java-version: ${{ env.CI_JDK_MAJOR_VERSION }} - name: Install gh-actions-artifact-client.js uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master - name: restore maven build results from Github artifact cache run: | cd $HOME $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries - name: Pick ubuntu mirror for the docker image build run: | # pick the closest ubuntu mirror and set it to UBUNTU_MIRROR environment variable $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh pick_ubuntu_mirror - name: Build latest-version-image docker image run: | # build docker image # include building of Pulsar SQL, Connectors, Offloaders and server distros mvn -B -am -pl pulsar-sql/presto-distribution,distribution/io,distribution/offloaders,distribution/server,distribution/shell,tests/docker-images/latest-version-image install \ -DUBUNTU_MIRROR="${UBUNTU_MIRROR}" -DUBUNTU_SECURITY_MIRROR="${UBUNTU_SECURITY_MIRROR}" -DIMAGE_JDK_MAJOR_VERSION="${IMAGE_JDK_MAJOR_VERSION}" \ -Pmain,docker -Dmaven.test.skip=true -Ddocker.squash=true \ -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true # check full build artifacts licenses - name: Check binary licenses run: src/check-binary-license.sh ./distribution/server/target/apache-pulsar-*-bin.tar.gz && src/check-binary-license.sh ./distribution/shell/target/apache-pulsar-shell-*-bin.tar.gz - name: Clean up disk space run: | # release disk space since saving docker image consumes local disk space # echo "::group::Available diskspace before cleaning" time df -BM / /mnt echo "::endgroup::" echo "::group::Clean build directory" # docker build changes some files to root ownership, fix this before deleting files sudo chown -R $USER:$GROUP . # clean build directories time git clean -fdx echo "::endgroup::" echo "::group::Available diskspace after cleaning build directory" time df -BM / /mnt echo "::endgroup::" echo "::group::Delete maven repository" # delete maven repository time rm -rf ~/.m2/repository echo "::endgroup::" echo "::group::Available diskspace after cleaning maven repository" time df -BM / /mnt echo "::endgroup::" - name: save docker image apachepulsar/pulsar-test-latest-version:latest to Github artifact cache run: | $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_save_image_to_github_actions_artifacts apachepulsar/pulsar-test-latest-version:latest pulsar-test-latest-version-image - name: Wait for ssh connection when build fails # ssh access is enabled for builds in own forks uses: ./.github/actions/ssh-access if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} continue-on-error: true with: action: wait system-tests: name: CI - System - ${{ matrix.name }} runs-on: ubuntu-22.04 timeout-minutes: 60 needs: ['preconditions', 'pulsar-test-latest-version-image'] if: ${{ needs.preconditions.outputs.docs_only != 'true' }} env: JOB_NAME: CI - System - ${{ matrix.name }} PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} strategy: fail-fast: false matrix: include: - name: Tiered FileSystem group: TIERED_FILESYSTEM - name: Tiered JCloud group: TIERED_JCLOUD - name: Function group: FUNCTION - name: Schema group: SCHEMA - name: Pulsar Connectors - Thread group: PULSAR_CONNECTORS_THREAD - name: Pulsar Connectors - Process group: PULSAR_CONNECTORS_PROCESS - name: Pulsar IO group: PULSAR_IO clean_disk: true - name: Sql group: SQL steps: - name: checkout uses: actions/checkout@v3 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Clean Disk when needed if: ${{ matrix.clean_disk }} uses: ./.github/actions/clean-disk - name: Setup ssh access to build runner VM # ssh access is enabled for builds in own forks if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} uses: ./.github/actions/ssh-access continue-on-error: true with: limit-access-to-actor: true - name: Cache local Maven repository uses: actions/cache@v3 timeout-minutes: 5 with: path: | ~/.m2/repository/*/*/* !~/.m2/repository/org/apache/pulsar key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} ${{ runner.os }}-m2-dependencies-core-modules- - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }} uses: actions/setup-java@v3 with: distribution: 'temurin' java-version: ${{ env.CI_JDK_MAJOR_VERSION }} - name: Install gh-actions-artifact-client.js uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master - name: Restore maven build results from Github artifact cache run: | cd $HOME $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts - name: Load docker image apachepulsar/pulsar-test-latest-version:latest from Github artifact cache run: | $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-test-latest-version-image - name: Run setup commands if: ${{ matrix.setup }} run: | ${{ matrix.setup }} - name: Run system test group '${{ matrix.group }}' run: | if [[ "${{ matrix.no_coverage }}" != "true" && "${{ needs.preconditions.outputs.collect_coverage }}" == "true" ]]; then coverage_args="--coverage" fi ./build/run_integration_group.sh ${{ matrix.group }} $coverage_args - name: Upload coverage to build artifacts if: ${{ !matrix.no_coverage && needs.preconditions.outputs.collect_coverage == 'true' }} run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh upload_systest_coverage_files ${{ matrix.group }} - name: print JVM thread dumps when cancelled if: cancelled() run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories if: ${{ always() }} uses: ./.github/actions/copy-test-reports - name: Publish Test Report uses: apache/pulsar-test-infra/action-junit-report@master if: ${{ always() }} with: report_paths: 'test-reports/TEST-*.xml' annotate_only: 'true' - name: Upload container logs uses: actions/upload-artifact@v3 if: ${{ !success() }} continue-on-error: true with: name: System-${{ matrix.group }}-container-logs path: tests/integration/target/container-logs retention-days: 7 - name: Upload Surefire reports uses: actions/upload-artifact@v3 if: ${{ !success() }} with: name: System-${{ matrix.name }}-surefire-reports path: surefire-reports retention-days: 7 - name: Wait for ssh connection when build fails # ssh access is enabled for builds in own forks uses: ./.github/actions/ssh-access if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} continue-on-error: true with: action: wait system-tests-upload-coverage: name: CI - System - Upload Coverage runs-on: ubuntu-22.04 timeout-minutes: 30 needs: ['preconditions', 'system-tests'] if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }} env: PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} steps: - name: checkout uses: actions/checkout@v3 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Setup ssh access to build runner VM # ssh access is enabled for builds in own forks if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} uses: ./.github/actions/ssh-access continue-on-error: true with: limit-access-to-actor: true - name: Cache local Maven repository uses: actions/cache@v3 timeout-minutes: 5 with: path: | ~/.m2/repository/*/*/* !~/.m2/repository/org/apache/pulsar key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} ${{ runner.os }}-m2-dependencies-core-modules- - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }} uses: actions/setup-java@v3 with: distribution: 'temurin' java-version: ${{ env.CI_JDK_MAJOR_VERSION }} - name: Install gh-actions-artifact-client.js uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master - name: Restore maven build results from Github artifact cache run: | cd $HOME $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries - name: Load docker image apachepulsar/pulsar-test-latest-version:latest from Github artifact cache run: | $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-test-latest-version-image - name: Restore coverage files from build artifacts and create Jacoco reports run: | $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_systest_coverage_files $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_inttest_coverage_report $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_test_coverage_report cd $GITHUB_WORKSPACE/target zip -qr jacoco_test_coverage_report_systests.zip jacoco_test_coverage_report jacoco_inttest_coverage_report || true - name: Upload Jacoco report files to build artifacts uses: actions/upload-artifact@v3 with: name: Jacoco-coverage-report-systests path: target/jacoco_test_coverage_report_systests.zip retention-days: 3 - name: Upload to Codecov uses: ./.github/actions/upload-coverage with: flags: systests - name: Delete coverage files from build artifacts run: | $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh delete_systest_coverage_files - name: Wait for ssh connection when build fails # ssh access is enabled for builds in own forks uses: ./.github/actions/ssh-access if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} continue-on-error: true with: action: wait flaky-system-tests: name: CI Flaky - System - ${{ matrix.name }} runs-on: ubuntu-22.04 timeout-minutes: 60 needs: [ 'preconditions', 'pulsar-test-latest-version-image' ] if: ${{ needs.preconditions.outputs.docs_only != 'true' }} env: JOB_NAME: CI Flaky - System - ${{ matrix.name }} PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} strategy: fail-fast: false matrix: include: - name: Plugin group: PLUGIN - name: Pulsar IO - Oracle group: PULSAR_IO_ORA clean_disk: true steps: - name: checkout uses: actions/checkout@v3 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Clean Disk when needed if: ${{ matrix.clean_disk }} uses: ./.github/actions/clean-disk - name: Setup ssh access to build runner VM # ssh access is enabled for builds in own forks if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} uses: ./.github/actions/ssh-access continue-on-error: true with: limit-access-to-actor: true - name: Cache local Maven repository uses: actions/cache@v3 timeout-minutes: 5 with: path: | ~/.m2/repository/*/*/* !~/.m2/repository/org/apache/pulsar key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} ${{ runner.os }}-m2-dependencies-core-modules- - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }} uses: actions/setup-java@v3 with: distribution: 'temurin' java-version: ${{ env.CI_JDK_MAJOR_VERSION }} - name: Install gh-actions-artifact-client.js uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master - name: Restore maven build results from Github artifact cache run: | cd $HOME $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries - name: Load docker image apachepulsar/pulsar-test-latest-version:latest from Github artifact cache run: | $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-test-latest-version-image - name: Run setup commands if: ${{ matrix.setup }} run: | ${{ matrix.setup }} - name: Run system test group '${{ matrix.group }}' run: | ./build/run_integration_group.sh ${{ matrix.group }} - name: print JVM thread dumps when cancelled if: cancelled() run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories if: ${{ always() }} uses: ./.github/actions/copy-test-reports - name: Publish Test Report uses: apache/pulsar-test-infra/action-junit-report@master if: ${{ always() }} with: report_paths: 'test-reports/TEST-*.xml' annotate_only: 'true' - name: Upload container logs uses: actions/upload-artifact@v3 if: ${{ !success() }} continue-on-error: true with: name: System-${{ matrix.group }}-container-logs path: tests/integration/target/container-logs retention-days: 7 - name: Upload Surefire reports uses: actions/upload-artifact@v3 if: ${{ !success() }} with: name: System-${{ matrix.name }}-surefire-reports path: surefire-reports retention-days: 7 - name: Wait for ssh connection when build fails # ssh access is enabled for builds in own forks uses: ./.github/actions/ssh-access if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} continue-on-error: true with: action: wait delete-system-test-docker-image-artifact: name: "Delete system test docker image artifact" runs-on: ubuntu-22.04 timeout-minutes: 10 needs: [ 'preconditions', 'system-tests', 'system-tests-upload-coverage', 'flaky-system-tests' ] if: ${{ needs.preconditions.outputs.docs_only != 'true' }} steps: - name: checkout uses: actions/checkout@v3 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Install gh-actions-artifact-client.js uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master - name: Delete docker image from GitHub Actions Artifacts run: | gh-actions-artifact-client.js delete pulsar-test-latest-version-image.zst macos-build: name: Build Pulsar on MacOS runs-on: macos-11 timeout-minutes: 120 needs: ['preconditions', 'integration-tests'] if: ${{ needs.preconditions.outputs.docs_only != 'true' }} env: GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} steps: - name: checkout uses: actions/checkout@v3 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Cache Maven dependencies uses: actions/cache@v3 timeout-minutes: 5 with: path: | ~/.m2/repository/*/*/* !~/.m2/repository/org/apache/pulsar key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-m2-dependencies-all- - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }} uses: actions/setup-java@v3 with: distribution: 'temurin' java-version: ${{ env.CI_JDK_MAJOR_VERSION }} - name: build package run: mvn -B clean package -DskipTests -T 1C -ntp owasp-dep-check: name: OWASP dependency check runs-on: ubuntu-22.04 timeout-minutes: 120 needs: [ 'preconditions', 'integration-tests' ] if: ${{ needs.preconditions.outputs.need_owasp == 'true' }} env: GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} steps: - name: checkout uses: actions/checkout@v3 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Setup ssh access to build runner VM # ssh access is enabled for builds in own forks if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} uses: ./.github/actions/ssh-access continue-on-error: true with: limit-access-to-actor: true - name: Cache Maven dependencies uses: actions/cache@v3 timeout-minutes: 5 with: path: | ~/.m2/repository/*/*/* !~/.m2/repository/org/apache/pulsar key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-m2-dependencies-core-modules- - name: Set up JDK ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }} uses: actions/setup-java@v3 with: distribution: 'temurin' java-version: ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }} - name: Clean Disk uses: ./.github/actions/clean-disk - name: Install gh-actions-artifact-client.js uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master - name: Restore maven build results from Github artifact cache run: | cd $HOME $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries # Projects dependent on flume, hdfs, hbase, and presto currently excluded from the scan. - name: trigger dependency check run: | mvn -B -ntp verify -PskipDocker,skip-all,owasp-dependency-check -Dcheckstyle.skip=true -DskipTests \ -pl '!pulsar-sql,!distribution/server,!distribution/io,!distribution/offloaders,!pulsar-sql/presto-distribution,!tiered-storage/file-system,!pulsar-io/flume,!pulsar-io/hbase,!pulsar-io/hdfs2,!pulsar-io/hdfs3,!pulsar-io/docs,!pulsar-io/jdbc/openmldb' - name: Upload report uses: actions/upload-artifact@v3 if: ${{ cancelled() || failure() }} continue-on-error: true with: name: dependency report path: target/dependency-check-report.html - name: Wait for ssh connection when build fails # ssh access is enabled for builds in own forks uses: ./.github/actions/ssh-access if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} continue-on-error: true with: action: wait # This job is required for pulls to be merged. # It depends on all other jobs in this workflow. # It cleans up the binaries in the same job in order to not spin up another runner for basically doing nothing. pulsar-ci-checks-completed: name: "Pulsar CI checks completed" if: ${{ always() && ((github.event_name != 'schedule') || (github.repository == 'apache/pulsar')) }} runs-on: ubuntu-22.04 timeout-minutes: 10 needs: [ 'preconditions', 'unit-tests', 'integration-tests', 'system-tests', 'flaky-system-tests', 'macos-build', 'unit-tests-upload-coverage', 'integration-tests-upload-coverage', 'system-tests-upload-coverage', 'owasp-dep-check' ] steps: - name: Check that all required jobs were completed successfully if: ${{ needs.preconditions.outputs.docs_only != 'true' }} run: | if [[ ! ( \ "${{ needs.unit-tests.result }}" == "success" \ && "${{ needs.integration-tests.result }}" == "success" \ && "${{ needs.system-tests.result }}" == "success" \ && "${{ needs.macos-build.result }}" == "success" \ ) ]]; then echo "Required jobs haven't been completed successfully." exit 1 fi - name: checkout if: ${{ needs.preconditions.outputs.docs_only != 'true' }} uses: actions/checkout@v3 - name: Tune Runner VM if: ${{ needs.preconditions.outputs.docs_only != 'true' }} uses: ./.github/actions/tune-runner-vm - name: Install gh-actions-artifact-client.js if: ${{ needs.preconditions.outputs.docs_only != 'true' }} uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master - name: Delete maven repository binaries from GitHub Actions Artifacts if: ${{ needs.preconditions.outputs.docs_only != 'true' && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }} run: | gh-actions-artifact-client.js delete pulsar-maven-repository-binaries.tar.zst || true gh-actions-artifact-client.js delete pulsar-server-distribution.tar.zst || true